diff --git a/.github/workflows/correction_multistream_ztf_step.yaml b/.github/workflows/correction_multistream_ztf_step.yaml index ed0ac391c..e9a4dc792 100644 --- a/.github/workflows/correction_multistream_ztf_step.yaml +++ b/.github/workflows/correction_multistream_ztf_step.yaml @@ -14,6 +14,15 @@ jobs: with: base-folder: "correction_multistream_ztf_step" sources-folder: "correction_multistream_ztf_step" + correction_step_integration: + uses: ./.github/workflows/poetry-tests-template.yaml + with: + base-folder: "correction_multistream_ztf_step" + sources-folder: "correction_multistream_ztf_step" + test-folder: "tests/integration" + codecov-flags: "" # Do not upload + secrets: + GH_TOKEN: "${{ secrets.ADMIN_TOKEN }}" build-correction-multistream-ztf-dagger: uses: ./.github/workflows/template_build_with_dagger.yaml with: @@ -21,4 +30,4 @@ jobs: stage: staging extra-args: correction_multistream_ztf_step --dry-run secrets: - GH_TOKEN: ${{ secrets.ADMIN_TOKEN }} + GH_TOKEN: ${{ secrets.ADMIN_TOKEN }} \ No newline at end of file diff --git a/correction_multistream_ztf_step/.gitignore b/correction_multistream_ztf_step/.gitignore new file mode 100644 index 000000000..c92073b63 --- /dev/null +++ b/correction_multistream_ztf_step/.gitignore @@ -0,0 +1 @@ +correction_venv \ No newline at end of file diff --git a/correction_multistream_ztf_step/core/DB/database_sql.py b/correction_multistream_ztf_step/core/DB/database_sql.py index 65c49bf95..e83cda46a 100644 --- a/correction_multistream_ztf_step/core/DB/database_sql.py +++ b/correction_multistream_ztf_step/core/DB/database_sql.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine, select from sqlalchemy.orm import sessionmaker, Session -from db_plugins.db.sql.models_new import ( +from db_plugins.db.sql.models import ( Base, Detection, ZtfForcedPhotometry, diff --git a/correction_multistream_ztf_step/core/parsers/parser_sql.py b/correction_multistream_ztf_step/core/parsers/parser_sql.py index 99c47ac75..6773750c5 100644 --- a/correction_multistream_ztf_step/core/parsers/parser_sql.py +++ b/correction_multistream_ztf_step/core/parsers/parser_sql.py @@ -1,4 +1,4 @@ -from db_plugins.db.sql.models_new import ( +from db_plugins.db.sql.models import ( ZtfDetection, ZtfForcedPhotometry, ForcedPhotometry, diff --git a/correction_multistream_ztf_step/tests/integration/conftest.py b/correction_multistream_ztf_step/tests/integration/conftest.py new file mode 100644 index 000000000..de21a344b --- /dev/null +++ b/correction_multistream_ztf_step/tests/integration/conftest.py @@ -0,0 +1,68 @@ +import os + +import psycopg2 +import pytest +from db_plugins.db.sql._connection import PsqlDatabase + +psql_config = { + "ENGINE": "postgresql", + "HOST": "localhost", + "USER": "postgres", + "PASSWORD": "postgres", + "PORT": 5432, + "DB_NAME": "postgres", +} + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + return os.path.join( + str(pytestconfig.rootdir), "tests/integration", "docker-compose.yml" + ) + + +@pytest.fixture(scope="session") +def docker_compose_command(): + version = os.getenv("COMPOSE", "v2") + return "docker compose" if version == "v2" else "docker-compose" + + +def is_responsive_psql(host, port): + try: + conn = psycopg2.connect( + f"dbname='postgres' user='postgres' host={host} port={port} password='postgres'" + ) + conn.close() + return True + except Exception as e: + print(f"Connection failed: {e}") + return False + + +@pytest.fixture(scope="session") +def psql_service(docker_ip, docker_services): + """Ensure that PSQL service is up and responsive.""" + # `port_for` takes a container port and returns the corresponding host port + port = docker_services.port_for("postgres", 5432) + docker_services.wait_until_responsive( + timeout=30.0, + pause=0.1, + check=lambda: is_responsive_psql(docker_ip, port), + ) + + +@pytest.fixture(scope="session") +def psql_db(docker_ip, docker_services): + port = docker_services.port_for("postgres", 5432) + docker_services.wait_until_responsive( + timeout=30.0, + pause=0.1, + check=lambda: is_responsive_psql(docker_ip, port), + ) + + psql_db = PsqlDatabase(psql_config) + psql_db.create_db() + + yield psql_db + + psql_db.drop_db() diff --git a/correction_multistream_ztf_step/tests/integration/data/data_input_prv_candidates_staging.json b/correction_multistream_ztf_step/tests/integration/data/data_input_prv_candidates_staging.json new file mode 100644 index 000000000..3a4be4a3c --- /dev/null +++ b/correction_multistream_ztf_step/tests/integration/data/data_input_prv_candidates_staging.json @@ -0,0 +1,30 @@ +[ + {"oid": 812282744, "measurement_id": 3002139200815010001, "detections": [{"oid": 812282744, "sid": "ZTF", "pid": 3002139200815, "tid": "ZTF", "band": 2, "measurement_id": "3002139200815010001", "mjd": 60756.13920139879453281, "ra": 102.4124856, "e_ra": 0.06505842506885529, "dec": 2.428348, "e_dec": 0.06499999761581421, "mag": 18.211593627929688, "e_mag": 0.13416489958763661678953, "isdiffpos": -1, "has_stamp": true, "forced": false, "parent_candid": 4444444444, "extra_fields": {"diffmaglim": 20.361549377441406, "pdiffimfilename": "ztf_20250322139201_000461_zg_c03_o_q1_scimrefdiffimg.fits", "programpi": "Kulkarni", "programid": 1, "tblid": 1, "nid": 3002, "rcid": 8, "field": 461, "xpos": 3055.9765625, "ypos": 486.32110595703125, "chipsf": 21.012855529785156, "magap": 18.452800750732422, "sigmagap": 0.07240000367164612, "distnr": 0.6040137410163879, "magnr": 15.531000137329102, "sigmagnr": 0.014000000432133675, "chinr": 0.7239999771118164, "sharpnr": -0.013000000268220901, "sky": -0.8485742807388306, "magdiff": 0.24120700359344482, "fwhm": 1.4315736293792725, "classtar": 0.8849999904632568, "mindtoedge": 16.523399353027344, "magfromlim": 1.9087491035461426, "seeratio": 2.0, "aimage": 0.7509998794532811253967, "bimage": 0.6259998794532811253967, "aimagerat": 0.5245975255966187, "bimagerat": 0.4372810125350952, "elong": 1.1996805667877197, "nneg": 4, "nbad": 0, "rb": 0.6557142734527588, "ssdistnr": -999.0, "ssmagnr": -999.0, "ssnamenr": "null", "sumrat": 0.9966363310813904, "magapbig": 18.455699920654297, "sigmagapbig": 0.0868000015616417, "ranr": 102.46616789531536865234, "decnr": 2.428351640701294, "sgmag1": 15.458499908447266, "srmag1": 13.499699592590332, "simag1": 12.519000053405762, "szmag1": 11.694000244140625, "sgscore1": 0.9637920260429382, "distpsnr1": 0.6792148351669312, "ndethist": 141, "ncovhist": 852, "jdstarthist": 2458426.0, "jdendhist": 2460756.75, "scorr": 8.626946449279785, "tooflag": 0, "objectidps1": 1.1091102783871386e+17, "objectidps2": 1.1091102783871386e+17, "sgmag2": -999.0, "srmag2": 21.4768009185791, "simag2": -999.0, "szmag2": -999.0, "sgscore2": 0.09989289939403534, "distpsnr2": 9.005562782287598, "objectidps3": 1.1091102783871386e+17, "sgmag3": 19.660999298095703, "srmag3": 18.793399810791016, "simag3": 18.318500518798828, "szmag3": 18.097000122070312, "sgscore3": 0.4681670069694519, "distpsnr3": 9.469527244567871, "nmtchps": 17, "rband": 461120108, "jdstartref": 2458369.0, "jdendref": 2458423.0, "nframesref": 15, "rbversion": "t17_f5_c3", "dsnrms": 8.453817367553711, "ssnrms": 46.838356018066406, "dsdiff": -38.38454055786133, "magzpsci": 26.404592514038086, "magzpsciunc": 3.219299969714484e-06, "magzpscirms": 0.031863998621702194, "nmatches": 2005, "clrcoeff": -0.1269499957561493, "clrcounc": 6.498699804069474e-06, "zpclrcov": -4.410000201460207e-06, "zpmed": 26.322999954223633, "clrmed": 0.656000018119812, "clrrms": 0.18664300441741943, "neargaia": 0.689031720161438, "neargaiabright": 0.689031720161438, "maggaia": 12.743537902832031, "maggaiabright": 12.743537902832031, "exptime": 30.0, "drb": 0.011009779758751392, "drbversion": "d6_m7", "brokerIngestTimestamp": 1742614285, "surveyPublishTimestamp": 1742614298624.0, "parent_candid": 4444444444}}, {"oid": 812282744, "sid": "ZTF", "pid": 2974191730815, "tid": "ZTF", "band": 1, "measurement_id": "10", "mjd": 60728.191736099776, "ra": 102.4124856, "e_ra": 0.0, "dec": 2.428348, "e_dec": 0.0, "mag": 17.095138549804688, "e_mag": 0.011105540208518505, "isdiffpos": -1, "has_stamp": false, "forced": true, "parent_candid": "3002139200815010001", "extra_fields": {"field": 461, "rcid": 8, "rband": 461120208, "sciinpseeing": 2.894200086593628, "scibckgnd": 251.1649932861328, "scisigpix": 9.40414047241211, "magzpsci": 26.312000274658203, "magzpsciunc": 3.8145999496919103e-06, "magzpscirms": 0.023059900850057602, "clrcoeff": 0.08249370008707047, "clrcounc": 7.810180250089616e-06, "exptime": 30.0, "adpctdif1": 0.06852299720048904, "adpctdif2": 0.06517399847507477, "diffmaglim": 19.89859962463379, "programid": 1, "forcediffimfluxunc": 49.72494888305664, "procstatus": "0", "distnr": 0.6907779574394226, "ranr": 102.41229248046875, "decnr": 2.4283649921417236, "magnr": 13.352999687194824, "sigmagnr": 0.014999999664723873, "chinr": 0.574999988079071, "sharpnr": -0.01600000075995922}}], "non_detections": [{"oid": 812282744, "sid": "ZTF", "tid": "ZTF", "band": 1, "mjd": 60726.0, "diffmaglim": 20.403099060058594}, {"oid": 812282744, "sid": "ZTF", "tid": "ZTF", "band": 1, "mjd": 60728.191736099776, "diffmaglim": 19.89859962463379}, {"oid": 812282744, "sid": "ZTF", "tid": "ZTF", "band": 2, "mjd": 60728.33390050009, "diffmaglim": 19.596900939941406}], "timestamp": 1742614288323}, + + {"oid": 1111111111, "measurement_id": 111, + "detections": [{"oid": 1111111111, "sid": "ZTF", "pid": 3002139200815, "tid": "ZTF", "band": 2, "measurement_id": 0, "mjd": 60750.13920139879453281, "ra": 102.4124856, "e_ra": 0.06505842506885529, "dec": 2.428348, "e_dec": 0.06499999761581421, "mag": 18.211593627929688, "e_mag": 0.13416489958763661678953, "isdiffpos": -1, "has_stamp": true, "forced": false, "parent_candid": 4444444444, "extra_fields": {"diffmaglim": 20.361549377441406, "pdiffimfilename": "ztf_20250322139201_000461_zg_c03_o_q1_scimrefdiffimg.fits", "programpi": "Kulkarni", "programid": 1, "tblid": 1, "nid": 3002, "rcid": 8, "field": 461, "xpos": 3055.9765625, "ypos": 486.32110595703125, "chipsf": 21.012855529785156, "magap": 18.452800750732422, "sigmagap": 0.07240000367164612, "distnr": 0.6040137410163879, "magnr": 15.531000137329102, "sigmagnr": 0.014000000432133675, "chinr": 0.7239999771118164, "sharpnr": -0.013000000268220901, "sky": -0.8485742807388306, "magdiff": 0.24120700359344482, "fwhm": 1.4315736293792725, "classtar": 0.8849999904632568, "mindtoedge": 16.523399353027344, "magfromlim": 1.9087491035461426, "seeratio": 2.0, "aimage": 0.7509998794532811253967, "bimage": 0.6259998794532811253967, "aimagerat": 0.5245975255966187, "bimagerat": 0.4372810125350952, "elong": 1.1996805667877197, "nneg": 4, "nbad": 0, "rb": 0.6557142734527588, "ssdistnr": -999.0, "ssmagnr": -999.0, "ssnamenr": "null", "sumrat": 0.9966363310813904, "magapbig": 18.455699920654297, "sigmagapbig": 0.0868000015616417, "ranr": 102.46616789531536865234, "decnr": 2.428351640701294, "sgmag1": 15.458499908447266, "srmag1": 13.499699592590332, "simag1": 12.519000053405762, "szmag1": 11.694000244140625, "sgscore1": 0.9637920260429382, "distpsnr1": 0.6792148351669312, "ndethist": 141, "ncovhist": 852, "jdstarthist": 2458426.0, "jdendhist": 2460756.75, "scorr": 8.626946449279785, "tooflag": 0, "objectidps1": 1.1091102783871386e+17, "objectidps2": 1.1091102783871386e+17, "sgmag2": -999.0, "srmag2": 21.4768009185791, "simag2": -999.0, "szmag2": -999.0, "sgscore2": 0.09989289939403534, "distpsnr2": 9.005562782287598, "objectidps3": 1.1091102783871386e+17, "sgmag3": 19.660999298095703, "srmag3": 18.793399810791016, "simag3": 18.318500518798828, "szmag3": 18.097000122070312, "sgscore3": 0.4681670069694519, "distpsnr3": 9.469527244567871, "nmtchps": 17, "rband": 461120108, "jdstartref": 2458369.0, "jdendref": 2458423.0, "nframesref": 15, "rbversion": "t17_f5_c3", "dsnrms": 8.453817367553711, "ssnrms": 46.838356018066406, "dsdiff": -38.38454055786133, "magzpsci": 26.404592514038086, "magzpsciunc": 3.219299969714484e-06, "magzpscirms": 0.031863998621702194, "nmatches": 2005, "clrcoeff": -0.1269499957561493, "clrcounc": 6.498699804069474e-06, "zpclrcov": -4.410000201460207e-06, "zpmed": 26.322999954223633, "clrmed": 0.656000018119812, "clrrms": 0.18664300441741943, "neargaia": 0.689031720161438, "neargaiabright": 0.689031720161438, "maggaia": 12.743537902832031, "maggaiabright": 12.743537902832031, "exptime": 30.0, "drb": 0.011009779758751392, "drbversion": "d6_m7", "brokerIngestTimestamp": 1742614285, "surveyPublishTimestamp": 1742614298624.0, "parent_candid": 4444444444}}, + {"oid": 1111111111,"sid": "ZTF","pid": 2975147530415, "tid": "ZTF","band": 1, "measurement_id": 0,"mjd": 60729.14753470011, "ra": 80.9235609, "e_ra": 0.09017135947942734,"dec": -19.4986049, "e_dec": 0.08500000089406967, "mag": 17.592100143432617, "e_mag": 0.06143999844789505, "isdiffpos": -1, "has_stamp": false, "forced": false, "parent_candid": "2991131970415010002","extra_fields": { "diffmaglim": 19.903900146484375,"pdiffimfilename": "/ztf/archive/sci/2025/0223/147350/ztf_20250223147350_000305_zr_c02_o_q1_scimrefdiffimg.fits.fz", "programpi": "Kulkarni", "programid": 1, "tblid": 20,"nid": 2975, "rcid": 4, "field": 305, "xpos": 1761.81005859375, "ypos": 1701.7099609375, "chipsf": 6.360939979553223, "magap": 17.629100799560547, "sigmagap": 0.04919999837875366, "distnr": 0.20032399892807007, "magnr": 15.486000061035156, "sigmagnr": 0.014000000432133675, "chinr": 0.6230000257492065, "sharpnr": -0.017999999225139618, "sky": 0.4195370078086853, "magdiff": 0.03695699945092201, "fwhm": 1.3672599792480469, "classtar": 0.9959999918937683, "mindtoedge": 1310.68994140625, "magfromlim": 2.274790048599243, "seeratio": 2.0, "aimage": 0.7369999885559082, "bimage": 0.656000018119812, "aimagerat": 0.5390329957008362, "bimagerat": 0.47979000210762024, "elong": 1.661678953479962348938, "nneg": 3, "nbad": 0, "rb": 0.9528570175170898, "ssdistnr": null, "ssmagnr": null, "ssnamenr": null, "sumrat": 0.9951059818267822, "magapbig": 17.643199920654297, "sigmagapbig": 0.06129999831318855, "ranr": 80.92357635498047, "decnr": -19.498550415039062, "scorr": 17.82550048828125, "magzpsci": 26.23710060119629, "magzpsciunc": 3.371399998286506e-06, "magzpscirms": 0.02283800020813942, "clrcoeff": 0.07911709696054459, "clrcounc": 5.949420028628083e-06, "rbversion": "t17_f5_c3"}}, + {"oid": 1111111111, "sid": "ZTF", "pid": 2974191730815, "tid": "ZTF", "band": 1, "measurement_id": 1, "mjd": 60728.191736099776, "ra": 102.4124856, "e_ra": 0.0, "dec": 2.428348, "e_dec": 0.0, "mag": 17.095138549804688, "e_mag": 0.011105540208518505, "isdiffpos": -1, "has_stamp": false, "forced": true, "parent_candid": 4444444444, "extra_fields": {"field": 461, "rcid": 8, "rband": 461120208, "sciinpseeing": 2.894200086593628, "scibckgnd": 251.1649932861328, "scisigpix": 9.40414047241211, "magzpsci": 26.312000274658203, "magzpsciunc": 3.8145999496919103e-06, "magzpscirms": 0.023059900850057602, "clrcoeff": 0.08249370008707047, "clrcounc": 7.810180250089616e-06, "exptime": 30.0, "adpctdif1": 0.06852299720048904, "adpctdif2": 0.06517399847507477, "diffmaglim": 19.89859962463379, "programid": 1, "forcediffimfluxunc": 49.72494888305664, "procstatus": "0", "distnr": 0.6907779574394226, "ranr": 102.41229248046875, "decnr": 2.4283649921417236, "magnr": 13.352999687194824, "sigmagnr": 0.014999999664723873, "chinr": 0.574999988079071, "sharpnr": -0.01600000075995922}}, + {"oid": 1111111111,"sid": "ZTF","pid": 2989194170415,"tid": "ZTF","band": 2,"measurement_id": 2,"mjd": 60743.19417819986,"ra": 80.9235912,"e_ra": 0.0,"dec": -19.498545,"e_dec": 0.0,"mag": 21.11893081665039,"e_mag": 0.9244838356971741,"isdiffpos": 1,"has_stamp": false,"forced": true,"parent_candid": "2991131970415010002","extra_fields": {"field": 305,"rcid": 4,"rband": 305120104,"sciinpseeing": 2.8603999614715576,"scibckgnd": 118.7030029296875,"scisigpix": 15.75469970703125,"magzpsci": 26.10740089416504,"magzpsciunc": 1.607850026630331e-05,"magzpscirms": 0.0383777990937233,"clrcoeff": -0.05813689902424812,"clrcounc": 3.839499913738109e-05,"exptime": 30.0,"adpctdif1": 0.07247500121593475,"adpctdif2": 0.0905120000243187,"diffmaglim": 18.98080062866211,"programid": 1,"forcediffimfluxunc": 84.2514419555664,"procstatus": "0","distnr": 0.07343738526105881,"ranr": 80.92357635498047,"decnr": -19.49855613708496,"magnr": 15.998000144958496,"sigmagnr": 0.009999999776482582,"chinr": 0.36899987945328184065247,"sharpnr": -0.008999999612569809}} + ], + "non_detections": [{"oid": 1111111111, "sid": "ZTF", "tid": "ZTF", "band": 1, "mjd": 60726.0, "diffmaglim": 20.403099060058594}, + {"oid": 1111111111, "sid": "ZTF", "tid": "ZTF", "band": 1, "mjd": 60727.0, "diffmaglim": 19.89859962463379}, + {"oid": 1111111111, "sid": "ZTF", "tid": "ZTF", "band": 2, "mjd": 60728.0, "diffmaglim": 19.596900939941406}, + {"oid": 1111111111, "sid": "ZTF", "tid": "ZTF", "band": 2, "mjd": 60728.33390050009, "diffmaglim": 19.596900939941406} + ] + , "timestamp": 1742614288323}, + + {"oid": 2222222222, "measurement_id": 0, + "detections": [{"oid": 2222222222, "sid": "ZTF", "pid": 3002139200815, "tid": "ZTF", "band": 2, "measurement_id": 4, "mjd": 60750.13920139879453281, "ra": 102.4124856, "e_ra": 0.06505842506885529, "dec": 2.428348, "e_dec": 0.06499999761581421, "mag": 18.211593627929688, "e_mag": 0.13416489958763661678953, "isdiffpos": -1, "has_stamp": true, "forced": false, "parent_candid": 4444444444, "extra_fields": {"diffmaglim": 20.361549377441406, "pdiffimfilename": "ztf_20250322139201_000461_zg_c03_o_q1_scimrefdiffimg.fits", "programpi": "Kulkarni", "programid": 1, "tblid": 1, "nid": 3002, "rcid": 8, "field": 461, "xpos": 3055.9765625, "ypos": 486.32110595703125, "chipsf": 21.012855529785156, "magap": 18.452800750732422, "sigmagap": 0.07240000367164612, "distnr": 0.6040137410163879, "magnr": 15.531000137329102, "sigmagnr": 0.014000000432133675, "chinr": 0.7239999771118164, "sharpnr": -0.013000000268220901, "sky": -0.8485742807388306, "magdiff": 0.24120700359344482, "fwhm": 1.4315736293792725, "classtar": 0.8849999904632568, "mindtoedge": 16.523399353027344, "magfromlim": 1.9087491035461426, "seeratio": 2.0, "aimage": 0.7509998794532811253967, "bimage": 0.6259998794532811253967, "aimagerat": 0.5245975255966187, "bimagerat": 0.4372810125350952, "elong": 1.1996805667877197, "nneg": 4, "nbad": 0, "rb": 0.6557142734527588, "ssdistnr": -999.0, "ssmagnr": -999.0, "ssnamenr": "null", "sumrat": 0.9966363310813904, "magapbig": 18.455699920654297, "sigmagapbig": 0.0868000015616417, "ranr": 102.46616789531536865234, "decnr": 2.428351640701294, "sgmag1": 15.458499908447266, "srmag1": 13.499699592590332, "simag1": 12.519000053405762, "szmag1": 11.694000244140625, "sgscore1": 0.9637920260429382, "distpsnr1": 0.6792148351669312, "ndethist": 141, "ncovhist": 852, "jdstarthist": 2458426.0, "jdendhist": 2460756.75, "scorr": 8.626946449279785, "tooflag": 0, "objectidps1": 1.1091102783871386e+17, "objectidps2": 1.1091102783871386e+17, "sgmag2": -999.0, "srmag2": 21.4768009185791, "simag2": -999.0, "szmag2": -999.0, "sgscore2": 0.09989289939403534, "distpsnr2": 9.005562782287598, "objectidps3": 1.1091102783871386e+17, "sgmag3": 19.660999298095703, "srmag3": 18.793399810791016, "simag3": 18.318500518798828, "szmag3": 18.097000122070312, "sgscore3": 0.4681670069694519, "distpsnr3": 9.469527244567871, "nmtchps": 17, "rband": 461120108, "jdstartref": 2458369.0, "jdendref": 2458423.0, "nframesref": 15, "rbversion": "t17_f5_c3", "dsnrms": 8.453817367553711, "ssnrms": 46.838356018066406, "dsdiff": -38.38454055786133, "magzpsci": 26.404592514038086, "magzpsciunc": 3.219299969714484e-06, "magzpscirms": 0.031863998621702194, "nmatches": 2005, "clrcoeff": -0.1269499957561493, "clrcounc": 6.498699804069474e-06, "zpclrcov": -4.410000201460207e-06, "zpmed": 26.322999954223633, "clrmed": 0.656000018119812, "clrrms": 0.18664300441741943, "neargaia": 0.689031720161438, "neargaiabright": 0.689031720161438, "maggaia": 12.743537902832031, "maggaiabright": 12.743537902832031, "exptime": 30.0, "drb": 0.011009779758751392, "drbversion": "d6_m7", "brokerIngestTimestamp": 1742614285, "surveyPublishTimestamp": 1742614298624.0, "parent_candid": 4444444444}}, + {"oid": 2222222222,"sid": "ZTF","pid": 2975147530415, "tid": "ZTF","band": 1, "measurement_id": 5,"mjd": 60729.14753470011, "ra": 80.9235609, "e_ra": 0.09017135947942734,"dec": -19.4986049, "e_dec": 0.08500000089406967, "mag": 17.592100143432617, "e_mag": 0.06143999844789505, "isdiffpos": -1, "has_stamp": false, "forced": false, "parent_candid": "2991131970415010002","extra_fields": { "diffmaglim": 19.903900146484375,"pdiffimfilename": "/ztf/archive/sci/2025/0223/147350/ztf_20250223147350_000305_zr_c02_o_q1_scimrefdiffimg.fits.fz", "programpi": "Kulkarni", "programid": 1, "tblid": 20,"nid": 2975, "rcid": 4, "field": 305, "xpos": 1761.81005859375, "ypos": 1701.7099609375, "chipsf": 6.360939979553223, "magap": 17.629100799560547, "sigmagap": 0.04919999837875366, "distnr": 0.20032399892807007, "magnr": 15.486000061035156, "sigmagnr": 0.014000000432133675, "chinr": 0.6230000257492065, "sharpnr": -0.017999999225139618, "sky": 0.4195370078086853, "magdiff": 0.03695699945092201, "fwhm": 1.3672599792480469, "classtar": 0.9959999918937683, "mindtoedge": 1310.68994140625, "magfromlim": 2.274790048599243, "seeratio": 2.0, "aimage": 0.7369999885559082, "bimage": 0.656000018119812, "aimagerat": 0.5390329957008362, "bimagerat": 0.47979000210762024, "elong": 1.661678953479962348938, "nneg": 3, "nbad": 0, "rb": 0.9528570175170898, "ssdistnr": null, "ssmagnr": null, "ssnamenr": null, "sumrat": 0.9951059818267822, "magapbig": 17.643199920654297, "sigmagapbig": 0.06129999831318855, "ranr": 80.92357635498047, "decnr": -19.498550415039062, "scorr": 17.82550048828125, "magzpsci": 26.23710060119629, "magzpsciunc": 3.371399998286506e-06, "magzpscirms": 0.02283800020813942, "clrcoeff": 0.07911709696054459, "clrcounc": 5.949420028628083e-06, "rbversion": "t17_f5_c3"}}, + {"oid": 2222222222, "sid": "ZTF", "pid": 2974191730815, "tid": "ZTF", "band": 1, "measurement_id": 6, "mjd": 60728.191736099776, "ra": 102.4124856, "e_ra": 0.0, "dec": 2.428348, "e_dec": 0.0, "mag": 17.095138549804688, "e_mag": 0.011105540208518505, "isdiffpos": -1, "has_stamp": false, "forced": true, "parent_candid": 4444444444, "extra_fields": {"field": 461, "rcid": 8, "rband": 461120208, "sciinpseeing": 2.894200086593628, "scibckgnd": 251.1649932861328, "scisigpix": 9.40414047241211, "magzpsci": 26.312000274658203, "magzpsciunc": 3.8145999496919103e-06, "magzpscirms": 0.023059900850057602, "clrcoeff": 0.08249370008707047, "clrcounc": 7.810180250089616e-06, "exptime": 30.0, "adpctdif1": 0.06852299720048904, "adpctdif2": 0.06517399847507477, "diffmaglim": 19.89859962463379, "programid": 1, "forcediffimfluxunc": 49.72494888305664, "procstatus": "0", "distnr": 0.6907779574394226, "ranr": 102.41229248046875, "decnr": 2.4283649921417236, "magnr": 13.352999687194824, "sigmagnr": 0.014999999664723873, "chinr": 0.574999988079071, "sharpnr": -0.01600000075995922}}, + {"oid": 2222222222,"sid": "ZTF","pid": 2989194170415,"tid": "ZTF","band": 2,"measurement_id": 7,"mjd": 60743.19417819986,"ra": 80.9235912,"e_ra": 0.0,"dec": -19.498545,"e_dec": 0.0,"mag": 21.11893081665039,"e_mag": 0.9244838356971741,"isdiffpos": 1,"has_stamp": false,"forced": true,"parent_candid": "2991131970415010002","extra_fields": {"field": 305,"rcid": 4,"rband": 305120104,"sciinpseeing": 2.8603999614715576,"scibckgnd": 118.7030029296875,"scisigpix": 15.75469970703125,"magzpsci": 26.10740089416504,"magzpsciunc": 1.607850026630331e-05,"magzpscirms": 0.0383777990937233,"clrcoeff": -0.05813689902424812,"clrcounc": 3.839499913738109e-05,"exptime": 30.0,"adpctdif1": 0.07247500121593475,"adpctdif2": 0.0905120000243187,"diffmaglim": 18.98080062866211,"programid": 1,"forcediffimfluxunc": 84.2514419555664,"procstatus": "0","distnr": 0.07343738526105881,"ranr": 80.92357635498047,"decnr": -19.49855613708496,"magnr": 15.998000144958496,"sigmagnr": 0.009999999776482582,"chinr": 0.36899987945328184065247,"sharpnr": -0.008999999612569809}} + ], + "non_detections": [{"oid": 2222222222, "sid": "ZTF", "tid": "ZTF", "band": 1, "mjd": 60726.0, "diffmaglim": 20.403099060058594}, + {"oid": 2222222222, "sid": "ZTF", "tid": "ZTF", "band": 1, "mjd": 60727.0, "diffmaglim": 19.89859962463379}, + {"oid": 2222222222, "sid": "ZTF", "tid": "ZTF", "band": 2, "mjd": 60728.0, "diffmaglim": 19.596900939941406}, + {"oid": 2222222222, "sid": "ZTF", "tid": "ZTF", "band": 2, "mjd": 60720.0, "diffmaglim": 19.69406} + ] + , "timestamp": 1742614288323} + +] \ No newline at end of file diff --git a/correction_multistream_ztf_step/tests/integration/data/ztf_messages.py b/correction_multistream_ztf_step/tests/integration/data/ztf_messages.py new file mode 100644 index 000000000..83c3da3b1 --- /dev/null +++ b/correction_multistream_ztf_step/tests/integration/data/ztf_messages.py @@ -0,0 +1,417 @@ +objects = [ + { + "oid": 1234567890, + "tid": 1, + "sid": 2, + "meanra": 5.8, + "meandec": -11.1, + "sigmara": 0.066, + "sigmadec": 0.065, + "firstmjd": 60684.1, + "lastmjd": 60684.1, + "deltamjd": 0.0, + "n_det": 1, + "n_forced": 0, + "n_non_det": 1480, + "corrected": False, + "stellar": False, + }, + { + "oid": 1111111111, + "tid": 1, + "sid": 2, + "meanra": 5.8, + "meandec": -11.1, + "sigmara": 0.066, + "sigmadec": 0.065, + "firstmjd": 60684.1, + "lastmjd": 60694.1, + "deltamjd": 0.0, + "n_det": 1, + "n_forced": 0, + "n_non_det": 1480, + "corrected": False, + "stellar": False, + }, + { + "oid": 2222222222, + "tid": 1, + "sid": 2, + "meanra": 5.8, + "meandec": -11.1, + "sigmara": 0.056, + "sigmadec": 0.065, + "firstmjd": 60684.1, + "lastmjd": 60684.1, + "deltamjd": 0.0, + "n_det": 1, + "n_forced": 0, + "n_non_det": 1480, + "corrected": False, + "stellar": False, + }, +] + +detections = [ + { + "oid": 1234567890, + "measurement_id": 1, + "mjd": 0.4124512, + "ra": 152.3524, + "dec": -89.51242, + "band": 1, + }, + { + "oid": 2222222222, + "measurement_id": 2, + "mjd": 0.4124512, + "ra": 152.3524, + "dec": -89.51242, + "band": 2, + }, + { + "oid": 1111111111, + "measurement_id": 0, + "mjd": 0.5124513, + "ra": 152.3524, + "dec": -89.51242, + "band": 1, + }, + { + "oid": 1111111111, + "measurement_id": 1, + "mjd": 0.4124512, + "ra": 152.3524, + "dec": -89.51242, + "band": 1, + }, +] + + +ztf_detections = [ + { + "oid": 1234567890, + "measurement_id": 1, + "pid": 6677889, + "diffmaglim": 19.8, + "isdiffpos": -1, + "nid": 2002, + "magpsf": 17.5, + "sigmapsf": 0.12, + "magap": 17.4, + "sigmagap": 0.18, + "distnr": 2.5, + "rb": 0.9, + "rbversion": "v1.2", + "drb": 0.95, + "drbversion": "v1.3", + "magapbig": 17.3, + "sigmagapbig": 0.22, + "rfid": 2, + "magpsf_corr": 17.45, + "sigmapsf_corr": 0.11, + "sigmapsf_corr_ext": 0.10, + "corrected": True, + "dubious": True, + "parent_candid": 9876543211, + "has_stamp": False, + }, + { + "oid": 2222222222, + "measurement_id": 2, + "pid": 6677889, + "diffmaglim": 19.8, + "isdiffpos": -1, + "nid": 2002, + "magpsf": 17.5, + "sigmapsf": 0.12, + "magap": 17.4, + "sigmagap": 0.18, + "distnr": 2.5, + "rb": 0.9, + "rbversion": "v1.2", + "drb": 0.95, + "drbversion": "v1.3", + "magapbig": 17.3, + "sigmagapbig": 0.22, + "rfid": 2, + "magpsf_corr": 17.45, + "sigmapsf_corr": 0.11, + "sigmapsf_corr_ext": 0.10, + "corrected": True, + "dubious": True, + "parent_candid": 9876543211, + "has_stamp": False, + }, + { + "oid": 1111111111, + "measurement_id": 0, + "pid": 6677889, + "diffmaglim": 19.8, + "isdiffpos": -1, + "nid": 2002, + "magpsf": 17.5, + "sigmapsf": 0.12, + "magap": 17.4, + "sigmagap": 0.18, + "distnr": 2.5, + "rb": 0.9, + "rbversion": "v1.2", + "drb": 0.95, + "drbversion": "v1.3", + "magapbig": 17.3, + "sigmagapbig": 0.22, + "rfid": 2, + "magpsf_corr": 17.45, + "sigmapsf_corr": 0.11, + "sigmapsf_corr_ext": 0.10, + "corrected": False, + "dubious": True, + "parent_candid": 9876543211, + "has_stamp": False, + }, + { + "oid": 1111111111, + "measurement_id": 1, + "pid": 6677889, + "diffmaglim": 19.8, + "isdiffpos": -1, + "nid": 2002, + "magpsf": 17.5, + "sigmapsf": 0.12, + "magap": 17.4, + "sigmagap": 0.18, + "distnr": 2.5, + "rb": 0.9, + "rbversion": "v1.2", + "drb": 0.95, + "drbversion": "v1.3", + "magapbig": 17.3, + "sigmagapbig": 0.22, + "rfid": 2, + "magpsf_corr": 17.45, + "sigmapsf_corr": 0.11, + "sigmapsf_corr_ext": 0.10, + "corrected": False, + "dubious": True, + "parent_candid": 9876543211, + "has_stamp": False, + }, +] + +non_detection = [ + {"oid": 1234567890, "band": 1, "mjd": 60655.155775499996, "diffmaglim": 17.6075}, + {"oid": 1234567890, "band": 2, "mjd": 60658.11518519977, "diffmaglim": 18.2779}, + {"oid": 1234567890, "band": 1, "mjd": 60658.19327550009, "diffmaglim": 19.0953}, + {"oid": 1234567890, "band": 2, "mjd": 60660.149386600126, "diffmaglim": 19.4483}, + {"oid": 1234567890, "band": 1, "mjd": 60660.2327430998, "diffmaglim": 18.7138}, + {"oid": 1234567890, "band": 2, "mjd": 60663.106655099895, "diffmaglim": 19.9582}, + {"oid": 1234567890, "band": 1, "mjd": 60663.22942129988, "diffmaglim": 18.8907}, + {"oid": 1234567890, "band": 1, "mjd": 60664.11018519988, "diffmaglim": 20.1121}, + {"oid": 1234567890, "band": 2, "mjd": 60664.14053239999, "diffmaglim": 20.2564}, + {"oid": 1234567890, "band": 1, "mjd": 60666.120231499895, "diffmaglim": 19.9874}, + {"oid": 1234567890, "band": 2, "mjd": 60666.12964119995, "diffmaglim": 20.0196}, + {"oid": 1234567890, "band": 1, "mjd": 60668.110752299894, "diffmaglim": 18.8554}, + {"oid": 1234567890, "band": 2, "mjd": 60668.180081000086, "diffmaglim": 17.0589}, + {"oid": 1234567890, "band": 2, "mjd": 60672.10907409992, "diffmaglim": 19.6796}, + {"oid": 1234567890, "band": 1, "mjd": 60672.1296064998, "diffmaglim": 19.6858}, + {"oid": 1234567890, "band": 1, "mjd": 60674.09282410005, "diffmaglim": 20.1046}, + {"oid": 1234567890, "band": 2, "mjd": 60674.155729200225, "diffmaglim": 19.8949}, + {"oid": 1234567890, "band": 1, "mjd": 60677.194907399826, "diffmaglim": 19.6374}, + {"oid": 1234567890, "band": 1, "mjd": 60679.10571759986, "diffmaglim": 19.8679}, + {"oid": 1234567890, "band": 2, "mjd": 60679.185705999844, "diffmaglim": 19.6428}, + {"oid": 2222222222, "band": 2, "mjd": 60679.185705999844, "diffmaglim": 19.6428}, + {"oid": 2222222222, "band": 1, "mjd": 60679.285705999844, "diffmaglim": 20.4587}, + {"oid": 1111111111, "band": 1, "mjd": 60726.0, "diffmaglim": 20.403099060058594}, + {"oid": 1111111111, "band": 2, "mjd": 60728.0, "diffmaglim": 19.596900939941406}, +] + +ztf_forced_photometry = [ + { + "oid": 1234567890, + "measurement_id": 1, + "mag": 18.45, + "e_mag": 0.12, + "mag_corr": 18.37, + "e_mag_corr": 0.09, + "e_mag_corr_ext": 0.11, + "isdiffpos": 1, + "corrected": True, + "dubious": False, + "parent_candid": 4444444444, + "has_stamp": True, + "field": 456, + "rcid": 3, + "rfid": 2, + "sciinpseeing": 1.23, + "scibckgnd": 25.67, + "scisigpix": 8.97, + "magzpsci": 26.54, + "magzpsciunc": 0.03, + "magzpscirms": 0.02, + "clrcoeff": 0.15, + "clrcounc": 0.02, + "exptime": 30.0, + "adpctdif1": 0.35, + "adpctdif2": 0.42, + "diffmaglim": 19.23, + "programid": 1, + "procstatus": "SUCCESS", + "distnr": 0.76, + "ranr": 182.45678, + "decnr": 23.12345, + "magnr": 17.89, + "sigmagnr": 0.05, + "chinr": 1.12, + "sharpnr": 0.23, + }, + { + "oid": 2222222222, + "measurement_id": 10, + "mag": 18.45, + "e_mag": 0.12, + "mag_corr": 18.37, + "e_mag_corr": 0.09, + "e_mag_corr_ext": 0.11, + "isdiffpos": 1, + "corrected": True, + "dubious": False, + "parent_candid": 4444444444, + "has_stamp": True, + "field": 456, + "rcid": 3, + "rfid": 2, + "sciinpseeing": 1.23, + "scibckgnd": 25.67, + "scisigpix": 8.97, + "magzpsci": 26.54, + "magzpsciunc": 0.03, + "magzpscirms": 0.02, + "clrcoeff": 0.15, + "clrcounc": 0.02, + "exptime": 30.0, + "adpctdif1": 0.35, + "adpctdif2": 0.42, + "diffmaglim": 19.23, + "programid": 1, + "procstatus": "SUCCESS", + "distnr": 0.76, + "ranr": 182.45678, + "decnr": 23.12345, + "magnr": 17.89, + "sigmagnr": 0.05, + "chinr": 1.12, + "sharpnr": 0.23, + }, + { + "oid": 1111111111, + "measurement_id": 2, + "mag": 10.45, + "e_mag": 0.12, + "mag_corr": 18.37, + "e_mag_corr": 0.09, + "e_mag_corr_ext": 0.11, + "isdiffpos": 1, + "corrected": True, + "dubious": False, + "parent_candid": 4444444444, + "has_stamp": True, + "field": 456, + "rcid": 3, + "rfid": 2, + "sciinpseeing": 1.23, + "scibckgnd": 25.67, + "scisigpix": 8.97, + "magzpsci": 26.54, + "magzpsciunc": 0.03, + "magzpscirms": 0.02, + "clrcoeff": 0.15, + "clrcounc": 0.02, + "exptime": 30.0, + "adpctdif1": 0.35, + "adpctdif2": 0.42, + "diffmaglim": 19.23, + "programid": 1, + "procstatus": "SUCCESS", + "distnr": 0.76, + "ranr": 182.45678, + "decnr": 23.12345, + "magnr": 17.89, + "sigmagnr": 0.05, + "chinr": 1.12, + "sharpnr": 0.23, + }, + { + "oid": 1111111111, + "measurement_id": 1, + "mag": 10.45, + "e_mag": 0.12, + "mag_corr": 18.37, + "e_mag_corr": 0.09, + "e_mag_corr_ext": 0.11, + "isdiffpos": 1, + "corrected": True, + "dubious": False, + "parent_candid": 4444444444, + "has_stamp": True, + "field": 456, + "rcid": 3, + "rfid": 2, + "sciinpseeing": 1.23, + "scibckgnd": 25.67, + "scisigpix": 8.97, + "magzpsci": 26.54, + "magzpsciunc": 0.03, + "magzpscirms": 0.02, + "clrcoeff": 0.15, + "clrcounc": 0.02, + "exptime": 30.0, + "adpctdif1": 0.35, + "adpctdif2": 0.42, + "diffmaglim": 19.23, + "programid": 1, + "procstatus": "SUCCESS", + "distnr": 0.76, + "ranr": 182.45678, + "decnr": 23.12345, + "magnr": 17.89, + "sigmagnr": 0.05, + "chinr": 1.12, + "sharpnr": 0.23, + }, +] + +forced_photometry = [ + { + "oid": 1234567890, + "measurement_id": 1, + "mjd": 60671.155729200225, + "ra": 150.3524, + "dec": -19.51242, + "band": 1, + }, + { + "oid": 2222222222, + "measurement_id": 10, + "mjd": 60671.155729200225, + "ra": 150.3524, + "dec": -19.51242, + "band": 1, + }, + { + "oid": 1111111111, + "measurement_id": 2, + "mjd": 60571.155729200225, + "ra": 154.3524, + "dec": 21.51242, + "band": 1, + }, + { + "oid": 1111111111, + "measurement_id": 1, + "mjd": 60571.155729200225, + "ra": 154.3524, + "dec": 21.51242, + "band": 1, + }, +] \ No newline at end of file diff --git a/correction_multistream_ztf_step/tests/integration/docker-compose.yml b/correction_multistream_ztf_step/tests/integration/docker-compose.yml new file mode 100644 index 000000000..771149a3f --- /dev/null +++ b/correction_multistream_ztf_step/tests/integration/docker-compose.yml @@ -0,0 +1,10 @@ +version: "3" +services: + postgres: + image: postgres + environment: + - POSTGRES_USER=postgres + - POSTGRES_DB=postgres + - POSTGRES_PASSWORD=postgres + ports: + - "5432:5432" diff --git a/correction_multistream_ztf_step/tests/integration/test_db_operation.py b/correction_multistream_ztf_step/tests/integration/test_db_operation.py new file mode 100644 index 000000000..b80ce1bc7 --- /dev/null +++ b/correction_multistream_ztf_step/tests/integration/test_db_operation.py @@ -0,0 +1,331 @@ +import unittest +import logging +import pytest +from apf.core.settings import config_from_yaml_file + +import json + +from db_plugins.db.sql._connection import PsqlDatabase +from correction_multistream_ztf_step.step import CorrectionMultistreamZTFStep +from db_plugins.db.sql.models import ( + Detection, + ZtfDetection, + ZtfForcedPhotometry, + ForcedPhotometry, + NonDetection, + Object, +) +from tests.integration.data.ztf_messages import ( + detections, + ztf_detections, + non_detection as non_detections, + ztf_forced_photometry, + forced_photometry, + objects, +) +from sqlalchemy import insert + +psql_config = { + "ENGINE": "postgresql", + "HOST": "localhost", + "USER": "postgres", + "PASSWORD": "postgres", + "PORT": 5432, + "DB_NAME": "postgres", +} + +with open('tests/integration/data/data_input_prv_candidates_staging.json', 'r') as file: + data_consumer = json.load(file) + + +@pytest.mark.usefixtures("psql_service") # Usando el mismo fixture que el primer script +class TestCorrectionMultistreamZTF(unittest.TestCase): + """Test class for Correction Multistream ZTF Step.""" + + @classmethod + def setUpClass(cls): + """Set up the test environment once for all test methods.""" + cls.settings = config_from_yaml_file("tests/test_utils/config.yaml") + cls.logger = cls._set_logger(cls.settings) + cls.db_sql = PsqlDatabase(psql_config) + cls.step_params = {"config": cls.settings, "db_sql": cls.db_sql} + + def setUp(self): + # crear db + self.db_sql = PsqlDatabase(psql_config) + self.db_sql.create_db() + self.insert_test_data() + + self.step = self._step_creator() + + def insert_test_data(self): + """Insert test data using the InsertData class pattern.""" + with self.db_sql.session() as session: + # Insert objects first + for obj_data in objects: + obj = Object(**obj_data) + session.add(obj) + session.commit() # commit objects before inserting other data. + print("Object listo!") + + # Insertar detecciones + for detection_data in detections: + detection = Detection(**detection_data) + session.add(detection) + print("Detection listo!") + + # Insertar ztf_detecciones + for ztf_detection_data in ztf_detections: + ztf_detection = ZtfDetection(**ztf_detection_data) + session.add(ztf_detection) + print("ZtfDetection listo!") + + # Insertar no-detecciones + for non_detection_data in non_detections: + non_detection = NonDetection(**non_detection_data) + session.add(non_detection) + print("NonDetection listo!") + + # Insertar fotometría forzada ZTF + if isinstance(ztf_forced_photometry, dict): + forced_phot = ZtfForcedPhotometry(**ztf_forced_photometry) + session.add(forced_phot) + else: + for fp_data in ztf_forced_photometry: + forced_phot = ZtfForcedPhotometry(**fp_data) + session.add(forced_phot) + print("ZTFFP listo!") + + # Insertar fotometría forzada + if isinstance(forced_photometry, dict): + forced_phot = ForcedPhotometry(**forced_photometry) + session.add(forced_phot) + else: + for fp_data in forced_photometry: + forced_phot = ForcedPhotometry(**fp_data) + session.add(forced_phot) + print("FP listo!") + + session.commit() + + def tearDown(self): + """Clean up after each test.""" + # Limpiar la base de datos + self.db_sql.drop_db() + + @classmethod + def _set_logger(cls, settings): + """Set up the logger with the appropriate level and format.""" + level = logging.INFO + if settings.get("LOGGING_DEBUG"): + level = logging.DEBUG + + logger = logging.getLogger("alerce") + logger.setLevel(level) + + fmt = logging.Formatter( + "%(asctime)s %(levelname)7s %(name)36s: %(message)s", + "%Y-%m-%d %H:%M:%S", + ) + handler = logging.StreamHandler() + handler.setFormatter(fmt) + handler.setLevel(level) + + logger.addHandler(handler) + return logger + + def _step_creator(self): + """Create an instance of the CorrectionMultistreamZTFStep.""" + step = CorrectionMultistreamZTFStep(**self.step_params) + + if self.settings["FEATURE_FLAGS"]["SKIP_MJD_FILTER"]: + self.logger.info( + "This step won't filter detections by MJD. \ + Keep this in mind when using for ELAsTiCC" + ) + return step + + @staticmethod + def validate_alert_fields_message(data): + """Validate that all required top-level fields are present in the messages.""" + for message in data: + required_top_fields = [ + "oid", + "measurement_id", + "meanra", + "meandec", + "detections", + "non_detections", + ] + missing_top_fields = [field for field in required_top_fields if field not in message] + if missing_top_fields: + return False + + unexpected_fields = [field for field in message if field not in required_top_fields] + if unexpected_fields: + return False + + return True + + @staticmethod + def validate_non_detection_fields_message(data): + """Validate that all required non-detection fields are present in the messages.""" + for message in data: + if "non_detections" in message: + if not isinstance(message["non_detections"], list): + return False + + if message["non_detections"]: + required_non_det_fields = ["oid", "sid", "tid", "band", "mjd", "diffmaglim"] + for non_det in message["non_detections"]: + missing_non_det_fields = [ + field for field in required_non_det_fields if field not in non_det + ] + if missing_non_det_fields: + return False + + unexpected_non_det_fields = [ + field for field in non_det if field not in required_non_det_fields + ] + if unexpected_non_det_fields: + print('aqui3') + return False + + return True + + @staticmethod + def validate_detection_fields_message(data): + """Validate that all required detection fields are present in the messages.""" + for message in data: + if "detections" in message: + if not isinstance(message["detections"], list): + return False + + if message["detections"]: + required_det_fields = [ + "oid", + "sid", + "tid", + "pid", + "band", + "measurement_id", + "mjd", + "ra", + "e_ra", + "dec", + "e_dec", + "mag", + "e_mag", + "mag_corr", + "e_mag_corr", + "e_mag_corr_ext", + "isdiffpos", + "parent_candid", + "has_stamp", + "corrected", + "dubious", + "stellar", + "forced", + "new", + "extra_fields", + ] + for det in message["detections"]: + missing_det_fields = [ + field for field in required_det_fields if field not in det + ] + if missing_det_fields: + return False + + unexpected_det_fields = [ + field for field in det if field not in required_det_fields + ] + if unexpected_det_fields: + return False + + return True + + @staticmethod + def message_validation(data): + """Combine all validation methods to validate the entire message structure.""" + print(TestCorrectionMultistreamZTF.validate_alert_fields_message(data), TestCorrectionMultistreamZTF.validate_non_detection_fields_message(data), TestCorrectionMultistreamZTF.validate_detection_fields_message(data)) + return ( + TestCorrectionMultistreamZTF.validate_alert_fields_message(data) # True + and TestCorrectionMultistreamZTF.validate_non_detection_fields_message(data) # False + and TestCorrectionMultistreamZTF.validate_detection_fields_message(data) # False + ) + + @staticmethod + def output_expected_count(data, oid, expected_dets, expected_non_dets): + """Check if a message with the given OID has the expected number of detections and non-detections.""" + matching_dicts = [item for item in data if item.get("oid") == oid] + if not matching_dicts: + return False + + matching_dict = matching_dicts[0] + len_detections = len(matching_dict["detections"]) + len_non_detections = len(matching_dict["non_detections"]) + return len_detections == expected_dets and len_non_detections == expected_non_dets + + def test_correction_step_execution(self): + """Test the full execution of the correction step.""" + + # Configurar el consumer de prueba para que devuelva nuestros mensajes + self.step.consumer.messages = data_consumer + + original_consume = self.step.consumer.consume + self.step.consumer.consume = lambda: data_consumer + + try: + # Pre-consume y configuración + self.step._pre_consume() + + # Procesar todos los mensajes en el consumer + processed_messages = [] + for message in self.step.consumer.consume(): + preprocessed_msg = self.step._pre_execute(message) + if len(preprocessed_msg) == 0: + self.logger.info("Message of len zero after pre_execute") + continue + + try: + result = self.step.execute(preprocessed_msg) + result = self.step._post_execute(result) + result = self.step._pre_produce(result) + self.step.producer.produce(result) + processed_messages.extend(result) + except Exception as error: + self.logger.error(f"Error during execution: {error}") + raise + + # Si llegamos aquí sin errores, la prueba básica pasó + print('processed_messages', len(processed_messages)) + self.assertTrue(len(processed_messages) > 0, "No se procesaron mensajes") + + # Verificaciones adicionales si los mensajes se procesaron correctamente + if hasattr(self.step.producer, "pre_produce_message") and self.step.producer.pre_produce_message: + messages_produce = self.step.producer.pre_produce_message[0] + + # Verificar la estructura del mensaje + self.assertTrue( + self.message_validation(messages_produce), + "Message validation failed" + ) + + # Si hay suficientes datos, verificar conteos específicos + if len(messages_produce) >= 2: + self.assertTrue( + self.output_expected_count( + messages_produce, "661678953", expected_dets=3, expected_non_dets=3 + ), + "Failed count validation for OID 661678953" + ) + self.assertTrue( + self.output_expected_count( + messages_produce, "879453281", expected_dets=6, expected_non_dets=6 + ), + "Failed count validation for OID 879453281" + ) + + finally: + # Restaurar el método original + self.step.consumer.consume = original_consume \ No newline at end of file diff --git a/correction_multistream_ztf_step/tests/test_utils/.gitignore b/correction_multistream_ztf_step/tests/test_utils/.gitignore deleted file mode 100644 index 2a6160516..000000000 --- a/correction_multistream_ztf_step/tests/test_utils/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.yaml \ No newline at end of file diff --git a/correction_multistream_ztf_step/tests/test_utils/config.yaml b/correction_multistream_ztf_step/tests/test_utils/config.yaml new file mode 100644 index 000000000..c5ca4e408 --- /dev/null +++ b/correction_multistream_ztf_step/tests/test_utils/config.yaml @@ -0,0 +1,13 @@ +CONSUMER_CONFIG: + CLASS: "apf.consumers.test_consumer.TestConsumer" + SCHEMA_PATH: "schemas/prv_candidate_step/input.avsc" + PARAMS: + input_format: "json" + input_path: "tests/data_input_prv_candidates_staging.json" + auto.offset.reset: beginning + consume.messages: 10 + consume.timeout: 10 +FEATURE_FLAGS: + SKIP_MJD_FILTER: true +PRODUCER_CONFIG: + CLASS: "apf.producers.test_producer.TestProducer" \ No newline at end of file diff --git a/correction_multistream_ztf_step/tests/test_utils/test_utils.py b/correction_multistream_ztf_step/tests/test_utils/test_utils.py index dc93c9b06..f7db1fddf 100644 --- a/correction_multistream_ztf_step/tests/test_utils/test_utils.py +++ b/correction_multistream_ztf_step/tests/test_utils/test_utils.py @@ -4,7 +4,6 @@ from core.DB.database_sql import PSQLConnection from correction_multistream_ztf_step.step import CorrectionMultistreamZTFStep - def set_logger(settings): level = logging.INFO if settings.get("LOGGING_DEBUG"): diff --git a/correction_multistream_ztf_step/tests/write_db/_connection.py b/correction_multistream_ztf_step/tests/write_db/_connection.py index 28c54c233..60fb468a1 100644 --- a/correction_multistream_ztf_step/tests/write_db/_connection.py +++ b/correction_multistream_ztf_step/tests/write_db/_connection.py @@ -1,6 +1,6 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session -from db_plugins.db.sql.models_new import Base +from db_plugins.db.sql.models import Base from contextlib import contextmanager import logging