feat(pipelines): add v4 lof curations pipeline

Author: rileyhgrant

data-pipeline/src/data_pipeline/datasets/gnomad_v2/gnomad_v2_lof_curation.py

@@ -30,8 +30,10 @@
     "not_lof": "Not LoF",
 }
 
+VERDICT_MAPPINGS_CLEAN = VERDICT_MAPPING.values()
 
-def import_gnomad_v2_lof_curation_results(curation_result_paths, genes_path):
+
+def import_gnomad_lof_curation_results(curation_result_paths, genes_path, reference_genome="GRCh37"):
     all_flags = set()
 
     with hl.hadoop_open("/tmp/import_temp.tsv", "w") as temp_output_file:
@@ -52,8 +54,13 @@ def import_gnomad_v2_lof_curation_results(curation_result_paths, genes_path):
 
                 for row in reader:
                     [chrom, pos, ref, alt] = row["Variant ID"].split("-")
+                    chrom = f"chr{chrom}" if reference_genome == "GRCh38" else chrom
 
-                    variant_flags = [FLAG_MAPPING.get(f, f) for f in raw_dataset_flags if row[f"Flag {f}"] == "TRUE"]
+                    variant_flags = [
+                        FLAG_MAPPING.get(f, f)
+                        for f in raw_dataset_flags
+                        if row.get(f"Flag {f}") == "TRUE" or row.get(f"FLAG {f}") == "1"
+                    ]
 
                     genes = [gene_id for (gene_id, gene_symbol) in (gene.split(":") for gene in row["Gene"].split(";"))]
 
@@ -62,7 +69,8 @@ def import_gnomad_v2_lof_curation_results(curation_result_paths, genes_path):
                     if verdict == "inufficient_evidence":
                         verdict = "insufficient_evidence"
 
-                    verdict = VERDICT_MAPPING[verdict]
+                    if verdict not in VERDICT_MAPPINGS_CLEAN:
+                        verdict = VERDICT_MAPPING[verdict]
 
                     output_row = [
                         chrom,
@@ -81,7 +89,7 @@ def import_gnomad_v2_lof_curation_results(curation_result_paths, genes_path):
     ds = hl.import_table("/tmp/import_temp.tsv")
 
     ds = ds.transmute(
-        locus=hl.locus(ds.chrom, hl.int(ds.position)),
+        locus=hl.locus(ds.chrom, hl.int(ds.position), reference_genome),
         alleles=[ds.ref, ds.alt],
     )
 

data-pipeline/src/data_pipeline/pipelines/export_to_elasticsearch.py

@@ -38,8 +38,8 @@
 from data_pipeline.pipelines.gnomad_v3_short_tandem_repeats import pipeline as gnomad_v3_short_tandem_repeats_pipeline
 from data_pipeline.pipelines.gnomad_v4_variants import pipeline as gnomad_v4_variants_pipeline
 from data_pipeline.pipelines.gnomad_v4_coverage import pipeline as gnomad_v4_coverage_pipeline
-
 from data_pipeline.pipelines.gnomad_v4_cnvs import pipeline as gnomad_v4_cnvs_pipeline
+from data_pipeline.pipelines.gnomad_v4_lof_curation_results import pipeline as gnomad_v4_lof_curation_results_pipeline
 
 
 logger = logging.getLogger("gnomad_data_pipeline")
@@ -145,6 +145,18 @@ def add_liftover_document_id(ds):
     #     ),
     #     "args": {"index": "gnomad_v4_genome_coverage", "id_field": "xpos", "num_shards": 2, "block_size": 10_000},
     # },
+    "gnomad_v4_lof_curation_results": {
+        "get_table": lambda: add_variant_document_id(
+            hl.read_table(gnomad_v4_lof_curation_results_pipeline.get_output("lof_curation_results").get_output_path())
+        ),
+        "args": {
+            "index": "gnomad_v4_lof_curation_results",
+            "index_fields": ["document_id", "variant_id", "locus", "lof_curations.gene_id"],
+            "id_field": "document_id",
+            "num_shards": 1,
+            "block_size": 1_000,
+        },
+    },
     ##############################################################################################################
     # gnomAD v4 CNVs
     ##############################################################################################################

data-pipeline/src/data_pipeline/pipelines/gnomad_v2_lof_curation_results.py

@@ -1,6 +1,6 @@
 from data_pipeline.pipeline import Pipeline, run_pipeline
 
-from data_pipeline.datasets.gnomad_v2.gnomad_v2_lof_curation import import_gnomad_v2_lof_curation_results
+from data_pipeline.datasets.gnomad_v2.gnomad_v2_lof_curation import import_gnomad_lof_curation_results
 
 from data_pipeline.pipelines.genes import pipeline as genes_pipeline
 
@@ -9,7 +9,7 @@
 
 pipeline.add_task(
     "prepare_gnomad_v2_lof_curation_results",
-    import_gnomad_v2_lof_curation_results,
+    import_gnomad_lof_curation_results,
     "/gnomad_v2/gnomad_v2_lof_curation_results.ht",
     {"genes_path": genes_pipeline.get_output("genes_grch37")},
     {

data-pipeline/src/data_pipeline/pipelines/gnomad_v4_lof_curation_results.py

@@ -0,0 +1,36 @@
+from data_pipeline.pipeline import Pipeline, run_pipeline
+
+from data_pipeline.datasets.gnomad_v2.gnomad_v2_lof_curation import import_gnomad_lof_curation_results
+
+from data_pipeline.pipelines.genes import pipeline as genes_pipeline
+
+
+pipeline = Pipeline()
+
+pipeline.add_task(
+    "prepare_gnomad_v4_lof_curation_results",
+    import_gnomad_lof_curation_results,
+    "/gnomad_v4/gnomad_v4_lof_curation_results.ht",
+    {"genes_path": genes_pipeline.get_output("genes_grch38")},
+    {
+        # If a result for a variant/gene pair is present in more than one file,
+        # the result in the first file in this list takes precedence.
+        "curation_result_paths": [
+            "gs://gnomad-v4-data-pipeline/inputs/lof_curation/gnomAD_v4/gnomAD_incomplete_penetrance_final_results.csv",
+        ],
+        "reference_genome": "GRCh38",
+    },
+)
+
+###############################################
+# Outputs
+###############################################
+
+pipeline.set_outputs({"lof_curation_results": "prepare_gnomad_v4_lof_curation_results"})
+
+###############################################
+# Run
+###############################################
+
+if __name__ == "__main__":
+    run_pipeline(pipeline)