[feature] sr2silo in V-pipe

Author: DrYak

workflow/envs/sr2silo.yaml

@@ -1,6 +1,5 @@
 channels:
-  - bioconda
   - conda-forge
-  - defaults
+  - bioconda
 dependencies:
-  - sr2silo=0.0.4
+  - sr2silo=1.0.1

workflow/rules/publish.smk

@@ -189,41 +189,87 @@ rule checksum:
 ruleorder: unfiltered_cram > checksum
 
 
-rule transform_upload_to_SILO:
+rule silo_preprocess:
     """
     Handles the wrangleing and upload to SILO.
     
     Takes the primer trimmed nudlitide aligments, wrangles them 
     into cleartext sequences, transalted to amino acids, aligns them, 
-    then uploads the result to SILO, with all the metadata.
+    and combine with all the metadata.
     """
     input:
-        nudlitide_alignment="{alignment_wildcard}",
+        nuc_alignment=alignment_wildcard,
     output:
-        silo_input= "{dataset}/alignments/silo_input.ndjson.zst",
-
-    # TO EDIT WIP
+        silo_input="{dataset}/alignments/silo_input.ndjson.zst",
     params:
-        sample_id = rebase_datadir(config.input["datadir"], wildcards.dataset),
-        batch_id=lambda wildcards: wildcards.batch_id,
-        timeline_file=config["TIMELINE_FILE"],
-        primers_file=config["PRIMERS_FILE"],
-        nuc_reference=config["NUC_REFERENCE"],
-        aa_reference=config["NUC_REFERENCE"],
+        s_rec=get_s_rec,
+        timeline_file=(
+            config.tallymut.get("timeline_file", None) or cohortdir("timeline.tsv")
+        ),
+        primers_file=config["input"]["protocols_file"],
+        nextclade_reference=config["loculus"]["nextclade_reference"],
+        SR2SILO=config["applications"]["sr2silo"],
     log:
-        # ???
+        outlog="{dataset}/alignments/silo_input.out.log",
+        errlog="{dataset}/alignments/silo_input.out.err",
     conda:
-        "envs/sr2silo.yaml"
+        config["loculus"]["conda"]
+    benchmark:
+        "{dataset}/alignments/silo_input.benchmark"
+    resources:
+        disk_mb=65536,
+        mem_mb=config["loculus"]["mem"],
+        runtime=config["loculus"]["time"],
+    threads: config["loculus"]["threads"]
+    shell:
+        """
+        {params.SR2SILO} process-from-vpipe \
+            --input-file "{input.nuc_alignment}" \
+            --sample-id "{params.s_rec.sample_id}" \
+            --batch-id "{params.s_rec.date}" \
+            --timeline-file "{params.timeline_file}" \
+            --primer-file "{params.primers_file}" \
+            --output-fp "{output.silo_input}" \
+            --reference "{params.nextclade_reference}" \
+            > {log.outfile} 2> >(tee {log.errfile} >&2)
+        """
+
 
+if config.loculus["local"]:
+
+    localrules:
+        silo_upload,
+
+
+rule silo_upload:
+    input:
+        silo_input="{dataset}/alignments/silo_input.ndjson.zst",
+    output:
+        flag="{dataset}/alignments/silo_input.uploaded",
+    params:
+        upload_name=get_s_rec,
+        keycloak_token_url=config["loculus"]["keycloak_token_url"],
+        submission_url=config["loculus"]["submission_url"],
+        SR2SILO=config["applications"]["sr2silo"],
+    log:
+        outlog="{dataset}/alignments/silo_upload.out.log",
+        errlog="{dataset}/alignments/silo_upload.out.err",
+    conda:
+        config["loculus"]["conda"]
+    benchmark:
+        "{dataset}/alignments/silo_upload.benchmark"
+    resources:
+        disk_mb=65536,
+        mem_mb=config["loculus"]["mem"],
+        runtime=config["loculus"]["time"],
+    threads: config["loculus"]["threads"]
     shell:
         """
-        sr2silo import-to-loculus \
-            --input-file {input.nudlitide_alignment} \
-            --sample-id {params.sample_id} \
-            --batch-id {params.batch_id} \
-            --timeline-file {params.timeline_file} \
-            --primer-file {params.primers_file} \
-            --output-fp {output.silo_input} \
-            --reference {params.nuc_reference} \
-            --no-upload 2>&1 | tee -a {log}
+        {params.SR2SILO} submit-to-loculus \
+            --keycloak-token-url "{params.keycloak_token_url}" \
+            --submission-url "{params.submission_url}" \
+            --processed-file "{input.silo_input}" \
+            --sample-id "{params.s_rec.sample_id}" && \
+        mkdir -p $(dirname {output.flag}) && \
+        touch {output.flag}
         """

workflow/schemas/config_schema.json

@@ -440,6 +440,10 @@
                 "lollipop": {
                     "type": "string",
                     "default": "lollipop"
+                },
+                "sr2silo": {
+                    "type": "string",
+                    "default": "sr2silo"
                 }
             },
             "default": {},
@@ -1933,6 +1937,53 @@
             },
             "default": {},
             "type": "object"
+        },
+        "loculus": {
+            "description": "This section is used to assist and prepare uploads of the data to instances of [Loculus](https://loculus.org/).\n\nNOTE: The upload package for Loculus requires metadata, currently this metadata is fetched from the same timeline.tsv file as used by tallymut, see [`tallymut` => `timeline_file`](#tallymut_timeline_file)",
+            "properties": {
+                "conda": {
+                    "type": "string",
+                    "default": "{VPIPE_BASEDIR}/envs/sr2silo.yaml"
+                },
+                "mem": {
+                    "type": "integer",
+                    "default": 8192
+                },
+                "time": {
+                    "type": "integer",
+                    "default": 120
+                },
+                "threads": {
+                    "type": "integer",
+                    "default": 1
+                },
+                "nextclade_reference": {
+                    "type": "string",
+                    "default": "",
+                    "description": "Name of the reference. Loculus uses the same names as nextclade datasets.",
+                    "example": ["sars-cov-2"]
+                },
+                "local": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "Don't dispatch the upload rule to the cluster for execution, run the upload locally.\n\nThis is useful for execution on clusters where the compute nodes are isolated from internet.",
+                    "examples": [false]
+                },
+                "keycloak_token_url": {
+                    "type": "string",
+                    "default": "",
+                    "description": "Keycloak authentication URL to use during submission",
+                    "example": ["https://auth.example.com/token"]
+                },
+                "submission_url": {
+                    "type": "string",
+                    "default": "",
+                    "description": "Loculus submission URL to use during submission",
+                    "example": ["https://backend.example.com/api"]
+                }
+            },
+            "default": {},
+            "type": "object"
         }
     }
 }