set things up for the real test

nf-core · Jan 12, 2024 · 2e36bb2 · 2e36bb2
1 parent ed486b6
commit 2e36bb2
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 36 deletions.
diff --git a/.github/workflows/ncbench.yml b/.github/workflows/ncbench.yml
@@ -60,14 +60,14 @@ jobs:
         if: ${{ matrix.enabled }}
         with:
           # TODO: fix path to agilent test path
-          source: s3://nf-core-awsmegatests/sarek/results-test-59026dc07633edb83aab3bfb2f65f79db38437a1/test_full/variant_calling
+          source: "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/test_full_germline_ncbench_agilent/variant_calling"
           command: sync
           aws_region: eu-west-1
           destination: ./variant_calling
           flags: --no-sign-request --include ".vcf.gz" --exclude "g.vcf.gz"
 
       - name: Install requests
-        run: pip install requests # version pinning?
+        run: pip install requests
 
       - name: Upload to zenodo
         id: upload_zenodo

diff --git a/.github/workflows/set_ncbench_config.sh b/.github/workflows/set_ncbench_config.sh
@@ -7,7 +7,6 @@ declare -A variant_callers=(
     ["strelka2"]="strelka/NA12878_%sM/NA12878_%sM.strelka.variants.vcf.gz"
 )
 
-
 declare -A variant_versions=(
     ["deepvariant"]="${DEEPVARIANT_VERSION}"
     ["freebayes"]="${FREEBAYES_VERSION}"

diff --git a/.github/workflows/upload.py b/.github/workflows/upload.py
@@ -8,48 +8,37 @@
 workspace_directory = os.environ["GITHUB_WORKSPACE"]
 pipeline_version = os.environ["PIPELINE_VERSION"]
 
-# TODO: replace sandbox link https://zenodo.org/api/deposit/depositions
-# https://sandbox.zenodo.org/api/deposit/depositions?access_token={access_token}
-url = f"https://sandbox.zenodo.org/api/deposit/depositions"
+# TODO: replace sandbox link
+url = f"https://zenodo.org/api/deposit/depositions" # f"https://sandbox.zenodo.org/api/deposit/depositions"
 
 # Create empty upload
 r = requests.post(url,
                 params=params,
                 json={},
                 headers=headers)
 
-# Add DEPOSITION ID to environment variables and make it available there
-#os.environ["DEPOSITION_ID"] = str(r.json()["id"])
-deposition_id = r.json()["id"]
+print("Create empty upload:\n")
+print(r.json())
 
-#print("Create empty upload:\n")
-#print(r.json())
-#print("Deposition id: ")
+deposition_id = r.json()["id"]
 
-## Print deposition ID to stderr to allow multiple print statement but capture the right one for later use
-#print(str(r.json()["id"]), file=sys.stderr)
+## Store deposition ID
 with open('deposition_id.txt', 'w') as f:
-    f.write(str(r.json()["id"]))
+    f.write(str(deposition_id))
 
 # Upload a new file
 bucket_url = r.json()["links"]["bucket"]
 
-#print(os.listdir('./variant_calling/strelka/HCC1395N/'))
-filenames = [ "strelka/HCC1395N/HCC1395N.strelka.genome.vcf.gz"]
-
-# filenames = [ "deepvariant/NA12878_75M/NA12878_75M.deepvariant.vcf.gz",
-#               "freebayes/NA12878_75M/NA12878_75M.freebayes.vcf.gz",
-#               "haplotypecaller/NA12878_75M/NA12878_75M.haplotypecaller.filtered.vcf.gz",
-#               "haplotypecaller/NA12878_75M/NA12878_75M.freebayes.vcf.gz",
-#               "strelka/NA12878_75M/NA12878_75M.strelka.variants.vcf.gz",
-#               "strelka/NA12878_75M/NA12878_75M.strelka.genome.vcf.gz",
-
-#               "deepvariant/NA12878_200M/NA12878_200M.deepvariant.vcf.gz",
-#               "freebayes/NA12878_200M/NA12878_200M.freebayes.vcf.gz",
-#               "haplotypecaller/NA12878_200M/NA12878_200M.haplotypecaller.filtered.vcf.gz",
-#               "haplotypecaller/NA12878_200M/NA12878_200M.freebayes.vcf.gz",
-#               "strelka/NA12878_200M/NA12878_200M.strelka.variants.vcf.gz",
-#               "strelka/NA12878_200M/NA12878_200M.strelka.genome.vcf.gz"]
+filenames = [ "deepvariant/NA12878_75M/NA12878_75M.deepvariant.vcf.gz",
+            "freebayes/NA12878_75M/NA12878_75M.freebayes.vcf.gz",
+            "haplotypecaller/NA12878_75M/NA12878_75M.haplotypecaller.filtered.vcf.gz",
+            "haplotypecaller/NA12878_75M/NA12878_75M.freebayes.vcf.gz",
+            "strelka/NA12878_75M/NA12878_75M.strelka.variants.vcf.gz",
+            "deepvariant/NA12878_200M/NA12878_200M.deepvariant.vcf.gz",
+            "freebayes/NA12878_200M/NA12878_200M.freebayes.vcf.gz",
+            "haplotypecaller/NA12878_200M/NA12878_200M.haplotypecaller.filtered.vcf.gz",
+            "haplotypecaller/NA12878_200M/NA12878_200M.freebayes.vcf.gz",
+            "strelka/NA12878_200M/NA12878_200M.strelka.variants.vcf.gz"]
 
 for file in filenames:
     path = "./variant_calling/%s" % file
@@ -59,11 +48,8 @@
             data=fp,
             params=params,
         )
-#print("Upload new files")
-#print(r.json())
 
 # Add metadata to uploaded file
-
 title = 'WES benchmark results nf-core/sarek v{}'.format(pipeline_version)
 data = {
     'metadata': {
@@ -75,15 +61,14 @@
     }
 }
 
-r = requests.put('https://sandbox.zenodo.org/api/deposit/depositions/%s' % deposition_id,
+r = requests.put('https://zenodo.org/api/deposit/depositions/%s' % deposition_id,
                 params=params,
                 data=json.dumps(data),
                 headers=headers)
 
 print("Add metadata: ")
 print(r.status_code)
 print(r.json())
-#print()
 
 # Publish this