diff --git a/.github/workflows/ncbench.yml b/.github/workflows/ncbench.yml index 6854358f96..b5ff8947f0 100644 --- a/.github/workflows/ncbench.yml +++ b/.github/workflows/ncbench.yml @@ -60,14 +60,14 @@ jobs: if: ${{ matrix.enabled }} with: # TODO: fix path to agilent test path - source: s3://nf-core-awsmegatests/sarek/results-test-59026dc07633edb83aab3bfb2f65f79db38437a1/test_full/variant_calling + source: "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/test_full_germline_ncbench_agilent/variant_calling" command: sync aws_region: eu-west-1 destination: ./variant_calling flags: --no-sign-request --include ".vcf.gz" --exclude "g.vcf.gz" - name: Install requests - run: pip install requests # version pinning? + run: pip install requests - name: Upload to zenodo id: upload_zenodo diff --git a/.github/workflows/set_ncbench_config.sh b/.github/workflows/set_ncbench_config.sh index 165dfde8a5..a82d61ff49 100644 --- a/.github/workflows/set_ncbench_config.sh +++ b/.github/workflows/set_ncbench_config.sh @@ -7,7 +7,6 @@ declare -A variant_callers=( ["strelka2"]="strelka/NA12878_%sM/NA12878_%sM.strelka.variants.vcf.gz" ) - declare -A variant_versions=( ["deepvariant"]="${DEEPVARIANT_VERSION}" ["freebayes"]="${FREEBAYES_VERSION}" diff --git a/.github/workflows/upload.py b/.github/workflows/upload.py index 2e91fc6f88..cd896c473d 100644 --- a/.github/workflows/upload.py +++ b/.github/workflows/upload.py @@ -8,9 +8,8 @@ workspace_directory = os.environ["GITHUB_WORKSPACE"] pipeline_version = os.environ["PIPELINE_VERSION"] -# TODO: replace sandbox link https://zenodo.org/api/deposit/depositions -# https://sandbox.zenodo.org/api/deposit/depositions?access_token={access_token} -url = f"https://sandbox.zenodo.org/api/deposit/depositions" +# TODO: replace sandbox link +url = f"https://zenodo.org/api/deposit/depositions" # f"https://sandbox.zenodo.org/api/deposit/depositions" # Create empty upload r = requests.post(url, @@ -18,38 +17,28 @@ json={}, headers=headers) -# Add DEPOSITION ID to environment variables and make it available there -#os.environ["DEPOSITION_ID"] = str(r.json()["id"]) -deposition_id = r.json()["id"] +print("Create empty upload:\n") +print(r.json()) -#print("Create empty upload:\n") -#print(r.json()) -#print("Deposition id: ") +deposition_id = r.json()["id"] -## Print deposition ID to stderr to allow multiple print statement but capture the right one for later use -#print(str(r.json()["id"]), file=sys.stderr) +## Store deposition ID with open('deposition_id.txt', 'w') as f: - f.write(str(r.json()["id"])) + f.write(str(deposition_id)) # Upload a new file bucket_url = r.json()["links"]["bucket"] -#print(os.listdir('./variant_calling/strelka/HCC1395N/')) -filenames = [ "strelka/HCC1395N/HCC1395N.strelka.genome.vcf.gz"] - -# filenames = [ "deepvariant/NA12878_75M/NA12878_75M.deepvariant.vcf.gz", -# "freebayes/NA12878_75M/NA12878_75M.freebayes.vcf.gz", -# "haplotypecaller/NA12878_75M/NA12878_75M.haplotypecaller.filtered.vcf.gz", -# "haplotypecaller/NA12878_75M/NA12878_75M.freebayes.vcf.gz", -# "strelka/NA12878_75M/NA12878_75M.strelka.variants.vcf.gz", -# "strelka/NA12878_75M/NA12878_75M.strelka.genome.vcf.gz", - -# "deepvariant/NA12878_200M/NA12878_200M.deepvariant.vcf.gz", -# "freebayes/NA12878_200M/NA12878_200M.freebayes.vcf.gz", -# "haplotypecaller/NA12878_200M/NA12878_200M.haplotypecaller.filtered.vcf.gz", -# "haplotypecaller/NA12878_200M/NA12878_200M.freebayes.vcf.gz", -# "strelka/NA12878_200M/NA12878_200M.strelka.variants.vcf.gz", -# "strelka/NA12878_200M/NA12878_200M.strelka.genome.vcf.gz"] +filenames = [ "deepvariant/NA12878_75M/NA12878_75M.deepvariant.vcf.gz", + "freebayes/NA12878_75M/NA12878_75M.freebayes.vcf.gz", + "haplotypecaller/NA12878_75M/NA12878_75M.haplotypecaller.filtered.vcf.gz", + "haplotypecaller/NA12878_75M/NA12878_75M.freebayes.vcf.gz", + "strelka/NA12878_75M/NA12878_75M.strelka.variants.vcf.gz", + "deepvariant/NA12878_200M/NA12878_200M.deepvariant.vcf.gz", + "freebayes/NA12878_200M/NA12878_200M.freebayes.vcf.gz", + "haplotypecaller/NA12878_200M/NA12878_200M.haplotypecaller.filtered.vcf.gz", + "haplotypecaller/NA12878_200M/NA12878_200M.freebayes.vcf.gz", + "strelka/NA12878_200M/NA12878_200M.strelka.variants.vcf.gz"] for file in filenames: path = "./variant_calling/%s" % file @@ -59,11 +48,8 @@ data=fp, params=params, ) -#print("Upload new files") -#print(r.json()) # Add metadata to uploaded file - title = 'WES benchmark results nf-core/sarek v{}'.format(pipeline_version) data = { 'metadata': { @@ -75,7 +61,7 @@ } } -r = requests.put('https://sandbox.zenodo.org/api/deposit/depositions/%s' % deposition_id, +r = requests.put('https://zenodo.org/api/deposit/depositions/%s' % deposition_id, params=params, data=json.dumps(data), headers=headers) @@ -83,7 +69,6 @@ print("Add metadata: ") print(r.status_code) print(r.json()) -#print() # Publish this