From e50417eb4b0294c1674a85af73b0a1a0ec63c5e9 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 11:19:39 +0100 Subject: [PATCH 01/11] Does not validate Azure or Google blob storage Azure and Google blob storage failed to validate if it was a directory or path. This PR skips validating directories so it does raise an inappropriate error. --- .../validation/FormatValidators/DirectoryPathValidator.groovy | 2 +- .../validation/FormatValidators/FilePathPatternValidator.groovy | 2 +- .../validation/FormatValidators/FilePathValidator.groovy | 2 +- .../nextflow/validation/FormatValidators/PathValidator.groovy | 2 +- .../src/main/nextflow/validation/SchemaValidator.groovy | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy index 0d935b5f..fd1ea9f5 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy @@ -11,7 +11,7 @@ public class DirectoryPathValidator implements FormatValidator { @Override public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { + if (subject.matches("(s3://|az://|gs://).*")) { log.debug("S3 paths are not supported by 'DirectoryPathValidator': '${subject}'") return Optional.empty() } diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy index 4ddaaea0..f05ed8ed 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy @@ -11,7 +11,7 @@ public class FilePathPatternValidator implements FormatValidator { @Override public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { + if (subject.matches("(s3://|az://|gs://).*")) { log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'") return Optional.empty() } diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy index a49ec6c7..4f510656 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy @@ -11,7 +11,7 @@ public class FilePathValidator implements FormatValidator { @Override public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { + if (subject.matches("(s3://|az://|gs://).*")) { log.debug("S3 paths are not supported by 'FilePathValidator': '${subject}'") return Optional.empty() } diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy index afe82e02..4156668f 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy @@ -11,7 +11,7 @@ public class PathValidator implements FormatValidator { @Override public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { + if (subject.matches("(s3://|az://|gs://).*")) { log.debug("S3 paths are not supported by 'PathValidator': '${subject}'") return Optional.empty() } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 9d701125..71f5c8e9 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -615,7 +615,7 @@ class SchemaValidator extends PluginExtensionPoint { // Function to check if a file or directory exists // List pathExists(String path, String paramName, Boolean s3PathCheck) { - if (path.startsWith('s3://') && !s3PathCheck) { + if (path.matches("(s3://|az://|gs://).*") && !s3PathCheck) { log.debug "Ignoring validation of S3 URL path '${path}'".toString() } else { def Path file = Nextflow.file(path) as Path From f70cecc136f80c3c312fa27690b2edd88e445af8 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:03:30 +0100 Subject: [PATCH 02/11] Add tests ignoring for az/s3/gs paths --- .../PluginExtensionMethodsTest.groovy | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy index 6b66c2ad..4ee4c451 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy @@ -445,6 +445,72 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ !stdout } + def 'should ignore s3 path' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.glob = 's3://fake/path' + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema') + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } + + def 'should ignore az path' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.glob = 'az://fake/path' + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema') + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } + + def 'should ignore gs path' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.glob = 'gs://fake/path' + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema') + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } + def 'correct validation of numbers with lenient mode' () { given: def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() From 38b48b78bc3c6468a9c9daa580c9aae39b936f81 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:08:54 +0100 Subject: [PATCH 03/11] Correct tests --- .../validation/PluginExtensionMethodsTest.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy index 4ee4c451..5073d653 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy @@ -447,9 +447,9 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def 'should ignore s3 path' () { given: - def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() + def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ - params.glob = 's3://fake/path' + params.input = 's3://fake/path' include { validateParameters } from 'plugin/nf-validation' validateParameters(parameters_schema: '$schema') @@ -469,9 +469,9 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def 'should ignore az path' () { given: - def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() + def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ - params.glob = 'az://fake/path' + params.input = 'az://fake/path' include { validateParameters } from 'plugin/nf-validation' validateParameters(parameters_schema: '$schema') @@ -491,9 +491,9 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def 'should ignore gs path' () { given: - def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() + def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ - params.glob = 'gs://fake/path' + params.input = 'gs://fake/path' include { validateParameters } from 'plugin/nf-validation' validateParameters(parameters_schema: '$schema') From de1fcda34a14ad31bcfb94b00854a802e022a836 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:26:29 +0100 Subject: [PATCH 04/11] Correct tests AGAIN --- .../nextflow/validation/PluginExtensionMethodsTest.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy index 5073d653..b5cd882a 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy @@ -447,7 +447,7 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def 'should ignore s3 path' () { given: - def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString() + def schema = Path.of('src/testResources/nextflow_schema_file_cloud_path.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ params.input = 's3://fake/path' include { validateParameters } from 'plugin/nf-validation' @@ -469,7 +469,7 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def 'should ignore az path' () { given: - def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString() + def schema = Path.of('src/testResources/nextflow_schema_file_cloud_path.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ params.input = 'az://fake/path' include { validateParameters } from 'plugin/nf-validation' @@ -491,7 +491,7 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def 'should ignore gs path' () { given: - def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString() + def schema = Path.of('src/testResources/nextflow_schema_file_cloud_path.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ params.input = 'gs://fake/path' include { validateParameters } from 'plugin/nf-validation' From 8ab72826a2ae4a58d3382aaa0ca4170de62c87f1 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:30:57 +0100 Subject: [PATCH 05/11] CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f425546..8f771770 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # nextflow-io/nf-validation: Changelog +# Version 1.1.4 + +## Improvements + +- No longer does false validation on Azure and GCP cloud storage paths. + # Version 1.1.3 - Asahikawa ## Improvements From 7e47cf2abf6bc92a50d96efc5d4b122bedb6cdeb Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:31:19 +0100 Subject: [PATCH 06/11] Actually add path --- .../nextflow_schema_file_cloud_path.json | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_file_cloud_path.json diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_file_cloud_path.json b/plugins/nf-validation/src/testResources/nextflow_schema_file_cloud_path.json new file mode 100644 index 00000000..69860f0b --- /dev/null +++ b/plugins/nf-validation/src/testResources/nextflow_schema_file_cloud_path.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "definitions": { + "file_patterns": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "properties": { + "input": { + "type": "string", + "format": "file-path", + "exists": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/file_patterns" + } + ] +} \ No newline at end of file From a22b3f0c0927cecf54e0ff3ca9443020a9da1a0b Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:33:29 +0100 Subject: [PATCH 07/11] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f771770..b5c3ef92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## Improvements -- No longer does false validation on Azure and GCP cloud storage paths. +- No longer does false validation on Azure and GCP cloud storage paths ([#171](https://github.com/nextflow-io/nf-validation/pull/171)) # Version 1.1.3 - Asahikawa From 9c9a1fe812f5642ff01847a3bc2a692bc8cafa45 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:35:26 +0100 Subject: [PATCH 08/11] Bump nf-validation version --- plugins/nf-validation/src/resources/META-INF/MANIFEST.MF | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/resources/META-INF/MANIFEST.MF b/plugins/nf-validation/src/resources/META-INF/MANIFEST.MF index 16d6abc2..b2fc81ca 100644 --- a/plugins/nf-validation/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-validation/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Id: nf-validation -Plugin-Version: 1.1.3 +Plugin-Version: 1.1.4 Plugin-Class: nextflow.validation.ValidationPlugin Plugin-Provider: nextflow Plugin-Requires: >=22.10.0 From 85057afdd0b79477e7e967bea262bea64644793b Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:56:16 +0100 Subject: [PATCH 09/11] Update plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> --- .../validation/FormatValidators/DirectoryPathValidator.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy index fd1ea9f5..1c64a7c3 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy @@ -12,7 +12,7 @@ public class DirectoryPathValidator implements FormatValidator { @Override public Optional validate(final String subject) { if (subject.matches("(s3://|az://|gs://).*")) { - log.debug("S3 paths are not supported by 'DirectoryPathValidator': '${subject}'") + log.debug("Cloud storage paths are not supported by 'DirectoryPathValidator': '${subject}'") return Optional.empty() } Path file = Nextflow.file(subject) as Path From 5a99d9636d0338ce0d82a1bbfa92365668c990b6 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:56:37 +0100 Subject: [PATCH 10/11] Apply suggestions from code review Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> --- .../validation/FormatValidators/FilePathValidator.groovy | 2 +- .../nextflow/validation/FormatValidators/PathValidator.groovy | 2 +- .../src/main/nextflow/validation/SchemaValidator.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy index 4f510656..089e3420 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy @@ -12,7 +12,7 @@ public class FilePathValidator implements FormatValidator { @Override public Optional validate(final String subject) { if (subject.matches("(s3://|az://|gs://).*")) { - log.debug("S3 paths are not supported by 'FilePathValidator': '${subject}'") + log.debug("Cloud storage paths are not supported by 'FilePathValidator': '${subject}'") return Optional.empty() } Path file = Nextflow.file(subject) as Path diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy index 4156668f..800f05b8 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy @@ -12,7 +12,7 @@ public class PathValidator implements FormatValidator { @Override public Optional validate(final String subject) { if (subject.matches("(s3://|az://|gs://).*")) { - log.debug("S3 paths are not supported by 'PathValidator': '${subject}'") + log.debug("Cloud storage paths are not supported by 'PathValidator': '${subject}'") return Optional.empty() } Path file = Nextflow.file(subject) as Path diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 71f5c8e9..9a4c3ec2 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -616,7 +616,7 @@ class SchemaValidator extends PluginExtensionPoint { // List pathExists(String path, String paramName, Boolean s3PathCheck) { if (path.matches("(s3://|az://|gs://).*") && !s3PathCheck) { - log.debug "Ignoring validation of S3 URL path '${path}'".toString() + log.debug "Ignoring validation of cloud storage path '${path}'".toString() } else { def Path file = Nextflow.file(path) as Path if (!file.exists()) { From 1a93ea9c3f4b26faffab1ec4e4467391d1e4fa56 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:56:45 +0100 Subject: [PATCH 11/11] Update plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> --- .../validation/FormatValidators/FilePathPatternValidator.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy index f05ed8ed..84dbf7f6 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy @@ -12,7 +12,7 @@ public class FilePathPatternValidator implements FormatValidator { @Override public Optional validate(final String subject) { if (subject.matches("(s3://|az://|gs://).*")) { - log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'") + log.debug("Cloud storage paths are not supported by 'FilePathPatternValidator': '${subject}'") return Optional.empty() } ArrayList files = Nextflow.files(subject)