From 1dfa11aeeb4c74ddce27ad9546579bee0dd7341b Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 12 Sep 2023 12:47:04 +0100 Subject: [PATCH 01/16] Extend with additional fuzzing probes Signed-off-by: David Korczynski --- checks/evaluation/fuzzing.go | 10 ++ checks/evaluation/fuzzing_test.go | 40 +++++ checks/fileparser/listing.go | 4 - checks/fuzzing_test.go | 8 +- checks/raw/fuzzing.go | 98 +++++++++--- checks/raw/fuzzing_test.go | 9 +- probes/entries.go | 10 ++ probes/fuzzedWithCLibFuzzer/def.yml | 32 ++++ probes/fuzzedWithCLibFuzzer/impl.go | 39 +++++ probes/fuzzedWithCLibFuzzer/impl_test.go | 144 ++++++++++++++++++ probes/fuzzedWithCppLibFuzzer/def.yml | 32 ++++ probes/fuzzedWithCppLibFuzzer/impl.go | 39 +++++ probes/fuzzedWithCppLibFuzzer/impl_test.go | 144 ++++++++++++++++++ probes/fuzzedWithJavaJazzerFuzzer/def.yml | 32 ++++ probes/fuzzedWithJavaJazzerFuzzer/impl.go | 39 +++++ .../fuzzedWithJavaJazzerFuzzer/impl_test.go | 144 ++++++++++++++++++ probes/fuzzedWithPythonAtheris/def.yml | 32 ++++ probes/fuzzedWithPythonAtheris/impl.go | 39 +++++ probes/fuzzedWithPythonAtheris/impl_test.go | 144 ++++++++++++++++++ probes/fuzzedWithRustCargofuzz/def.yml | 32 ++++ probes/fuzzedWithRustCargofuzz/impl.go | 39 +++++ probes/fuzzedWithRustCargofuzz/impl_test.go | 144 ++++++++++++++++++ 22 files changed, 1222 insertions(+), 32 deletions(-) create mode 100644 probes/fuzzedWithCLibFuzzer/def.yml create mode 100644 probes/fuzzedWithCLibFuzzer/impl.go create mode 100644 probes/fuzzedWithCLibFuzzer/impl_test.go create mode 100644 probes/fuzzedWithCppLibFuzzer/def.yml create mode 100644 probes/fuzzedWithCppLibFuzzer/impl.go create mode 100644 probes/fuzzedWithCppLibFuzzer/impl_test.go create mode 100644 probes/fuzzedWithJavaJazzerFuzzer/def.yml create mode 100644 probes/fuzzedWithJavaJazzerFuzzer/impl.go create mode 100644 probes/fuzzedWithJavaJazzerFuzzer/impl_test.go create mode 100644 probes/fuzzedWithPythonAtheris/def.yml create mode 100644 probes/fuzzedWithPythonAtheris/impl.go create mode 100644 probes/fuzzedWithPythonAtheris/impl_test.go create mode 100644 probes/fuzzedWithRustCargofuzz/def.yml create mode 100644 probes/fuzzedWithRustCargofuzz/impl.go create mode 100644 probes/fuzzedWithRustCargofuzz/impl_test.go diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go index 8fe621eda85..cd1b014a0cc 100644 --- a/checks/evaluation/fuzzing.go +++ b/checks/evaluation/fuzzing.go @@ -20,6 +20,11 @@ import ( "github.com/ossf/scorecard/v4/finding" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" + "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" @@ -35,6 +40,11 @@ func Fuzzing(name string, expectedProbes := []string{ fuzzedWithClusterFuzzLite.Probe, fuzzedWithGoNative.Probe, + fuzzedWithPythonAtheris.Probe, + fuzzedWithCLibFuzzer.Probe, + fuzzedWithCppLibFuzzer.Probe, + fuzzedWithRustCargofuzz.Probe, + fuzzedWithJavaJazzerFuzzer.Probe, fuzzedWithOneFuzz.Probe, fuzzedWithOSSFuzz.Probe, fuzzedWithPropertyBasedHaskell.Probe, diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go index 1300f1e2e01..86b65a68fc3 100644 --- a/checks/evaluation/fuzzing_test.go +++ b/checks/evaluation/fuzzing_test.go @@ -47,6 +47,26 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithGoNative", Outcome: finding.OutcomeNegative, }, + { + Probe: "fuzzedWithPythonAtheris", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCppLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithRustCargofuzz", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithJavaJazzerFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithOneFuzz", Outcome: finding.OutcomeNegative, @@ -89,6 +109,26 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithGoNative", Outcome: finding.OutcomePositive, }, + { + Probe: "fuzzedWithPythonAtheris", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCppLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithRustCargofuzz", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithJavaJazzerFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithOneFuzz", Outcome: finding.OutcomeNegative, diff --git a/checks/fileparser/listing.go b/checks/fileparser/listing.go index 3efb8208991..55cf58601b1 100644 --- a/checks/fileparser/listing.go +++ b/checks/fileparser/listing.go @@ -76,10 +76,6 @@ func OnMatchingFileContentDo(repoClient clients.RepoClient, matchPathTo PathMatc onFileContent DoWhileTrueOnFileContent, args ...interface{}, ) error { predicate := func(filepath string) (bool, error) { - // Filter out test files. - if isTestdataFile(filepath) { - return false, nil - } // Filter out files based on path/names using the pattern. b, err := isMatchingPath(filepath, matchPathTo) if err != nil { diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go index 5fd34bbdb9d..a4dcb40fe4d 100644 --- a/checks/fuzzing_test.go +++ b/checks/fuzzing_test.go @@ -53,7 +53,7 @@ func TestFuzzing(t *testing.T) { wantErr: false, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 7, + NumberOfWarn: 12, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, @@ -76,7 +76,7 @@ func TestFuzzing(t *testing.T) { }, wantErr: false, expected: scut.TestReturn{ - NumberOfWarn: 6, + NumberOfWarn: 11, NumberOfDebug: 0, NumberOfInfo: 1, Score: 10, @@ -110,7 +110,7 @@ func TestFuzzing(t *testing.T) { wantFuzzErr: false, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 7, + NumberOfWarn: 12, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, @@ -121,7 +121,7 @@ func TestFuzzing(t *testing.T) { wantFuzzErr: true, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 7, + NumberOfWarn: 12, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index a8a161d9847..9ba4fe13bc8 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -35,6 +35,11 @@ const ( fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" fuzzerPropertyBasedJavaScript = "JavaScriptPropertyBasedTesting" fuzzerPropertyBasedTypeScript = "TypeScriptPropertyBasedTesting" + fuzzerPythonAtheris = "PythonAtherisFuzzer" + fuzzerCLibFuzzer = "CLibFuzzer" + fuzzerCppLibFuzzer = "CppLibFuzzer" + fuzzerRustCargoFuzz = "RustCargoFuzzer" + fuzzerJavaJazzerFuzzer = "JavaJazzerFuzzer" // TODO: add more fuzzing check supports. ) @@ -47,8 +52,8 @@ type filesWithPatternStr struct { type languageFuzzConfig struct { URL, Desc *string - // Pattern is according to path.Match. - filePattern string + // Patterns are according to path.Match. + filePatterns []string funcPattern, Name string // TODO: add more language fuzzing-related fields. @@ -59,10 +64,10 @@ type languageFuzzConfig struct { var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // Default fuzz patterns for Go. clients.Go: { - filePattern: "*_test.go", - funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`, - Name: fuzzerBuiltInGo, - URL: asPointer("https://go.dev/doc/fuzz/"), + filePatterns: []string{"*_test.go"}, + funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`, + Name: fuzzerBuiltInGo, + URL: asPointer("https://go.dev/doc/fuzz/"), Desc: asPointer( "Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."), }, @@ -80,7 +85,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // // This is not an exhaustive list. clients.Haskell: { - filePattern: "*.hs", + filePatterns: []string{"*.hs"}, // Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck, // or their indirect imports through the higher-level Hspec or Tasty testing frameworks. funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, @@ -96,7 +101,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // // This is not an exhaustive list. clients.JavaScript: { - filePattern: "*.js", + filePatterns: []string{"*.js"}, // Look for direct imports of fast-check. funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, Name: fuzzerPropertyBasedJavaScript, @@ -105,7 +110,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ "and test that specific properties are satisfied."), }, clients.TypeScript: { - filePattern: "*.ts", + filePatterns: []string{"*.ts"}, // Look for direct imports of fast-check. funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, Name: fuzzerPropertyBasedTypeScript, @@ -113,6 +118,47 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ "Property-based testing in TypeScript generates test instances randomly or exhaustively " + "and test that specific properties are satisfied."), }, + clients.Python: { + filePatterns: []string{"*.py"}, + // Look for direct imports of fast-check. + funcPattern: `import atheris`, + Name: fuzzerPythonAtheris, + Desc: asPointer( + "Python fuzzing by way of Atheris"), + }, + clients.C: { + filePatterns: []string{"*.c"}, + // Look for direct imports of fast-check. + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerCLibFuzzer, + Desc: asPointer( + "Fuzzed with C LibFuzzer"), + }, + clients.Cpp: { + filePatterns: []string{"*.cc", "*.cpp"}, + // Look for direct imports of fast-check. + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerCppLibFuzzer, + Desc: asPointer( + "Fuzzed with cpp LibFuzzer"), + }, + clients.Rust: { + filePatterns: []string{"*.rs"}, + // Look for direct imports of fast-check. + funcPattern: `libfuzzer_sys`, + Name: fuzzerRustCargoFuzz, + Desc: asPointer( + "Fuzzed with Cargo-fuzz"), + }, + clients.Java: { + filePatterns: []string{"*.java"}, + // Look for direct imports of fast-check. + funcPattern: `com.code_intelligence.jazzer.api.FuzzedDataProvider;`, + // funcPattern: `jazze`, + Name: fuzzerJavaJazzerFuzzer, + Desc: asPointer( + "Fuzzed with Jazzer fuzzer"), + }, // TODO: add more language-specific fuzz patterns & configs. } @@ -171,6 +217,7 @@ func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) { prominentLangs := getProminentLanguages(langs) for _, lang := range prominentLangs { usingFuzzFunc, files, e := checkFuzzFunc(c, lang) + fmt.Print(files) if e != nil { return checker.FuzzingData{}, fmt.Errorf("%w", e) } @@ -254,22 +301,31 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] // Get patterns for file and func. // We use the file pattern in the matcher to match the test files, // and put the func pattern in var data to match file contents (func names). - filePattern, funcPattern := pattern.filePattern, pattern.funcPattern - matcher := fileparser.PathMatcher{ - Pattern: filePattern, - CaseSensitive: false, - } - data.pattern = funcPattern - err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) - if err != nil { - return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) + filePatterns, funcPattern := pattern.filePatterns, pattern.funcPattern + dataFiles := []checker.File{} + + for _, filePattern := range filePatterns { + matcher := fileparser.PathMatcher{ + Pattern: filePattern, + CaseSensitive: false, + } + data.pattern = funcPattern + err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) + if err != nil { + return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) + } + if len(data.files) != 0 { + dataFiles = append(dataFiles, data.files...) + // This means no fuzz funcs matched for this language. + //return false, nil, nil + } } - if len(data.files) == 0 { + if len(dataFiles) == 0 { // This means no fuzz funcs matched for this language. return false, nil, nil } - return true, data.files, nil + return true, dataFiles, nil } // This is the callback func for interface OnMatchingFileContentDo @@ -322,7 +378,7 @@ func getProminentLanguages(langs []clients.Language) []clients.LanguageName { // Languages that have lines of code above average will be considered prominent. ret := []clients.LanguageName{} for lName, loC := range langMap { - if loC >= avgLoC { + if loC >= avgLoC/4.0 { lang := clients.LanguageName(strings.ToLower(string(lName))) ret = append(ret, lang) } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index c0a06edfe72..2e413be6935 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -261,8 +261,8 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { expectedFileMatch: false, expectedFuncMatch: false, lang: clients.LanguageName("not_a_supported_one"), - fileName: "a_fuzz_test.py", - fileContent: `def NotSupported (foo)`, + fileName: "a_fuzz_test.php", + fileContent: `function function-not-supported (foo)`, wantErr: true, }, } @@ -274,7 +274,10 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { if !ok && !tt.wantErr { t.Errorf("retrieve supported language error") } - fileMatchPattern := langSpecs.filePattern + fileMatchPattern := ""; + if (len(langSpecs.filePatterns) > 0) { + fileMatchPattern = langSpecs.filePatterns[0] + } fileMatch, err := path.Match(fileMatchPattern, tt.fileName) if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr { t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) diff --git a/probes/entries.go b/probes/entries.go index 7e7dd72944a..a6caa4b39e4 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -19,6 +19,11 @@ import ( "github.com/ossf/scorecard/v4/finding" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" + "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" @@ -60,6 +65,11 @@ var ( fuzzedWithOSSFuzz.Run, fuzzedWithOneFuzz.Run, fuzzedWithGoNative.Run, + fuzzedWithPythonAtheris.Run, + fuzzedWithCLibFuzzer.Run, + fuzzedWithCppLibFuzzer.Run, + fuzzedWithRustCargofuzz.Run, + fuzzedWithJavaJazzerFuzzer.Run, fuzzedWithClusterFuzzLite.Run, fuzzedWithPropertyBasedHaskell.Run, fuzzedWithPropertyBasedTypescript.Run, diff --git a/probes/fuzzedWithCLibFuzzer/def.yml b/probes/fuzzedWithCLibFuzzer/def.yml new file mode 100644 index 00000000000..132f11af466 --- /dev/null +++ b/probes/fuzzedWithCLibFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithCLibFuzzer +short: Check that the project is fuzzed using LibFuzzer +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of functions with the signature 'LLVMFuzzerTestOneInput' in .c files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://llvm.org/docs/LibFuzzer.html to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithCLibFuzzer/impl.go b/probes/fuzzedWithCLibFuzzer/impl.go new file mode 100644 index 00000000000..b2d198c033c --- /dev/null +++ b/probes/fuzzedWithCLibFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCLibFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithCLibFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "CLibFuzzer") +} diff --git a/probes/fuzzedWithCLibFuzzer/impl_test.go b/probes/fuzzedWithCLibFuzzer/impl_test.go new file mode 100644 index 00000000000..e4fe0dce5bf --- /dev/null +++ b/probes/fuzzedWithCLibFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCLibFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CLibFuzzer", + }, + { + Name: "CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CLibFuzzer", + }, + { + Name: "not-CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-CLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithCppLibFuzzer/def.yml b/probes/fuzzedWithCppLibFuzzer/def.yml new file mode 100644 index 00000000000..bb35db9f4ca --- /dev/null +++ b/probes/fuzzedWithCppLibFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithCppLibFuzzer +short: Check that the project is fuzzed using LibFuzzer +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of functions with the signature 'LLVMFuzzerTestOneInput' in .cpp or .cc files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://llvm.org/docs/LibFuzzer.html to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithCppLibFuzzer/impl.go b/probes/fuzzedWithCppLibFuzzer/impl.go new file mode 100644 index 00000000000..cbbb3146243 --- /dev/null +++ b/probes/fuzzedWithCppLibFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCppLibFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithCppLibFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "CppLibFuzzer") +} diff --git a/probes/fuzzedWithCppLibFuzzer/impl_test.go b/probes/fuzzedWithCppLibFuzzer/impl_test.go new file mode 100644 index 00000000000..48ea6bae6d9 --- /dev/null +++ b/probes/fuzzedWithCppLibFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithCppLibFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CppLibFuzzer", + }, + { + Name: "CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "CppLibFuzzer", + }, + { + Name: "not-CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-CppLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithJavaJazzerFuzzer/def.yml b/probes/fuzzedWithJavaJazzerFuzzer/def.yml new file mode 100644 index 00000000000..6b71fd132b1 --- /dev/null +++ b/probes/fuzzedWithJavaJazzerFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithJavaJazzerFuzzer +short: Check that the project is fuzzed using the Jazzer Java fuzzing framework +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of an import of 'com.code_intelligence.jazzer.api.FuzzedDataProvider' in .java files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://github.com/CodeIntelligenceTesting/jazzer to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://github.com/CodeIntelligenceTesting/jazzer](https://github.com/CodeIntelligenceTesting/jazzer) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithJavaJazzerFuzzer/impl.go b/probes/fuzzedWithJavaJazzerFuzzer/impl.go new file mode 100644 index 00000000000..4fb46f0b113 --- /dev/null +++ b/probes/fuzzedWithJavaJazzerFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithJavaJazzerFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithJavaJazzerFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "JavaJazzerFuzzer") +} diff --git a/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go b/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go new file mode 100644 index 00000000000..3dcb5bb3549 --- /dev/null +++ b/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithJavaJazzerFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "JavaJazzerFuzzer", + }, + { + Name: "JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "JavaJazzerFuzzer", + }, + { + Name: "not-JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-JavaJazzerFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithPythonAtheris/def.yml b/probes/fuzzedWithPythonAtheris/def.yml new file mode 100644 index 00000000000..73eb086897d --- /dev/null +++ b/probes/fuzzedWithPythonAtheris/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithPythonAtheris +short: Check that the project is fuzzed using Python Atheris fuzzing framework +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of 'import atheris' in .py files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://github.com/google/atheris to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://github.com/google/atheris](https://github.com/google/atheris) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithPythonAtheris/impl.go b/probes/fuzzedWithPythonAtheris/impl.go new file mode 100644 index 00000000000..eb26c2981a0 --- /dev/null +++ b/probes/fuzzedWithPythonAtheris/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithPythonAtheris + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithPythonAtheris" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "PythonAtherisFuzzer") +} diff --git a/probes/fuzzedWithPythonAtheris/impl_test.go b/probes/fuzzedWithPythonAtheris/impl_test.go new file mode 100644 index 00000000000..34c918bb102 --- /dev/null +++ b/probes/fuzzedWithPythonAtheris/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithPythonAtheris + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "PythonAtherisFuzzer", + }, + { + Name: "PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "PythonAtherisFuzzer", + }, + { + Name: "not-PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-PythonAtherisFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/fuzzedWithRustCargofuzz/def.yml b/probes/fuzzedWithRustCargofuzz/def.yml new file mode 100644 index 00000000000..cf7550584d1 --- /dev/null +++ b/probes/fuzzedWithRustCargofuzz/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithRustCargofuzz +short: Check that the project is fuzzed using Rust Cargo-fuzz framework. +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of 'libfuzzer_sys' in .rs files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://rust-fuzz.github.io/book/cargo-fuzz.html to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://rust-fuzz.github.io/book/cargo-fuzz.html](https://rust-fuzz.github.io/book/cargo-fuzz.html) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithRustCargofuzz/impl.go b/probes/fuzzedWithRustCargofuzz/impl.go new file mode 100644 index 00000000000..df531b4af25 --- /dev/null +++ b/probes/fuzzedWithRustCargofuzz/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithRustCargofuzz + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithRustCargofuzz" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "RustCargoFuzzer") +} diff --git a/probes/fuzzedWithRustCargofuzz/impl_test.go b/probes/fuzzedWithRustCargofuzz/impl_test.go new file mode 100644 index 00000000000..bfcfb052692 --- /dev/null +++ b/probes/fuzzedWithRustCargofuzz/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithRustCargofuzz + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "RustCargoFuzzer", + }, + { + Name: "RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "RustCargoFuzzer", + }, + { + Name: "not-RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-RustCargoFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} From cfeef59fb6587273f11f32b0bc3038d7e21dc966 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 12 Sep 2023 13:39:23 +0100 Subject: [PATCH 02/16] fix formatting Signed-off-by: David Korczynski --- checks/evaluation/fuzzing.go | 9 +++++---- checks/evaluation/fuzzing_test.go | 20 ++++++++++---------- checks/raw/fuzzing.go | 2 +- checks/raw/fuzzing_test.go | 4 ++-- probes/entries.go | 9 +++++---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go index cd1b014a0cc..26a468ae870 100644 --- a/checks/evaluation/fuzzing.go +++ b/checks/evaluation/fuzzing.go @@ -18,18 +18,19 @@ import ( "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" - "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" - "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" - "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" "github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" + "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" + "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" ) // Fuzzing applies the score policy for the Fuzzing check. diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go index 86b65a68fc3..208defe79ca 100644 --- a/checks/evaluation/fuzzing_test.go +++ b/checks/evaluation/fuzzing_test.go @@ -48,23 +48,23 @@ func TestFuzzing(t *testing.T) { Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithPythonAtheris", + Probe: "fuzzedWithPythonAtheris", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithCLibFuzzer", + Probe: "fuzzedWithCLibFuzzer", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithCppLibFuzzer", + Probe: "fuzzedWithCppLibFuzzer", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithRustCargofuzz", + Probe: "fuzzedWithRustCargofuzz", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithJavaJazzerFuzzer", + Probe: "fuzzedWithJavaJazzerFuzzer", Outcome: finding.OutcomeNegative, }, { @@ -110,23 +110,23 @@ func TestFuzzing(t *testing.T) { Outcome: finding.OutcomePositive, }, { - Probe: "fuzzedWithPythonAtheris", + Probe: "fuzzedWithPythonAtheris", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithCLibFuzzer", + Probe: "fuzzedWithCLibFuzzer", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithCppLibFuzzer", + Probe: "fuzzedWithCppLibFuzzer", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithRustCargofuzz", + Probe: "fuzzedWithRustCargofuzz", Outcome: finding.OutcomeNegative, }, { - Probe: "fuzzedWithJavaJazzerFuzzer", + Probe: "fuzzedWithJavaJazzerFuzzer", Outcome: finding.OutcomeNegative, }, { diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 9ba4fe13bc8..4eec38d8ea4 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -317,7 +317,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] if len(data.files) != 0 { dataFiles = append(dataFiles, data.files...) // This means no fuzz funcs matched for this language. - //return false, nil, nil + } } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index 2e413be6935..0a266d9a1f3 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -274,8 +274,8 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { if !ok && !tt.wantErr { t.Errorf("retrieve supported language error") } - fileMatchPattern := ""; - if (len(langSpecs.filePatterns) > 0) { + fileMatchPattern := "" + if len(langSpecs.filePatterns) > 0 { fileMatchPattern = langSpecs.filePatterns[0] } fileMatch, err := path.Match(fileMatchPattern, tt.fileName) diff --git a/probes/entries.go b/probes/entries.go index a6caa4b39e4..70d0a937dda 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -17,18 +17,19 @@ package probes import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" - "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" - "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" + "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" - "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" "github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" + "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" + "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" From b5a81d8c7983f032e0bfd95364148eae4d26f35e Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 12 Sep 2023 13:43:31 +0100 Subject: [PATCH 03/16] cleanup formatting Signed-off-by: David Korczynski --- checks/evaluation/fuzzing.go | 1 - checks/raw/fuzzing.go | 1 - probes/entries.go | 1 - 3 files changed, 3 deletions(-) diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go index 26a468ae870..830b49ec827 100644 --- a/checks/evaluation/fuzzing.go +++ b/checks/evaluation/fuzzing.go @@ -19,7 +19,6 @@ import ( sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" - "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 4eec38d8ea4..28576a2d8a7 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -317,7 +317,6 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] if len(data.files) != 0 { dataFiles = append(dataFiles, data.files...) // This means no fuzz funcs matched for this language. - } } diff --git a/probes/entries.go b/probes/entries.go index 70d0a937dda..de8fcc34e74 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -18,7 +18,6 @@ import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" - "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative" From 2c6472f93c3b803485f6608190e45293225f528b Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 12 Sep 2023 20:53:59 +0100 Subject: [PATCH 04/16] make skip testing optional Signed-off-by: David Korczynski --- checks/fileparser/listing.go | 8 +++++++- checks/fileparser/listing_test.go | 2 +- checks/raw/binary_artifact.go | 4 ++-- checks/raw/branch_protection.go | 2 +- checks/raw/dangerous_workflow.go | 2 +- checks/raw/fuzzing.go | 6 +++--- checks/raw/permissions.go | 2 +- checks/raw/pinned_dependencies.go | 10 +++++----- checks/raw/security_policy.go | 4 ++-- checks/sast.go | 4 ++-- 10 files changed, 25 insertions(+), 19 deletions(-) diff --git a/checks/fileparser/listing.go b/checks/fileparser/listing.go index 55cf58601b1..25bcda63902 100644 --- a/checks/fileparser/listing.go +++ b/checks/fileparser/listing.go @@ -73,9 +73,15 @@ type DoWhileTrueOnFileContent func(path string, content []byte, args ...interfac // Continues iterating along the matched files until onFileContent returns // either a false value or an error. func OnMatchingFileContentDo(repoClient clients.RepoClient, matchPathTo PathMatcher, - onFileContent DoWhileTrueOnFileContent, args ...interface{}, + onFileContent DoWhileTrueOnFileContent, skipTests bool, args ...interface{}, ) error { predicate := func(filepath string) (bool, error) { + if skipTests { + // Filter out test files. + if isTestdataFile(filepath) { + return false, nil + } + } // Filter out files based on path/names using the pattern. b, err := isMatchingPath(filepath, matchPathTo) if err != nil { diff --git a/checks/fileparser/listing_test.go b/checks/fileparser/listing_test.go index 7d3e7cbfacb..264bcc3b742 100644 --- a/checks/fileparser/listing_test.go +++ b/checks/fileparser/listing_test.go @@ -532,7 +532,7 @@ func TestOnMatchingFileContent(t *testing.T) { result := OnMatchingFileContentDo(mockRepo, PathMatcher{ Pattern: tt.shellPattern, CaseSensitive: tt.caseSensitive, - }, x) + }, x, true) if tt.wantErr && result == nil { t.Errorf("OnMatchingFileContentDo() = %v, want %v test name %v", result, tt.wantErr, tt.name) diff --git a/checks/raw/binary_artifact.go b/checks/raw/binary_artifact.go index 03c129b5727..8fc1259297a 100644 --- a/checks/raw/binary_artifact.go +++ b/checks/raw/binary_artifact.go @@ -54,7 +54,7 @@ func BinaryArtifacts(c clients.RepoClient) (checker.BinaryArtifactData, error) { err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: "*", CaseSensitive: false, - }, checkBinaryFileContent, &files) + }, checkBinaryFileContent, true, &files) if err != nil { return checker.BinaryArtifactData{}, fmt.Errorf("%w", err) } @@ -199,7 +199,7 @@ func gradleWrapperValidated(c clients.RepoClient) (bool, error) { err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, checkWorkflowValidatesGradleWrapper, &gradleWrapperValidatingWorkflowFile) + }, checkWorkflowValidatesGradleWrapper, true, &gradleWrapperValidatingWorkflowFile) if err != nil { return false, fmt.Errorf("%w", err) } diff --git a/checks/raw/branch_protection.go b/checks/raw/branch_protection.go index 6e58bd2f2f8..1f577461b7a 100644 --- a/checks/raw/branch_protection.go +++ b/checks/raw/branch_protection.go @@ -123,7 +123,7 @@ func collectCodeownersFiles(c clients.RepoClient, codeownersFiles *[]string) err return fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: "CODEOWNERS", CaseSensitive: true, - }, addCodeownersFile, codeownersFiles) + }, addCodeownersFile, true, codeownersFiles) } var addCodeownersFile fileparser.DoWhileTrueOnFileContent = func( diff --git a/checks/raw/dangerous_workflow.go b/checks/raw/dangerous_workflow.go index 1da80f0e2eb..9c92eaaea22 100644 --- a/checks/raw/dangerous_workflow.go +++ b/checks/raw/dangerous_workflow.go @@ -72,7 +72,7 @@ func DangerousWorkflow(c clients.RepoClient) (checker.DangerousWorkflowData, err err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, validateGitHubActionWorkflowPatterns, &data) + }, validateGitHubActionWorkflowPatterns, true, &data) return data, err } diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 28576a2d8a7..edd51bee9f1 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -243,7 +243,7 @@ func checkCFLite(c *checker.CheckRequest) (bool, error) { }, func(path string, content []byte, args ...interface{}) (bool, error) { result = fileparser.CheckFileContainsCommands(content, "#") return false, nil - }, nil) + }, false, nil) if e != nil { return result, fmt.Errorf("%w", e) } @@ -259,7 +259,7 @@ func checkOneFuzz(c *checker.CheckRequest) (bool, error) { }, func(path string, content []byte, args ...interface{}) (bool, error) { result = true return false, nil - }, nil) + }, false, nil) if e != nil { return result, fmt.Errorf("%w", e) } @@ -310,7 +310,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] CaseSensitive: false, } data.pattern = funcPattern - err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) + err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, false, &data) if err != nil { return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) } diff --git a/checks/raw/permissions.go b/checks/raw/permissions.go index c80e45fdeab..5c67a55d282 100644 --- a/checks/raw/permissions.go +++ b/checks/raw/permissions.go @@ -57,7 +57,7 @@ func TokenPermissions(c *checker.CheckRequest) (checker.TokenPermissionsData, er err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, validateGitHubActionTokenPermissions, &data) + }, validateGitHubActionTokenPermissions, true, &data) return data.results, err } diff --git a/checks/raw/pinned_dependencies.go b/checks/raw/pinned_dependencies.go index 0b93181443e..90446096fbf 100644 --- a/checks/raw/pinned_dependencies.go +++ b/checks/raw/pinned_dependencies.go @@ -74,7 +74,7 @@ func collectShellScriptInsecureDownloads(c *checker.CheckRequest, r *checker.Pin return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*", CaseSensitive: false, - }, validateShellScriptIsFreeOfInsecureDownloads, r) + }, validateShellScriptIsFreeOfInsecureDownloads, true, r) } var validateShellScriptIsFreeOfInsecureDownloads fileparser.DoWhileTrueOnFileContent = func( @@ -106,7 +106,7 @@ func collectDockerfileInsecureDownloads(c *checker.CheckRequest, r *checker.Pinn return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*Dockerfile*", CaseSensitive: false, - }, validateDockerfileInsecureDownloads, r) + }, validateDockerfileInsecureDownloads, true, r) } var validateDockerfileInsecureDownloads fileparser.DoWhileTrueOnFileContent = func( @@ -190,7 +190,7 @@ func collectDockerfilePinning(c *checker.CheckRequest, r *checker.PinningDepende return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*Dockerfile*", CaseSensitive: false, - }, validateDockerfilesPinning, r) + }, validateDockerfilesPinning, true, r) } var validateDockerfilesPinning fileparser.DoWhileTrueOnFileContent = func( @@ -321,7 +321,7 @@ func collectGitHubWorkflowScriptInsecureDownloads(c *checker.CheckRequest, r *ch return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, validateGitHubWorkflowIsFreeOfInsecureDownloads, r) + }, validateGitHubWorkflowIsFreeOfInsecureDownloads, true, r) } // validateGitHubWorkflowIsFreeOfInsecureDownloads checks if the workflow file downloads dependencies that are unpinned. @@ -409,7 +409,7 @@ func collectGitHubActionsWorkflowPinning(c *checker.CheckRequest, r *checker.Pin return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: true, - }, validateGitHubActionWorkflow, r) + }, validateGitHubActionWorkflow, true, r) } // validateGitHubActionWorkflow checks if the workflow file contains unpinned actions. Returns true if the check diff --git a/checks/raw/security_policy.go b/checks/raw/security_policy.go index c4da3f29a39..a5f2fbbe933 100644 --- a/checks/raw/security_policy.go +++ b/checks/raw/security_policy.go @@ -50,7 +50,7 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: data.files[idx].File.Path, CaseSensitive: false, - }, checkSecurityPolicyFileContent, &data.files[idx].File, &data.files[idx].Information) + }, checkSecurityPolicyFileContent, true, &data.files[idx].File, &data.files[idx].Information) if err != nil { return checker.SecurityPolicyData{}, err } @@ -89,7 +89,7 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) err := fileparser.OnMatchingFileContentDo(client, fileparser.PathMatcher{ Pattern: filePattern, CaseSensitive: false, - }, checkSecurityPolicyFileContent, &data.files[idx].File, &data.files[idx].Information) + }, checkSecurityPolicyFileContent, true, &data.files[idx].File, &data.files[idx].Information) if err != nil { return checker.SecurityPolicyData{}, err } diff --git a/checks/sast.go b/checks/sast.go index d7ed2991c35..e2f3523550c 100644 --- a/checks/sast.go +++ b/checks/sast.go @@ -204,7 +204,7 @@ func codeQLInCheckDefinitions(c *checker.CheckRequest) (int, error) { err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, searchGitHubActionWorkflowCodeQL, &workflowPaths) + }, searchGitHubActionWorkflowCodeQL, true, &workflowPaths) if err != nil { return checker.InconclusiveResultScore, err } @@ -286,7 +286,7 @@ func sonarEnabled(c *checker.CheckRequest) (int, error) { err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*", CaseSensitive: false, - }, validateSonarConfig, &config) + }, validateSonarConfig, true, &config) if err != nil { return checker.InconclusiveResultScore, err } From 8e6166379e144c90a59bf6f551233f50c9638aa7 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 19 Sep 2023 11:45:40 +0100 Subject: [PATCH 05/16] address reviews Signed-off-by: David Korczynski --- checks/raw/fuzzing.go | 32 ++++++++++++-------------------- checks/raw/fuzzing_test.go | 2 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index edd51bee9f1..10499c71c15 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -85,7 +85,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // // This is not an exhaustive list. clients.Haskell: { - filePatterns: []string{"*.hs"}, + filePatterns: []string{"*.hs", "*.lhs"}, // Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck, // or their indirect imports through the higher-level Hspec or Tasty testing frameworks. funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, @@ -120,42 +120,36 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ }, clients.Python: { filePatterns: []string{"*.py"}, - // Look for direct imports of fast-check. - funcPattern: `import atheris`, - Name: fuzzerPythonAtheris, + funcPattern: `import atheris`, + Name: fuzzerPythonAtheris, Desc: asPointer( "Python fuzzing by way of Atheris"), }, clients.C: { filePatterns: []string{"*.c"}, - // Look for direct imports of fast-check. - funcPattern: `LLVMFuzzerTestOneInput`, - Name: fuzzerCLibFuzzer, + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerCLibFuzzer, Desc: asPointer( "Fuzzed with C LibFuzzer"), }, clients.Cpp: { filePatterns: []string{"*.cc", "*.cpp"}, - // Look for direct imports of fast-check. - funcPattern: `LLVMFuzzerTestOneInput`, - Name: fuzzerCppLibFuzzer, + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerCppLibFuzzer, Desc: asPointer( "Fuzzed with cpp LibFuzzer"), }, clients.Rust: { filePatterns: []string{"*.rs"}, - // Look for direct imports of fast-check. - funcPattern: `libfuzzer_sys`, - Name: fuzzerRustCargoFuzz, + funcPattern: `libfuzzer_sys`, + Name: fuzzerRustCargoFuzz, Desc: asPointer( "Fuzzed with Cargo-fuzz"), }, clients.Java: { filePatterns: []string{"*.java"}, - // Look for direct imports of fast-check. - funcPattern: `com.code_intelligence.jazzer.api.FuzzedDataProvider;`, - // funcPattern: `jazze`, - Name: fuzzerJavaJazzerFuzzer, + funcPattern: `com.code_intelligence.jazzer.api.FuzzedDataProvider;`, + Name: fuzzerJavaJazzerFuzzer, Desc: asPointer( "Fuzzed with Jazzer fuzzer"), }, @@ -217,7 +211,6 @@ func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) { prominentLangs := getProminentLanguages(langs) for _, lang := range prominentLangs { usingFuzzFunc, files, e := checkFuzzFunc(c, lang) - fmt.Print(files) if e != nil { return checker.FuzzingData{}, fmt.Errorf("%w", e) } @@ -302,8 +295,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] // We use the file pattern in the matcher to match the test files, // and put the func pattern in var data to match file contents (func names). filePatterns, funcPattern := pattern.filePatterns, pattern.funcPattern - dataFiles := []checker.File{} - + var dataFiles []checker.File for _, filePattern := range filePatterns { matcher := fileparser.PathMatcher{ Pattern: filePattern, diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index 0a266d9a1f3..68ae3cb0356 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -274,7 +274,7 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { if !ok && !tt.wantErr { t.Errorf("retrieve supported language error") } - fileMatchPattern := "" + var fileMatchPattern string if len(langSpecs.filePatterns) > 0 { fileMatchPattern = langSpecs.filePatterns[0] } From e8e3e59be6b69517d7d61129e1ad76938e3d1536 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 19 Sep 2023 12:02:33 +0100 Subject: [PATCH 06/16] add todo Signed-off-by: David Korczynski --- checks/evaluation/fuzzing.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go index 830b49ec827..d694c0d6ac6 100644 --- a/checks/evaluation/fuzzing.go +++ b/checks/evaluation/fuzzing.go @@ -51,6 +51,8 @@ func Fuzzing(name string, fuzzedWithPropertyBasedJavascript.Probe, fuzzedWithPropertyBasedTypescript.Probe, } + // TODO: other packages to consider: + // - github.com/google/fuzztest if !finding.UniqueProbesEqual(findings, expectedProbes) { e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") From 709c8e2b3050be31b3ce58efbaf1fcc620ba5daa Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 19 Sep 2023 12:24:58 +0100 Subject: [PATCH 07/16] nit Signed-off-by: David Korczynski --- checks/evaluation/fuzzing_test.go | 150 ++++++++---------------------- checks/fuzzing_test.go | 2 +- 2 files changed, 42 insertions(+), 110 deletions(-) diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go index 35738d7e95d..7f3cef9c199 100644 --- a/checks/evaluation/fuzzing_test.go +++ b/checks/evaluation/fuzzing_test.go @@ -31,59 +31,6 @@ func TestFuzzing(t *testing.T) { }{ { name: "Fuzzing - no fuzzing", -<<<<<<< HEAD - args: args{ - name: "Fuzzing", - findings: []finding.Finding{ - { - Probe: "fuzzedWithClusterFuzzLite", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithGoNative", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPythonAtheris", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithCLibFuzzer", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithCppLibFuzzer", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithRustCargofuzz", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithJavaJazzerFuzzer", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithOneFuzz", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithOSSFuzz", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPropertyBasedHaskell", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPropertyBasedJavascript", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPropertyBasedTypescript", - Outcome: finding.OutcomeNegative, - }, -======= findings: []finding.Finding{ { Probe: "fuzzedWithClusterFuzzLite", @@ -93,6 +40,26 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithGoNative", Outcome: finding.OutcomeNegative, }, + { + Probe: "fuzzedWithPythonAtheris", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCppLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithRustCargofuzz", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithJavaJazzerFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithOneFuzz", Outcome: finding.OutcomeNegative, @@ -112,69 +79,15 @@ func TestFuzzing(t *testing.T) { { Probe: "fuzzedWithPropertyBasedTypescript", Outcome: finding.OutcomeNegative, ->>>>>>> ac13ac7c016d82cb962ea7b0858656e875cb0829 }, }, result: scut.TestReturn{ Score: checker.MinResultScore, - NumberOfWarn: 7, + NumberOfWarn: 12, }, }, { name: "Fuzzing - fuzzing GoNative", -<<<<<<< HEAD - args: args{ - name: "Fuzzing", - findings: []finding.Finding{ - { - Probe: "fuzzedWithClusterFuzzLite", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithGoNative", - Outcome: finding.OutcomePositive, - }, - { - Probe: "fuzzedWithPythonAtheris", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithCLibFuzzer", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithCppLibFuzzer", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithRustCargofuzz", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithJavaJazzerFuzzer", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithOneFuzz", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithOSSFuzz", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPropertyBasedHaskell", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPropertyBasedJavascript", - Outcome: finding.OutcomeNegative, - }, - { - Probe: "fuzzedWithPropertyBasedTypescript", - Outcome: finding.OutcomeNegative, - }, -======= findings: []finding.Finding{ { Probe: "fuzzedWithClusterFuzzLite", @@ -184,6 +97,26 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithGoNative", Outcome: finding.OutcomePositive, }, + { + Probe: "fuzzedWithPythonAtheris", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithCppLibFuzzer", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithRustCargofuzz", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "fuzzedWithJavaJazzerFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithOneFuzz", Outcome: finding.OutcomeNegative, @@ -203,7 +136,6 @@ func TestFuzzing(t *testing.T) { { Probe: "fuzzedWithPropertyBasedTypescript", Outcome: finding.OutcomeNegative, ->>>>>>> ac13ac7c016d82cb962ea7b0858656e875cb0829 }, }, result: scut.TestReturn{ diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go index a4dcb40fe4d..6b031fa5c30 100644 --- a/checks/fuzzing_test.go +++ b/checks/fuzzing_test.go @@ -76,7 +76,7 @@ func TestFuzzing(t *testing.T) { }, wantErr: false, expected: scut.TestReturn{ - NumberOfWarn: 11, + NumberOfWarn: 0, NumberOfDebug: 0, NumberOfInfo: 1, Score: 10, From a13d68bb777002997c2d1a665161e04cc5f678ba Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 19 Sep 2023 13:41:52 +0100 Subject: [PATCH 08/16] nit Signed-off-by: David Korczynski --- checks/raw/fuzzing.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 10499c71c15..b1573bd87d1 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -306,10 +306,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] if err != nil { return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) } - if len(data.files) != 0 { - dataFiles = append(dataFiles, data.files...) - // This means no fuzz funcs matched for this language. - } + dataFiles = append(dataFiles, data.files...) } if len(dataFiles) == 0 { From fd2138e7626b9491f781f286f9fcdfcc835da828 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 19 Sep 2023 16:58:58 +0100 Subject: [PATCH 09/16] add swift fuzzing probe Signed-off-by: David Korczynski --- checks/evaluation/fuzzing.go | 2 + checks/evaluation/fuzzing_test.go | 10 +- checks/fuzzing_test.go | 6 +- checks/raw/fuzzing.go | 8 ++ probes/entries.go | 2 + probes/fuzzedWithSwiftLibFuzzer/def.yml | 32 +++++ probes/fuzzedWithSwiftLibFuzzer/impl.go | 39 +++++ probes/fuzzedWithSwiftLibFuzzer/impl_test.go | 144 +++++++++++++++++++ 8 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 probes/fuzzedWithSwiftLibFuzzer/def.yml create mode 100644 probes/fuzzedWithSwiftLibFuzzer/impl.go create mode 100644 probes/fuzzedWithSwiftLibFuzzer/impl_test.go diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go index 6922f9069ae..132dda90df3 100644 --- a/checks/evaluation/fuzzing.go +++ b/checks/evaluation/fuzzing.go @@ -30,6 +30,7 @@ import ( "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer" ) // Fuzzing applies the score policy for the Fuzzing check. @@ -44,6 +45,7 @@ func Fuzzing(name string, fuzzedWithCLibFuzzer.Probe, fuzzedWithCppLibFuzzer.Probe, fuzzedWithRustCargofuzz.Probe, + fuzzedWithSwiftLibFuzzer.Probe, fuzzedWithJavaJazzerFuzzer.Probe, fuzzedWithOneFuzz.Probe, fuzzedWithOSSFuzz.Probe, diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go index 7f3cef9c199..85018eadb25 100644 --- a/checks/evaluation/fuzzing_test.go +++ b/checks/evaluation/fuzzing_test.go @@ -56,6 +56,10 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithRustCargofuzz", Outcome: finding.OutcomeNegative, }, + { + Probe: "fuzzedWithSwiftLibFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithJavaJazzerFuzzer", Outcome: finding.OutcomeNegative, @@ -83,7 +87,7 @@ func TestFuzzing(t *testing.T) { }, result: scut.TestReturn{ Score: checker.MinResultScore, - NumberOfWarn: 12, + NumberOfWarn: 13, }, }, { @@ -113,6 +117,10 @@ func TestFuzzing(t *testing.T) { Probe: "fuzzedWithRustCargofuzz", Outcome: finding.OutcomeNegative, }, + { + Probe: "fuzzedWithSwiftLibFuzzer", + Outcome: finding.OutcomeNegative, + }, { Probe: "fuzzedWithJavaJazzerFuzzer", Outcome: finding.OutcomeNegative, diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go index 6b031fa5c30..dd2f3a17994 100644 --- a/checks/fuzzing_test.go +++ b/checks/fuzzing_test.go @@ -53,7 +53,7 @@ func TestFuzzing(t *testing.T) { wantErr: false, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 12, + NumberOfWarn: 13, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, @@ -110,7 +110,7 @@ func TestFuzzing(t *testing.T) { wantFuzzErr: false, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 12, + NumberOfWarn: 13, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, @@ -121,7 +121,7 @@ func TestFuzzing(t *testing.T) { wantFuzzErr: true, expected: scut.TestReturn{ Error: nil, - NumberOfWarn: 12, + NumberOfWarn: 13, NumberOfDebug: 0, NumberOfInfo: 0, Score: 0, diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index b1573bd87d1..a5ebe2b703a 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -38,6 +38,7 @@ const ( fuzzerPythonAtheris = "PythonAtherisFuzzer" fuzzerCLibFuzzer = "CLibFuzzer" fuzzerCppLibFuzzer = "CppLibFuzzer" + fuzzerSwiftLibFuzzer = "SwiftLibFuzzer" fuzzerRustCargoFuzz = "RustCargoFuzzer" fuzzerJavaJazzerFuzzer = "JavaJazzerFuzzer" // TODO: add more fuzzing check supports. @@ -153,6 +154,13 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ Desc: asPointer( "Fuzzed with Jazzer fuzzer"), }, + clients.Swift: { + filePatterns: []string{"*.swift"}, + funcPattern: `LLVMFuzzerTestOneInput`, + Name: fuzzerSwiftLibFuzzer, + Desc: asPointer( + "Fuzzed with Swift LibFuzzer"), + }, // TODO: add more language-specific fuzz patterns & configs. } diff --git a/probes/entries.go b/probes/entries.go index de8fcc34e74..4d96d051f65 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -29,6 +29,7 @@ import ( "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris" "github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz" + "github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" @@ -68,6 +69,7 @@ var ( fuzzedWithPythonAtheris.Run, fuzzedWithCLibFuzzer.Run, fuzzedWithCppLibFuzzer.Run, + fuzzedWithSwiftLibFuzzer.Run, fuzzedWithRustCargofuzz.Run, fuzzedWithJavaJazzerFuzzer.Run, fuzzedWithClusterFuzzLite.Run, diff --git a/probes/fuzzedWithSwiftLibFuzzer/def.yml b/probes/fuzzedWithSwiftLibFuzzer/def.yml new file mode 100644 index 00000000000..288e5ee91f1 --- /dev/null +++ b/probes/fuzzedWithSwiftLibFuzzer/def.yml @@ -0,0 +1,32 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: fuzzedWithSwiftLibFuzzer +short: Check that the project is fuzzed using the LibFuzzer for Swift code. +motivation: > + Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. + Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws. +implementation: > + The implementation checks whether fo the presence of an import of '@_cdecl("LLVMFuzzerTestOneInput")' in .swift files. +outcome: + - If fuzzing functions are found, each finding is returned with OutcomePositive (1). + - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - Follow the steps in https://google.github.io/oss-fuzz/getting-started/new-project-guide/swift-lang/ to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. + markdown: + - Follow the steps in [https://google.github.io/oss-fuzz/getting-started/new-project-guide/swift-lang/](https://google.github.io/oss-fuzz/getting-started/new-project-guide/swift-lang/) to enable fuzzing on your project. + - Over time, try to add fuzzing for more functionalities of your project. diff --git a/probes/fuzzedWithSwiftLibFuzzer/impl.go b/probes/fuzzedWithSwiftLibFuzzer/impl.go new file mode 100644 index 00000000000..7236d18e870 --- /dev/null +++ b/probes/fuzzedWithSwiftLibFuzzer/impl.go @@ -0,0 +1,39 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithSwiftLibFuzzer + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/fuzzing" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "fuzzedWithSwiftLibFuzzer" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + //nolint:wrapcheck + return fuzzing.Run(raw, fs, Probe, "SwiftLibFuzzer") +} diff --git a/probes/fuzzedWithSwiftLibFuzzer/impl_test.go b/probes/fuzzedWithSwiftLibFuzzer/impl_test.go new file mode 100644 index 00000000000..5f3be6a4b65 --- /dev/null +++ b/probes/fuzzedWithSwiftLibFuzzer/impl_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package fuzzedWithSwiftLibFuzzer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "fuzzer present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present twice", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "SwiftLibFuzzer", + }, + { + Name: "SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "fuzzer present and other present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "SwiftLibFuzzer", + }, + { + Name: "not-SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "fuzzer not present", + raw: &checker.RawResults{ + FuzzingResults: checker.FuzzingData{ + Fuzzers: []checker.Tool{ + { + Name: "not-SwiftLibFuzzer", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "no fuzzer", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: uerror.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function. + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} From 3081c02fcd366e1e11f0617589a955fa95f8e522 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Thu, 28 Sep 2023 17:39:30 +0100 Subject: [PATCH 10/16] avoid changing OnMatchingFileContentDo Signed-off-by: David Korczynski --- checks/fileparser/listing.go | 10 ++++------ checks/fileparser/listing_test.go | 2 +- checks/raw/binary_artifact.go | 4 ++-- checks/raw/branch_protection.go | 2 +- checks/raw/dangerous_workflow.go | 2 +- checks/raw/fuzzing.go | 6 +++--- checks/raw/permissions.go | 2 +- checks/raw/pinned_dependencies.go | 10 +++++----- checks/raw/security_policy.go | 4 ++-- checks/sast.go | 4 ++-- 10 files changed, 22 insertions(+), 24 deletions(-) diff --git a/checks/fileparser/listing.go b/checks/fileparser/listing.go index 25bcda63902..3efb8208991 100644 --- a/checks/fileparser/listing.go +++ b/checks/fileparser/listing.go @@ -73,14 +73,12 @@ type DoWhileTrueOnFileContent func(path string, content []byte, args ...interfac // Continues iterating along the matched files until onFileContent returns // either a false value or an error. func OnMatchingFileContentDo(repoClient clients.RepoClient, matchPathTo PathMatcher, - onFileContent DoWhileTrueOnFileContent, skipTests bool, args ...interface{}, + onFileContent DoWhileTrueOnFileContent, args ...interface{}, ) error { predicate := func(filepath string) (bool, error) { - if skipTests { - // Filter out test files. - if isTestdataFile(filepath) { - return false, nil - } + // Filter out test files. + if isTestdataFile(filepath) { + return false, nil } // Filter out files based on path/names using the pattern. b, err := isMatchingPath(filepath, matchPathTo) diff --git a/checks/fileparser/listing_test.go b/checks/fileparser/listing_test.go index 264bcc3b742..7d3e7cbfacb 100644 --- a/checks/fileparser/listing_test.go +++ b/checks/fileparser/listing_test.go @@ -532,7 +532,7 @@ func TestOnMatchingFileContent(t *testing.T) { result := OnMatchingFileContentDo(mockRepo, PathMatcher{ Pattern: tt.shellPattern, CaseSensitive: tt.caseSensitive, - }, x, true) + }, x) if tt.wantErr && result == nil { t.Errorf("OnMatchingFileContentDo() = %v, want %v test name %v", result, tt.wantErr, tt.name) diff --git a/checks/raw/binary_artifact.go b/checks/raw/binary_artifact.go index 8fc1259297a..03c129b5727 100644 --- a/checks/raw/binary_artifact.go +++ b/checks/raw/binary_artifact.go @@ -54,7 +54,7 @@ func BinaryArtifacts(c clients.RepoClient) (checker.BinaryArtifactData, error) { err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: "*", CaseSensitive: false, - }, checkBinaryFileContent, true, &files) + }, checkBinaryFileContent, &files) if err != nil { return checker.BinaryArtifactData{}, fmt.Errorf("%w", err) } @@ -199,7 +199,7 @@ func gradleWrapperValidated(c clients.RepoClient) (bool, error) { err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, checkWorkflowValidatesGradleWrapper, true, &gradleWrapperValidatingWorkflowFile) + }, checkWorkflowValidatesGradleWrapper, &gradleWrapperValidatingWorkflowFile) if err != nil { return false, fmt.Errorf("%w", err) } diff --git a/checks/raw/branch_protection.go b/checks/raw/branch_protection.go index 1f577461b7a..6e58bd2f2f8 100644 --- a/checks/raw/branch_protection.go +++ b/checks/raw/branch_protection.go @@ -123,7 +123,7 @@ func collectCodeownersFiles(c clients.RepoClient, codeownersFiles *[]string) err return fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: "CODEOWNERS", CaseSensitive: true, - }, addCodeownersFile, true, codeownersFiles) + }, addCodeownersFile, codeownersFiles) } var addCodeownersFile fileparser.DoWhileTrueOnFileContent = func( diff --git a/checks/raw/dangerous_workflow.go b/checks/raw/dangerous_workflow.go index 9c92eaaea22..1da80f0e2eb 100644 --- a/checks/raw/dangerous_workflow.go +++ b/checks/raw/dangerous_workflow.go @@ -72,7 +72,7 @@ func DangerousWorkflow(c clients.RepoClient) (checker.DangerousWorkflowData, err err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, validateGitHubActionWorkflowPatterns, true, &data) + }, validateGitHubActionWorkflowPatterns, &data) return data, err } diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index a5ebe2b703a..6f9d4f1c9f7 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -244,7 +244,7 @@ func checkCFLite(c *checker.CheckRequest) (bool, error) { }, func(path string, content []byte, args ...interface{}) (bool, error) { result = fileparser.CheckFileContainsCommands(content, "#") return false, nil - }, false, nil) + }, nil) if e != nil { return result, fmt.Errorf("%w", e) } @@ -260,7 +260,7 @@ func checkOneFuzz(c *checker.CheckRequest) (bool, error) { }, func(path string, content []byte, args ...interface{}) (bool, error) { result = true return false, nil - }, false, nil) + }, nil) if e != nil { return result, fmt.Errorf("%w", e) } @@ -310,7 +310,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] CaseSensitive: false, } data.pattern = funcPattern - err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, false, &data) + err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) if err != nil { return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) } diff --git a/checks/raw/permissions.go b/checks/raw/permissions.go index 5c67a55d282..c80e45fdeab 100644 --- a/checks/raw/permissions.go +++ b/checks/raw/permissions.go @@ -57,7 +57,7 @@ func TokenPermissions(c *checker.CheckRequest) (checker.TokenPermissionsData, er err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, validateGitHubActionTokenPermissions, true, &data) + }, validateGitHubActionTokenPermissions, &data) return data.results, err } diff --git a/checks/raw/pinned_dependencies.go b/checks/raw/pinned_dependencies.go index 90446096fbf..0b93181443e 100644 --- a/checks/raw/pinned_dependencies.go +++ b/checks/raw/pinned_dependencies.go @@ -74,7 +74,7 @@ func collectShellScriptInsecureDownloads(c *checker.CheckRequest, r *checker.Pin return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*", CaseSensitive: false, - }, validateShellScriptIsFreeOfInsecureDownloads, true, r) + }, validateShellScriptIsFreeOfInsecureDownloads, r) } var validateShellScriptIsFreeOfInsecureDownloads fileparser.DoWhileTrueOnFileContent = func( @@ -106,7 +106,7 @@ func collectDockerfileInsecureDownloads(c *checker.CheckRequest, r *checker.Pinn return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*Dockerfile*", CaseSensitive: false, - }, validateDockerfileInsecureDownloads, true, r) + }, validateDockerfileInsecureDownloads, r) } var validateDockerfileInsecureDownloads fileparser.DoWhileTrueOnFileContent = func( @@ -190,7 +190,7 @@ func collectDockerfilePinning(c *checker.CheckRequest, r *checker.PinningDepende return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*Dockerfile*", CaseSensitive: false, - }, validateDockerfilesPinning, true, r) + }, validateDockerfilesPinning, r) } var validateDockerfilesPinning fileparser.DoWhileTrueOnFileContent = func( @@ -321,7 +321,7 @@ func collectGitHubWorkflowScriptInsecureDownloads(c *checker.CheckRequest, r *ch return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, validateGitHubWorkflowIsFreeOfInsecureDownloads, true, r) + }, validateGitHubWorkflowIsFreeOfInsecureDownloads, r) } // validateGitHubWorkflowIsFreeOfInsecureDownloads checks if the workflow file downloads dependencies that are unpinned. @@ -409,7 +409,7 @@ func collectGitHubActionsWorkflowPinning(c *checker.CheckRequest, r *checker.Pin return fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: true, - }, validateGitHubActionWorkflow, true, r) + }, validateGitHubActionWorkflow, r) } // validateGitHubActionWorkflow checks if the workflow file contains unpinned actions. Returns true if the check diff --git a/checks/raw/security_policy.go b/checks/raw/security_policy.go index a5f2fbbe933..c4da3f29a39 100644 --- a/checks/raw/security_policy.go +++ b/checks/raw/security_policy.go @@ -50,7 +50,7 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: data.files[idx].File.Path, CaseSensitive: false, - }, checkSecurityPolicyFileContent, true, &data.files[idx].File, &data.files[idx].Information) + }, checkSecurityPolicyFileContent, &data.files[idx].File, &data.files[idx].Information) if err != nil { return checker.SecurityPolicyData{}, err } @@ -89,7 +89,7 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) err := fileparser.OnMatchingFileContentDo(client, fileparser.PathMatcher{ Pattern: filePattern, CaseSensitive: false, - }, checkSecurityPolicyFileContent, true, &data.files[idx].File, &data.files[idx].Information) + }, checkSecurityPolicyFileContent, &data.files[idx].File, &data.files[idx].Information) if err != nil { return checker.SecurityPolicyData{}, err } diff --git a/checks/sast.go b/checks/sast.go index e2f3523550c..d7ed2991c35 100644 --- a/checks/sast.go +++ b/checks/sast.go @@ -204,7 +204,7 @@ func codeQLInCheckDefinitions(c *checker.CheckRequest) (int, error) { err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".github/workflows/*", CaseSensitive: false, - }, searchGitHubActionWorkflowCodeQL, true, &workflowPaths) + }, searchGitHubActionWorkflowCodeQL, &workflowPaths) if err != nil { return checker.InconclusiveResultScore, err } @@ -286,7 +286,7 @@ func sonarEnabled(c *checker.CheckRequest) (int, error) { err := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "*", CaseSensitive: false, - }, validateSonarConfig, true, &config) + }, validateSonarConfig, &config) if err != nil { return checker.InconclusiveResultScore, err } From e2a24017b51c48cccca9051b0fecb87bced5e329 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Thu, 28 Sep 2023 18:12:28 +0100 Subject: [PATCH 11/16] nit Signed-off-by: David Korczynski --- checks/fileparser/listing.go | 46 ++++++++++++++++++++++++++++++++++++ checks/raw/fuzzing.go | 10 ++++---- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/checks/fileparser/listing.go b/checks/fileparser/listing.go index 3efb8208991..cb70b52fefd 100644 --- a/checks/fileparser/listing.go +++ b/checks/fileparser/listing.go @@ -112,6 +112,52 @@ func OnMatchingFileContentDo(repoClient clients.RepoClient, matchPathTo PathMatc return nil } +func OnMatchingFileContentWithTestDataDo(repoClient clients.RepoClient, matchPathTo PathMatcher, + onFileContent DoWhileTrueOnFileContent, args ...interface{}, +) error { + predicate := func(filepath string) (bool, error) { + // Filter out test files. + if isTestdataFile(filepath) { + return false, nil + } + // Filter out files based on path/names using the pattern. + b, err := isMatchingPath(filepath, matchPathTo) + if err != nil { + return false, err + } + return b, nil + } + return OnMatchDo(repoClient, onFileContent, predicate, args) +} + + +func OnMatchDo(repoClient clients.RepoClient, + onFileContent DoWhileTrueOnFileContent, predicate func(filepath string) (bool, error), args ...interface{}, +) error { + matchedFiles, err := repoClient.ListFiles(predicate) + if err != nil { + return fmt.Errorf("error during ListFiles: %w", err) + } + + for _, file := range matchedFiles { + content, err := repoClient.GetFileContent(file) + if err != nil { + return fmt.Errorf("error during GetFileContent: %w", err) + } + + continueIter, err := onFileContent(file, content, args...) + if err != nil { + return err + } + + if !continueIter { + break + } + } + + return nil +} + // DoWhileTrueOnFilename takes a filename and optional variadic args and returns // true if the next filename should continue to be processed. type DoWhileTrueOnFilename func(path string, args ...interface{}) (bool, error) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 6f9d4f1c9f7..0b5410ddcf3 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -238,7 +238,7 @@ func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) { func checkCFLite(c *checker.CheckRequest) (bool, error) { result := false - e := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ + e := fileparser.OnMatchingFileContentWithTestDataDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".clusterfuzzlite/Dockerfile", CaseSensitive: true, }, func(path string, content []byte, args ...interface{}) (bool, error) { @@ -254,7 +254,7 @@ func checkCFLite(c *checker.CheckRequest) (bool, error) { func checkOneFuzz(c *checker.CheckRequest) (bool, error) { result := false - e := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ + e := fileparser.OnMatchingFileContentWithTestDataDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "^\\.onefuzz$", CaseSensitive: true, }, func(path string, content []byte, args ...interface{}) (bool, error) { @@ -310,9 +310,9 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] CaseSensitive: false, } data.pattern = funcPattern - err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) + err := fileparser.OnMatchingFileContentWithTestDataDo(c.RepoClient, matcher, getFuzzFunc, &data) if err != nil { - return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) + return false, nil, fmt.Errorf("error when OnMatchingFileContentWithTestDataDo: %w", err) } dataFiles = append(dataFiles, data.files...) } @@ -324,7 +324,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] return true, dataFiles, nil } -// This is the callback func for interface OnMatchingFileContentDo +// This is the callback func for interface OnMatchingFileContentWithTestDataDo // used for matching fuzz functions in the file content, // and return a list of files (or nil for not found). var getFuzzFunc fileparser.DoWhileTrueOnFileContent = func( From 383ef043e89579c41596babaf793a91d7d603140 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 3 Oct 2023 17:09:10 +0100 Subject: [PATCH 12/16] undo matching file content extension Signed-off-by: David Korczynski --- checks/fileparser/listing.go | 46 ------------------------------------ checks/raw/fuzzing.go | 10 ++++---- 2 files changed, 5 insertions(+), 51 deletions(-) diff --git a/checks/fileparser/listing.go b/checks/fileparser/listing.go index cb70b52fefd..3efb8208991 100644 --- a/checks/fileparser/listing.go +++ b/checks/fileparser/listing.go @@ -112,52 +112,6 @@ func OnMatchingFileContentDo(repoClient clients.RepoClient, matchPathTo PathMatc return nil } -func OnMatchingFileContentWithTestDataDo(repoClient clients.RepoClient, matchPathTo PathMatcher, - onFileContent DoWhileTrueOnFileContent, args ...interface{}, -) error { - predicate := func(filepath string) (bool, error) { - // Filter out test files. - if isTestdataFile(filepath) { - return false, nil - } - // Filter out files based on path/names using the pattern. - b, err := isMatchingPath(filepath, matchPathTo) - if err != nil { - return false, err - } - return b, nil - } - return OnMatchDo(repoClient, onFileContent, predicate, args) -} - - -func OnMatchDo(repoClient clients.RepoClient, - onFileContent DoWhileTrueOnFileContent, predicate func(filepath string) (bool, error), args ...interface{}, -) error { - matchedFiles, err := repoClient.ListFiles(predicate) - if err != nil { - return fmt.Errorf("error during ListFiles: %w", err) - } - - for _, file := range matchedFiles { - content, err := repoClient.GetFileContent(file) - if err != nil { - return fmt.Errorf("error during GetFileContent: %w", err) - } - - continueIter, err := onFileContent(file, content, args...) - if err != nil { - return err - } - - if !continueIter { - break - } - } - - return nil -} - // DoWhileTrueOnFilename takes a filename and optional variadic args and returns // true if the next filename should continue to be processed. type DoWhileTrueOnFilename func(path string, args ...interface{}) (bool, error) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 0b5410ddcf3..6f9d4f1c9f7 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -238,7 +238,7 @@ func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) { func checkCFLite(c *checker.CheckRequest) (bool, error) { result := false - e := fileparser.OnMatchingFileContentWithTestDataDo(c.RepoClient, fileparser.PathMatcher{ + e := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: ".clusterfuzzlite/Dockerfile", CaseSensitive: true, }, func(path string, content []byte, args ...interface{}) (bool, error) { @@ -254,7 +254,7 @@ func checkCFLite(c *checker.CheckRequest) (bool, error) { func checkOneFuzz(c *checker.CheckRequest) (bool, error) { result := false - e := fileparser.OnMatchingFileContentWithTestDataDo(c.RepoClient, fileparser.PathMatcher{ + e := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{ Pattern: "^\\.onefuzz$", CaseSensitive: true, }, func(path string, content []byte, args ...interface{}) (bool, error) { @@ -310,9 +310,9 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] CaseSensitive: false, } data.pattern = funcPattern - err := fileparser.OnMatchingFileContentWithTestDataDo(c.RepoClient, matcher, getFuzzFunc, &data) + err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data) if err != nil { - return false, nil, fmt.Errorf("error when OnMatchingFileContentWithTestDataDo: %w", err) + return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err) } dataFiles = append(dataFiles, data.files...) } @@ -324,7 +324,7 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, [] return true, dataFiles, nil } -// This is the callback func for interface OnMatchingFileContentWithTestDataDo +// This is the callback func for interface OnMatchingFileContentDo // used for matching fuzz functions in the file content, // and return a list of files (or nil for not found). var getFuzzFunc fileparser.DoWhileTrueOnFileContent = func( From b04351e99e74d88d48a65d4bc038d25868327645 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Thu, 5 Oct 2023 15:54:39 +0100 Subject: [PATCH 13/16] nit: fix constant Signed-off-by: David Korczynski --- checks/raw/fuzzing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 6f9d4f1c9f7..8354b72ccf0 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -42,6 +42,7 @@ const ( fuzzerRustCargoFuzz = "RustCargoFuzzer" fuzzerJavaJazzerFuzzer = "JavaJazzerFuzzer" // TODO: add more fuzzing check supports. + fourPointZero = 4.0 ) type filesWithPatternStr struct { @@ -374,7 +375,7 @@ func getProminentLanguages(langs []clients.Language) []clients.LanguageName { // Languages that have lines of code above average will be considered prominent. ret := []clients.LanguageName{} for lName, loC := range langMap { - if loC >= avgLoC/4.0 { + if loC >= avgLoC/fourPointZero { lang := clients.LanguageName(strings.ToLower(string(lName))) ret = append(ret, lang) } From 480b9434c74f4325775b76af9bacba64d4bce526 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Thu, 5 Oct 2023 16:10:01 +0100 Subject: [PATCH 14/16] test all fileMatchPatterns per client Signed-off-by: David Korczynski --- checks/raw/fuzzing_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index 68ae3cb0356..976ce65f4a4 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -274,19 +274,19 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) { if !ok && !tt.wantErr { t.Errorf("retrieve supported language error") } - var fileMatchPattern string - if len(langSpecs.filePatterns) > 0 { - fileMatchPattern = langSpecs.filePatterns[0] - } - fileMatch, err := path.Match(fileMatchPattern, tt.fileName) - if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr { - t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) + var found bool + for _, fileMatchPattern := range langSpecs.filePatterns { + fileMatch, err := path.Match(fileMatchPattern, tt.fileName) + if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr { + t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) + } + funcRegexPattern := langSpecs.funcPattern + r := regexp.MustCompile(funcRegexPattern) + found = found || r.MatchString(tt.fileContent) } - funcRegexPattern := langSpecs.funcPattern - r := regexp.MustCompile(funcRegexPattern) - found := r.MatchString(tt.fileContent) + if (found != tt.expectedFuncMatch) && !tt.wantErr { - t.Errorf("funcMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name) + t.Errorf("found = %v, want %v for %v", found, tt.expectedFileMatch, tt.name) } }) } From 0fb89e4e24950a7239089ac52aa3c1c78c3255db Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Thu, 5 Oct 2023 21:47:11 +0100 Subject: [PATCH 15/16] fix test logging counts Signed-off-by: David Korczynski --- e2e/fuzzing_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index cf94f297688..83d6e720123 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, - NumberOfInfo: 1, + NumberOfInfo: 12, NumberOfDebug: 0, } result := checks.Fuzzing(&req) @@ -192,7 +192,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { expected := scut.TestReturn{ Error: nil, Score: checker.MinResultScore, - NumberOfWarn: 7, + NumberOfWarn: 13, NumberOfInfo: 0, NumberOfDebug: 0, } From 9e0c744b22078332eff8dd720392db236448141a Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Mon, 9 Oct 2023 21:59:25 +0100 Subject: [PATCH 16/16] nit Signed-off-by: David Korczynski --- checks/raw/fuzzing.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 8354b72ccf0..9e92767d5a7 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -42,7 +42,6 @@ const ( fuzzerRustCargoFuzz = "RustCargoFuzzer" fuzzerJavaJazzerFuzzer = "JavaJazzerFuzzer" // TODO: add more fuzzing check supports. - fourPointZero = 4.0 ) type filesWithPatternStr struct { @@ -373,9 +372,10 @@ func getProminentLanguages(langs []clients.Language) []clients.LanguageName { // This var can stay as an int, no need for a precise float value. avgLoC := totalLoC / numLangs // Languages that have lines of code above average will be considered prominent. + prominentThreshold := avgLoC / 4.0 ret := []clients.LanguageName{} for lName, loC := range langMap { - if loC >= avgLoC/fourPointZero { + if loC >= prominentThreshold { lang := clients.LanguageName(strings.ToLower(string(lName))) ret = append(ret, lang) }