cron: add gitlab projects

* support gitlab client * simplify gitlab detection Signed-off-by: Raghav Kaul <[email protected]>
ossf · May 10, 2023 · a5d1ac6 · a5d1ac6
1 parent 7736f36
commit a5d1ac6
Show file tree

Hide file tree

Showing 18 changed files with 31,492 additions and 34 deletions.
diff --git a/checker/client.go b/checker/client.go
@@ -58,7 +58,7 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
 	var repoClient clients.RepoClient
 
 	//nolint:nestif
-	if experimental && glrepo.DetectGitLab(repoURI) {
+	if experimental {
 		repo, makeRepoError = glrepo.MakeGitlabRepo(repoURI)
 		if makeRepoError != nil {
 			return repo,
@@ -70,7 +70,7 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
 		}
 
 		var err error
-		repoClient, err = glrepo.CreateGitlabClientWithToken(ctx, os.Getenv("GITLAB_AUTH_TOKEN"), repo)
+		repoClient, err = glrepo.CreateGitlabClient(ctx, repo)
 		if err != nil {
 			return repo,
 				nil,

diff --git a/clients/gitlabrepo/client.go b/clients/gitlabrepo/client.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"os"
 	"time"
 
 	"github.com/xanzy/go-gitlab"
@@ -228,6 +229,22 @@ func (client *Client) Close() error {
 	return nil
 }
 
+func CreateGitlabDotComClient(ctx context.Context) (clients.RepoClient, error) {
+	var err error
+	var gitlabComRepo clients.Repo
+	if gitlabComRepo, err = MakeGitlabRepo("https://gitlab.com/gitlab-org/gitlab"); err != nil {
+		return nil, fmt.Errorf("gitlabrepo.MakeGitlabRepo: %w", err)
+	}
+
+	token := os.Getenv("GITLAB_AUTH_TOKEN")
+	return CreateGitlabClientWithToken(ctx, token, gitlabComRepo)
+}
+
+func CreateGitlabClient(ctx context.Context, repo clients.Repo) (clients.RepoClient, error) {
+	token := os.Getenv("GITLAB_AUTH_TOKEN")
+	return CreateGitlabClientWithToken(ctx, token, repo)
+}
+
 func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients.Repo) (clients.RepoClient, error) {
 	client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.Host()))
 	if err != nil {
@@ -285,14 +302,3 @@ func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients
 func CreateOssFuzzRepoClient(ctx context.Context, logger *log.Logger) (clients.RepoClient, error) {
 	return nil, fmt.Errorf("%w, oss fuzz currently only supported for github repos", clients.ErrUnsupportedFeature)
 }
-
-// DetectGitLab: check whether the repoURI is a GitLab URI
-// Makes HTTP request to GitLab API.
-func DetectGitLab(repoURI string) bool {
-	var repo repoURL
-	if err := repo.parse(repoURI); err != nil {
-		return false
-	}
-
-	return repo.IsValid() == nil
-}
diff --git a/clients/gitlabrepo/repo.go b/clients/gitlabrepo/repo.go
@@ -140,7 +140,7 @@ func MakeGitlabRepo(input string) (clients.Repo, error) {
 		return nil, fmt.Errorf("error during parse: %w", err)
 	}
 	if err := repo.IsValid(); err != nil {
-		return nil, fmt.Errorf("error n IsValid: %w", err)
+		return nil, fmt.Errorf("error in IsValid: %w", err)
 	}
 	return &repo, nil
 }
diff --git a/clients/gitlabrepo/repo_test.go b/clients/gitlabrepo/repo_test.go
@@ -157,9 +157,13 @@ func TestRepoURL_DetectGitlab(t *testing.T) {
 		if tt.flagRequired && os.Getenv("TEST_GITLAB_EXTERNAL") == "" {
 			continue
 		}
-		g := DetectGitLab(tt.repouri)
-		if g != tt.expected {
-			t.Errorf("got %s isgitlab: %t expected %t", tt.repouri, g, tt.expected)
+		g, err := MakeGitlabRepo(tt.repouri)
+		if (g != nil) != (err == nil) {
+			t.Errorf("got gitlabrepo: %s with err %s", g, err)
+		}
+		isGitlab := g != nil && err == nil
+		if isGitlab != tt.expected {
+			t.Errorf("got %s isgitlab: %t expected %t", tt.repouri, isGitlab, tt.expected)
 		}
 	}
 }
diff --git a/cron/data/iterator.go b/cron/data/iterator.go
@@ -24,6 +24,7 @@ import (
 	"github.com/jszwec/csvutil"
 
 	"github.com/ossf/scorecard/v4/clients/githubrepo"
+	"github.com/ossf/scorecard/v4/clients/gitlabrepo"
 )
 
 // Iterator interface is used to iterate through list of input repos for the cron job.
@@ -83,9 +84,13 @@ func (reader *csvIterator) Next() (RepoFormat, error) {
 	if reader.err != nil {
 		return reader.next, fmt.Errorf("reader has error: %w", reader.err)
 	}
+
+	repoURI := reader.next.Repo
 	// Sanity check valid GitHub URL.
-	if _, err := githubrepo.MakeGithubRepo(reader.next.Repo); err != nil {
-		return reader.next, fmt.Errorf("invalid GitHub URL: %w", err)
+	if _, err := gitlabrepo.MakeGitlabRepo(repoURI); err != nil {
+		if _, err := githubrepo.MakeGithubRepo(reader.next.Repo); err != nil {
+			return reader.next, fmt.Errorf("invalid URL, neither github nor gitlab: %w", err)
+		}
 	}
 	return reader.next, nil
 }

diff --git a/cron/data/iterator_test.go b/cron/data/iterator_test.go
@@ -64,6 +64,63 @@ func TestCsvIterator(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:     "BasicGitlabOnly",
+			filename: "testdata/basic-gitlab-only.csv",
+			outcomes: []outcome{
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo: "gitlab.com/owner1/repo1",
+					},
+				},
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo:     "gitlab.com/owner3/path1/repo2",
+						Metadata: []string{"meta"},
+					},
+				},
+			},
+		},
+		{
+			name:     "BasicWithGitlab",
+			filename: "testdata/basic-with-gitlab.csv",
+			outcomes: []outcome{
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo: "github.com/owner1/repo1",
+					},
+				},
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo: "github.com/owner2/repo2",
+					},
+				},
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo:     "github.com/owner3/repo3",
+						Metadata: []string{"meta"},
+					},
+				},
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo: "gitlab.com/owner1/repo1",
+					},
+				},
+				{
+					hasError: false,
+					repo: RepoFormat{
+						Repo:     "gitlab.com/owner3/path1/repo2",
+						Metadata: []string{"meta"},
+					},
+				},
+			},
+		},
 		{
 			name:     "Comment",
 			filename: "testdata/comment.csv",
@@ -95,7 +152,7 @@ func TestCsvIterator(t *testing.T) {
 			outcomes: []outcome{
 				{
 					hasError:    true,
-					expectedErr: sce.ErrorUnsupportedHost,
+					expectedErr: sce.ErrorInvalidURL,
 				},
 				{
 					hasError:    true,

diff --git a/cron/data/testdata/basic-gitlab-only.csv b/cron/data/testdata/basic-gitlab-only.csv
@@ -0,0 +1,3 @@
+repo,metadata
+gitlab.com/owner1/repo1,
+gitlab.com/owner3/path1/repo2,meta
diff --git a/cron/data/testdata/basic-with-gitlab.csv b/cron/data/testdata/basic-with-gitlab.csv
@@ -0,0 +1,6 @@
+repo,metadata
+github.com/owner1/repo1,
+github.com/owner2/repo2,
+github.com/owner3/repo3,meta
+gitlab.com/owner1/repo1,
+gitlab.com/owner3/path1/repo2,meta
diff --git a/cron/data/testdata/failing_urls.csv b/cron/data/testdata/failing_urls.csv
@@ -1,4 +1,4 @@
 repo,metadata
-gitlab.com/owner1/repo1,
+gitlab.com//repo1,
 github.com/owner2/,
 github.com//repo3,meta
diff --git a/cron/data/writer_test.go b/cron/data/writer_test.go
@@ -34,16 +34,26 @@ func TestCsvWriter(t *testing.T) {
 					Repo:     "github.com/owner1/repo1",
 					Metadata: []string{"meta1"},
 				},
+				{
+					Repo:     "gitlab.com/owner3/repo3",
+					Metadata: []string{"meta3"},
+				},
 			},
 			newRepos: []RepoFormat{
 				{
 					Repo:     "github.com/owner2/repo2",
 					Metadata: []string{"meta2"},
 				},
+				{
+					Repo:     "gitlab.com/owner4/repo4",
+					Metadata: []string{"meta4"},
+				},
 			},
 			out: `repo,metadata
 github.com/owner1/repo1,meta1
 github.com/owner2/repo2,meta2
+gitlab.com/owner3/repo3,meta3
+gitlab.com/owner4/repo4,meta4
 `,
 		},
 	}

diff --git a/cron/internal/data/gitlab-projects-selected.csv b/cron/internal/data/gitlab-projects-selected.csv
@@ -0,0 +1,100 @@
+https://gitlab.com/gitlab-org/gitlab-foss,
+https://gitlab.com/gitlab-org/gitlab,
+https://gitlab.com/CalcProgrammer1/OpenRGB,
+https://gitlab.com/gitlab-org/gitlab-runner,
+https://gitlab.com/fdroid/fdroidclient,
+https://gitlab.com/bramw/baserow,
+https://gitlab.com/AuroraOSS/AuroraStore,
+https://gitlab.com/graphviz/graphviz,
+https://gitlab.com/pgjones/quart,
+https://gitlab.com/libeigen/eigen,
+https://gitlab.com/gitlab-org/gitlab-development-kit,
+https://gitlab.com/gitlab-org/omnibus-gitlab,
+https://gitlab.com/tezos/tezos,
+https://gitlab.com/mayan-edms/mayan-edms,
+https://gitlab.com/meltano/meltano,
+https://gitlab.com/gitlab-com/runbooks,
+https://gitlab.com/antora/antora,
+https://gitlab.com/pycqa/flake8,
+https://gitlab.com/meno/dropzone,
+https://gitlab.com/pages/hugo,
+https://gitlab.com/sequoia-pgp/sequoia,
+https://gitlab.com/gableroux/unity3d-gitlab-ci-example,
+https://gitlab.com/gitlab-org/gitaly,
+https://gitlab.com/gitlab-org/cli,
+https://gitlab.com/postgres-ai/database-lab,
+https://gitlab.com/timvisee/ffsend,
+https://gitlab.com/leanlabsio/kanban,
+https://gitlab.com/pgjones/hypercorn,
+https://gitlab.com/Rich-Harris/buble,
+https://gitlab.com/cznic/sqlite,
+https://gitlab.com/postgres-ai/postgres-checkup,
+https://gitlab.com/mojo42/Jirafeau,
+https://gitlab.com/eidheim/Simple-Web-Server,
+https://gitlab.com/NebulousLabs/Sia,
+https://gitlab.com/akihe/radamsa,
+https://gitlab.com/procps-ng/procps,
+https://gitlab.com/jam-systems/jam,
+https://gitlab.com/catamphetamine/libphonenumber-js,
+https://gitlab.com/olaris/olaris-server,
+https://gitlab.com/stavros/harbormaster,
+https://gitlab.com/conradsnicta/armadillo-code,
+https://gitlab.com/gitlab-org/gitlab-shell,
+https://gitlab.com/dalibo/postgresql_anonymizer,
+https://gitlab.com/fatihacet/gitlab-vscode-extension,
+https://gitlab.com/brinkervii/grapejuice,
+https://gitlab.com/gitlab-org/gitlab-runner-docker-cleanup,
+https://gitlab.com/gitlab-org/gitlab-ui,
+https://gitlab.com/ajak/tuir,
+https://gitlab.com/kornelski/babel-preset-php,
+https://gitlab.com/mailman/hyperkitty,
+https://gitlab.com/wg1/jpeg-xl,
+https://gitlab.com/nsnam/ns-3-dev,
+https://gitlab.com/axet/android-book-reader,
+https://gitlab.com/shodan-public/nrich,
+https://gitlab.com/bloom42/bloom,
+https://gitlab.com/lfortran/lfortran,
+https://gitlab.com/gitlab-org/gitlab-triage,
+https://gitlab.com/esr/reposurgeon,
+https://gitlab.com/leinardi/gkraken,
+https://gitlab.com/QEF/q-e,
+https://gitlab.com/eidheim/Simple-WebSocket-Server,
+https://gitlab.com/signald/signald,
+https://gitlab.com/chaica/feed2toot,
+https://gitlab.com/gitlab-org/gitlab-pages,
+https://gitlab.com/pulsechaincom/go-pulse,
+https://gitlab.com/GoogleDriveIndex/Google-Drive-Index,
+https://gitlab.com/antonok/enum_dispatch,
+https://gitlab.com/gitlab-org/gitlab-workhorse,
+https://gitlab.com/petsc/petsc,
+https://gitlab.com/eternal-twin/etwin,
+https://gitlab.com/mattbas/python-lottie,
+https://gitlab.com/gitlab-org/docker-distribution-pruner,
+https://gitlab.com/rosie-pattern-language/rosie,
+https://gitlab.com/BuildStream/buildstream,
+https://gitlab.com/kicad/libraries/kicad-footprints,
+https://gitlab.com/dmfay/massive-js,
+https://gitlab.com/nanuchi/go-full-course-youtube,
+https://gitlab.com/sublime-music/sublime-music,
+https://gitlab.com/gitlab-org/opstrace/opstrace,
+https://gitlab.com/gitlab-org/release-cli,
+https://gitlab.com/gitlab-org/ci-cd/docker-machine,
+https://gitlab.com/catamphetamine/react-phone-number-input,
+https://gitlab.com/IvanSanchez/Leaflet.GridLayer.GoogleMutant,
+https://gitlab.com/klamonte/jexer,
+https://gitlab.com/woob/woob,
+https://gitlab.com/crates.rs/crates.rs,
+https://gitlab.com/stavros/python-yeelight,
+https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image,
+https://gitlab.com/dbsystel/gitlab-ci-python-library,
+https://gitlab.com/DerManu/QCustomPlot,
+https://gitlab.com/juhani/go-semrel-gitlab,
+https://gitlab.com/postgres-ai/joe,
+https://gitlab.com/altek/accountant,
+https://gitlab.com/formschema/native,
+https://gitlab.com/gardenappl/readability-cli,
+https://gitlab.com/doctormo/python-crontab,
+https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent,
+https://gitlab.com/mattbas/glaxnimate,
+https://gitlab.com/mailman/postorius,
+https://gitlab.com/cznic/ql,