From 74869d615bf470d45cc19d65c782f1a230216693 Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Tue, 13 Jul 2021 14:20:38 +0200 Subject: [PATCH] Local source code feature (using bundle images) Reference https://github.com/shipwright-io/build/pull/717 Add `bundle` package that contains convenience code to push a local source code directory as a bundle image to a container registry. Add new flags for container image and local directory settings. --- pkg/shp/bundle/bundle.go | 255 +++++++++++++++++++++++++++++++++ pkg/shp/cmd/build/create.go | 27 +++- pkg/shp/cmd/build/delete.go | 2 +- pkg/shp/cmd/build/list.go | 7 +- pkg/shp/cmd/build/run.go | 169 ++++------------------ pkg/shp/cmd/buildrun/create.go | 234 +++++++++++++++++++++++++----- pkg/shp/cmd/buildrun/list.go | 1 - pkg/shp/cmd/buildrun/logs.go | 2 +- pkg/shp/flags/build.go | 18 ++- pkg/shp/flags/build_test.go | 14 +- pkg/shp/flags/buildrun.go | 31 +++- pkg/shp/flags/flags.go | 10 +- 12 files changed, 559 insertions(+), 211 deletions(-) create mode 100644 pkg/shp/bundle/bundle.go diff --git a/pkg/shp/bundle/bundle.go b/pkg/shp/bundle/bundle.go new file mode 100644 index 000000000..24bfdbf46 --- /dev/null +++ b/pkg/shp/bundle/bundle.go @@ -0,0 +1,255 @@ +package bundle + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "k8s.io/cli-runtime/pkg/genericclioptions" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/schollz/progressbar/v3" + buildbundle "github.com/shipwright-io/build/pkg/bundle" +) + +// Push bundles the provided local directory into a container image and pushes +// it to the given registry. For this to work, it relies on valid and working +// container registry access credentials and tokens to be available in the +// local system, for example logins done by `docker login` or similar. +func Push(ctx context.Context, io *genericclioptions.IOStreams, localDirectory string, targetImage string) (name.Digest, error) { + tag, err := name.NewTag(targetImage) + if err != nil { + return name.Digest{}, err + } + + // The default keychain resolver takes the provided image reference and + // checks it against the available login credentials in the system. The + // needs to have done a `docker login` or similar to the respective + // registry before using the Shipwright CLI. + auth, err := authn.DefaultKeychain.Resolve(tag.Context()) + if err != nil { + return name.Digest{}, err + } + + updates := make(chan v1.Update, 1) + done := make(chan struct{}, 1) + go func() { + var progress *progressbar.ProgressBar + for { + select { + case <-ctx.Done(): + return + + case <-done: + return + + case update, ok := <-updates: + if !ok { + return + } + + if progress == nil { + progress = progressbar.NewOptions(int(update.Total), + progressbar.OptionSetWriter(io.ErrOut), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowBytes(true), + progressbar.OptionSetWidth(15), + progressbar.OptionSetPredictTime(false), + progressbar.OptionSetDescription("Uploading local source..."), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]"}), + progressbar.OptionOnCompletion(func() { + fmt.Fprintln(io.Out) + }), + ) + defer progress.Close() + } + + progress.ChangeMax64(update.Total) + _ = progress.Set64(update.Complete) + } + } + }() + + fmt.Fprintf(io.Out, "Bundling %q as %q ...\n", localDirectory, targetImage) + digest, err := buildbundle.PackAndPush( + tag, + localDirectory, + remote.WithContext(ctx), + remote.WithAuth(auth), + remote.WithProgress(updates), + ) + + done <- struct{}{} + return digest, err +} + +// Prune removes the image from the container registry +// +// Deleting a tag, or a whole repo is not as straightforward as initially +// planned as DockerHub seems to restrict deleting a single tag for +// standard users. This might be subject to change, but as of September +// 2021 it is limited to the business tier. However, there is an API call +// to delete the whole repository. In case there is only one tag used in +// a repository, the effect is pretty much the same. For convenience, there +// is a provider switch to deal with images on DockerHub differently. +// +// DockerHub images: +// - In case the repository only has one tag, the repository is deleted. +// - If there are multiple tags, the tag to be deleted is overwritten +// with an empty image (to remove the content, and save quota). +// - Edge case would be no tags in the repository, which is ignored. +// +// Other registries: +// Use standard spec delete API request to delete the provided tag. +// +func Prune(ctx context.Context, io *genericclioptions.IOStreams, image string) error { + ref, err := name.ParseReference(image) + if err != nil { + return err + } + + auth, err := authn.DefaultKeychain.Resolve(ref.Context()) + if err != nil { + return err + } + + switch ref.Context().RegistryStr() { + case "index.docker.io": + list, err := remote.List(ref.Context(), remote.WithContext(ctx), remote.WithAuth(auth)) + if err != nil { + return err + } + + switch len(list) { + case 0: + return nil + + case 1: + authr, err := auth.Authorization() + if err != nil { + return err + } + + token, err := dockerHubLogin(authr.Username, authr.Password) + if err != nil { + return err + } + + return dockerHubRepoDelete(token, ref) + + default: + fmt.Fprintf(io.ErrOut, "Removing a specific image tag is not supported on %s, the respective image tag will be overwritten with an empty image.\n", ref.Context().RegistryStr()) + + // In case the input argument included a digest, the reference + // needs to be updated to exclude the digest for the empty image + // override to succeed. + switch ref.(type) { + case name.Digest: + tmp := strings.SplitN(image, "@", 2) + ref, err = name.NewTag(tmp[0]) + if err != nil { + return err + } + } + + return remote.Write(ref, empty.Image, remote.WithContext(ctx), remote.WithAuth(auth)) + } + + default: + return remote.Delete( + ref, + remote.WithContext(ctx), + remote.WithAuth(auth), + ) + } +} + +func dockerHubLogin(username string, password string) (string, error) { + type LoginData struct { + Username string `json:"username"` + Password string `json:"password"` + } + + loginData, err := json.Marshal(LoginData{Username: username, Password: password}) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", "https://hub.docker.com/v2/users/login/", bytes.NewReader(loginData)) + if err != nil { + return "", err + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + bodyData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + switch resp.StatusCode { + case http.StatusOK: + type LoginToken struct { + Token string `json:"token"` + } + + var loginToken LoginToken + if err := json.Unmarshal(bodyData, &loginToken); err != nil { + return "", err + } + + return loginToken.Token, nil + + default: + return "", fmt.Errorf(string(bodyData)) + } +} + +func dockerHubRepoDelete(token string, ref name.Reference) error { + req, err := http.NewRequest("DELETE", fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/", ref.Context().RepositoryStr()), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", "JWT "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + switch resp.StatusCode { + case http.StatusAccepted: + return nil + + default: + return fmt.Errorf("failed with HTTP status code %d: %s", resp.StatusCode, string(respData)) + } +} diff --git a/pkg/shp/cmd/build/create.go b/pkg/shp/cmd/build/create.go index 0128e8031..cb4255e5a 100644 --- a/pkg/shp/cmd/build/create.go +++ b/pkg/shp/cmd/build/create.go @@ -3,14 +3,15 @@ package build import ( "fmt" - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - "github.com/shipwright-io/cli/pkg/shp/cmd/runner" - "github.com/shipwright-io/cli/pkg/shp/flags" - "github.com/shipwright-io/cli/pkg/shp/params" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + "github.com/shipwright-io/cli/pkg/shp/cmd/runner" + "github.com/shipwright-io/cli/pkg/shp/flags" + "github.com/shipwright-io/cli/pkg/shp/params" ) // CreateCommand contains data input from user @@ -48,6 +49,19 @@ func (c *CreateCommand) Validate() error { if c.name == "" { return fmt.Errorf("name must be provided") } + + if c.buildSpec.Source.URL != "" && + c.buildSpec.Source.BundleContainer != nil && + c.buildSpec.Source.BundleContainer.Image != "" { + return fmt.Errorf("both source URL and container image are specified, only one can be used at the same time") + } + + if c.buildSpec.Source.URL == "" && + c.buildSpec.Source.BundleContainer != nil && + c.buildSpec.Source.BundleContainer.Image == "" { + return fmt.Errorf("no input source was specified, either source URL or container image needs to be provided") + } + return nil } @@ -56,6 +70,8 @@ func (c *CreateCommand) Run(params *params.Params, io *genericclioptions.IOStrea b := &buildv1alpha1.Build{Spec: *c.buildSpec} flags.SanitizeBuildSpec(&b.Spec) + b.Name = c.name + clientset, err := params.ShipwrightClientSet() if err != nil { return err @@ -78,9 +94,6 @@ func createCmd() runner.SubCommand { // instantiating command-line flags and the build-spec structure which receives the informed flag // values, also marking certain flags as mandatory buildSpecFlags := flags.BuildSpecFromFlags(cmd.Flags()) - if err := cmd.MarkFlagRequired(flags.SourceURLFlag); err != nil { - panic(err) - } if err := cmd.MarkFlagRequired(flags.OutputImageFlag); err != nil { panic(err) } diff --git a/pkg/shp/cmd/build/delete.go b/pkg/shp/cmd/build/delete.go index 6f4ed9205..cff7ecf62 100644 --- a/pkg/shp/cmd/build/delete.go +++ b/pkg/shp/cmd/build/delete.go @@ -3,12 +3,12 @@ package build import ( "fmt" - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/spf13/cobra" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/shipwright-io/cli/pkg/shp/cmd/runner" "github.com/shipwright-io/cli/pkg/shp/params" ) diff --git a/pkg/shp/cmd/build/list.go b/pkg/shp/cmd/build/list.go index 420bfda41..9cd15b464 100644 --- a/pkg/shp/cmd/build/list.go +++ b/pkg/shp/cmd/build/list.go @@ -4,13 +4,14 @@ import ( "fmt" "text/tabwriter" - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - "github.com/shipwright-io/cli/pkg/shp/cmd/runner" - "github.com/shipwright-io/cli/pkg/shp/params" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + "github.com/shipwright-io/cli/pkg/shp/cmd/runner" + "github.com/shipwright-io/cli/pkg/shp/params" ) // ListCommand struct contains user input to the List subcommand of Build diff --git a/pkg/shp/cmd/build/run.go b/pkg/shp/cmd/build/run.go index 3bc33641d..56818a900 100644 --- a/pkg/shp/cmd/build/run.go +++ b/pkg/shp/cmd/build/run.go @@ -3,39 +3,26 @@ package build import ( "errors" "fmt" - "time" - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - buildclientset "github.com/shipwright-io/build/pkg/client/clientset/versioned" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + "github.com/shipwright-io/cli/pkg/shp/cmd/buildrun" "github.com/shipwright-io/cli/pkg/shp/cmd/runner" "github.com/shipwright-io/cli/pkg/shp/flags" "github.com/shipwright-io/cli/pkg/shp/params" - "github.com/shipwright-io/cli/pkg/shp/reactor" - "github.com/shipwright-io/cli/pkg/shp/tail" - - "github.com/spf13/cobra" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/cli-runtime/pkg/genericclioptions" ) // RunCommand represents the `build run` sub-command, which creates a unique BuildRun instance to run // the build process, informed via arguments. type RunCommand struct { - cmd *cobra.Command // cobra command instance - - ioStreams *genericclioptions.IOStreams // io-streams instance - pw *reactor.PodWatcher // pod-watcher instance - logTail *tail.Tail // follow container logs - tailLogsStarted map[string]bool // controls tail instance per container - - buildName string // build name - buildRunName string - buildRunSpec *buildv1alpha1.BuildRunSpec // stores command-line flags - shpClientset buildclientset.Interface - follow bool // flag to tail pod logs + cmd *cobra.Command + + buildName string + buildRunSpec *buildv1alpha1.BuildRunSpec + buildRunOpts *flags.BuildRunOpts } const buildRunLongDesc = ` @@ -55,16 +42,11 @@ func (r *RunCommand) Complete(params *params.Params, args []string) error { switch len(args) { case 1: r.buildName = args[0] + default: return errors.New("Build name is not informed") } - clientset, err := params.ClientSet() - if err != nil { - return err - } - r.logTail = tail.NewTail(r.Cmd().Context(), clientset) - // overwriting build-ref name to use what's on arguments return r.Cmd().Flags().Set(flags.BuildrefNameFlag, r.buildName) } @@ -77,120 +59,16 @@ func (r *RunCommand) Validate() error { return nil } -// tailLogs start tailing logs for each container name in init-containers and containers, if not -// started already. -func (r *RunCommand) tailLogs(pod *corev1.Pod) { - containers := append(pod.Spec.InitContainers, pod.Spec.Containers...) - for _, container := range containers { - if _, exists := r.tailLogsStarted[container.Name]; exists { - continue - } - r.tailLogsStarted[container.Name] = true - r.logTail.Start(pod.GetNamespace(), pod.GetName(), container.Name) - } -} - -// onEvent reacts on pod state changes, to start and stop tailing container logs. -func (r *RunCommand) onEvent(pod *corev1.Pod) error { - switch pod.Status.Phase { - case corev1.PodRunning: - // graceful time to wait for container start - time.Sleep(3 * time.Second) - // start tailing container logs - r.tailLogs(pod) - case corev1.PodFailed: - msg := "" - br, err := r.shpClientset.ShipwrightV1alpha1().BuildRuns(pod.Namespace).Get(r.cmd.Context(), r.buildRunName, metav1.GetOptions{}) - switch { - case err == nil && br.IsCanceled(): - msg = fmt.Sprintf("BuildRun '%s' has been canceled.\n", br.Name) - case err == nil && br.DeletionTimestamp != nil: - msg = fmt.Sprintf("BuildRun '%s' has been deleted.\n", br.Name) - case pod.DeletionTimestamp != nil: - msg = fmt.Sprintf("Pod '%s' has been deleted.\n", pod.GetName()) - default: - msg = fmt.Sprintf("Pod '%s' has failed!\n", pod.GetName()) - err = fmt.Errorf("build pod '%s' has failed", pod.GetName()) - } - // see if because of deletion or cancelation - fmt.Fprintf(r.ioStreams.Out, msg) - r.stop() - return err - case corev1.PodSucceeded: - fmt.Fprintf(r.ioStreams.Out, "Pod '%s' has succeeded!\n", pod.GetName()) - r.stop() - default: - fmt.Fprintf(r.ioStreams.Out, "Pod '%s' is in state %q...\n", pod.GetName(), string(pod.Status.Phase)) - // handle any issues with pulling images that may fail - for _, c := range pod.Status.Conditions { - if c.Type == corev1.PodInitialized || c.Type == corev1.ContainersReady { - if c.Status == corev1.ConditionUnknown { - return fmt.Errorf(c.Message) - } - } - } - } - return nil -} - -// stop invoke stop on streaming components. -func (r *RunCommand) stop() { - r.logTail.Stop() - r.pw.Stop() -} - // Run creates a BuildRun resource based on Build's name informed on arguments. func (r *RunCommand) Run(params *params.Params, ioStreams *genericclioptions.IOStreams) error { - // resource using GenerateName, which will provice a unique instance - br := &buildv1alpha1.BuildRun{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("%s-", r.buildName), - }, - Spec: *r.buildRunSpec, - } - flags.SanitizeBuildRunSpec(&br.Spec) - - clientset, err := params.ShipwrightClientSet() - if err != nil { - return err - } - br, err = clientset.ShipwrightV1alpha1().BuildRuns(params.Namespace()).Create(r.cmd.Context(), br, metav1.CreateOptions{}) - if err != nil { - return err - } - - if !r.follow { - fmt.Fprintf(ioStreams.Out, "BuildRun created %q for build %q\n", br.GetName(), r.buildName) - return nil - } - - r.buildRunName = br.Name - if r.shpClientset, err = params.ShipwrightClientSet(); err != nil { - return err - } - - kclientset, err := params.ClientSet() - if err != nil { - return err - } - - // instantiating a pod watcher with a specific label-selector to find the indented pod where the - // actual build started by this subcommand is being executed, including the randomized buildrun - // name - listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf( - "build.shipwright.io/name=%s,buildrun.shipwright.io/name=%s", - r.buildName, - br.GetName(), - )} - r.pw, err = reactor.NewPodWatcher(r.Cmd().Context(), kclientset, listOpts, params.Namespace()) - if err != nil { - return err - } - - r.ioStreams = ioStreams - r.pw.WithOnPodModifiedFn(r.onEvent) - _, err = r.pw.Start() - return err + return buildrun.CreateBuildRun( + r.cmd.Context(), + "", + params, + ioStreams, + r.buildRunSpec, + r.buildRunOpts, + ) } // runCmd instantiate the "build run" sub-command using common BuildRun flags. @@ -200,11 +78,12 @@ func runCmd() runner.SubCommand { Short: "Start a build specified by 'name'", Long: buildRunLongDesc, } + runCommand := &RunCommand{ - cmd: cmd, - buildRunSpec: flags.BuildRunSpecFromFlags(cmd.Flags()), - tailLogsStarted: make(map[string]bool), + cmd: cmd, + buildRunSpec: flags.BuildRunSpecFromFlags(cmd.Flags()), + buildRunOpts: flags.BuildRunOptsFromFlags(cmd.Flags()), } - cmd.Flags().BoolVarP(&runCommand.follow, "follow", "F", runCommand.follow, "Start a build and watch its log until it completes or fails.") + return runCommand } diff --git a/pkg/shp/cmd/buildrun/create.go b/pkg/shp/cmd/buildrun/create.go index 4922812ad..6b6869359 100644 --- a/pkg/shp/cmd/buildrun/create.go +++ b/pkg/shp/cmd/buildrun/create.go @@ -1,26 +1,33 @@ package buildrun import ( + "context" "fmt" - "strings" + "time" + "github.com/google/go-containerregistry/pkg/name" + "github.com/spf13/cobra" + + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/rand" "k8s.io/cli-runtime/pkg/genericclioptions" buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + "github.com/shipwright-io/cli/pkg/shp/bundle" "github.com/shipwright-io/cli/pkg/shp/cmd/runner" "github.com/shipwright-io/cli/pkg/shp/flags" "github.com/shipwright-io/cli/pkg/shp/params" - "github.com/spf13/cobra" + "github.com/shipwright-io/cli/pkg/shp/reactor" + "github.com/shipwright-io/cli/pkg/shp/tail" ) // CreateCommand reprents the build's create subcommand. type CreateCommand struct { - cmd *cobra.Command // cobra command instance + cmd *cobra.Command - name string // buildrun name - buildRunSpec *buildv1alpha1.BuildRunSpec // stores command-line flags + name string + buildRunSpec *buildv1alpha1.BuildRunSpec + buildRunOpts *flags.BuildRunOpts } const buildRunCreateLongDesc = ` @@ -39,23 +46,13 @@ func (c *CreateCommand) Cmd() *cobra.Command { func (c *CreateCommand) Complete(params *params.Params, args []string) error { switch len(args) { case 0: - var randomName = func(name, suffix string) string { - return fmt.Sprintf("%s%s-%s", name, suffix, rand.String(5)) - } - - switch { - case strings.HasSuffix(c.buildRunSpec.BuildRef.Name, "build"): - c.name = randomName(c.buildRunSpec.BuildRef.Name, "run") - - default: - c.name = randomName(c.buildRunSpec.BuildRef.Name, "-buildrun") - } + c.name = "" case 1: c.name = args[0] default: - return fmt.Errorf("wrong amount of arguments, expected only one (specific name) or none (random name)") + return fmt.Errorf("wrong amount of arguments, expected only one (specific name) or none (generated name)") } return nil @@ -63,26 +60,23 @@ func (c *CreateCommand) Complete(params *params.Params, args []string) error { // Validate makes sure a name is informed. func (c *CreateCommand) Validate() error { - if c.name == "" { - return fmt.Errorf("name is not informed") + if c.buildRunSpec.BuildRef == nil || c.buildRunSpec.BuildRef.Name == "" { + return fmt.Errorf("build name is not informed") } + return nil } // Run executes the creation of BuildRun object. func (c *CreateCommand) Run(params *params.Params, ioStreams *genericclioptions.IOStreams) error { - br := &buildv1alpha1.BuildRun{Spec: *c.buildRunSpec} - flags.SanitizeBuildRunSpec(&br.Spec) - - clientset, err := params.ShipwrightClientSet() - if err != nil { - return err - } - if _, err = clientset.ShipwrightV1alpha1().BuildRuns(params.Namespace()).Create(c.cmd.Context(), br, metav1.CreateOptions{}); err != nil { - return err - } - fmt.Fprintf(ioStreams.Out, "BuildRun created %q for Build %q\n", c.name, br.Spec.BuildRef.Name) - return nil + return CreateBuildRun( + c.cmd.Context(), + c.name, + params, + ioStreams, + c.buildRunSpec, + c.buildRunOpts, + ) } // createCmd instantiate a new CreateCommand, by wiring it as a cobra.Command and registering the @@ -94,15 +88,179 @@ func createCmd() runner.SubCommand { Long: buildRunCreateLongDesc, } - // instantiating command-line flags, using an actual BuildRunSpec object to receive the flags - // issued on command-line, also marking flags as required - buildRunSpecFlags := flags.BuildRunSpecFromFlags(cmd.Flags()) + createCommand := &CreateCommand{ + cmd: cmd, + buildRunSpec: flags.BuildRunSpecFromFlags(cmd.Flags()), + buildRunOpts: flags.BuildRunOptsFromFlags(cmd.Flags()), + } + if err := cmd.MarkFlagRequired(flags.BuildrefNameFlag); err != nil { panic(err) } - return &CreateCommand{ - cmd: cmd, - buildRunSpec: buildRunSpecFlags, + return createCommand +} + +// CreateBuildRun creates a BuildRun based on the given name and spec using the +// provided end-user options/settings, for example printing the pod container +// logs of the build. +func CreateBuildRun( + ctx context.Context, + buildRunName string, + params *params.Params, + ioStreams *genericclioptions.IOStreams, + buildRunSpec *buildv1alpha1.BuildRunSpec, + buildRunOpts *flags.BuildRunOpts) error { + + // The prune-bundle, and follow flags implicitly enable the wait flag + if buildRunOpts.PruneBundle || buildRunOpts.Follow { + buildRunOpts.Wait = true + } + + shpClient, err := params.ShipwrightClientSet() + if err != nil { + return err + } + + // Make sure the referenced build of the buildrun does exist + build, err := shpClient.ShipwrightV1alpha1().Builds(params.Namespace()).Get(ctx, buildRunSpec.BuildRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get referenced build %q: %w", buildRunSpec.BuildRef.Name, err) + } + + // Setup the buildrun to be created + buildRun := &buildv1alpha1.BuildRun{Spec: *buildRunSpec} + flags.SanitizeBuildRunSpec(&buildRun.Spec) + + // Enable usage of generated buildrun names in case there is none given + if buildRunName == "" { + buildRun.GenerateName = buildRunSpec.BuildRef.Name + } + + // Local source code mode: + // Make sure to bundle the configured local source directory into an image + // bundle and push it to the provided target registry. + var digest name.Digest + if build.Spec.Source.BundleContainer.Image != "" { + if digest, err = bundle.Push(ctx, ioStreams, buildRunOpts.LocalSourceDir, build.Spec.Source.BundleContainer.Image); err != nil { + return err + } + } + + // Initiate the actual create command in the cluster + buildRun, err = shpClient.ShipwrightV1alpha1().BuildRuns(params.Namespace()).Create(ctx, buildRun, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create BuildRun: %v", err) + } + + buildRunName = buildRun.Name + fmt.Fprintf(ioStreams.Out, "BuildRun created %q for Build %q\n", buildRun.Name, buildRun.Spec.BuildRef.Name) + if !buildRunOpts.Wait { + return nil + } + + kubeClient, err := params.ClientSet() + if err != nil { + return err + } + + podWatcher, err := reactor.NewPodWatcher(ctx, kubeClient, podByLabelSelector(build, buildRun), params.Namespace()) + if err != nil { + return err + } + + tailLogsStarted := make(map[string]bool) + logTail := tail.NewTail(ctx, kubeClient) + podPrevState := "" + + podWatcher.WithOnPodModifiedFn(func(pod *v1.Pod) error { + if podPrevState != string(pod.Status.Phase) { + fmt.Fprintf(ioStreams.Out, "Pod %q is in phase %q ...\n", pod.GetName(), string(pod.Status.Phase)) + podPrevState = string(pod.Status.Phase) + } + + switch pod.Status.Phase { + case v1.PodRunning: + if buildRunOpts.Follow { + // graceful time to wait for container start + time.Sleep(3 * time.Second) + + // start tailing container logs + for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { + if _, exists := tailLogsStarted[container.Name]; exists { + continue + } + + tailLogsStarted[container.Name] = true + logTail.Start( + pod.GetNamespace(), + pod.GetName(), + container.Name, + ) + } + } + + case v1.PodFailed: + br, err := shpClient.ShipwrightV1alpha1().BuildRuns(params.Namespace()).Get(ctx, buildRunName, metav1.GetOptions{}) + switch { + case err == nil && br.IsCanceled(): + fmt.Fprintf(ioStreams.Out, "BuildRun %q has been canceled.\n", br.Name) + + case err == nil && br.DeletionTimestamp != nil: + fmt.Fprintf(ioStreams.Out, "BuildRun %q has been deleted.\n", br.Name) + + case pod.DeletionTimestamp != nil: + fmt.Fprintf(ioStreams.Out, "Pod %q has been deleted.\n", pod.GetName()) + + default: + fmt.Fprintf(ioStreams.Out, "Pod %q has failed!\n", pod.GetName()) + err = fmt.Errorf("build pod %q has failed", pod.GetName()) + } + + logTail.Stop() + podWatcher.Stop() + return err + + case v1.PodSucceeded: + logTail.Stop() + podWatcher.Stop() + + default: + // handle any issues with pulling images that may fail + for _, c := range pod.Status.Conditions { + if c.Type == v1.PodInitialized || c.Type == v1.ContainersReady { + if c.Status == v1.ConditionUnknown { + return fmt.Errorf(c.Message) + } + } + } + } + + return nil + }) + + _, err = podWatcher.Start() + if err != nil { + return err + } + + // If configured, delete the source code bundle image once buildrun is done + if buildRunOpts.PruneBundle { + fmt.Fprintf(ioStreams.Out, "Pruning source code bundle %q\n", digest.String()) + if err := bundle.Prune(ctx, ioStreams, digest.String()); err != nil { + return err + } + } + + return nil +} + +func podByLabelSelector(build *buildv1alpha1.Build, buildRun *buildv1alpha1.BuildRun) metav1.ListOptions { + return metav1.ListOptions{ + LabelSelector: fmt.Sprintf( + "build.shipwright.io/name=%s,buildrun.shipwright.io/name=%s", + build.Name, + buildRun.Name, + ), } } diff --git a/pkg/shp/cmd/buildrun/list.go b/pkg/shp/cmd/buildrun/list.go index 3c1539917..e6f37b394 100644 --- a/pkg/shp/cmd/buildrun/list.go +++ b/pkg/shp/cmd/buildrun/list.go @@ -11,7 +11,6 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - "github.com/shipwright-io/cli/pkg/shp/cmd/runner" "github.com/shipwright-io/cli/pkg/shp/params" ) diff --git a/pkg/shp/cmd/buildrun/logs.go b/pkg/shp/cmd/buildrun/logs.go index 06dc7a03b..74ec7d579 100644 --- a/pkg/shp/cmd/buildrun/logs.go +++ b/pkg/shp/cmd/buildrun/logs.go @@ -4,13 +4,13 @@ import ( "fmt" "strings" - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/shipwright-io/cli/pkg/shp/cmd/runner" "github.com/shipwright-io/cli/pkg/shp/params" "github.com/shipwright-io/cli/pkg/shp/util" diff --git a/pkg/shp/flags/build.go b/pkg/shp/flags/build.go index 2e64e6f99..33e95f715 100644 --- a/pkg/shp/flags/build.go +++ b/pkg/shp/flags/build.go @@ -1,11 +1,13 @@ package flags import ( - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + + "github.com/spf13/pflag" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" ) // BuildSpecFromFlags creates a BuildSpec instance based on command-line flags. @@ -13,9 +15,10 @@ func BuildSpecFromFlags(flags *pflag.FlagSet) *buildv1alpha1.BuildSpec { clusterBuildStrategyKind := buildv1alpha1.ClusterBuildStrategyKind spec := &buildv1alpha1.BuildSpec{ Source: buildv1alpha1.Source{ - Credentials: &corev1.LocalObjectReference{}, - Revision: pointer.String(""), - ContextDir: pointer.String(""), + Credentials: &corev1.LocalObjectReference{}, + Revision: pointer.String(""), + ContextDir: pointer.String(""), + BundleContainer: &buildv1alpha1.BundleContainer{}, }, Strategy: &buildv1alpha1.Strategy{ Kind: &clusterBuildStrategyKind, @@ -49,6 +52,11 @@ func SanitizeBuildSpec(b *buildv1alpha1.BuildSpec) { if b.Source.Credentials != nil && b.Source.Credentials.Name == "" { b.Source.Credentials = nil } + + if b.Source.BundleContainer != nil && b.Source.BundleContainer.Image == "" { + b.Source.BundleContainer = nil + } + if b.Builder != nil { if b.Builder.Credentials != nil && b.Builder.Credentials.Name == "" { b.Builder.Credentials = nil diff --git a/pkg/shp/flags/build_test.go b/pkg/shp/flags/build_test.go index d78153cc6..b55406ea0 100644 --- a/pkg/shp/flags/build_test.go +++ b/pkg/shp/flags/build_test.go @@ -17,14 +17,14 @@ import ( func TestBuildSpecFromFlags(t *testing.T) { g := gomega.NewGomegaWithT(t) - credentials := corev1.LocalObjectReference{Name: "name"} buildStrategyKind := buildv1alpha1.ClusterBuildStrategyKind expected := &buildv1alpha1.BuildSpec{ Source: buildv1alpha1.Source{ - Credentials: &credentials, - URL: "https://some.url", - Revision: pointer.String("some-rev"), - ContextDir: pointer.String("some-contextdir"), + Credentials: &corev1.LocalObjectReference{Name: "source-credentials"}, + URL: "https://some.url", + Revision: pointer.String("some-rev"), + ContextDir: pointer.String("some-contextdir"), + BundleContainer: &buildv1alpha1.BundleContainer{}, }, Strategy: &buildv1alpha1.Strategy{ Name: "strategy-name", @@ -33,11 +33,11 @@ func TestBuildSpecFromFlags(t *testing.T) { }, Dockerfile: pointer.String("some-dockerfile"), Builder: &buildv1alpha1.Image{ - Credentials: &credentials, + Credentials: &corev1.LocalObjectReference{Name: "builder-credentials"}, Image: "builder-image", }, Output: buildv1alpha1.Image{ - Credentials: &credentials, + Credentials: &corev1.LocalObjectReference{Name: "output-credentials"}, Image: "output-image", }, Timeout: &metav1.Duration{ diff --git a/pkg/shp/flags/buildrun.go b/pkg/shp/flags/buildrun.go index fff124ca9..e5c59176b 100644 --- a/pkg/shp/flags/buildrun.go +++ b/pkg/shp/flags/buildrun.go @@ -1,11 +1,13 @@ package flags import ( - buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - "github.com/spf13/pflag" + "os" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + "github.com/spf13/pflag" ) // BuildRunSpecFromFlags creates a BuildRun spec from command-line flags. @@ -50,3 +52,28 @@ func SanitizeBuildRunSpec(br *buildv1alpha1.BuildRunSpec) { } } } + +// BuildRunOpts contain the end-user settings for creating buildruns +type BuildRunOpts struct { + LocalSourceDir string + PruneBundle bool + Wait bool + Follow bool +} + +// BuildRunOptsFromFlags sets a BuildRunOpts based on the command-line flags +func BuildRunOptsFromFlags(flags *pflag.FlagSet) *BuildRunOpts { + opts := &BuildRunOpts{} + + cwd, err := os.Getwd() + if err != nil { + cwd = "." + } + + flags.BoolVar(&opts.Wait, "wait", false, "wait until BuildRun runs to completion") + flags.BoolVarP(&opts.Follow, "follow", "F", false, "creates buildrun and watch its log until it completes or fails (this implies wait)") + flags.BoolVar(&opts.PruneBundle, "prune-bundle", false, "prune source code bundle after use (this implies wait)") + flags.StringVar(&opts.LocalSourceDir, "source-directory", cwd, "directory to be used for local source code") + + return opts +} diff --git a/pkg/shp/flags/flags.go b/pkg/shp/flags/flags.go index 84b56718a..15d9182f0 100644 --- a/pkg/shp/flags/flags.go +++ b/pkg/shp/flags/flags.go @@ -22,6 +22,8 @@ const ( SourceURLFlag = "source-url" // SourceRevisionFlag command-line flag. SourceRevisionFlag = "source-revision" + // SourceContainerImageFlag command-line flag. + SourceContainerImageFlag = "source-bundle-image" // SourceContextDirFlag command-line flag. SourceContextDirFlag = "source-context-dir" // SourceCredentialsSecretFlag command-line flag. @@ -58,6 +60,12 @@ func sourceFlags(flags *pflag.FlagSet, source *buildv1alpha1.Source) { "", "git repository source revision", ) + flags.StringVar( + &source.BundleContainer.Image, + SourceContainerImageFlag, + "", + "source code bundle image", + ) flags.StringVar( source.ContextDir, SourceContextDirFlag, @@ -68,7 +76,7 @@ func sourceFlags(flags *pflag.FlagSet, source *buildv1alpha1.Source) { &source.Credentials.Name, SourceCredentialsSecretFlag, "", - "name of the secret with git repository credentials", + "name of the secret with credentials for Git repository/container registry", ) }