Skip to content

Commit

Permalink
simplifying matrix combinations
Browse files Browse the repository at this point in the history 
Matrix combinations is created in [][]Param format which is very hard
to process with matrix.Include. The parameters value can change in a combination
or a new combination can be added based on matrix.Include.Params.

Its easy to compare and work with map[string]string instead of []Param.

Signed-off-by: pritidesai <[email protected]>
  • Loading branch information
@pritidesai
pritidesai committed Mar 9, 2023
1 parent e8f2637 commit 50e272d
Show file tree
Hide file tree
Showing 16 changed files with 693 additions and 452 deletions.
123 changes: 57 additions & 66 deletions docs/pipeline-api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1320,6 +1320,19 @@ string
</tr>
</tbody>
</table>
<h3 id="tekton.dev/v1.Combination">Combination
(<code>map[string]string</code> alias)</h3>
<div>
<p>Combination holds a single combination from a Matrix with key as param.Name and value as param.Value</p>
</div>
<h3 id="tekton.dev/v1.Combinations">Combinations
(<code>[]github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Combination</code> alias)</h3>
<p>
(<em>Appears on:</em><a href="#tekton.dev/v1.Matrix">Matrix</a>)
</p>
<div>
<p>Combinations holds a list of combination for a give Matrix</p>
</div>
<h3 id="tekton.dev/v1.ConfigSource">ConfigSource
</h3>
<p>
Expand Down Expand Up @@ -1515,6 +1528,21 @@ The names of the <code>params</code> in the <code>Matrix</code> must match the n
Note that Include is in preview mode and not yet supported.</p>
</td>
</tr>
<tr>
<td>
<code>-</code><br/>
<em>
<a href="#tekton.dev/v1.Combinations">
Combinations
</a>
</em>
</td>
<td>
<p>combinations is a list of combinations created based on a given Matrix configurations including
Matrix.Params and Matrix.Include.Params
Combinations is skipped while encoding JSON since they are calculated based on the specified params</p>
</td>
</tr>
</tbody>
</table>
<h3 id="tekton.dev/v1.MatrixInclude">MatrixInclude
Expand Down Expand Up @@ -8176,6 +8204,19 @@ int32
</tr>
</tbody>
</table>
<h3 id="tekton.dev/v1beta1.Combination">Combination
(<code>map[string]string</code> alias)</h3>
<div>
<p>Combination holds a single combination from a Matrix with key as param.Name and value as param.Value</p>
</div>
<h3 id="tekton.dev/v1beta1.Combinations">Combinations
(<code>[]github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1.Combination</code> alias)</h3>
<p>
(<em>Appears on:</em><a href="#tekton.dev/v1beta1.Matrix">Matrix</a>)
</p>
<div>
<p>Combinations holds a list of combination for a give Matrix</p>
</div>
<h3 id="tekton.dev/v1beta1.ConfigSource">ConfigSource
</h3>
<p>
Expand Down Expand Up @@ -8601,6 +8642,21 @@ The names of the <code>params</code> in the <code>Matrix</code> must match the n
Note that Include is in preview mode and not yet supported.</p>
</td>
</tr>
<tr>
<td>
<code>-</code><br/>
<em>
<a href="#tekton.dev/v1beta1.Combinations">
Combinations
</a>
</em>
</td>
<td>
<p>combinations is a list of combinations created based on a given Matrix configurations including
Matrix.Params and Matrix.Include.Params
Combinations is skipped while encoding JSON since they are calculated based on the specified params</p>
</td>
</tr>
</tbody>
</table>
<h3 id="tekton.dev/v1beta1.MatrixInclude">MatrixInclude
Expand Down Expand Up @@ -8658,11 +8714,7 @@ The names of the <code>params</code> must match the names of the <code>params</c
<h3 id="tekton.dev/v1beta1.Param">Param
</h3>
<p>
<<<<<<< HEAD
(<em>Appears on:</em><a href="#tekton.dev/v1alpha1.RunSpec">RunSpec</a>, <a href="#tekton.dev/v1beta1.CustomRunSpec">CustomRunSpec</a>, <a href="#tekton.dev/v1beta1.PipelineRunSpec">PipelineRunSpec</a>, <a href="#tekton.dev/v1beta1.ResolverRef">ResolverRef</a>, <a href="#tekton.dev/v1beta1.TaskRunInputs">TaskRunInputs</a>, <a href="#tekton.dev/v1beta1.TaskRunSpec">TaskRunSpec</a>, <a href="#resolution.tekton.dev/v1beta1.ResolutionRequestSpec">ResolutionRequestSpec</a>)
=======
(<em>Appears on:</em><a href="#tekton.dev/v1alpha1.RunSpec">RunSpec</a>, <a href="#tekton.dev/v1beta1.CustomRunSpec">CustomRunSpec</a>, <a href="#tekton.dev/v1beta1.Matrix">Matrix</a>, <a href="#tekton.dev/v1beta1.MatrixInclude">MatrixInclude</a>, <a href="#tekton.dev/v1beta1.PipelineRunSpec">PipelineRunSpec</a>, <a href="#tekton.dev/v1beta1.PipelineTask">PipelineTask</a>, <a href="#tekton.dev/v1beta1.ResolverRef">ResolverRef</a>, <a href="#tekton.dev/v1beta1.TaskRunSpec">TaskRunSpec</a>, <a href="#resolution.tekton.dev/v1beta1.ResolutionRequestSpec">ResolutionRequestSpec</a>)
>>>>>>> 88e27b27f (Remove Git, Storage and Generic PipelineResources)
(<em>Appears on:</em><a href="#tekton.dev/v1alpha1.RunSpec">RunSpec</a>, <a href="#tekton.dev/v1beta1.CustomRunSpec">CustomRunSpec</a>, <a href="#tekton.dev/v1beta1.PipelineRunSpec">PipelineRunSpec</a>, <a href="#tekton.dev/v1beta1.ResolverRef">ResolverRef</a>, <a href="#tekton.dev/v1beta1.TaskRunSpec">TaskRunSpec</a>, <a href="#resolution.tekton.dev/v1beta1.ResolutionRequestSpec">ResolutionRequestSpec</a>)
</p>
<div>
<p>Param declares an ParamValues to use for the parameter called name.</p>
Expand Down Expand Up @@ -8863,7 +8915,6 @@ map[string]string
</tr>
</tbody>
</table>
<<<<<<< HEAD
<h3 id="tekton.dev/v1beta1.Params">Params
(<code>[]github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1.Param</code> alias)</h3>
<p>
Expand All @@ -8872,66 +8923,6 @@ map[string]string
<div>
<p>Params is a list of Param</p>
</div>
<h3 id="tekton.dev/v1beta1.PipelineDeclaredResource">PipelineDeclaredResource
</h3>
<p>
(<em>Appears on:</em><a href="#tekton.dev/v1beta1.PipelineSpec">PipelineSpec</a>)
</p>
<div>
<p>PipelineDeclaredResource is used by a Pipeline to declare the types of the
PipelineResources that it will required to run and names which can be used to
refer to these PipelineResources in PipelineTaskResourceBindings.</p>
</div>
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>name</code><br/>
<em>
string
</em>
</td>
<td>
<p>Name is the name that will be used by the Pipeline to refer to this resource.
It does not directly correspond to the name of any PipelineResources Task
inputs or outputs, and it does not correspond to the actual names of the
PipelineResources that will be bound in the PipelineRun.</p>
</td>
</tr>
<tr>
<td>
<code>type</code><br/>
<em>
string
</em>
</td>
<td>
<p>Type is the type of the PipelineResource.</p>
</td>
</tr>
<tr>
<td>
<code>optional</code><br/>
<em>
bool
</em>
</td>
<td>
<p>Optional declares the resource as optional.
optional: true - the resource is considered optional
optional: false - the resource is considered required (default/equivalent of not specifying it)</p>
</td>
</tr>
</tbody>
</table>
=======
>>>>>>> 88e27b27f (Remove Git, Storage and Generic PipelineResources)
<h3 id="tekton.dev/v1beta1.PipelineObject">PipelineObject
</h3>
<div>
Expand Down
25 changes: 13 additions & 12 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/tektoncd/plumbing v0.0.0-20220817140952-3da8ce01aeeb
go.opencensus.io v0.24.0
go.uber.org/zap v1.24.0
golang.org/x/exp v0.0.0-20230307190834-24139beb5833
golang.org/x/oauth2 v0.6.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0
gopkg.in/square/go-jose.v2 v2.6.0
Expand All @@ -41,9 +42,19 @@ require github.com/benbjohnson/clock v1.1.0 // indirect

require (
code.gitea.io/sdk/gitea v0.15.1
github.com/cenkalti/backoff/v3 v3.2.2
github.com/goccy/kpoward v0.1.0
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221030203717-1711cefd7eec
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-retryablehttp v0.7.2
github.com/hashicorp/go-rootcerts v1.0.2
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2
github.com/hashicorp/go-sockaddr v1.0.2
github.com/hashicorp/hcl v1.0.0
github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf
github.com/mitchellh/mapstructure v1.5.0
github.com/ryanuber/go-glob v1.0.0
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399
go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/exporters/jaeger v1.14.0
Expand Down Expand Up @@ -73,7 +84,6 @@ require (
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/aws/aws-sdk-go-v2/service/kms v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/fatih/color v1.13.0 // indirect
Expand All @@ -83,21 +93,12 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/vault/api v1.9.0 // indirect
github.com/jellydator/ttlcache/v2 v2.11.1 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/skeema/knownhosts v1.1.0 // indirect
github.com/theupdateframework/go-tuf v0.5.2 // indirect
github.com/zeebo/errs v1.3.0 // indirect
Expand Down Expand Up @@ -193,12 +194,12 @@ require (
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.6.0
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/net v0.8.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/time v0.3.0
golang.org/x/tools v0.6.0 // indirect
google.golang.org/api v0.110.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

86 changes: 65 additions & 21 deletions pkg/apis/pipeline/v1/matrix_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ package v1
import (
"context"
"fmt"
"sort"

"github.com/tektoncd/pipeline/pkg/apis/config"
"golang.org/x/exp/maps"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/strings/slices"
"knative.dev/pkg/apis"
Expand All @@ -37,6 +39,11 @@ type Matrix struct {
// +optional
// +listType=atomic
Include []MatrixInclude `json:"include,omitempty"`

// combinations is a list of combinations created based on a given Matrix configurations including
// Matrix.Params and Matrix.Include.Params
// Combinations is skipped while encoding JSON since they are calculated based on the specified params
Combinations Combinations `json:"-"`
}

// MatrixInclude allows passing in a specific combinations of Parameters into the Matrix.
Expand All @@ -51,49 +58,86 @@ type MatrixInclude struct {
Params Params `json:"params,omitempty"`
}

// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array.
func (m *Matrix) FanOut() []Params {
var combinations []Params
for _, parameter := range m.Params {
combinations = fanOut(parameter, combinations)
// Combination holds a single combination from a Matrix with key as param.Name and value as param.Value
type Combination map[string]string

// Combinations holds a list of combination for a give Matrix
type Combinations []Combination

// ToParams transforms Combinations from a slice of map[string]string to a slice of Params
// such that, these combinations can be directly consumed in creating taskRun/run object
func (cs Combinations) ToParams() []Params {
listOfParams := make([]Params, len(cs))
for i := range cs {
var params Params
combination := cs[i]
order, _ := combination.sortCombination()
for _, key := range order {
params = append(params, Param{
Name: key,
Value: ParamValue{Type: ParamTypeString, StringVal: combination[key]},
})
}
listOfParams[i] = params
}
return combinations
return listOfParams
}

// fanOut generates a new combination based on a given Parameter in the Matrix.
func fanOut(param Param, combinations []Params) []Params {
if len(combinations) == 0 {
func (cs Combinations) fanOut(param Param) Combinations {
if len(cs) == 0 {
return initializeCombinations(param)
}
return distribute(param, combinations)
return cs.distribute(param)
}

// distribute generates a new combination of Parameters by adding a new Parameter to an existing list of Combinations.
func distribute(param Param, combinations []Params) []Params {
var expandedCombinations []Params
func (cs Combinations) distribute(param Param) Combinations {
var expandedCombinations Combinations
for _, value := range param.Value.ArrayVal {
for _, combination := range combinations {
expandedCombinations = append(expandedCombinations, createCombination(param.Name, value, combination))
for _, combination := range cs {
newCombination := make(Combination)
maps.Copy(newCombination, combination)
newCombination[param.Name] = value
_, orderedCombination := newCombination.sortCombination()
expandedCombinations = append(expandedCombinations, orderedCombination)
}
}
return expandedCombinations
}

// initializeCombinations generates a new combination based on the first Parameter in the Matrix.
func initializeCombinations(param Param) []Params {
var combinations []Params
func initializeCombinations(param Param) Combinations {
var combinations Combinations
for _, value := range param.Value.ArrayVal {
combinations = append(combinations, createCombination(param.Name, value, []Param{}))
combinations = append(combinations, Combination{param.Name: value})
}
return combinations
}

func createCombination(name string, value string, combination Params) Params {
combination = append(combination, Param{
Name: name,
Value: ParamValue{Type: ParamTypeString, StringVal: value},
// sortCombination sorts the given Combination based on the param names to produce a deterministic ordering
func (c Combination) sortCombination() ([]string, Combination) {
sortedCombination := make(Combination, len(c))
order := make([]string, 0, len(c))
for key := range c {
order = append(order, key)
}
sort.Slice(order, func(i, j int) bool {
return order[i] <= order[j]
})
return combination
for _, key := range order {
sortedCombination[key] = c[key]
}
return order, sortedCombination
}

// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array.
func (m *Matrix) FanOut() {
var combinations Combinations
for _, parameter := range m.Params {
combinations = combinations.fanOut(parameter)
}
m.Combinations = combinations
}

// CountCombinations returns the count of combinations of Parameters generated from the Matrix in PipelineTask.
Expand Down
Loading

0 comments on commit 50e272d

Please sign in to comment.