Skip to content

Commit

Permalink
Merge pull request #467 from zhangtbj/buildrunname_label
Browse files Browse the repository at this point in the history 
Add buildrunname label for custom metric
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot authored Nov 12, 2020
2 parents c2e796a + 76d3882 commit 3036828
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 50 deletions.
13 changes: 7 additions & 6 deletions docs/metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ Following build metrics are exposed at service `build-operator-metrics` on port
| ---- | ---- | ----------- | ------ | ------ |
| `build_builds_registered_total` | Counter | Number of total registered Builds. | buildstrategy=<build_buildstrategy_name> | experimental |
| `build_buildruns_completed_total` | Counter | Number of total completed BuildRuns. | buildstrategy=<build_buildstrategy_name> | experimental |
| `build_buildrun_establish_duration_seconds` | Histogram | BuildRun establish duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup> | experimental |
| `build_buildrun_completion_duration_seconds` | Histogram | BuildRun completion duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup> | experimental |
| `build_buildrun_rampup_duration_seconds` | Histogram | BuildRun ramp-up duration in seconds | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup> | experimental |
| `build_buildrun_taskrun_rampup_duration_seconds` | Histogram | BuildRun taskrun ramp-up duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup> | experimental |
| `build_buildrun_taskrun_pod_rampup_duration_seconds` | Histogram | BuildRun taskrun pod ramp-up duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup> | experimental |
| `build_buildrun_establish_duration_seconds` | Histogram | BuildRun establish duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup><br>buildrun=<buildrun_name> <sup>1</sup> | experimental |
| `build_buildrun_completion_duration_seconds` | Histogram | BuildRun completion duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup><br>buildrun=<buildrun_name> <sup>1</sup> | experimental |
| `build_buildrun_rampup_duration_seconds` | Histogram | BuildRun ramp-up duration in seconds | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup><br>buildrun=<buildrun_name> <sup>1</sup> | experimental |
| `build_buildrun_taskrun_rampup_duration_seconds` | Histogram | BuildRun taskrun ramp-up duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup><br>buildrun=<buildrun_name> <sup>1</sup> | experimental |
| `build_buildrun_taskrun_pod_rampup_duration_seconds` | Histogram | BuildRun taskrun pod ramp-up duration in seconds. | buildstrategy=<build_buildstrategy_name> <sup>1</sup><br>namespace=<buildrun_namespace> <sup>1</sup><br>buildrun=<buildrun_name> <sup>1</sup> | experimental |

<sup>1</sup> Labels for histograms are disabled by default. See [Configuration of histogram labels](#configuration-of-histogram-labels) to enable them.

Expand Down Expand Up @@ -57,6 +57,7 @@ As the amount of buckets and labels has a direct impact on the number of Prometh

* buildstrategy
* namespace
* buildrun

Use a comma-separated value to enable multiple labels. For example:

Expand All @@ -68,7 +69,7 @@ make local
or

```bash
export PROMETHEUS_HISTOGRAM_ENABLED_LABELS=buildstrategy,namespace
export PROMETHEUS_HISTOGRAM_ENABLED_LABELS=buildstrategy,namespace,buildrun
make local
```

Expand Down
5 changes: 5 additions & 0 deletions pkg/controller/buildrun/buildrun_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,7 @@ func (r *ReconcileBuildRun) Reconcile(request reconcile.Request) (reconcile.Resu
buildmetrics.BuildRunRampUpDurationObserve(
buildRun.Status.BuildSpec.StrategyRef.Name,
buildRun.Namespace,
buildRun.Name,
generatedTaskRun.CreationTimestamp.Time.Sub(buildRun.CreationTimestamp.Time),
)
} else {
Expand Down Expand Up @@ -401,6 +402,7 @@ func (r *ReconcileBuildRun) Reconcile(request reconcile.Request) (reconcile.Resu
buildmetrics.BuildRunEstablishObserve(
buildRun.Status.BuildSpec.StrategyRef.Name,
buildRun.Namespace,
buildRun.Name,
buildRun.Status.StartTime.Time.Sub(buildRun.CreationTimestamp.Time),
)
}
Expand All @@ -413,6 +415,7 @@ func (r *ReconcileBuildRun) Reconcile(request reconcile.Request) (reconcile.Resu
buildmetrics.BuildRunCompletionObserve(
buildRun.Status.BuildSpec.StrategyRef.Name,
buildRun.Namespace,
buildRun.Name,
buildRun.Status.CompletionTime.Time.Sub(buildRun.CreationTimestamp.Time),
)

Expand All @@ -429,6 +432,7 @@ func (r *ReconcileBuildRun) Reconcile(request reconcile.Request) (reconcile.Resu
buildmetrics.TaskRunPodRampUpDurationObserve(
buildRun.Status.BuildSpec.StrategyRef.Name,
buildRun.Namespace,
buildRun.Name,
lastInitPod.State.Terminated.FinishedAt.Sub(pod.CreationTimestamp.Time),
)
}
Expand All @@ -438,6 +442,7 @@ func (r *ReconcileBuildRun) Reconcile(request reconcile.Request) (reconcile.Resu
buildmetrics.TaskRunRampUpDurationObserve(
buildRun.Status.BuildSpec.StrategyRef.Name,
buildRun.Namespace,
buildRun.Name,
pod.CreationTimestamp.Time.Sub(lastTaskRun.CreationTimestamp.Time),
)

Expand Down
51 changes: 30 additions & 21 deletions pkg/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ import (
)

const (
buildStrategyLabel string = "buildstrategy"
namespaceLabel string = "namespace"
BuildStrategyLabel string = "buildstrategy"
NamespaceLabel string = "namespace"
BuildRunLabel string = "buildrun"
)

var (
Expand All @@ -25,14 +26,14 @@ var (
Name: "build_builds_registered_total",
Help: "Number of total registered Builds.",
},
[]string{buildStrategyLabel})
[]string{BuildStrategyLabel})

buildRunCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "build_buildruns_completed_total",
Help: "Number of total completed BuildRuns.",
},
[]string{buildStrategyLabel})
[]string{BuildStrategyLabel})

buildRunEstablishDuration *prometheus.HistogramVec
buildRunCompletionDuration *prometheus.HistogramVec
Expand All @@ -43,6 +44,7 @@ var (

histogramBuildStrategyLabelEnabled = false
histogramNamespaceLabelEnabled = false
histogramBuildRunLabelEnabled = false

initialized = false
)
Expand All @@ -56,14 +58,18 @@ func InitPrometheus(config *config.Config) {
initialized = true

var histogramLabels []string
if contains(config.Prometheus.HistogramEnabledLabels, buildStrategyLabel) {
histogramLabels = append(histogramLabels, buildStrategyLabel)
if contains(config.Prometheus.HistogramEnabledLabels, BuildStrategyLabel) {
histogramLabels = append(histogramLabels, BuildStrategyLabel)
histogramBuildStrategyLabelEnabled = true
}
if contains(config.Prometheus.HistogramEnabledLabels, namespaceLabel) {
histogramLabels = append(histogramLabels, namespaceLabel)
if contains(config.Prometheus.HistogramEnabledLabels, NamespaceLabel) {
histogramLabels = append(histogramLabels, NamespaceLabel)
histogramNamespaceLabelEnabled = true
}
if contains(config.Prometheus.HistogramEnabledLabels, BuildRunLabel) {
histogramLabels = append(histogramLabels, BuildRunLabel)
histogramBuildRunLabelEnabled = true
}

buildRunEstablishDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Expand Down Expand Up @@ -126,14 +132,17 @@ func contains(slice []string, element string) bool {
return false
}

func createHistogramLabels(buildStrategy string, namespace string) prometheus.Labels {
func createHistogramLabels(buildStrategy string, namespace string, buildRun string) prometheus.Labels {
labels := prometheus.Labels{}

if histogramBuildStrategyLabelEnabled {
labels[buildStrategyLabel] = buildStrategy
labels[BuildStrategyLabel] = buildStrategy
}
if histogramNamespaceLabelEnabled {
labels[namespaceLabel] = namespace
labels[NamespaceLabel] = namespace
}
if histogramBuildRunLabelEnabled {
labels[BuildRunLabel] = buildRun
}

return labels
Expand All @@ -152,36 +161,36 @@ func BuildRunCountInc(buildStrategy string) {
}

// BuildRunEstablishObserve sets the build run establish time
func BuildRunEstablishObserve(buildStrategy string, namespace string, duration time.Duration) {
func BuildRunEstablishObserve(buildStrategy string, namespace string, buildRun string, duration time.Duration) {
if buildRunEstablishDuration != nil {
buildRunEstablishDuration.With(createHistogramLabels(buildStrategy, namespace)).Observe(duration.Seconds())
buildRunEstablishDuration.With(createHistogramLabels(buildStrategy, namespace, buildRun)).Observe(duration.Seconds())
}
}

// BuildRunCompletionObserve sets the build run completion time
func BuildRunCompletionObserve(buildStrategy string, namespace string, duration time.Duration) {
func BuildRunCompletionObserve(buildStrategy string, namespace string, buildRun string, duration time.Duration) {
if buildRunCompletionDuration != nil {
buildRunCompletionDuration.With(createHistogramLabels(buildStrategy, namespace)).Observe(duration.Seconds())
buildRunCompletionDuration.With(createHistogramLabels(buildStrategy, namespace, buildRun)).Observe(duration.Seconds())
}
}

// BuildRunRampUpDurationObserve processes the observation of a new buildrun ramp-up duration
func BuildRunRampUpDurationObserve(buildStrategy string, namespace string, duration time.Duration) {
func BuildRunRampUpDurationObserve(buildStrategy string, namespace string, buildRun string, duration time.Duration) {
if buildRunRampUpDuration != nil {
buildRunRampUpDuration.With(createHistogramLabels(buildStrategy, namespace)).Observe(duration.Seconds())
buildRunRampUpDuration.With(createHistogramLabels(buildStrategy, namespace, buildRun)).Observe(duration.Seconds())
}
}

// TaskRunRampUpDurationObserve processes the observation of a new taskrun ramp-up duration
func TaskRunRampUpDurationObserve(buildStrategy string, namespace string, duration time.Duration) {
func TaskRunRampUpDurationObserve(buildStrategy string, namespace string, buildRun string, duration time.Duration) {
if taskRunRampUpDuration != nil {
taskRunRampUpDuration.With(createHistogramLabels(buildStrategy, namespace)).Observe(duration.Seconds())
taskRunRampUpDuration.With(createHistogramLabels(buildStrategy, namespace, buildRun)).Observe(duration.Seconds())
}
}

// TaskRunPodRampUpDurationObserve processes the observation of a new taskrun pod ramp-up duration
func TaskRunPodRampUpDurationObserve(buildStrategy string, namespace string, duration time.Duration) {
func TaskRunPodRampUpDurationObserve(buildStrategy string, namespace string, buildRun string, duration time.Duration) {
if taskRunPodRampUpDuration != nil {
taskRunPodRampUpDuration.With(createHistogramLabels(buildStrategy, namespace)).Observe(duration.Seconds())
taskRunPodRampUpDuration.With(createHistogramLabels(buildStrategy, namespace, buildRun)).Observe(duration.Seconds())
}
}
48 changes: 25 additions & 23 deletions pkg/metrics/metrics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@ import (

var _ = Describe("Custom Metrics", func() {
type buildRunLabels struct {
namespace string
buildStrategy string
namespace string
buildRun string
}

var (
Expand All @@ -30,11 +31,12 @@ var _ = Describe("Custom Metrics", func() {
var result = buildRunLabels{}
for _, label := range in {
switch *label.Name {
case "buildstrategy":
case BuildStrategyLabel:
result.buildStrategy = *label.Value

case "namespace":
case NamespaceLabel:
result.namespace = *label.Value
case BuildRunLabel:
result.buildRun = *label.Value
}
}

Expand All @@ -45,8 +47,8 @@ var _ = Describe("Custom Metrics", func() {
BeforeSuite(func() {
var (
testLabels = []buildRunLabels{
{namespace: "default", buildStrategy: "kaniko"},
{namespace: "default", buildStrategy: "buildpacks"},
{buildStrategy: "kaniko", namespace: "default", buildRun: "kaniko-buildrun"},
{buildStrategy: "buildpacks", namespace: "default", buildRun: "buildpacks-buildrun"},
}

knownCounterMetrics = []string{
Expand Down Expand Up @@ -75,22 +77,22 @@ var _ = Describe("Custom Metrics", func() {

// initialize prometheus (second init should be no-op)
config := config.NewDefaultConfig()
config.Prometheus.HistogramEnabledLabels = []string{"buildstrategy", "namespace"}
config.Prometheus.HistogramEnabledLabels = []string{BuildStrategyLabel, NamespaceLabel, BuildRunLabel}
InitPrometheus(config)
InitPrometheus(config)

// and fire some examples
for _, entry := range testLabels {
buildStrategy, namespace := entry.buildStrategy, entry.namespace
buildStrategy, namespace, buildRun := entry.buildStrategy, entry.namespace, entry.buildRun

// tell prometheus some things have happened
BuildCountInc(buildStrategy)
BuildRunCountInc(buildStrategy)
BuildRunEstablishObserve(buildStrategy, namespace, time.Duration(1)*time.Second)
BuildRunCompletionObserve(buildStrategy, namespace, time.Duration(200)*time.Second)
BuildRunRampUpDurationObserve(buildStrategy, namespace, time.Duration(1)*time.Second)
TaskRunRampUpDurationObserve(buildStrategy, namespace, time.Duration(2)*time.Second)
TaskRunPodRampUpDurationObserve(buildStrategy, namespace, time.Duration(3)*time.Second)
BuildRunEstablishObserve(buildStrategy, namespace, buildRun, time.Duration(1)*time.Second)
BuildRunCompletionObserve(buildStrategy, namespace, buildRun, time.Duration(200)*time.Second)
BuildRunRampUpDurationObserve(buildStrategy, namespace, buildRun, time.Duration(1)*time.Second)
TaskRunRampUpDurationObserve(buildStrategy, namespace, buildRun, time.Duration(2)*time.Second)
TaskRunPodRampUpDurationObserve(buildStrategy, namespace, buildRun, time.Duration(3)*time.Second)
}

// gather metrics from prometheus and fill the result maps
Expand Down Expand Up @@ -130,22 +132,22 @@ var _ = Describe("Custom Metrics", func() {

It("should record the kaniko buildrun establish time", func() {
Expect(histogramMetrics).To(HaveKey("build_buildrun_establish_duration_seconds"))
Expect(histogramMetrics["build_buildrun_establish_duration_seconds"][buildRunLabels{"default", "kaniko"}]).To(Equal(1.0))
Expect(histogramMetrics["build_buildrun_establish_duration_seconds"][buildRunLabels{"kaniko", "default", "kaniko-buildrun"}]).To(Equal(1.0))
})

It("should record the kaniko buildrun completion time", func() {
Expect(histogramMetrics).To(HaveKey("build_buildrun_completion_duration_seconds"))
Expect(histogramMetrics["build_buildrun_completion_duration_seconds"][buildRunLabels{"default", "kaniko"}]).To(Equal(200.0))
Expect(histogramMetrics["build_buildrun_completion_duration_seconds"][buildRunLabels{"kaniko", "default", "kaniko-buildrun"}]).To(Equal(200.0))
})

It("should record the kaniko ramp-up durations", func() {
Expect(histogramMetrics).To(HaveKey("build_buildrun_rampup_duration_seconds"))
Expect(histogramMetrics).To(HaveKey("build_buildrun_taskrun_rampup_duration_seconds"))
Expect(histogramMetrics).To(HaveKey("build_buildrun_taskrun_pod_rampup_duration_seconds"))

Expect(histogramMetrics["build_buildrun_rampup_duration_seconds"][buildRunLabels{"default", "kaniko"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_rampup_duration_seconds"][buildRunLabels{"default", "kaniko"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_pod_rampup_duration_seconds"][buildRunLabels{"default", "kaniko"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_rampup_duration_seconds"][buildRunLabels{"kaniko", "default", "kaniko-buildrun"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_rampup_duration_seconds"][buildRunLabels{"kaniko", "default", "kaniko-buildrun"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_pod_rampup_duration_seconds"][buildRunLabels{"kaniko", "default", "kaniko-buildrun"}]).To(BeNumerically(">", 0.0))
})
})

Expand All @@ -163,22 +165,22 @@ var _ = Describe("Custom Metrics", func() {

It("should record the buildpacks buildrun establish time", func() {
Expect(histogramMetrics).To(HaveKey("build_buildrun_establish_duration_seconds"))
Expect(histogramMetrics["build_buildrun_establish_duration_seconds"][buildRunLabels{"default", "buildpacks"}]).To(Equal(1.0))
Expect(histogramMetrics["build_buildrun_establish_duration_seconds"][buildRunLabels{"buildpacks", "default", "buildpacks-buildrun"}]).To(Equal(1.0))
})

It("should record the buildpacks buildrun completion time", func() {
Expect(histogramMetrics).To(HaveKey("build_buildrun_completion_duration_seconds"))
Expect(histogramMetrics["build_buildrun_completion_duration_seconds"][buildRunLabels{"default", "buildpacks"}]).To(Equal(200.0))
Expect(histogramMetrics["build_buildrun_completion_duration_seconds"][buildRunLabels{"buildpacks", "default", "buildpacks-buildrun"}]).To(Equal(200.0))
})

It("should record the buildpacks ramp-up durations", func() {
Expect(histogramMetrics).To(HaveKey("build_buildrun_rampup_duration_seconds"))
Expect(histogramMetrics).To(HaveKey("build_buildrun_taskrun_rampup_duration_seconds"))
Expect(histogramMetrics).To(HaveKey("build_buildrun_taskrun_pod_rampup_duration_seconds"))

Expect(histogramMetrics["build_buildrun_rampup_duration_seconds"][buildRunLabels{"default", "buildpacks"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_rampup_duration_seconds"][buildRunLabels{"default", "buildpacks"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_pod_rampup_duration_seconds"][buildRunLabels{"default", "buildpacks"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_rampup_duration_seconds"][buildRunLabels{"buildpacks", "default", "buildpacks-buildrun"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_rampup_duration_seconds"][buildRunLabels{"buildpacks", "default", "buildpacks-buildrun"}]).To(BeNumerically(">", 0.0))
Expect(histogramMetrics["build_buildrun_taskrun_pod_rampup_duration_seconds"][buildRunLabels{"buildpacks", "default", "buildpacks-buildrun"}]).To(BeNumerically(">", 0.0))
})
})
})

0 comments on commit 3036828

Please sign in to comment.