fix: Introduce a Go routine to remove no longer existing Kymas from t…

…he metrics (#1273) * import gocron package * Start the goroutine with flag * routine implementation * add unit test for fetching lifecycle manager metrics * Add the second unit test * Add the second unit test * fix e2e * try time.sleep in the kyma metrics test * Add a check for the error * Add a check for the error * Comments
kyma-project · Feb 6, 2024 · d1a203c · d1a203c
1 parent cc95495
commit d1a203c
Show file tree

Hide file tree

Showing 10 changed files with 279 additions and 24 deletions.
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
@@ -141,6 +141,18 @@ jobs:
           cat purge_finalizer.yaml
           kustomize edit add patch --path purge_finalizer.yaml --kind Deployment
           popd
+      - name: Patch metrics cleanup interval
+        if : ${{ matrix.e2e-test == 'kyma-metrics' }}
+        working-directory: lifecycle-manager
+        run: |
+          pushd config/watcher_local_test
+          echo \
+          "- op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --metrics-cleanup-interval=1" >> metrics_cleanup.yaml
+          cat metrics_cleanup.yaml
+          kustomize edit add patch --path metrics_cleanup.yaml --kind Deployment
+          popd
       - name: Patch self signed certificate lifetime
         if: ${{matrix.e2e-test == 'self-signed-certificate-rotation'}}
         working-directory: lifecycle-manager

diff --git a/.golangci.yaml b/.golangci.yaml
@@ -128,6 +128,8 @@ linters-settings:
         alias: ctrlmetrics
       - pkg: sigs.k8s.io/controller-runtime/pkg/metrics/server
         alias: metricsserver
+      - pkg: github.com/prometheus/client_model/go
+        alias: prometheusclient
   ireturn:
     allow:
       - anon

diff --git a/cmd/main.go b/cmd/main.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
+	"github.com/go-co-op/gocron"
 	"go.uber.org/zap/zapcore"
 	"golang.org/x/time/rate"
 	istioclientapiv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
@@ -66,6 +67,8 @@ import (
 	// +kubebuilder:scaffold:imports
 )
 
+const metricCleanupTimeout = 5 * time.Minute
+
 var (
 	scheme   = machineryruntime.NewScheme() //nolint:gochecknoglobals // scheme used to add CRDs
 	setupLog = ctrl.Log.WithName("setup")   //nolint:gochecknoglobals // logger used for setup
@@ -157,19 +160,34 @@ func setupManager(flagVar *flags.FlagVar, cacheOptions cache.Options, scheme *ma
 	remoteClientCache := remote.NewClientCache()
 	sharedMetrics := metrics.NewSharedMetrics()
 	descriptorProvider := provider.NewCachedDescriptorProvider(nil)
-	setupKymaReconciler(mgr, remoteClientCache, descriptorProvider, flagVar, options, skrWebhookManager, sharedMetrics)
+	kymaMetrics := metrics.NewKymaMetrics(sharedMetrics)
+	setupKymaReconciler(mgr, remoteClientCache, descriptorProvider, flagVar, options, skrWebhookManager, kymaMetrics)
 	setupManifestReconciler(mgr, flagVar, options, sharedMetrics)
 	setupMandatoryModuleReconciler(mgr, descriptorProvider, flagVar, options)
 	setupMandatoryModuleDeletionReconciler(mgr, descriptorProvider, flagVar, options)
 
 	if flagVar.EnablePurgeFinalizer {
 		setupPurgeReconciler(mgr, remoteClientCache, flagVar, options)
 	}
-
 	if flagVar.EnableWebhooks {
 		enableWebhooks(mgr)
 	}
 
+	addHealthChecks(mgr)
+	if flagVar.DropStoredVersion != "" {
+		go func(version string) {
+			dropStoredVersion(mgr, version)
+		}(flagVar.DropStoredVersion)
+	}
+	go runKymaMetricsCleanup(kymaMetrics, mgr.GetClient(), flagVar.MetricsCleanupIntervalInMinutes)
+
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		setupLog.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+}
+
+func addHealthChecks(mgr manager.Manager) {
 	// +kubebuilder:scaffold:builder
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
@@ -179,15 +197,24 @@ func setupManager(flagVar *flags.FlagVar, cacheOptions cache.Options, scheme *ma
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
-	if flagVar.DropStoredVersion != "" {
-		go func(version string) {
-			dropStoredVersion(mgr, version)
-		}(flagVar.DropStoredVersion)
-	}
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
+}
+
+func runKymaMetricsCleanup(kymaMetrics *metrics.KymaMetrics, kcpClient client.Client,
+	cleanupIntervalInMinutes int,
+) {
+	scheduler := gocron.NewScheduler(time.UTC)
+	_, scheduleErr := scheduler.Every(cleanupIntervalInMinutes).Minutes().Do(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), metricCleanupTimeout)
+		defer cancel()
+		if err := kymaMetrics.CleanupNonExistingKymaCrsMetrics(ctx, kcpClient); err != nil {
+			setupLog.Info(fmt.Sprintf("failed to cleanup non existing kyma crs metrics, err: %s", err))
+		}
+	})
+	if scheduleErr != nil {
+		setupLog.Info(fmt.Sprintf("failed to setup cleanup routine for non existing kyma crs metrics, err: %s",
+			scheduleErr))
 	}
+	scheduler.StartAsync()
 }
 
 func enableWebhooks(mgr manager.Manager) {
@@ -225,9 +252,10 @@ func controllerOptionsFromFlagVar(flagVar *flags.FlagVar) ctrlruntime.Options {
 	}
 }
 
-func setupKymaReconciler(mgr ctrl.Manager, remoteClientCache *remote.ClientCache, descriptorProvider *provider.CachedDescriptorProvider,
+func setupKymaReconciler(mgr ctrl.Manager, remoteClientCache *remote.ClientCache,
+	descriptorProvider *provider.CachedDescriptorProvider,
 	flagVar *flags.FlagVar, options ctrlruntime.Options, skrWebhookManager *watcher.SKRWebhookManifestManager,
-	sharedMetrics *metrics.SharedMetrics,
+	kymaMetrics *metrics.KymaMetrics,
 ) {
 	options.MaxConcurrentReconciles = flagVar.MaxConcurrentKymaReconciles
 	kcpRestConfig := mgr.GetConfig()
@@ -248,7 +276,7 @@ func setupKymaReconciler(mgr ctrl.Manager, remoteClientCache *remote.ClientCache
 		InKCPMode:           flagVar.InKCPMode,
 		RemoteSyncNamespace: flagVar.RemoteSyncNamespace,
 		IsManagedKyma:       flagVar.IsKymaManaged,
-		Metrics:             metrics.NewKymaMetrics(sharedMetrics),
+		Metrics:             kymaMetrics,
 	}).SetupWithManager(
 		mgr, options, controller.SetupUpSetting{
 			ListenerAddr:                 flagVar.KymaListenerAddr,

diff --git a/go.mod b/go.mod
@@ -34,6 +34,8 @@ require (
 )
 
 require (
+	github.com/go-co-op/gocron v1.37.0
+	github.com/prometheus/client_model v0.5.0
 	k8s.io/api v0.29.1
 	k8s.io/apiextensions-apiserver v0.29.1
 	k8s.io/apimachinery v0.29.1
@@ -105,6 +107,7 @@ require (
 	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231024185945-8841054dbdb8 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/buildkite/agent/v3 v3.58.0 // indirect
 	github.com/buildkite/interpolate v0.0.0-20200526001904-07f35b4ae251 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -249,10 +252,10 @@ require (
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/puzpuzpuz/xsync/v2 v2.5.1 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -337,6 +340,7 @@ require (
 	inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a // indirect
 	k8s.io/component-base v0.29.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/kube-aggregator v0.28.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/gateway-api v0.8.0 // indirect

diff --git a/go.sum b/go.sum
@@ -327,6 +327,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
@@ -683,6 +685,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0=
+github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -1122,6 +1126,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -1426,11 +1431,15 @@ github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2z
 github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA=
 github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
@@ -2453,6 +2462,8 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
+k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs=
+k8s.io/kube-aggregator v0.28.1/go.mod h1:JaLizMe+AECSpO2OmrWVsvnG0V3dX1RpW+Wq/QHbu18=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=

diff --git a/internal/pkg/flags/flags.go b/internal/pkg/flags/flags.go
@@ -51,6 +51,7 @@ const (
 	DefaultWatcherResourceLimitsCPU                                     = "0.1"
 	DefaultWatcherResourceLimitsMemory                                  = "200Mi"
 	DefaultDropStoredVersion                                            = "v1alpha1"
+	DefaultMetricsCleanupIntervalInMinutes                              = 15
 )
 
 var (
@@ -188,6 +189,9 @@ func DefineFlagVar() *FlagVar {
 		"The resources.limits.cpu for skr webhook.")
 	flag.StringVar(&flagVar.WatcherResourcesPath, "skr-watcher-path", DefaultWatcherResourcesPath,
 		"The path to the skr watcher resources.")
+	flag.IntVar(&flagVar.MetricsCleanupIntervalInMinutes, "metrics-cleanup-interval",
+		DefaultMetricsCleanupIntervalInMinutes,
+		"The interval at which the cleanup of non-existing kyma CRs metrics runs.")
 	return flagVar
 }
 
@@ -246,6 +250,7 @@ type FlagVar struct {
 	WatcherResourceLimitsMemory            string
 	WatcherResourceLimitsCPU               string
 	WatcherResourcesPath                   string
+	MetricsCleanupIntervalInMinutes        int
 }
 
 func (f FlagVar) Validate() error {

diff --git a/internal/pkg/metrics/kyma.go b/internal/pkg/metrics/kyma.go
@@ -1,9 +1,13 @@
 package metrics
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/cert-manager/cert-manager/pkg/logs"
 	"github.com/prometheus/client_golang/prometheus"
+	prometheusclient "github.com/prometheus/client_model/go"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/kyma-project/lifecycle-manager/api/shared"
@@ -19,7 +23,7 @@ const (
 )
 
 type KymaMetrics struct {
-	kymaStateGauge   *prometheus.GaugeVec
+	KymaStateGauge   *prometheus.GaugeVec
 	moduleStateGauge *prometheus.GaugeVec
 	*SharedMetrics
 }
@@ -53,7 +57,7 @@ const (
 func NewKymaMetrics(sharedMetrics *SharedMetrics) *KymaMetrics {
 	kymaMetrics := &KymaMetrics{
 		SharedMetrics: sharedMetrics,
-		kymaStateGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		KymaStateGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: MetricKymaState,
 			Help: "Indicates the Status.state for a given Kyma object",
 		}, []string{KymaNameLabel, stateLabel, shootIDLabel, instanceIDLabel}),
@@ -63,7 +67,7 @@ func NewKymaMetrics(sharedMetrics *SharedMetrics) *KymaMetrics {
 			Help: "Indicates the Status.state for modules of Kyma",
 		}, []string{moduleNameLabel, KymaNameLabel, stateLabel, shootIDLabel, instanceIDLabel}),
 	}
-	ctrlmetrics.Registry.MustRegister(kymaMetrics.kymaStateGauge)
+	ctrlmetrics.Registry.MustRegister(kymaMetrics.KymaStateGauge)
 	ctrlmetrics.Registry.MustRegister(kymaMetrics.moduleStateGauge)
 	return kymaMetrics
 }
@@ -80,6 +84,7 @@ func (k *KymaMetrics) UpdateAll(kyma *v1beta2.Kyma) error {
 	}
 
 	k.setKymaStateGauge(kyma.Status.State, kyma.Name, shootID, instanceID)
+
 	for _, moduleStatus := range kyma.Status.Modules {
 		k.setModuleStateGauge(moduleStatus.State, moduleStatus.Name, kyma.Name, shootID, instanceID)
 	}
@@ -89,7 +94,7 @@ func (k *KymaMetrics) UpdateAll(kyma *v1beta2.Kyma) error {
 // CleanupMetrics deletes all 'lifecycle_mgr_kyma_state',
 // 'lifecycle_mgr_module_state' metrics for the matching Kyma.
 func (k *KymaMetrics) CleanupMetrics(kymaName string) {
-	k.kymaStateGauge.DeletePartialMatch(prometheus.Labels{
+	k.KymaStateGauge.DeletePartialMatch(prometheus.Labels{
 		KymaNameLabel: kymaName,
 	})
 	k.moduleStateGauge.DeletePartialMatch(prometheus.Labels{
@@ -109,7 +114,7 @@ func (k *KymaMetrics) setKymaStateGauge(newState shared.State, kymaName, shootID
 	states := shared.AllStates()
 	for _, state := range states {
 		newValue := calcStateValue(state, newState)
-		k.kymaStateGauge.With(prometheus.Labels{
+		k.KymaStateGauge.With(prometheus.Labels{
 			KymaNameLabel:   kymaName,
 			shootIDLabel:    shootID,
 			instanceIDLabel: instanceID,
@@ -142,3 +147,72 @@ func calcStateValue(state, newState shared.State) float64 {
 func (k *KymaMetrics) RecordRequeueReason(kymaRequeueReason KymaRequeueReason, requeueType RequeueType) {
 	k.requeueReasonCounter.WithLabelValues(string(kymaRequeueReason), string(requeueType)).Inc()
 }
+
+func (k *KymaMetrics) CleanupNonExistingKymaCrsMetrics(ctx context.Context, kcpClient client.Client) error {
+	currentLifecycleManagerMetrics, err := FetchLifecycleManagerMetrics()
+	if err != nil {
+		return err
+	}
+
+	if len(currentLifecycleManagerMetrics) == 0 {
+		return nil
+	}
+
+	kymaCrsList := &v1beta2.KymaList{}
+	err = kcpClient.List(ctx, kymaCrsList)
+	if err != nil {
+		return fmt.Errorf("failed to fetch Kyma CRs, %w", err)
+	}
+	kymaNames := getKymaNames(kymaCrsList)
+	for _, m := range currentLifecycleManagerMetrics {
+		currentKymaName := getKymaNameFromLabels(m)
+		if currentKymaName == "" {
+			continue
+		}
+		if _, exists := kymaNames[currentKymaName]; !exists {
+			logs.FromContext(ctx).Info(fmt.Sprintf("Deleting a metric for non-existing Kyma: %s", currentKymaName))
+			k.KymaStateGauge.DeletePartialMatch(prometheus.Labels{
+				KymaNameLabel: currentKymaName,
+			})
+		}
+	}
+
+	return nil
+}
+
+func FetchLifecycleManagerMetrics() ([]*prometheusclient.Metric, error) {
+	currentMetrics, err := ctrlmetrics.Registry.Gather()
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch current kyma metrics, %w", err)
+	}
+
+	for _, metric := range currentMetrics {
+		if metric.GetName() == MetricKymaState {
+			return metric.GetMetric(), nil
+		}
+	}
+
+	return nil, nil
+}
+
+func getKymaNameFromLabels(metric *prometheusclient.Metric) string {
+	for _, label := range metric.GetLabel() {
+		if label.GetName() == KymaNameLabel {
+			return label.GetValue()
+		}
+	}
+
+	return ""
+}
+
+func getKymaNames(kymaCrs *v1beta2.KymaList) map[string]bool {
+	if len(kymaCrs.Items) == 0 {
+		return nil
+	}
+
+	names := make(map[string]bool)
+	for _, kyma := range kymaCrs.Items {
+		names[kyma.GetName()] = true
+	}
+	return names
+}