From 1a332e340ab98a7b4bd0e17abae818b112a4d68a Mon Sep 17 00:00:00 2001 From: Zach Aller Date: Thu, 25 Jan 2024 10:09:09 -0600 Subject: [PATCH] chore: add logging context around replicaset updates (#3326) * chore: add more logging Signed-off-by: Zach Aller * chore: remove accidental files Signed-off-by: Zach Aller --------- Signed-off-by: Zach Aller --- rollout/bluegreen.go | 8 ++++++-- rollout/canary.go | 10 +++++++--- rollout/replicaset.go | 4 ++-- rollout/sync.go | 11 +++++++---- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/rollout/bluegreen.go b/rollout/bluegreen.go index f1bcf7a7bb..9904de9dab 100644 --- a/rollout/bluegreen.go +++ b/rollout/bluegreen.go @@ -1,6 +1,7 @@ package rollout import ( + "fmt" "math" "sort" @@ -22,7 +23,7 @@ func (c *rolloutContext) rolloutBlueGreen() error { } c.newRS, err = c.getAllReplicaSetsAndSyncRevision(true) if err != nil { - return err + return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutBlueGreen create true: %w", err) } // This must happen right after the new replicaset is created @@ -82,6 +83,9 @@ func (c *rolloutContext) reconcileBlueGreenStableReplicaSet(activeSvc *corev1.Se c.log.Infof("Reconciling stable ReplicaSet '%s'", activeRS.Name) _, _, err := c.scaleReplicaSetAndRecordEvent(activeRS, defaults.GetReplicasOrDefault(c.rollout.Spec.Replicas)) + if err != nil { + return fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileBlueGreenStableReplicaSet: %w", err) + } return err } @@ -243,7 +247,7 @@ func (c *rolloutContext) scaleDownOldReplicaSetsForBlueGreen(oldRSs []*appsv1.Re // Scale down. _, _, err = c.scaleReplicaSetAndRecordEvent(targetRS, desiredReplicaCount) if err != nil { - return false, err + return false, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in scaleDownOldReplicaSetsForBlueGreen: %w", err) } hasScaled = true } diff --git a/rollout/canary.go b/rollout/canary.go index f37e03bab1..a003b092a3 100644 --- a/rollout/canary.go +++ b/rollout/canary.go @@ -1,6 +1,7 @@ package rollout import ( + "fmt" "sort" appsv1 "k8s.io/api/apps/v1" @@ -21,14 +22,14 @@ func (c *rolloutContext) rolloutCanary() error { if replicasetutil.PodTemplateOrStepsChanged(c.rollout, c.newRS) { c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false) if err != nil { - return err + return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutCanary with PodTemplateOrStepsChanged: %w", err) } return c.syncRolloutStatusCanary() } c.newRS, err = c.getAllReplicaSetsAndSyncRevision(true) if err != nil { - return err + return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutCanary create true: %w", err) } err = c.podRestarter.Reconcile(c) @@ -110,6 +111,9 @@ func (c *rolloutContext) reconcileCanaryStableReplicaSet() (bool, error) { _, desiredStableRSReplicaCount = replicasetutil.CalculateReplicaCountsForTrafficRoutedCanary(c.rollout, c.rollout.Status.Canary.Weights) } scaled, _, err := c.scaleReplicaSetAndRecordEvent(c.stableRS, desiredStableRSReplicaCount) + if err != nil { + return scaled, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileCanaryStableReplicaSet:L %w", err) + } return scaled, err } @@ -230,7 +234,7 @@ func (c *rolloutContext) scaleDownOldReplicaSetsForCanary(oldRSs []*appsv1.Repli // Scale down. _, _, err = c.scaleReplicaSetAndRecordEvent(targetRS, desiredReplicaCount) if err != nil { - return totalScaledDown, err + return totalScaledDown, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in scaleDownOldReplicaSetsForCanary: %w", err) } scaleDownCount := *targetRS.Spec.Replicas - desiredReplicaCount maxScaleDown -= scaleDownCount diff --git a/rollout/replicaset.go b/rollout/replicaset.go index 912417ca1a..c16ce6f037 100644 --- a/rollout/replicaset.go +++ b/rollout/replicaset.go @@ -174,7 +174,7 @@ func (c *rolloutContext) reconcileNewReplicaSet() (bool, error) { scaled, _, err := c.scaleReplicaSetAndRecordEvent(c.newRS, newReplicasCount) if err != nil { - return scaled, err + return scaled, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileNewReplicaSet: %w", err) } revision, _ := replicasetutil.Revision(c.newRS) @@ -279,7 +279,7 @@ func (c *rolloutContext) cleanupUnhealthyReplicas(oldRSs []*appsv1.ReplicaSet) ( } _, updatedOldRS, err := c.scaleReplicaSetAndRecordEvent(targetRS, newReplicasCount) if err != nil { - return nil, totalScaledDown, err + return nil, totalScaledDown, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in cleanupUnhealthyReplicas: %w", err) } totalScaledDown += scaledDownCount oldRSs[i] = updatedOldRS diff --git a/rollout/sync.go b/rollout/sync.go index 7ec692154e..167430a702 100644 --- a/rollout/sync.go +++ b/rollout/sync.go @@ -283,7 +283,7 @@ func (c *rolloutContext) syncReplicasOnly() error { var err error c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false) if err != nil { - return err + return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in syncReplicasOnly: %w", err) } newStatus := c.rollout.Status.DeepCopy() @@ -296,7 +296,7 @@ func (c *rolloutContext) syncReplicasOnly() error { if err := c.reconcileBlueGreenReplicaSets(activeSvc); err != nil { // If we get an error while trying to scale, the rollout will be requeued // so we can abort this resync - return err + return fmt.Errorf("failed to reconcileBlueGreenReplicaSets in syncReplicasOnly: %w", err) } activeRS, _ := replicasetutil.GetReplicaSetByTemplateHash(c.allRSs, newStatus.BlueGreen.ActiveSelector) if activeRS != nil { @@ -314,7 +314,7 @@ func (c *rolloutContext) syncReplicasOnly() error { if _, err := c.reconcileCanaryReplicaSets(); err != nil { // If we get an error while trying to scale, the rollout will be requeued // so we can abort this resync - return err + return fmt.Errorf("failed to reconcileCanaryReplicaSets in syncReplicasOnly: %w", err) } newStatus.AvailableReplicas = replicasetutil.GetAvailableReplicaCountForReplicaSets(c.allRSs) newStatus.HPAReplicas = replicasetutil.GetActualReplicaCountForReplicaSets(c.allRSs) @@ -330,7 +330,7 @@ func (c *rolloutContext) isScalingEvent() (bool, error) { var err error c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false) if err != nil { - return false, err + return false, fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in isScalingEvent: %w", err) } for _, rs := range controller.FilterActiveReplicaSets(c.allRSs) { @@ -357,6 +357,9 @@ func (c *rolloutContext) scaleReplicaSetAndRecordEvent(rs *appsv1.ReplicaSet, ne scalingOperation = "down" } scaled, newRS, err := c.scaleReplicaSet(rs, newScale, c.rollout, scalingOperation) + if err != nil { + return scaled, newRS, fmt.Errorf("failed to scaleReplicaSet in scaleReplicaSetAndRecordEvent: %w", err) + } return scaled, newRS, err }