From 2cddb2ac08070a560714d309e6d3f9d155f6d9a0 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Thu, 19 Nov 2026 13:33:28 -0500 Subject: [PATCH 1/2] fix: report rollout completion in Bindplane status Assisted-by: Claude Opus 5.5 --- cmd/main.go | 5 +- docs/configuration/configuration.md | 85 +++- internal/controller/bindplane_controller.go | 201 ++++---- .../controller/bindplane_controller_test.go | 358 +++++++++++++- internal/controller/jobs.go | 7 +- internal/controller/workload_status.go | 262 +++++++++++ internal/controller/workload_status_test.go | 439 ++++++++++++++++++ 7 files changed, 1244 insertions(+), 113 deletions(-) create mode 100644 internal/controller/workload_status.go create mode 100644 internal/controller/workload_status_test.go diff --git a/cmd/main.go b/cmd/main.go index 9a82308..bb5b47d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -231,8 +231,9 @@ func main() { } if err := (&controller.BindplaneReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + APIReader: mgr.GetAPIReader(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Bindplane") os.Exit(1) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 46aac1e..f5fb5cf 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -46,6 +46,7 @@ Configuration is provided via the `spec.config` field of the `Bindplane` custom - [Pause annotation](#pause-annotation) - [Finalizer and garbage collection](#finalizer-and-garbage-collection) - [Conditions and phases](#conditions-and-phases) + - [Argo CD health check](#argo-cd-health-check) - [Migration contract](#migration-contract) - [Examples](#examples) - [Minimal configuration](#minimal-configuration) @@ -1375,11 +1376,11 @@ While the annotation is set: - The Jobs Migrate Job is **still created and managed exactly as before**: it runs on install and on every image change, stale Jobs are replaced, and the `k8s.bindplane.com/force-migrate` annotation still works. Only the gate is bypassed. - Downstream workloads are reconciled on every pass, regardless of whether the Job is running or has failed. - The `Reconciled` condition is `True` with `Reason: MigrationCheckSkipped` while the Job is failed or not yet complete. The message names the Job and its state. Once the Job succeeds the condition returns to `Reason: Reconciled`. -- `status.phase` reflects workload readiness as normal (`ApplyingChanges` or `Ready`); it is not set to `Degraded`. +- `status.phase` and the `Ready` condition include the Jobs Migrate Job: a running Job keeps the phase at `ApplyingChanges` (`Ready=False` with reason `WorkloadsProgressing` and a `jobs-migrate: ...` entry in the message), and a failed Job sets the phase to `Degraded` (`Ready=False` with reason `WorkloadFailed`). - `status.components.jobsMigrate.image` is **not** written by the bypass. It is only recorded when a Job actually succeeds, so it always reflects a confirmed migration. - While a Job is still running the operator keeps requeueing every 10 seconds so the success is recorded; a terminal `Failed` Job is not requeued. -The annotation is sticky: the operator never clears it. Removing it (or setting it to any value other than `"true"`) re-engages the gate on the next reconcile. If the Job is still failed at that point, the `MigrationFailed` condition and `Degraded` phase return. Workloads that were rolled out during the bypass are left in place; the gate only blocks further updates. +The annotation is sticky: the operator never clears it. Removing it (or setting it to any value other than `"true"`) re-engages the gate on the next reconcile. If the Job is still failed at that point, the `Reconciled` and `Ready` conditions are set to `False` with reason `MigrationFailed` and the phase stays `Degraded`. Workloads that were rolled out during the bypass are left in place; the gate only blocks further updates. The `k8s.bindplane.com/pause-reconciliation` annotation takes precedence: a paused resource is not reconciled at all, whether or not the skip annotation is present. @@ -1395,7 +1396,7 @@ Set the annotation `k8s.bindplane.com/pause-reconciliation: "true"` on the `Bind - No resources are created, updated, or deleted. - The `Reconciled` condition is set to `False` with `Reason: Paused`. -- The `status.phase` is set to `Paused`. +- `status.phase` is left at its current value (`Pending` if the CR has never been reconciled). Remove the annotation (or set it to `"false"`) to resume reconciliation. The operator will re-apply the current desired state on the next reconcile cycle. @@ -1434,19 +1435,85 @@ The operator reports overall state via `status.conditions` and the `status.phase | `Reconciled` | `False` | `Invalid` | CR failed validation; no resources were mutated | | `Reconciled` | `False` | `MigrationFailed` | The Jobs Migrate Job failed; downstream workloads are blocked | | `Reconciled` | `True` | `MigrationCheckSkipped` | The `k8s.bindplane.com/skip-migrate-check` annotation is set and the Jobs Migrate Job is failed or not yet complete; downstream workloads were reconciled without a confirmed migration | +| `Ready` | `True` | `WorkloadsReady` | Every workload has rolled out the spec at the condition's `observedGeneration` and all of its replicas are updated and available | +| `Ready` | `False` | `WorkloadsProgressing` | One or more workloads are still rolling out; the message lists each one as `: ` | +| `Ready` | `False` | `RolloutPaused` | The Bindplane Node Argo Rollout is paused, for example a blue-green rollout with `autoPromotionEnabled: false` awaiting promotion | +| `Ready` | `False` | `WorkloadFailed` | A workload failed: a Deployment exceeded its progress deadline, the Bindplane Node Argo Rollout is `Degraded`, or the Jobs Migrate Job failed while the migration check is skipped | +| `Ready` | `False` | `MigrationInProgress` | Downstream workloads are waiting for the Jobs Migrate Job (`-migrate`) to complete | +| `Ready` | `False` | `MigrationFailed` | The Jobs Migrate Job failed; downstream workloads are blocked | +| `Ready` | `False` | `Invalid` | CR failed validation; no resources were mutated | +| `Ready` | `False` | `ReconcileError` | The last reconcile returned an error; the message holds the error and the operator retries with backoff | + +The `Ready` condition's `observedGeneration` is the CR generation it describes. Each workload counts as rolled out using the same checks as `kubectl rollout status`: + +- **Deployments** (Jobs, Transform Agent, OpAMP, and Node when Argo Rollouts is disabled): the Deployment controller has observed the latest spec, the updated replica count matches the live `spec.replicas` (which an HPA may set), no old replicas remain, and every updated replica is available. A `Progressing` condition with reason `ProgressDeadlineExceeded` counts as failed. +- **StatefulSets** (NATS, and TSDB unless remote TSDB is enabled): the StatefulSet controller has observed the latest spec, every replica is ready and updated, and the current revision equals the update revision. +- **Argo Rollout** (Node when `spec.bindplane.argoRollout.enabled: true`): the Argo Rollouts controller has observed the latest spec and reports phase `Healthy`. Phase `Degraded` counts as failed and phase `Paused` counts as paused. + +The operator reads workload status directly from the API server rather than from its cache, so a workload updated earlier in the same reconcile is never reported from its previous state. **Phases** | Phase | Meaning | |-------|---------| | `Pending` | CR was just created; first reconcile has not run yet | -| `ApplyingChanges` | One or more workloads are not yet at their desired replica count | -| `Ready` | All required workloads have their desired replica count ready | -| `Degraded` | CR failed validation; no resources are being managed | -| `Paused` | Reconciliation is suspended via the pause annotation | +| `ApplyingChanges` | The Jobs Migrate Job is running, or one or more workloads are still rolling out or paused. The operator requeues at least every 30 seconds while in this phase (every 10 seconds while the Jobs Migrate Job is running) | +| `Ready` | Every workload has rolled out and is ready (`Ready=True`) | +| `Degraded` | The CR failed validation, the Jobs Migrate Job failed, or a workload failed. The `Ready` condition holds the reason. The operator requeues every 30 seconds while the Bindplane Node Argo Rollout is `Degraded` | | `Deleting` | CR deletion is in progress; finalizer is being removed | -The `status.observedGeneration` field is set to the `metadata.generation` of the CR after every successful reconcile, allowing GitOps tools and `kubectl wait` to detect when the operator has processed the latest spec version. +A reconcile error sets `Ready=False` with reason `ReconcileError`. It leaves the phase unchanged when the phase was set for the current generation, and otherwise sets it to `ApplyingChanges`. The pause annotation does not change the phase either; it sets `Reconciled=False` with reason `Paused`. + +The `status.observedGeneration` field is set to the `metadata.generation` of the CR when the operator writes status for that generation, including when it halts for a running Jobs Migrate Job or records a reconcile error. GitOps tools should require both `status.observedGeneration == metadata.generation` and `Ready=True` with a matching `observedGeneration` before treating the CR as healthy. + +### Argo CD health check + +Argo CD has no built-in health check for `k8s.bindplane.com/Bindplane`. Add this Lua health check to the `argocd-cm` ConfigMap under the key `resource.customizations.health.k8s.bindplane.com_Bindplane` so an Application reports `Healthy` only after every workload has rolled out the CR's current generation: + +```lua +local hs = {} +if obj.status == nil or obj.status.observedGeneration == nil + or obj.status.observedGeneration ~= obj.metadata.generation then + hs.status = "Progressing" + hs.message = "Waiting for the operator to observe generation " .. tostring(obj.metadata.generation) + return hs +end +local ready, reconciled = nil, nil +for _, c in ipairs(obj.status.conditions or {}) do + if c.type == "Ready" then ready = c end + if c.type == "Reconciled" then reconciled = c end +end +if reconciled ~= nil and reconciled.status == "False" and reconciled.reason == "Paused" then + hs.status = "Suspended"; hs.message = reconciled.message; return hs +end +if ready ~= nil and ready.status == "False" and ready.reason == "RolloutPaused" then + hs.status = "Suspended"; hs.message = ready.message; return hs +end +if obj.status.phase == "Degraded" then + hs.status = "Degraded" + hs.message = (ready ~= nil and ready.message) or (reconciled ~= nil and reconciled.message) or "Bindplane is Degraded" + return hs +end +if obj.status.phase == "Ready" and ready ~= nil and ready.status == "True" + and ready.observedGeneration == obj.metadata.generation then + hs.status = "Healthy"; hs.message = ready.message; return hs +end +hs.status = "Progressing" +hs.message = (ready ~= nil and ready.message) or ("Phase " .. tostring(obj.status.phase)) +return hs +``` + +| CR state | Argo CD health | +|----------|----------------| +| `status.observedGeneration` differs from `metadata.generation` | `Progressing` | +| `Reconciled=False` with reason `Paused` | `Suspended` | +| `Ready=False` with reason `RolloutPaused` | `Suspended` | +| Phase `Degraded` | `Degraded` | +| Phase `Ready` with `Ready=True` at the current generation | `Healthy` | +| `Ready=False` with reason `ReconcileError` | `Progressing`, or `Degraded` when the phase was already `Degraded` at the current generation | +| Anything else, including `MigrationInProgress` and `WorkloadsProgressing` | `Progressing` | + +A CR without a `Ready` condition never reports `Healthy`. ### Component status @@ -1471,7 +1538,7 @@ When `spec.version` changes (or the `k8s.bindplane.com/force-migrate` annotation 1. Creates a new `Jobs Migrate` (`batch/v1 Job`) before updating any long-running workloads (Jobs, NATS, Node). 2. Blocks all downstream workload updates until the Jobs Migrate Job completes successfully (requeues every 10 seconds while the Job is still running), unless the `k8s.bindplane.com/skip-migrate-check` annotation is set (see [Skip migration check](#skip-migration-check)). 3. On success, records the migrated image in `status.components.jobsMigrate.image` and proceeds to roll out updated workloads. -4. On failure (the Job reaches a terminal `Failed` state after exhausting `backoffLimit`), sets the `Reconciled` condition to `False` with `Reason: MigrationFailed`, sets `status.phase = Degraded`, and **halts the rollout without requeueing** — a terminal Job will not change state on its own, so the operator stops retrying to avoid log spam. The failure remains visible in status until you intervene: inspect the retained failed pods for logs, fix the underlying cause, then change `spec.version` or set the `k8s.bindplane.com/force-migrate` annotation (see [Force migration](#force-migration)) to retry. Either action triggers a fresh reconcile. To proceed with workload updates despite the failure, set the `k8s.bindplane.com/skip-migrate-check` annotation (see [Skip migration check](#skip-migration-check)). +4. On failure (the Job reaches a terminal `Failed` state after exhausting `backoffLimit`), sets the `Reconciled` and `Ready` conditions to `False` with `Reason: MigrationFailed`, sets `status.phase = Degraded`, and **halts the rollout without requeueing** — a terminal Job will not change state on its own, so the operator stops retrying to avoid log spam. The failure remains visible in status until you intervene: inspect the retained failed pods for logs, fix the underlying cause, then change `spec.version` or set the `k8s.bindplane.com/force-migrate` annotation (see [Force migration](#force-migration)) to retry. Either action triggers a fresh reconcile. To proceed with workload updates despite the failure, set the `k8s.bindplane.com/skip-migrate-check` annotation (see [Skip migration check](#skip-migration-check)). This ordering guarantees that the database schema is always compatible with all running workloads before any new binary version is activated. diff --git a/internal/controller/bindplane_controller.go b/internal/controller/bindplane_controller.go index 6f3be4c..efc8334 100644 --- a/internal/controller/bindplane_controller.go +++ b/internal/controller/bindplane_controller.go @@ -316,6 +316,10 @@ const ( // Job is still running. migrateJobPollInterval = 10 * time.Second + // applyingChangesPollInterval is how often the controller requeues while status.phase + // is ApplyingChanges. + applyingChangesPollInterval = 30 * time.Second + // annotationPreserveSelectorKeys is an internal operator annotation placed on Service objects // to signal which selector keys should be preserved from the live Service (e.g. // rollouts-pod-template-hash written by Argo Rollouts during a BlueGreen cutover). @@ -511,6 +515,9 @@ const ( type BindplaneReconciler struct { client.Client Scheme *runtime.Scheme + // APIReader reads child workload status directly from the API server. When nil, the + // reconciler's Client is used. + APIReader client.Reader } // +kubebuilder:rbac:groups=k8s.bindplane.com,resources=bindplanes,verbs=get;list;watch;create;update;patch;delete @@ -608,7 +615,8 @@ func (r *BindplaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( LastTransitionTime: metav1.Now(), } meta.SetStatusCondition(&bindplane.Status.Conditions, condition) - bindplane.Status.Phase = "Degraded" + setReadyCondition(bindplane, metav1.ConditionFalse, reasonInvalid, err.Error()) + bindplane.Status.Phase = phaseDegraded bindplane.Status.ObservedGeneration = bindplane.Generation if statusErr := r.Status().Update(ctx, bindplane); statusErr != nil { log.Error(statusErr, "failed to update Bindplane status") @@ -619,40 +627,34 @@ func (r *BindplaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Reconcile internal TLS certificates (cert-manager) before workloads that mount them. if err := r.reconcileInternalTLSCertificates(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile internal TLS certificates") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile internal TLS certificates") } // Reconcile session secret before workloads that consume it via BINDPLANE_SESSION_SECRET. if err := r.reconcileSessionSecret(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile session secret") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile session secret") } // Reconcile status secret before workloads that consume it via BINDPLANE_STATUS_KEYS. if err := r.reconcileStatusSecret(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile status secret") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile status secret") } // Reconcile Transform Agent resources if err := r.reconcileTransformAgent(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile Transform Agent") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile Transform Agent") } // Reconcile Prometheus resources if err := r.reconcileTSDB(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile Prometheus") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile Prometheus") } // Reconcile the Jobs Migrate batch/v1 Job; block downstream workloads until it completes, // unless the skip-migrate-check annotation bypasses the gate. state, err := r.reconcileMigrateJob(ctx, bindplane, log) if err != nil { - log.Error(err, "unable to reconcile Jobs Migrate Job") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile Jobs Migrate Job") } halt, requeueAfter, err := r.applyMigrateGate(ctx, bindplane, state, log) if err != nil { @@ -664,33 +666,29 @@ func (r *BindplaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Reconcile Bindplane Jobs resources if err := r.reconcileBindplaneJobsRegular(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile Bindplane Jobs") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile Bindplane Jobs") } // Reconcile NATS resources if err := r.reconcileNats(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile NATS") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile NATS") } // Reconcile Node resources if err := r.reconcileNode(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile Node") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile Node") } // Reconcile OpAMP resources (optional dedicated OpAMP deployment) if err := r.reconcileOpAMP(ctx, bindplane, log); err != nil { - log.Error(err, "unable to reconcile OpAMP") - return ctrl.Result{}, err + return r.reconcileError(ctx, bindplane, log, err, "unable to reconcile OpAMP") } // Mark as reconciled so any previous InvalidName condition is cleared meta.SetStatusCondition(&bindplane.Status.Conditions, reconciledCondition(bindplane, state)) - // Populate per-component ready replica counts and overall phase. - r.updateReadyReplicaStatus(ctx, bindplane) + // Populate per-component ready replica counts, overall phase and the Ready condition. + poll := r.updateReadyReplicaStatus(ctx, bindplane, state) // Populate resolved images for each deployed component. updateImageStatus(bindplane) bindplane.Status.ObservedGeneration = bindplane.Generation @@ -699,9 +697,38 @@ func (r *BindplaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( log.Error(err, "failed to update Bindplane status") return ctrl.Result{}, err } + // Poll while workloads roll out or the node Rollout is Degraded, so status converges + // even without a watch event from the workload, for example when the Rollout watch is + // not registered. + if poll && (requeueAfter == 0 || requeueAfter > applyingChangesPollInterval) { + requeueAfter = applyingChangesPollInterval + } return ctrl.Result{RequeueAfter: requeueAfter}, nil } +// reconcileError logs err, records it as Ready=False/ReconcileError on the Bindplane CR and +// returns err so the request is retried. status.phase is left unchanged when it was computed +// for the current generation (the Ready condition and status.observedGeneration both match +// it); otherwise it is set to ApplyingChanges, so a phase from an older generation is not +// reported for the current one. The pause path advances status.observedGeneration without +// computing a phase, which is why the Ready condition is checked too. The status write is +// best effort: a failed write is logged unless it is a conflict. +func (r *BindplaneReconciler) reconcileError(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, log logr.Logger, err error, msg string) (ctrl.Result, error) { + log.Error(err, msg) + ready := meta.FindStatusCondition(bindplane.Status.Conditions, conditionTypeReady) + stalePhase := bindplane.Status.ObservedGeneration != bindplane.Generation || + ready == nil || ready.ObservedGeneration != bindplane.Generation + setReadyCondition(bindplane, metav1.ConditionFalse, reasonReconcileError, fmt.Sprintf("%s: %v", msg, err)) + if stalePhase { + bindplane.Status.Phase = phaseApplyingChanges + bindplane.Status.ObservedGeneration = bindplane.Generation + } + if statusErr := r.Status().Update(ctx, bindplane); statusErr != nil && !errors.IsConflict(statusErr) { + log.Error(statusErr, "failed to record reconcile error in Bindplane status") + } + return ctrl.Result{}, err +} + // applyMigrateGate applies the migration gate policy to the state reported by // reconcileMigrateJob. When halt is non-nil the caller must stop and return it: downstream // workloads are blocked (default behavior). When halt is nil the caller proceeds to downstream @@ -738,6 +765,10 @@ func (r *BindplaneReconciler) applyMigrateGate(ctx context.Context, bindplane *b case migrateInProgress: if !skipMigrate { log.Info("waiting for Jobs Migrate Job to complete") + if err := r.setMigrationInProgressStatus(ctx, bindplane, migrateJobName); err != nil { + log.Error(err, "failed to update Bindplane status for migration in progress") + return nil, 0, err + } return &ctrl.Result{RequeueAfter: migrateJobPollInterval}, 0, nil } log.Info("Jobs Migrate Job not yet complete but the migration check is skipped via annotation; "+ @@ -750,6 +781,23 @@ func (r *BindplaneReconciler) applyMigrateGate(ctx context.Context, bindplane *b } } +// setMigrationInProgressStatus records that downstream workloads are waiting on the Jobs +// Migrate Job. It writes status only when the phase, observedGeneration or Ready condition +// changed, so polling a running Job does not rewrite an unchanged status. +func (r *BindplaneReconciler) setMigrationInProgressStatus(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, migrateJobName string) error { + changed := setReadyCondition(bindplane, metav1.ConditionFalse, reasonMigrationInProgress, + fmt.Sprintf("waiting for Jobs Migrate Job %s to complete", migrateJobName)) + if bindplane.Status.Phase != phaseApplyingChanges || bindplane.Status.ObservedGeneration != bindplane.Generation { + bindplane.Status.Phase = phaseApplyingChanges + bindplane.Status.ObservedGeneration = bindplane.Generation + changed = true + } + if !changed { + return nil + } + return r.Status().Update(ctx, bindplane) +} + // reconciledCondition builds the Reconciled=True condition written at the end of a // successful reconcile. When the migration gate was bypassed via the skip-migrate-check // annotation and the migrate Job is not complete, the Reason is MigrationCheckSkipped so @@ -777,53 +825,51 @@ func reconciledCondition(bindplane *bindplanev1alpha1.Bindplane, state migrateSt return condition } -// updateReadyReplicaStatus queries the owned workloads for their current ready -// replica counts and sets the Phase field accordingly. -func (r *BindplaneReconciler) updateReadyReplicaStatus(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane) { - nodeReady := r.deploymentReadyReplicas(ctx, getResourceName(bindplane, nodeComponent), bindplane.Namespace) - natsReady := r.statefulSetReadyReplicas(ctx, getResourceName(bindplane, natsComponent), bindplane.Namespace) - taReady := r.deploymentReadyReplicas(ctx, getResourceName(bindplane, transformAgentComponent), bindplane.Namespace) - jobsReady := r.deploymentReadyReplicas(ctx, getResourceName(bindplane, bindplaneJobsComponent), bindplane.Namespace) +// updateReadyReplicaStatus reads every owned workload, records its ready replica count in +// status.components and sets status.phase and the Ready condition from the rollout state of +// all of them. state is the Jobs Migrate state; a Job that is not complete counts as a +// workload that is still rolling out, or as a failed one when the Job failed. poll is true +// when the phase is ApplyingChanges or the node Rollout is Degraded. +func (r *BindplaneReconciler) updateReadyReplicaStatus(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, state migrateState) (poll bool) { + var node workloadProgress + nodeIsRollout := bindplane.Spec.Bindplane.ArgoRollout != nil && bindplane.Spec.Bindplane.ArgoRollout.Enabled + if nodeIsRollout { + node = r.getRolloutProgress(ctx, bindplane, nodeComponent) + } else { + node = r.getDeploymentProgress(ctx, bindplane, nodeComponent) + } + nats := r.getStatefulSetProgress(ctx, bindplane, natsComponent) + transformAgent := r.getDeploymentProgress(ctx, bindplane, transformAgentComponent) + jobs := r.getDeploymentProgress(ctx, bindplane, bindplaneJobsComponent) + progress := []workloadProgress{node, nats, transformAgent, jobs} - bindplane.Status.Components.Bindplane.ReadyReplicas = nodeReady - bindplane.Status.Components.Nats.ReadyReplicas = natsReady - bindplane.Status.Components.TransformAgent.ReadyReplicas = taReady - bindplane.Status.Components.Jobs.ReadyReplicas = jobsReady + bindplane.Status.Components.Bindplane.ReadyReplicas = node.ready + bindplane.Status.Components.Nats.ReadyReplicas = nats.ready + bindplane.Status.Components.TransformAgent.ReadyReplicas = transformAgent.ready + bindplane.Status.Components.Jobs.ReadyReplicas = jobs.ready - // Only report TSDB readiness when local TSDB is deployed (not when remote TSDB is enabled). - var tsdbReady int32 + // TSDB is a local StatefulSet unless remote TSDB is enabled. + bindplane.Status.Components.TSDB.ReadyReplicas = 0 if !isTSDBRemoteEnabled(bindplane) { - tsdbReady = r.statefulSetReadyReplicas(ctx, getResourceName(bindplane, tsdbComponent), bindplane.Namespace) + tsdb := r.getStatefulSetProgress(ctx, bindplane, tsdbComponent) + bindplane.Status.Components.TSDB.ReadyReplicas = tsdb.ready + progress = append(progress, tsdb) } - bindplane.Status.Components.TSDB.ReadyReplicas = tsdbReady - // OpAMP is an optional dedicated deployment; track its readiness separately. - var opampReady int32 + // OpAMP is an optional dedicated Deployment. + bindplane.Status.Components.OpAMP.ReadyReplicas = 0 if bindplane.Spec.OpAMP != nil && bindplane.Spec.OpAMP.Enabled { - opampReady = r.deploymentReadyReplicas(ctx, getResourceName(bindplane, opampComponent), bindplane.Namespace) + opamp := r.getDeploymentProgress(ctx, bindplane, opampComponent) + bindplane.Status.Components.OpAMP.ReadyReplicas = opamp.ready + progress = append(progress, opamp) } - bindplane.Status.Components.OpAMP.ReadyReplicas = opampReady - nodeDesired := *bindplane.Spec.Bindplane.Replicas - var natsDesired int32 - if bindplane.Spec.Nats != nil && bindplane.Spec.Nats.Replicas != nil { - natsDesired = *bindplane.Spec.Nats.Replicas - } - var taDesired int32 - if bindplane.Spec.TransformAgent != nil && bindplane.Spec.TransformAgent.Replicas != nil { - taDesired = *bindplane.Spec.TransformAgent.Replicas - } - // TSDB: desired is 1 replica when local, 0 when remote (already satisfied — remote means no local SS). - var tsdbDesired int32 - if !isTSDBRemoteEnabled(bindplane) { - tsdbDesired = 1 + if migrate, ok := migrateProgress(bindplane, state); ok { + progress = append(progress, migrate) } - if r.allWorkloadsReady(nodeReady, nodeDesired, natsReady, natsDesired, taReady, taDesired, jobsReady, tsdbReady, tsdbDesired) { - bindplane.Status.Phase = "Ready" - } else { - bindplane.Status.Phase = "ApplyingChanges" - } + setPhaseFromProgress(bindplane, progress) + return bindplane.Status.Phase == phaseApplyingChanges || (nodeIsRollout && node.failed) } // updateImageStatus records the resolved container image for each deployed component into @@ -848,41 +894,6 @@ func updateImageStatus(bindplane *bindplanev1alpha1.Bindplane) { } } -// allWorkloadsReady returns true when every workload has met its desired replica count. -// Optional workloads (nats, transformAgent, tsdb) are considered ready when their desired count is zero. -func (r *BindplaneReconciler) allWorkloadsReady( - nodeReady, nodeDesired int32, - natsReady, natsDesired int32, - taReady, taDesired int32, - jobsReady int32, - tsdbReady, tsdbDesired int32, -) bool { - const jobsDesired int32 = 1 - return nodeReady >= nodeDesired && - (natsDesired == 0 || natsReady >= natsDesired) && - (taDesired == 0 || taReady >= taDesired) && - jobsReady >= jobsDesired && - (tsdbDesired == 0 || tsdbReady >= tsdbDesired) -} - -// deploymentReadyReplicas returns the ready replica count for a Deployment, or 0 if not found. -func (r *BindplaneReconciler) deploymentReadyReplicas(ctx context.Context, name, namespace string) int32 { - dep := &appsv1.Deployment{} - if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, dep); err != nil { - return 0 - } - return dep.Status.ReadyReplicas -} - -// statefulSetReadyReplicas returns the ready replica count for a StatefulSet, or 0 if not found. -func (r *BindplaneReconciler) statefulSetReadyReplicas(ctx context.Context, name, namespace string) int32 { - ss := &appsv1.StatefulSet{} - if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, ss); err != nil { - return 0 - } - return ss.Status.ReadyReplicas -} - // SetupWithManager sets up the controller with the Manager. func (r *BindplaneReconciler) SetupWithManager(mgr ctrl.Manager) error { b := ctrl.NewControllerManagedBy(mgr). diff --git a/internal/controller/bindplane_controller_test.go b/internal/controller/bindplane_controller_test.go index 5556409..99b1654 100644 --- a/internal/controller/bindplane_controller_test.go +++ b/internal/controller/bindplane_controller_test.go @@ -65,8 +65,9 @@ func newTestBindplane(name, namespace string) *bindplanev1alpha1.Bindplane { // newReconciler returns a BindplaneReconciler wired to the envtest k8sClient. func newReconciler() *BindplaneReconciler { return &BindplaneReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + APIReader: k8sClient, } } @@ -186,6 +187,64 @@ func getReconciledCondition(ctx context.Context, name, namespace string) (*bindp return bp, metav1.Condition{} } +// getReadyCondition fetches the Bindplane CR and returns its Ready condition, failing the +// test when the condition is absent. +func getReadyCondition(ctx context.Context, name, namespace string) metav1.Condition { + bp := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, bp)).To(Succeed()) + for i := range bp.Status.Conditions { + if bp.Status.Conditions[i].Type == conditionTypeReady { + return bp.Status.Conditions[i] + } + } + Fail("Ready condition not found on Bindplane " + name) + return metav1.Condition{} +} + +// markWorkloadsRolledOut writes a fully rolled out status to every Deployment and +// StatefulSet in the namespace, as the Deployment and StatefulSet controllers would. +// envtest runs no workload controllers, so status is otherwise never written. +func markWorkloadsRolledOut(ctx context.Context, namespace string) { + deps := &appsv1.DeploymentList{} + Expect(k8sClient.List(ctx, deps, client.InNamespace(namespace))).To(Succeed()) + for i := range deps.Items { + markDeploymentRolledOut(ctx, deps.Items[i].Name, namespace) + } + sets := &appsv1.StatefulSetList{} + Expect(k8sClient.List(ctx, sets, client.InNamespace(namespace))).To(Succeed()) + for i := range sets.Items { + sts := &sets.Items[i] + replicas := replicasOrDefault(sts.Spec.Replicas) + sts.Status = appsv1.StatefulSetStatus{ + ObservedGeneration: sts.Generation, + Replicas: replicas, + ReadyReplicas: replicas, + CurrentReplicas: replicas, + UpdatedReplicas: replicas, + AvailableReplicas: replicas, + CurrentRevision: sts.Name + "-1", + UpdateRevision: sts.Name + "-1", + } + Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) + } +} + +// markDeploymentRolledOut writes a fully rolled out status for the Deployment's current +// generation and live spec.replicas. +func markDeploymentRolledOut(ctx context.Context, name, namespace string) { + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, dep)).To(Succeed()) + replicas := replicasOrDefault(dep.Spec.Replicas) + dep.Status = appsv1.DeploymentStatus{ + ObservedGeneration: dep.Generation, + Replicas: replicas, + UpdatedReplicas: replicas, + ReadyReplicas: replicas, + AvailableReplicas: replicas, + } + Expect(k8sClient.Status().Update(ctx, dep)).To(Succeed()) +} + var _ = Describe("Reconcile - finalizer lifecycle", func() { var ( testNamespace string @@ -373,6 +432,11 @@ var _ = Describe("Reconcile - validation failure", func() { Expect(reconciledCond).NotTo(BeNil()) Expect(reconciledCond.Status).To(Equal(metav1.ConditionFalse)) Expect(reconciledCond.Reason).To(Equal("Invalid")) + + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonInvalid)) + Expect(updated.Status.Phase).To(Equal(phaseDegraded)) }) }) @@ -620,7 +684,7 @@ var _ = Describe("Reconcile - Jobs Migrate", func() { result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(BeZero()) + Expect(result.RequeueAfter).To(Equal(applyingChangesPollInterval), "polls while workloads roll out") // Jobs Deployment should now exist dep := &appsv1.Deployment{} @@ -679,6 +743,11 @@ var _ = Describe("Reconcile - Jobs Migrate", func() { Expect(reconciledCond.Status).To(Equal(metav1.ConditionFalse)) Expect(reconciledCond.Reason).To(Equal("MigrationFailed")) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonMigrationFailed)) + Expect(updated.Status.Phase).To(Equal(phaseDegraded)) + dep := &appsv1.Deployment{} err = k8sClient.Get(testCtx, types.NamespacedName{Name: name + "-jobs", Namespace: testNamespace}, dep) Expect(errors.IsNotFound(err)).To(BeTrue()) @@ -793,7 +862,12 @@ var _ = Describe("Reconcile - Jobs Migrate", func() { Expect(cond.Reason).To(Equal("MigrationCheckSkipped")) Expect(cond.Message).To(ContainSubstring(jobName)) Expect(cond.Message).To(ContainSubstring("Failed")) - Expect(updated.Status.Phase).NotTo(Equal("Degraded")) + // The bypass reconciles workloads, but the failed Job is still reported as a failure. + Expect(updated.Status.Phase).To(Equal(phaseDegraded)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonWorkloadFailed)) + Expect(readyCond.Message).To(ContainSubstring(jobsMigrateProgressComponent + ": job " + jobName + " failed")) Expect(updated.Status.Components.JobsMigrate.Image).To(BeEmpty(), "bypass must not record a migrated image") // The Job itself is untouched: still present and still Failed. @@ -826,13 +900,15 @@ var _ = Describe("Reconcile - Jobs Migrate", func() { Expect(cond.Reason).To(Equal("MigrationCheckSkipped")) Expect(cond.Message).To(ContainSubstring("not yet complete")) Expect(updated.Status.Components.JobsMigrate.Image).To(BeEmpty()) + Expect(updated.Status.Phase).To(Equal(phaseApplyingChanges)) + Expect(getReadyCondition(testCtx, name, testNamespace).Message).To(ContainSubstring(jobsMigrateProgressComponent + ": job " + jobName + " is not complete")) // Once the Job succeeds the normal condition and migrated image are recorded, // even with the annotation still present. markJobComplete(testCtx, jobName, testNamespace) result, err = r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(BeZero()) + Expect(result.RequeueAfter).To(Equal(applyingChangesPollInterval), "polls while workloads roll out") updated, cond = getReconciledCondition(testCtx, name, testNamespace) Expect(cond.Status).To(Equal(metav1.ConditionTrue)) @@ -1082,6 +1158,182 @@ var _ = Describe("Reconcile - status", func() { // In envtest, no pods run, so ready replicas are 0 => ApplyingChanges Expect(updated.Status.Phase).To(Equal("ApplyingChanges")) Expect(updated.Status.Components.Bindplane.ReadyReplicas).To(BeNumerically("==", 0)) + + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonWorkloadsProgressing)) + Expect(readyCond.ObservedGeneration).To(Equal(updated.Generation)) + Expect(readyCond.Message).To(ContainSubstring("node: waiting for deployment spec update to be observed")) + }) + + // reconcileRolledOut drives the CR past migration, marks every workload rolled out and + // reconciles once more, returning the result of the last reconcile. + reconcileRolledOut := func(r *BindplaneReconciler, name string) reconcile.Result { + reconcilePastMigration(testCtx, r, name, testNamespace) + markWorkloadsRolledOut(testCtx, testNamespace) + result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + return result + } + + It("sets phase Ready and Ready=True when every workload is rolled out", func() { + name := "bp-status-ready" + Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) + + result := reconcileRolledOut(newReconciler(), name) + Expect(result.RequeueAfter).To(BeZero()) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseReady)) + Expect(updated.Status.ObservedGeneration).To(Equal(updated.Generation)) + Expect(updated.Status.Components.Bindplane.ReadyReplicas).To(BeNumerically("==", 3)) + + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionTrue)) + Expect(readyCond.Reason).To(Equal(reasonWorkloadsReady)) + Expect(readyCond.ObservedGeneration).To(Equal(updated.Generation)) + }) + + It("sets phase ApplyingChanges while old node replicas are pending termination", func() { + name := "bp-status-old-node" + Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) + + r := newReconciler() + reconcileRolledOut(r, name) + + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name + "-node", Namespace: testNamespace}, dep)).To(Succeed()) + dep.Status.Replicas = *dep.Spec.Replicas + 1 + Expect(k8sClient.Status().Update(testCtx, dep)).To(Succeed()) + + result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(applyingChangesPollInterval)) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseApplyingChanges)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Reason).To(Equal(reasonWorkloadsProgressing)) + Expect(readyCond.Message).To(ContainSubstring("node: 1 old replicas are pending termination")) + }) + + It("sets phase ApplyingChanges and names opamp while OpAMP is not rolled out", func() { + name := "bp-status-opamp" + Expect(k8sClient.Create(testCtx, newTestBindplaneWithOpAMP(name, testNamespace))).To(Succeed()) + + r := newReconciler() + reconcilePastMigration(testCtx, r, name, testNamespace) + markWorkloadsRolledOut(testCtx, testNamespace) + + // Leave OpAMP at its first generation unobserved. + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name + "-opamp", Namespace: testNamespace}, dep)).To(Succeed()) + dep.Status = appsv1.DeploymentStatus{} + Expect(k8sClient.Status().Update(testCtx, dep)).To(Succeed()) + + _, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseApplyingChanges)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Reason).To(Equal(reasonWorkloadsProgressing)) + Expect(readyCond.Message).To(HavePrefix("opamp: ")) + }) + + It("sets phase Degraded when a Deployment exceeds its progress deadline", func() { + name := "bp-status-deadline" + Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) + + r := newReconciler() + reconcileRolledOut(r, name) + + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name + "-jobs", Namespace: testNamespace}, dep)).To(Succeed()) + dep.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentProgressing, + Status: corev1.ConditionFalse, + Reason: "ProgressDeadlineExceeded", + }} + Expect(k8sClient.Status().Update(testCtx, dep)).To(Succeed()) + + result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeZero()) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseDegraded)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonWorkloadFailed)) + Expect(readyCond.Message).To(ContainSubstring("jobs: deployment " + name + "-jobs exceeded its progress deadline")) + }) + + It("reaches Ready at the replica count an autoscaler wrote", func() { + name := "bp-status-hpa" + bp := newTestBindplane(name, testNamespace) + bp.Spec.Bindplane.Autoscaling = &bindplanev1alpha1.NodeAutoscalingSpec{Enabled: true} + Expect(k8sClient.Create(testCtx, bp)).To(Succeed()) + + r := newReconciler() + reconcilePastMigration(testCtx, r, name, testNamespace) + + // Scale the node Deployment as the HPA would; the operator preserves the live count. + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name + "-node", Namespace: testNamespace}, dep)).To(Succeed()) + hpaReplicas := int32(2) + dep.Spec.Replicas = &hpaReplicas + Expect(k8sClient.Update(testCtx, dep)).To(Succeed()) + + markWorkloadsRolledOut(testCtx, testNamespace) + _, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseReady)) + Expect(updated.Status.Components.Bindplane.ReadyReplicas).To(BeNumerically("==", 2)) + Expect(getReadyCondition(testCtx, name, testNamespace).Status).To(Equal(metav1.ConditionTrue)) + }) + + It("reports MigrationInProgress at the new generation while the migrate Job runs", func() { + name := "bp-status-migrating" + Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) + + r := newReconciler() + reconcileRolledOut(r, name) + + bp := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, bp)).To(Succeed()) + Expect(bp.Status.Phase).To(Equal(phaseReady)) + bp.Spec.Version = "1.99.0" + Expect(k8sClient.Update(testCtx, bp)).To(Succeed()) + + result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(migrateJobPollInterval)) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Generation).To(BeNumerically(">", bp.Status.ObservedGeneration)) + Expect(updated.Status.Phase).To(Equal(phaseApplyingChanges)) + Expect(updated.Status.ObservedGeneration).To(Equal(updated.Generation)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonMigrationInProgress)) + Expect(readyCond.ObservedGeneration).To(Equal(updated.Generation)) + Expect(readyCond.Message).To(ContainSubstring(name + "-migrate")) + + // Polling the running Job again leaves the unchanged status alone. + _, err = r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + again := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, again)).To(Succeed()) + Expect(again.ResourceVersion).To(Equal(updated.ResourceVersion)) }) It("sets Reconciled=True on successful reconcile", func() { @@ -1107,8 +1359,104 @@ var _ = Describe("Reconcile - status", func() { Expect(reconciledCond.Reason).To(Equal("Reconciled")) Expect(reconciledCond.ObservedGeneration).To(Equal(updated.Generation)) }) + + It("records a reconcile error as Ready=False/ReconcileError", func() { + name := "bp-status-reconcile-error" + Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) + reconcileRolledOut(newReconciler(), name) + + // failing reads the node Deployment with an error, so the Node step fails. + failing := newReconciler() + failing.Client = failGetClient{Client: k8sClient, name: name + "-node"} + + _, err := failing.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).To(HaveOccurred()) + + // The phase was computed for the current generation and is left unchanged. + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseReady)) + Expect(updated.Status.ObservedGeneration).To(Equal(updated.Generation)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(readyCond.Reason).To(Equal(reasonReconcileError)) + Expect(readyCond.ObservedGeneration).To(Equal(updated.Generation)) + Expect(readyCond.Message).To(HavePrefix("unable to reconcile Node: ")) + Expect(readyCond.Message).To(ContainSubstring(errInjectedGet)) + + // At a new generation the phase from the older generation is replaced. + replicas := int32(2) + updated.Spec.Bindplane.Replicas = &replicas + Expect(k8sClient.Update(testCtx, updated)).To(Succeed()) + + _, err = failing.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).To(HaveOccurred()) + + bumped := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, bumped)).To(Succeed()) + Expect(bumped.Generation).To(BeNumerically(">", updated.Status.ObservedGeneration)) + Expect(bumped.Status.Phase).To(Equal(phaseApplyingChanges)) + Expect(bumped.Status.ObservedGeneration).To(Equal(bumped.Generation)) + readyCond = getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Reason).To(Equal(reasonReconcileError)) + Expect(readyCond.ObservedGeneration).To(Equal(bumped.Generation)) + }) + + It("replaces a phase from an older generation after a spec change made while paused", func() { + name := "bp-status-reconcile-error-paused" + Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) + reconcileRolledOut(newReconciler(), name) + + // Pause, change the spec, and reconcile: the pause path advances + // status.observedGeneration without computing a phase. + bp := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, bp)).To(Succeed()) + bp.Annotations = map[string]string{annotationPauseReconciliation: "true"} + replicas := int32(2) + bp.Spec.Bindplane.Replicas = &replicas + Expect(k8sClient.Update(testCtx, bp)).To(Succeed()) + _, err := newReconciler().Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).NotTo(HaveOccurred()) + + paused := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, paused)).To(Succeed()) + Expect(paused.Status.ObservedGeneration).To(Equal(paused.Generation)) + Expect(paused.Status.Phase).To(Equal(phaseReady)) + + // Unpause and fail the reconcile: the phase from the older generation is replaced. + paused.Annotations = nil + Expect(k8sClient.Update(testCtx, paused)).To(Succeed()) + failing := newReconciler() + failing.Client = failGetClient{Client: k8sClient, name: name + "-node"} + _, err = failing.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).To(HaveOccurred()) + + updated := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseApplyingChanges)) + readyCond := getReadyCondition(testCtx, name, testNamespace) + Expect(readyCond.Reason).To(Equal(reasonReconcileError)) + Expect(readyCond.ObservedGeneration).To(Equal(updated.Generation)) + }) }) +// errInjectedGet is the message of the error failGetClient returns. +const errInjectedGet = "injected get failure" + +// failGetClient returns an error for every Get of the Deployment with the given name and +// passes every other call to the embedded Client. +type failGetClient struct { + client.Client + name string +} + +func (c failGetClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.Deployment); ok && key.Name == c.name { + return errors.NewServiceUnavailable(errInjectedGet) + } + return c.Client.Get(ctx, key, obj, opts...) +} + var _ = Describe("Reconcile - idempotency", func() { var ( testNamespace string diff --git a/internal/controller/jobs.go b/internal/controller/jobs.go index b9e7d6e..3362334 100644 --- a/internal/controller/jobs.go +++ b/internal/controller/jobs.go @@ -464,7 +464,8 @@ func (r *BindplaneReconciler) deleteJobWithPods(ctx context.Context, job *batchv return nil } -// setMigrateFailureCondition records a MigrationFailed condition on the Bindplane CR. +// setMigrateFailureCondition records MigrationFailed on the Reconciled and Ready conditions +// and sets status.phase to Degraded. func setMigrateFailureCondition(bindplane *bindplanev1alpha1.Bindplane) { meta.SetStatusCondition(&bindplane.Status.Conditions, metav1.Condition{ Type: conditionTypeReconciled, @@ -474,7 +475,9 @@ func setMigrateFailureCondition(bindplane *bindplanev1alpha1.Bindplane) { ObservedGeneration: bindplane.Generation, LastTransitionTime: metav1.Now(), }) - bindplane.Status.Phase = "Degraded" + setReadyCondition(bindplane, metav1.ConditionFalse, reasonMigrationFailed, + "Jobs Migrate Job "+getResourceName(bindplane, bindplaneJobsMigrateComponent)+" failed") + bindplane.Status.Phase = phaseDegraded bindplane.Status.ObservedGeneration = bindplane.Generation } diff --git a/internal/controller/workload_status.go b/internal/controller/workload_status.go new file mode 100644 index 0000000..c3e2916 --- /dev/null +++ b/internal/controller/workload_status.go @@ -0,0 +1,262 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "strconv" + "strings" + + rolloutsv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + bindplanev1alpha1 "github.com/observiq/bindplane-operator/api/v1alpha1" +) + +// conditionTypeReady is the condition that reports whether every workload has finished +// rolling out the spec at the Bindplane CR's current generation. +const conditionTypeReady = "Ready" + +// Reasons used on the Ready condition. +const ( + reasonWorkloadsReady = "WorkloadsReady" + reasonWorkloadsProgressing = "WorkloadsProgressing" + reasonWorkloadFailed = "WorkloadFailed" + reasonRolloutPaused = "RolloutPaused" + reasonMigrationInProgress = "MigrationInProgress" + reasonReconcileError = "ReconcileError" + reasonInvalid = "Invalid" + reasonMigrationFailed = "MigrationFailed" +) + +// Values written to status.phase. +const ( + phaseReady = "Ready" + phaseApplyingChanges = "ApplyingChanges" + phaseDegraded = "Degraded" +) + +// jobsMigrateProgressComponent names the Jobs Migrate Job in Ready condition messages. +const jobsMigrateProgressComponent = "jobs-migrate" + +// nodeRolloutPausedMessage is the progress message for a node Rollout awaiting promotion. +const nodeRolloutPausedMessage = "node rollout paused: promote with kubectl argo rollouts promote" + +// workloadProgress is the rollout state of one workload. A workload that is neither done, +// failed nor paused is waiting, and message says why. +type workloadProgress struct { + component string + ready int32 + done bool + failed bool + paused bool + message string +} + +// waiting returns a workloadProgress that is still rolling out. +func waiting(component string, ready int32, format string, args ...any) workloadProgress { + return workloadProgress{component: component, ready: ready, message: fmt.Sprintf(format, args...)} +} + +// replicasOrDefault returns *replicas, or 1 when replicas is nil (the Kubernetes default). +func replicasOrDefault(replicas *int32) int32 { + if replicas == nil { + return 1 + } + return *replicas +} + +// deploymentProgress reports a Deployment's rollout state using the same checks as +// `kubectl rollout status`. The desired count is the live spec.replicas, which an HPA +// may have written. +func deploymentProgress(component string, dep *appsv1.Deployment) workloadProgress { + ready := dep.Status.ReadyReplicas + if dep.Generation > dep.Status.ObservedGeneration { + return waiting(component, ready, "waiting for deployment spec update to be observed") + } + for _, c := range dep.Status.Conditions { + if c.Type == appsv1.DeploymentProgressing && c.Reason == "ProgressDeadlineExceeded" { + return workloadProgress{component: component, ready: ready, failed: true, + message: fmt.Sprintf("deployment %s exceeded its progress deadline", dep.Name)} + } + } + desired := replicasOrDefault(dep.Spec.Replicas) + updated := dep.Status.UpdatedReplicas + switch { + case updated < desired: + return waiting(component, ready, "%d out of %d new replicas have been updated", updated, desired) + case dep.Status.Replicas > updated: + return waiting(component, ready, "%d old replicas are pending termination", dep.Status.Replicas-updated) + case dep.Status.AvailableReplicas < updated: + return waiting(component, ready, "%d of %d updated replicas are available", dep.Status.AvailableReplicas, updated) + } + return workloadProgress{component: component, ready: ready, done: true} +} + +// statefulSetProgress reports a StatefulSet's rollout state: the generation is observed, +// every replica is ready and updated, and currentRevision equals updateRevision. +func statefulSetProgress(component string, sts *appsv1.StatefulSet) workloadProgress { + ready := sts.Status.ReadyReplicas + if sts.Status.ObservedGeneration == 0 || sts.Generation > sts.Status.ObservedGeneration { + return waiting(component, ready, "waiting for statefulset spec update to be observed") + } + desired := replicasOrDefault(sts.Spec.Replicas) + switch { + case ready < desired: + return waiting(component, ready, "%d of %d pods are ready", ready, desired) + case sts.Status.UpdatedReplicas < desired: + return waiting(component, ready, "%d out of %d pods have been updated", sts.Status.UpdatedReplicas, desired) + case sts.Status.UpdateRevision != sts.Status.CurrentRevision: + return waiting(component, ready, "waiting for rolling update to revision %s to complete", sts.Status.UpdateRevision) + } + return workloadProgress{component: component, ready: ready, done: true} +} + +// rolloutProgress reports an Argo Rollout's state. It is waiting unless the string +// status.observedGeneration parses and equals metadata.generation, and while the controller +// is unpausing. Otherwise it maps status.phase Healthy to done, Degraded to failed and Paused +// to paused. +func rolloutProgress(component string, ro *rolloutsv1alpha1.Rollout) workloadProgress { + ready := ro.Status.ReadyReplicas + observed, err := strconv.ParseInt(ro.Status.ObservedGeneration, 10, 64) + if err != nil || observed != ro.Generation { + return waiting(component, ready, "waiting for rollout spec update to be observed") + } + if ro.Status.ControllerPause && len(ro.Status.PauseConditions) == 0 { + return waiting(component, ready, "waiting for rollout to unpause") + } + switch ro.Status.Phase { + case rolloutsv1alpha1.RolloutPhaseHealthy: + return workloadProgress{component: component, ready: ready, done: true} + case rolloutsv1alpha1.RolloutPhaseDegraded: + return workloadProgress{component: component, ready: ready, failed: true, message: ro.Status.Message} + case rolloutsv1alpha1.RolloutPhasePaused: + return workloadProgress{component: component, ready: ready, paused: true, message: nodeRolloutPausedMessage} + } + if ro.Status.Message != "" { + return waiting(component, ready, "%s", ro.Status.Message) + } + return waiting(component, ready, "rollout phase is %q", ro.Status.Phase) +} + +// migrateProgress returns a progress entry for the Jobs Migrate Job when the migration gate +// was bypassed and the Job is not complete. ok is false when there is nothing to report. +func migrateProgress(bindplane *bindplanev1alpha1.Bindplane, state migrateState) (progress workloadProgress, ok bool) { + jobName := getResourceName(bindplane, bindplaneJobsMigrateComponent) + switch state { + case migrateFailed: + return workloadProgress{component: jobsMigrateProgressComponent, failed: true, + message: fmt.Sprintf("job %s failed", jobName)}, true + case migrateInProgress: + return waiting(jobsMigrateProgressComponent, 0, "job %s is not complete", jobName), true + default: + return workloadProgress{}, false + } +} + +// setPhaseFromProgress sets status.phase and the Ready condition from the progress of +// every workload. A failed workload wins over a paused one, and a paused one wins over +// one that is still rolling out. +func setPhaseFromProgress(bindplane *bindplanev1alpha1.Bindplane, progress []workloadProgress) { + var failed, paused, pending []string + for _, p := range progress { + entry := fmt.Sprintf("%s: %s", p.component, p.message) + switch { + case p.failed: + failed = append(failed, entry) + case p.paused: + paused = append(paused, entry) + case !p.done: + pending = append(pending, entry) + } + } + + switch { + case len(failed) > 0: + bindplane.Status.Phase = phaseDegraded + setReadyCondition(bindplane, metav1.ConditionFalse, reasonWorkloadFailed, strings.Join(failed, "; ")) + case len(paused) > 0: + bindplane.Status.Phase = phaseApplyingChanges + setReadyCondition(bindplane, metav1.ConditionFalse, reasonRolloutPaused, strings.Join(paused, "; ")) + case len(pending) > 0: + bindplane.Status.Phase = phaseApplyingChanges + setReadyCondition(bindplane, metav1.ConditionFalse, reasonWorkloadsProgressing, strings.Join(pending, "; ")) + default: + bindplane.Status.Phase = phaseReady + setReadyCondition(bindplane, metav1.ConditionTrue, reasonWorkloadsReady, "All workloads are rolled out and ready") + } +} + +// setReadyCondition sets the Ready condition at the CR's current generation and reports +// whether the condition changed. +func setReadyCondition(bindplane *bindplanev1alpha1.Bindplane, status metav1.ConditionStatus, reason, message string) bool { + return meta.SetStatusCondition(&bindplane.Status.Conditions, metav1.Condition{ + Type: conditionTypeReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: bindplane.Generation, + }) +} + +// statusReader returns the reader used for child workload status. APIReader bypasses the +// informer cache, so a child updated earlier in the same reconcile is read at its new +// generation rather than as the cached copy from before the update. +func (r *BindplaneReconciler) statusReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +// getDeploymentProgress reads a Deployment and reports its rollout state. A read error, +// including NotFound, is reported as waiting with the error as the message. +func (r *BindplaneReconciler) getDeploymentProgress(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, component string) workloadProgress { + dep := &appsv1.Deployment{} + key := types.NamespacedName{Name: getResourceName(bindplane, component), Namespace: bindplane.Namespace} + if err := r.statusReader().Get(ctx, key, dep); err != nil { + return waiting(component, 0, "%s", err.Error()) + } + return deploymentProgress(component, dep) +} + +// getStatefulSetProgress reads a StatefulSet and reports its rollout state. A read error, +// including NotFound, is reported as waiting with the error as the message. +func (r *BindplaneReconciler) getStatefulSetProgress(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, component string) workloadProgress { + sts := &appsv1.StatefulSet{} + key := types.NamespacedName{Name: getResourceName(bindplane, component), Namespace: bindplane.Namespace} + if err := r.statusReader().Get(ctx, key, sts); err != nil { + return waiting(component, 0, "%s", err.Error()) + } + return statefulSetProgress(component, sts) +} + +// getRolloutProgress reads an Argo Rollout and reports its rollout state. A read error, +// including NotFound, is reported as waiting with the error as the message. +func (r *BindplaneReconciler) getRolloutProgress(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, component string) workloadProgress { + ro := &rolloutsv1alpha1.Rollout{} + key := types.NamespacedName{Name: getResourceName(bindplane, component), Namespace: bindplane.Namespace} + if err := r.statusReader().Get(ctx, key, ro); err != nil { + return waiting(component, 0, "%s", err.Error()) + } + return rolloutProgress(component, ro) +} diff --git a/internal/controller/workload_status_test.go b/internal/controller/workload_status_test.go new file mode 100644 index 0000000..dc827ad --- /dev/null +++ b/internal/controller/workload_status_test.go @@ -0,0 +1,439 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + rolloutsv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + bindplanev1alpha1 "github.com/observiq/bindplane-operator/api/v1alpha1" +) + +// progressState summarizes a workloadProgress for table assertions. +type progressState string + +const ( + stateDone progressState = "done" + stateWaiting progressState = "waiting" + stateFailed progressState = "failed" + statePaused progressState = "paused" +) + +func stateOf(p workloadProgress) progressState { + switch { + case p.failed: + return stateFailed + case p.paused: + return statePaused + case p.done: + return stateDone + default: + return stateWaiting + } +} + +// rolledOutDeployment returns a Deployment whose status matches its spec at generation 1. +func rolledOutDeployment(name string, replicas int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + Replicas: replicas, + UpdatedReplicas: replicas, + ReadyReplicas: replicas, + AvailableReplicas: replicas, + }, + } +} + +// rolledOutStatefulSet returns a StatefulSet whose status matches its spec at generation 1. +func rolledOutStatefulSet(name string, replicas int32) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1}, + Spec: appsv1.StatefulSetSpec{Replicas: &replicas}, + Status: appsv1.StatefulSetStatus{ + ObservedGeneration: 1, + Replicas: replicas, + ReadyReplicas: replicas, + UpdatedReplicas: replicas, + CurrentRevision: name + "-1", + UpdateRevision: name + "-1", + }, + } +} + +// healthyNodeRollout returns the "bp-node" Rollout that the Argo Rollouts controller reports +// Healthy at generation 1. +func healthyNodeRollout(replicas int32) *rolloutsv1alpha1.Rollout { + return &rolloutsv1alpha1.Rollout{ + ObjectMeta: metav1.ObjectMeta{Name: "bp-node", Namespace: "default", Generation: 1}, + Spec: rolloutsv1alpha1.RolloutSpec{Replicas: &replicas}, + Status: rolloutsv1alpha1.RolloutStatus{ + ObservedGeneration: "1", + Phase: rolloutsv1alpha1.RolloutPhaseHealthy, + ReadyReplicas: replicas, + }, + } +} + +var _ = Describe("deploymentProgress", func() { + DescribeTable("reports the rollout state", + func(mutate func(*appsv1.Deployment), want progressState, wantMessage string) { + dep := rolledOutDeployment("bp-jobs", 3) + mutate(dep) + p := deploymentProgress("jobs", dep) + Expect(stateOf(p)).To(Equal(want)) + Expect(p.component).To(Equal("jobs")) + Expect(p.message).To(ContainSubstring(wantMessage)) + }, + Entry("rolled out", func(*appsv1.Deployment) {}, stateDone, ""), + Entry("stale child: new generation not observed though fully rolled out", + func(d *appsv1.Deployment) { d.Generation = 2 }, stateWaiting, "spec update to be observed"), + Entry("HPA-written spec.replicas of 2 is done", + func(d *appsv1.Deployment) { + two := int32(2) + d.Spec.Replicas = &two + d.Status.Replicas, d.Status.UpdatedReplicas, d.Status.ReadyReplicas, d.Status.AvailableReplicas = 2, 2, 2, 2 + }, stateDone, ""), + Entry("updated replicas below desired", + func(d *appsv1.Deployment) { d.Status.UpdatedReplicas = 1 }, stateWaiting, "1 out of 3 new replicas have been updated"), + Entry("old replicas pending termination", + func(d *appsv1.Deployment) { d.Status.Replicas = 5 }, stateWaiting, "2 old replicas are pending termination"), + Entry("updated replicas not yet available", + func(d *appsv1.Deployment) { d.Status.AvailableReplicas = 2 }, stateWaiting, "2 of 3 updated replicas are available"), + Entry("progress deadline exceeded", + func(d *appsv1.Deployment) { + d.Status.UpdatedReplicas = 1 + d.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "ProgressDeadlineExceeded", + }} + }, stateFailed, "exceeded its progress deadline"), + ) + + It("reports ready replicas", func() { + dep := rolledOutDeployment("bp-jobs", 3) + dep.Status.ReadyReplicas = 2 + Expect(deploymentProgress("jobs", dep).ready).To(Equal(int32(2))) + }) +}) + +var _ = Describe("statefulSetProgress", func() { + DescribeTable("reports the rollout state", + func(mutate func(*appsv1.StatefulSet), want progressState, wantMessage string) { + sts := rolledOutStatefulSet("bp-nats", 3) + mutate(sts) + p := statefulSetProgress("nats", sts) + Expect(stateOf(p)).To(Equal(want)) + Expect(p.message).To(ContainSubstring(wantMessage)) + }, + Entry("rolled out", func(*appsv1.StatefulSet) {}, stateDone, ""), + Entry("never observed", func(s *appsv1.StatefulSet) { s.Status.ObservedGeneration = 0 }, stateWaiting, "spec update to be observed"), + Entry("stale child: new generation not observed though fully rolled out", + func(s *appsv1.StatefulSet) { s.Generation = 2 }, stateWaiting, "spec update to be observed"), + Entry("pods not ready", func(s *appsv1.StatefulSet) { s.Status.ReadyReplicas = 2 }, stateWaiting, "2 of 3 pods are ready"), + Entry("pods not updated", func(s *appsv1.StatefulSet) { s.Status.UpdatedReplicas = 1 }, stateWaiting, "1 out of 3 pods have been updated"), + Entry("revision rollout incomplete", func(s *appsv1.StatefulSet) { s.Status.UpdateRevision = "bp-nats-2" }, stateWaiting, "bp-nats-2"), + ) +}) + +var _ = Describe("rolloutProgress", func() { + DescribeTable("reports the rollout state", + func(mutate func(*rolloutsv1alpha1.Rollout), want progressState, wantMessage string) { + ro := healthyNodeRollout(3) + mutate(ro) + p := rolloutProgress("node", ro) + Expect(stateOf(p)).To(Equal(want)) + Expect(p.message).To(ContainSubstring(wantMessage)) + }, + Entry("healthy", func(*rolloutsv1alpha1.Rollout) {}, stateDone, ""), + Entry("stale child: healthy at an old generation", + func(r *rolloutsv1alpha1.Rollout) { r.Generation = 2 }, stateWaiting, "spec update to be observed"), + Entry("observedGeneration not numeric", + func(r *rolloutsv1alpha1.Rollout) { r.Status.ObservedGeneration = "5d8f7c9b" }, stateWaiting, "spec update to be observed"), + Entry("observedGeneration empty", + func(r *rolloutsv1alpha1.Rollout) { r.Status.ObservedGeneration = "" }, stateWaiting, "spec update to be observed"), + Entry("unpausing", func(r *rolloutsv1alpha1.Rollout) { r.Status.ControllerPause = true }, stateWaiting, "waiting for rollout to unpause"), + Entry("progressing", + func(r *rolloutsv1alpha1.Rollout) { + r.Status.Phase = rolloutsv1alpha1.RolloutPhaseProgressing + r.Status.Message = "active service cutover pending" + }, stateWaiting, "active service cutover pending"), + Entry("degraded", + func(r *rolloutsv1alpha1.Rollout) { + r.Status.Phase = rolloutsv1alpha1.RolloutPhaseDegraded + r.Status.Message = "ProgressDeadlineExceeded: ReplicaSet has timed out progressing." + }, stateFailed, "ProgressDeadlineExceeded"), + Entry("paused awaiting promotion", + func(r *rolloutsv1alpha1.Rollout) { + r.Status.Phase = rolloutsv1alpha1.RolloutPhasePaused + r.Status.ControllerPause = true + r.Status.PauseConditions = []rolloutsv1alpha1.PauseCondition{{Reason: rolloutsv1alpha1.PauseReasonBlueGreenPause}} + }, statePaused, nodeRolloutPausedMessage), + ) +}) + +var _ = Describe("migrateProgress", func() { + bp := newTestBindplane("bp", "default") + + It("reports nothing when migration is complete", func() { + _, ok := migrateProgress(bp, migrateComplete) + Expect(ok).To(BeFalse()) + }) + + It("reports a running Job as waiting", func() { + p, ok := migrateProgress(bp, migrateInProgress) + Expect(ok).To(BeTrue()) + Expect(stateOf(p)).To(Equal(stateWaiting)) + Expect(p.component).To(Equal(jobsMigrateProgressComponent)) + Expect(p.message).To(ContainSubstring("bp-migrate")) + }) + + It("reports a failed Job as failed", func() { + p, ok := migrateProgress(bp, migrateFailed) + Expect(ok).To(BeTrue()) + Expect(stateOf(p)).To(Equal(stateFailed)) + }) +}) + +var _ = Describe("setPhaseFromProgress", func() { + done := func(c string) workloadProgress { return workloadProgress{component: c, done: true} } + pending := func(c, msg string) workloadProgress { return workloadProgress{component: c, message: msg} } + failed := func(c, msg string) workloadProgress { + return workloadProgress{component: c, failed: true, message: msg} + } + paused := func(c string) workloadProgress { + return workloadProgress{component: c, paused: true, message: nodeRolloutPausedMessage} + } + + DescribeTable("sets the phase and Ready condition", + func(progress []workloadProgress, wantPhase string, wantStatus metav1.ConditionStatus, wantReason, wantMessage string) { + bp := newTestBindplane("bp", "default") + bp.Generation = 7 + setPhaseFromProgress(bp, progress) + Expect(bp.Status.Phase).To(Equal(wantPhase)) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(wantStatus)) + Expect(cond.Reason).To(Equal(wantReason)) + Expect(cond.ObservedGeneration).To(Equal(int64(7))) + Expect(cond.Message).To(ContainSubstring(wantMessage)) + }, + Entry("all done", []workloadProgress{done("node"), done("nats")}, + phaseReady, metav1.ConditionTrue, reasonWorkloadsReady, ""), + Entry("one waiting", []workloadProgress{done("node"), pending("opamp", "1 of 2 updated replicas are available")}, + phaseApplyingChanges, metav1.ConditionFalse, reasonWorkloadsProgressing, "opamp: 1 of 2 updated replicas are available"), + Entry("paused Rollout gives RolloutPaused", []workloadProgress{paused("node"), pending("nats", "x")}, + phaseApplyingChanges, metav1.ConditionFalse, reasonRolloutPaused, "node: "+nodeRolloutPausedMessage), + Entry("failed wins over paused and waiting", []workloadProgress{paused("node"), pending("nats", "x"), failed("jobs", "boom")}, + phaseDegraded, metav1.ConditionFalse, reasonWorkloadFailed, "jobs: boom"), + ) + + It("lists every waiting component", func() { + bp := newTestBindplane("bp", "default") + setPhaseFromProgress(bp, []workloadProgress{pending("node", "a"), pending("opamp", "b")}) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond.Message).To(Equal("node: a; opamp: b")) + }) +}) + +var _ = Describe("updateReadyReplicaStatus with an Argo Rollout node", func() { + newFakeReconciler := func(objs ...client.Object) *BindplaneReconciler { + c := newFakeClient(objs...) + return &BindplaneReconciler{Client: c, Scheme: c.Scheme()} + } + + newRolloutBindplane := func() *bindplanev1alpha1.Bindplane { + bp := newTestBindplane("bp", "default") + bp.Generation = 1 + bp.Spec.Bindplane.ArgoRollout = &bindplanev1alpha1.ArgoRolloutSpec{Enabled: true} + return bp + } + + // children returns the rolled out non-node workloads for the "bp" CR. + children := func() []client.Object { + return []client.Object{ + rolledOutStatefulSet("bp-nats", 3), + rolledOutStatefulSet("bp-tsdb", 1), + rolledOutDeployment("bp-transform-agent", 1), + rolledOutDeployment("bp-jobs", 1), + } + } + + It("reports ApplyingChanges while the node Rollout is Progressing", func() { + ro := healthyNodeRollout(2) + ro.Status.Phase = rolloutsv1alpha1.RolloutPhaseProgressing + ro.Status.Message = "active service cutover pending" + r := newFakeReconciler(append(children(), ro)...) + + bp := newRolloutBindplane() + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeTrue()) + Expect(bp.Status.Phase).To(Equal(phaseApplyingChanges)) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Reason).To(Equal(reasonWorkloadsProgressing)) + Expect(cond.Message).To(Equal("node: active service cutover pending")) + }) + + It("reports RolloutPaused while a blue-green node Rollout awaits promotion", func() { + ro := healthyNodeRollout(2) + ro.Status.Phase = rolloutsv1alpha1.RolloutPhasePaused + ro.Status.ControllerPause = true + ro.Status.PauseConditions = []rolloutsv1alpha1.PauseCondition{{ + Reason: rolloutsv1alpha1.PauseReasonBlueGreenPause, + StartTime: metav1.Now(), + }} + r := newFakeReconciler(append(children(), ro)...) + + bp := newRolloutBindplane() + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeTrue()) + Expect(bp.Status.Phase).To(Equal(phaseApplyingChanges)) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(reasonRolloutPaused)) + Expect(cond.Message).To(HavePrefix("node: node rollout paused")) + }) + + It("reports Degraded and polls while the node Rollout is Degraded", func() { + ro := healthyNodeRollout(2) + ro.Status.Phase = rolloutsv1alpha1.RolloutPhaseDegraded + ro.Status.Message = "ProgressDeadlineExceeded: rollout exceeded its progress deadline" + r := newFakeReconciler(append(children(), ro)...) + + bp := newRolloutBindplane() + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeTrue()) + Expect(bp.Status.Phase).To(Equal(phaseDegraded)) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Reason).To(Equal(reasonWorkloadFailed)) + Expect(cond.Message).To(Equal("node: " + ro.Status.Message)) + }) + + It("reports Ready when the node Rollout is Healthy", func() { + r := newFakeReconciler(append(children(), healthyNodeRollout(2))...) + + bp := newRolloutBindplane() + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeFalse()) + Expect(bp.Status.Phase).To(Equal(phaseReady)) + Expect(bp.Status.Components.Bindplane.ReadyReplicas).To(Equal(int32(2))) + Expect(bp.Status.Components.Nats.ReadyReplicas).To(Equal(int32(3))) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + }) + + It("does not read a node Deployment when the Rollout is enabled", func() { + objs := append(children(), rolledOutDeployment("bp-node", 3)) + r := newFakeReconciler(objs...) + + bp := newRolloutBindplane() + r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(bp.Status.Phase).To(Equal(phaseApplyingChanges)) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond.Message).To(HavePrefix("node: ")) + Expect(cond.Message).To(ContainSubstring("not found")) + }) +}) + +// newFakeClient returns a fake client holding objs, with the core, Argo Rollouts and +// Bindplane types registered. +func newFakeClient(objs ...client.Object) client.Client { + s := runtime.NewScheme() + Expect(clientgoscheme.AddToScheme(s)).To(Succeed()) + Expect(rolloutsv1alpha1.AddToScheme(s)).To(Succeed()) + Expect(bindplanev1alpha1.AddToScheme(s)).To(Succeed()) + return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build() +} + +var _ = Describe("updateReadyReplicaStatus with a Deployment node", func() { + newBindplane := func() *bindplanev1alpha1.Bindplane { + bp := newTestBindplane("bp", "default") + bp.Generation = 1 + return bp + } + + // children returns the rolled out non-node workloads for the "bp" CR, without TSDB. + children := func() []client.Object { + return []client.Object{ + rolledOutStatefulSet("bp-nats", 3), + rolledOutDeployment("bp-transform-agent", 1), + rolledOutDeployment("bp-jobs", 1), + } + } + + It("reads workload status through APIReader rather than Client", func() { + // Client holds the node Deployment as it was before the spec update; APIReader holds + // it at the new generation, not yet observed by the Deployment controller. + stale := rolledOutDeployment("bp-node", 3) + fresh := rolledOutDeployment("bp-node", 3) + fresh.Generation = 2 + cached := newFakeClient(append(children(), rolledOutStatefulSet("bp-tsdb", 1), stale)...) + live := newFakeClient(append(children(), rolledOutStatefulSet("bp-tsdb", 1), fresh)...) + r := &BindplaneReconciler{Client: cached, Scheme: cached.Scheme(), APIReader: live} + Expect(r.statusReader()).To(BeIdenticalTo(live)) + + bp := newBindplane() + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeTrue()) + Expect(bp.Status.Phase).To(Equal(phaseApplyingChanges)) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Reason).To(Equal(reasonWorkloadsProgressing)) + Expect(cond.Message).To(Equal("node: waiting for deployment spec update to be observed")) + }) + + It("does not read the TSDB StatefulSet when remote TSDB is enabled", func() { + c := newFakeClient(append(children(), rolledOutDeployment("bp-node", 3))...) + r := &BindplaneReconciler{Client: c, Scheme: c.Scheme(), APIReader: c} + + bp := newBindplane() + bp.Spec.Config.TSDB = &bindplanev1alpha1.TSDBConfig{ + Remote: &bindplanev1alpha1.TSDBRemoteConfig{Enable: true}, + } + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeFalse()) + Expect(bp.Status.Phase).To(Equal(phaseReady)) + Expect(bp.Status.Components.TSDB.ReadyReplicas).To(BeZero()) + cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Message).NotTo(ContainSubstring("tsdb:")) + }) +}) From 1c9f49d869c2ae5088bf5bf7b356264adcb96169 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Thu, 19 Nov 2026 14:53:51 -0500 Subject: [PATCH 2/2] fix: address status review feedback - reconcileError: drop a Ready phase to ApplyingChanges when writing Ready=False so phase and condition stay consistent; Degraded is kept. - Only poll an Argo Rollout node when the Rollout watch is not registered. Deployments, StatefulSets and the migrate Job are watched via Owns and no longer trigger a 30s requeue. - Ignore status-only updates on the Bindplane For watch so status writes do not requeue the CR and bypass the retry backoff. Generation, annotation, label, finalizer and deletion changes still reconcile. Assisted-by: Claude Opus 5.5 --- docs/configuration/configuration.md | 4 +- internal/controller/bindplane_controller.go | 65 ++++++++++--- .../controller/bindplane_controller_test.go | 28 ++++-- internal/controller/workload_status_test.go | 96 +++++++++++++++++-- 4 files changed, 161 insertions(+), 32 deletions(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index f5fb5cf..ce7d9a0 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -1457,12 +1457,12 @@ The operator reads workload status directly from the API server rather than from | Phase | Meaning | |-------|---------| | `Pending` | CR was just created; first reconcile has not run yet | -| `ApplyingChanges` | The Jobs Migrate Job is running, or one or more workloads are still rolling out or paused. The operator requeues at least every 30 seconds while in this phase (every 10 seconds while the Jobs Migrate Job is running) | +| `ApplyingChanges` | The Jobs Migrate Job is running, or one or more workloads are still rolling out or paused. Status updates as the operator observes changes to its workloads. It requeues every 10 seconds while the Jobs Migrate Job is running, and every 30 seconds while an Argo Rollout node rolls out when the Argo Rollouts CRD was not installed at operator startup (so Rollouts are not watched) | | `Ready` | Every workload has rolled out and is ready (`Ready=True`) | | `Degraded` | The CR failed validation, the Jobs Migrate Job failed, or a workload failed. The `Ready` condition holds the reason. The operator requeues every 30 seconds while the Bindplane Node Argo Rollout is `Degraded` | | `Deleting` | CR deletion is in progress; finalizer is being removed | -A reconcile error sets `Ready=False` with reason `ReconcileError`. It leaves the phase unchanged when the phase was set for the current generation, and otherwise sets it to `ApplyingChanges`. The pause annotation does not change the phase either; it sets `Reconciled=False` with reason `Paused`. +A reconcile error sets `Ready=False` with reason `ReconcileError` and sets the phase to `ApplyingChanges`, except that a `Degraded` phase set for the current generation is left unchanged. The pause annotation does not change the phase either; it sets `Reconciled=False` with reason `Paused`. The `status.observedGeneration` field is set to the `metadata.generation` of the CR when the operator writes status for that generation, including when it halts for a running Jobs Migrate Job or records a reconcile error. GitOps tools should require both `status.observedGeneration == metadata.generation` and `Ready=True` with a matching `observedGeneration` before treating the CR as healthy. diff --git a/internal/controller/bindplane_controller.go b/internal/controller/bindplane_controller.go index efc8334..718e348 100644 --- a/internal/controller/bindplane_controller.go +++ b/internal/controller/bindplane_controller.go @@ -41,9 +41,12 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" bindplanev1alpha1 "github.com/observiq/bindplane-operator/api/v1alpha1" "github.com/observiq/bindplane-operator/internal/validation" @@ -316,8 +319,8 @@ const ( // Job is still running. migrateJobPollInterval = 10 * time.Second - // applyingChangesPollInterval is how often the controller requeues while status.phase - // is ApplyingChanges. + // applyingChangesPollInterval is how often the controller requeues while an Argo + // Rollout node is rolling out or Degraded and the Rollout watch is not registered. applyingChangesPollInterval = 30 * time.Second // annotationPreserveSelectorKeys is an internal operator annotation placed on Service objects @@ -518,6 +521,9 @@ type BindplaneReconciler struct { // APIReader reads child workload status directly from the API server. When nil, the // reconciler's Client is used. APIReader client.Reader + // rolloutWatched records whether SetupWithManager registered a watch on Argo Rollouts. + // When it did, Rollout status changes trigger a reconcile and no polling is needed. + rolloutWatched bool } // +kubebuilder:rbac:groups=k8s.bindplane.com,resources=bindplanes,verbs=get;list;watch;create;update;patch;delete @@ -697,9 +703,8 @@ func (r *BindplaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( log.Error(err, "failed to update Bindplane status") return ctrl.Result{}, err } - // Poll while workloads roll out or the node Rollout is Degraded, so status converges - // even without a watch event from the workload, for example when the Rollout watch is - // not registered. + // Poll only when the node is an Argo Rollout that is not watched, so status converges + // without a watch event from it. Every other workload is watched via Owns. if poll && (requeueAfter == 0 || requeueAfter > applyingChangesPollInterval) { requeueAfter = applyingChangesPollInterval } @@ -707,20 +712,23 @@ func (r *BindplaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // reconcileError logs err, records it as Ready=False/ReconcileError on the Bindplane CR and -// returns err so the request is retried. status.phase is left unchanged when it was computed -// for the current generation (the Ready condition and status.observedGeneration both match -// it); otherwise it is set to ApplyingChanges, so a phase from an older generation is not -// reported for the current one. The pause path advances status.observedGeneration without -// computing a phase, which is why the Ready condition is checked too. The status write is -// best effort: a failed write is logged unless it is a conflict. +// returns err so the request is retried. status.phase is set to ApplyingChanges when it was +// not computed for the current generation (the Ready condition or status.observedGeneration +// does not match it), so a phase from an older generation is not reported for the current +// one. The pause path advances status.observedGeneration without computing a phase, which is +// why the Ready condition is checked too. A Ready phase is also set to ApplyingChanges, so +// phase and the Ready=False condition stay consistent; Degraded is left as is. The status +// write is best effort: a failed write is logged unless it is a conflict. func (r *BindplaneReconciler) reconcileError(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, log logr.Logger, err error, msg string) (ctrl.Result, error) { log.Error(err, msg) ready := meta.FindStatusCondition(bindplane.Status.Conditions, conditionTypeReady) stalePhase := bindplane.Status.ObservedGeneration != bindplane.Generation || ready == nil || ready.ObservedGeneration != bindplane.Generation setReadyCondition(bindplane, metav1.ConditionFalse, reasonReconcileError, fmt.Sprintf("%s: %v", msg, err)) - if stalePhase { + if stalePhase || bindplane.Status.Phase == phaseReady { bindplane.Status.Phase = phaseApplyingChanges + } + if stalePhase { bindplane.Status.ObservedGeneration = bindplane.Generation } if statusErr := r.Status().Update(ctx, bindplane); statusErr != nil && !errors.IsConflict(statusErr) { @@ -829,7 +837,9 @@ func reconciledCondition(bindplane *bindplanev1alpha1.Bindplane, state migrateSt // status.components and sets status.phase and the Ready condition from the rollout state of // all of them. state is the Jobs Migrate state; a Job that is not complete counts as a // workload that is still rolling out, or as a failed one when the Job failed. poll is true -// when the phase is ApplyingChanges or the node Rollout is Degraded. +// when the node is an Argo Rollout that is not watched (see rolloutWatched) and the phase is +// ApplyingChanges or the Rollout is Degraded. Deployments, StatefulSets and the migrate Job +// are watched via Owns, so they never need polling. func (r *BindplaneReconciler) updateReadyReplicaStatus(ctx context.Context, bindplane *bindplanev1alpha1.Bindplane, state migrateState) (poll bool) { var node workloadProgress nodeIsRollout := bindplane.Spec.Bindplane.ArgoRollout != nil && bindplane.Spec.Bindplane.ArgoRollout.Enabled @@ -869,7 +879,8 @@ func (r *BindplaneReconciler) updateReadyReplicaStatus(ctx context.Context, bind } setPhaseFromProgress(bindplane, progress) - return bindplane.Status.Phase == phaseApplyingChanges || (nodeIsRollout && node.failed) + return nodeIsRollout && !r.rolloutWatched && + (bindplane.Status.Phase == phaseApplyingChanges || node.failed) } // updateImageStatus records the resolved container image for each deployed component into @@ -897,7 +908,9 @@ func updateImageStatus(bindplane *bindplanev1alpha1.Bindplane) { // SetupWithManager sets up the controller with the Manager. func (r *BindplaneReconciler) SetupWithManager(mgr ctrl.Manager) error { b := ctrl.NewControllerManagedBy(mgr). - For(&bindplanev1alpha1.Bindplane{}). + // Ignore status-only updates to the CR: the controller writes status itself, and + // requeueing on those writes would bypass the rate-limited retry backoff. + For(&bindplanev1alpha1.Bindplane{}, builder.WithPredicates(ignoreStatusOnlyUpdates())). Owns(&appsv1.Deployment{}). Owns(&appsv1.StatefulSet{}). Owns(&batchv1.Job{}). @@ -912,12 +925,34 @@ func (r *BindplaneReconciler) SetupWithManager(mgr ctrl.Manager) error { if gvks, _, err := mgr.GetScheme().ObjectKinds(&rolloutsv1alpha1.Rollout{}); err == nil && len(gvks) > 0 { if _, err := mgr.GetRESTMapper().RESTMapping(gvks[0].GroupKind(), gvks[0].Version); err == nil { b = b.Owns(&rolloutsv1alpha1.Rollout{}) + r.rolloutWatched = true } } return b.Named("bindplane").Complete(r) } +// ignoreStatusOnlyUpdates returns a predicate that drops update events which change only the +// status (and resourceVersion/managedFields) of the object. Spec changes bump generation; +// annotations (pause, skip-migrate-check, force-migrate), labels, finalizers and the deletion +// timestamp are metadata changes that must still trigger a reconcile. In particular, the +// finalizer-add path returns without requeue and relies on its own update event. +func ignoreStatusOnlyUpdates() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return true + } + oldObj, newObj := e.ObjectOld, e.ObjectNew + return oldObj.GetGeneration() != newObj.GetGeneration() || + !maps.Equal(oldObj.GetAnnotations(), newObj.GetAnnotations()) || + !maps.Equal(oldObj.GetLabels(), newObj.GetLabels()) || + !slices.Equal(oldObj.GetFinalizers(), newObj.GetFinalizers()) || + !oldObj.GetDeletionTimestamp().Equal(newObj.GetDeletionTimestamp()) + }, + } +} + // getLabels returns the standard labels for Bindplane resources func getLabels(bindplane *bindplanev1alpha1.Bindplane, component string) map[string]string { return map[string]string{ diff --git a/internal/controller/bindplane_controller_test.go b/internal/controller/bindplane_controller_test.go index 99b1654..03445f9 100644 --- a/internal/controller/bindplane_controller_test.go +++ b/internal/controller/bindplane_controller_test.go @@ -684,7 +684,7 @@ var _ = Describe("Reconcile - Jobs Migrate", func() { result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(Equal(applyingChangesPollInterval), "polls while workloads roll out") + Expect(result.RequeueAfter).To(BeZero(), "watched workloads are not polled while they roll out") // Jobs Deployment should now exist dep := &appsv1.Deployment{} @@ -908,7 +908,7 @@ var _ = Describe("Reconcile - Jobs Migrate", func() { markJobComplete(testCtx, jobName, testNamespace) result, err = r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(Equal(applyingChangesPollInterval), "polls while workloads roll out") + Expect(result.RequeueAfter).To(BeZero(), "watched workloads are not polled while they roll out") updated, cond = getReconciledCondition(testCtx, name, testNamespace) Expect(cond.Status).To(Equal(metav1.ConditionTrue)) @@ -1209,7 +1209,7 @@ var _ = Describe("Reconcile - status", func() { result, err := r.Reconcile(testCtx, reconcileRequest(name, testNamespace)) Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(Equal(applyingChangesPollInterval)) + Expect(result.RequeueAfter).To(BeZero(), "the node Deployment is watched, so it is not polled") updated := &bindplanev1alpha1.Bindplane{} Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) @@ -1372,10 +1372,10 @@ var _ = Describe("Reconcile - status", func() { _, err := failing.Reconcile(testCtx, reconcileRequest(name, testNamespace)) Expect(err).To(HaveOccurred()) - // The phase was computed for the current generation and is left unchanged. + // A Ready phase is replaced so it stays consistent with Ready=False. updated := &bindplanev1alpha1.Bindplane{} Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) - Expect(updated.Status.Phase).To(Equal(phaseReady)) + Expect(updated.Status.Phase).To(Equal(phaseApplyingChanges)) Expect(updated.Status.ObservedGeneration).To(Equal(updated.Generation)) readyCond := getReadyCondition(testCtx, name, testNamespace) Expect(readyCond.Status).To(Equal(metav1.ConditionFalse)) @@ -1384,6 +1384,15 @@ var _ = Describe("Reconcile - status", func() { Expect(readyCond.Message).To(HavePrefix("unable to reconcile Node: ")) Expect(readyCond.Message).To(ContainSubstring(errInjectedGet)) + // A Degraded phase computed for the current generation is left unchanged. + updated.Status.Phase = phaseDegraded + Expect(k8sClient.Status().Update(testCtx, updated)).To(Succeed()) + _, err = failing.Reconcile(testCtx, reconcileRequest(name, testNamespace)) + Expect(err).To(HaveOccurred()) + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(phaseDegraded)) + Expect(getReadyCondition(testCtx, name, testNamespace).Reason).To(Equal(reasonReconcileError)) + // At a new generation the phase from the older generation is replaced. replicas := int32(2) updated.Spec.Bindplane.Replicas = &replicas @@ -1407,6 +1416,13 @@ var _ = Describe("Reconcile - status", func() { Expect(k8sClient.Create(testCtx, newTestBindplane(name, testNamespace))).To(Succeed()) reconcileRolledOut(newReconciler(), name) + // Record a Degraded phase for the current generation, which a reconcile error at + // that generation would leave unchanged. + degraded := &bindplanev1alpha1.Bindplane{} + Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, degraded)).To(Succeed()) + degraded.Status.Phase = phaseDegraded + Expect(k8sClient.Status().Update(testCtx, degraded)).To(Succeed()) + // Pause, change the spec, and reconcile: the pause path advances // status.observedGeneration without computing a phase. bp := &bindplanev1alpha1.Bindplane{} @@ -1421,7 +1437,7 @@ var _ = Describe("Reconcile - status", func() { paused := &bindplanev1alpha1.Bindplane{} Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: name, Namespace: testNamespace}, paused)).To(Succeed()) Expect(paused.Status.ObservedGeneration).To(Equal(paused.Generation)) - Expect(paused.Status.Phase).To(Equal(phaseReady)) + Expect(paused.Status.Phase).To(Equal(phaseDegraded)) // Unpause and fail the reconcile: the phase from the older generation is replaced. paused.Annotations = nil diff --git a/internal/controller/workload_status_test.go b/internal/controller/workload_status_test.go index dc827ad..fc81137 100644 --- a/internal/controller/workload_status_test.go +++ b/internal/controller/workload_status_test.go @@ -30,6 +30,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" bindplanev1alpha1 "github.com/observiq/bindplane-operator/api/v1alpha1" ) @@ -60,7 +61,7 @@ func stateOf(p workloadProgress) progressState { // rolledOutDeployment returns a Deployment whose status matches its spec at generation 1. func rolledOutDeployment(name string, replicas int32) *appsv1.Deployment { return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespaceDefault, Generation: 1}, Spec: appsv1.DeploymentSpec{Replicas: &replicas}, Status: appsv1.DeploymentStatus{ ObservedGeneration: 1, @@ -75,7 +76,7 @@ func rolledOutDeployment(name string, replicas int32) *appsv1.Deployment { // rolledOutStatefulSet returns a StatefulSet whose status matches its spec at generation 1. func rolledOutStatefulSet(name string, replicas int32) *appsv1.StatefulSet { return &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespaceDefault, Generation: 1}, Spec: appsv1.StatefulSetSpec{Replicas: &replicas}, Status: appsv1.StatefulSetStatus{ ObservedGeneration: 1, @@ -92,7 +93,7 @@ func rolledOutStatefulSet(name string, replicas int32) *appsv1.StatefulSet { // Healthy at generation 1. func healthyNodeRollout(replicas int32) *rolloutsv1alpha1.Rollout { return &rolloutsv1alpha1.Rollout{ - ObjectMeta: metav1.ObjectMeta{Name: "bp-node", Namespace: "default", Generation: 1}, + ObjectMeta: metav1.ObjectMeta{Name: "bp-node", Namespace: testNamespaceDefault, Generation: 1}, Spec: rolloutsv1alpha1.RolloutSpec{Replicas: &replicas}, Status: rolloutsv1alpha1.RolloutStatus{ ObservedGeneration: "1", @@ -199,7 +200,7 @@ var _ = Describe("rolloutProgress", func() { }) var _ = Describe("migrateProgress", func() { - bp := newTestBindplane("bp", "default") + bp := newTestBindplane("bp", testNamespaceDefault) It("reports nothing when migration is complete", func() { _, ok := migrateProgress(bp, migrateComplete) @@ -233,7 +234,7 @@ var _ = Describe("setPhaseFromProgress", func() { DescribeTable("sets the phase and Ready condition", func(progress []workloadProgress, wantPhase string, wantStatus metav1.ConditionStatus, wantReason, wantMessage string) { - bp := newTestBindplane("bp", "default") + bp := newTestBindplane("bp", testNamespaceDefault) bp.Generation = 7 setPhaseFromProgress(bp, progress) Expect(bp.Status.Phase).To(Equal(wantPhase)) @@ -255,7 +256,7 @@ var _ = Describe("setPhaseFromProgress", func() { ) It("lists every waiting component", func() { - bp := newTestBindplane("bp", "default") + bp := newTestBindplane("bp", testNamespaceDefault) setPhaseFromProgress(bp, []workloadProgress{pending("node", "a"), pending("opamp", "b")}) cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) Expect(cond.Message).To(Equal("node: a; opamp: b")) @@ -269,7 +270,7 @@ var _ = Describe("updateReadyReplicaStatus with an Argo Rollout node", func() { } newRolloutBindplane := func() *bindplanev1alpha1.Bindplane { - bp := newTestBindplane("bp", "default") + bp := newTestBindplane("bp", testNamespaceDefault) bp.Generation = 1 bp.Spec.Bindplane.ArgoRollout = &bindplanev1alpha1.ArgoRolloutSpec{Enabled: true} return bp @@ -356,6 +357,25 @@ var _ = Describe("updateReadyReplicaStatus with an Argo Rollout node", func() { Expect(cond.Status).To(Equal(metav1.ConditionTrue)) }) + It("does not poll a Rollout that is watched", func() { + for _, phase := range []rolloutsv1alpha1.RolloutPhase{ + rolloutsv1alpha1.RolloutPhaseProgressing, + rolloutsv1alpha1.RolloutPhasePaused, + rolloutsv1alpha1.RolloutPhaseDegraded, + } { + ro := healthyNodeRollout(2) + ro.Status.Phase = phase + r := newFakeReconciler(append(children(), ro)...) + r.rolloutWatched = true + + bp := newRolloutBindplane() + poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) + + Expect(poll).To(BeFalse(), "phase %s", phase) + Expect(bp.Status.Phase).NotTo(Equal(phaseReady), "phase %s", phase) + } + }) + It("does not read a node Deployment when the Rollout is enabled", func() { objs := append(children(), rolledOutDeployment("bp-node", 3)) r := newFakeReconciler(objs...) @@ -382,7 +402,7 @@ func newFakeClient(objs ...client.Object) client.Client { var _ = Describe("updateReadyReplicaStatus with a Deployment node", func() { newBindplane := func() *bindplanev1alpha1.Bindplane { - bp := newTestBindplane("bp", "default") + bp := newTestBindplane("bp", testNamespaceDefault) bp.Generation = 1 return bp } @@ -410,7 +430,7 @@ var _ = Describe("updateReadyReplicaStatus with a Deployment node", func() { bp := newBindplane() poll := r.updateReadyReplicaStatus(context.Background(), bp, migrateComplete) - Expect(poll).To(BeTrue()) + Expect(poll).To(BeFalse(), "a node Deployment is watched, so it is not polled") Expect(bp.Status.Phase).To(Equal(phaseApplyingChanges)) cond := meta.FindStatusCondition(bp.Status.Conditions, conditionTypeReady) Expect(cond).NotTo(BeNil()) @@ -437,3 +457,61 @@ var _ = Describe("updateReadyReplicaStatus with a Deployment node", func() { Expect(cond.Message).NotTo(ContainSubstring("tsdb:")) }) }) + +var _ = Describe("ignoreStatusOnlyUpdates", func() { + base := func() *bindplanev1alpha1.Bindplane { + bp := newTestBindplane("bp", testNamespaceDefault) + bp.Generation = 1 + bp.ResourceVersion = "1" + bp.Annotations = map[string]string{"a": "b"} + bp.Finalizers = []string{bindplaneFinalizer} + return bp + } + update := func(mutate func(*bindplanev1alpha1.Bindplane)) bool { + oldObj := base() + newObj := base() + newObj.ResourceVersion = "2" + mutate(newObj) + return ignoreStatusOnlyUpdates().Update(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObj}) + } + + It("ignores a status-only update", func() { + Expect(update(func(bp *bindplanev1alpha1.Bindplane) { + bp.Status.Phase = phaseApplyingChanges + bp.Status.Conditions = []metav1.Condition{{Type: conditionTypeReady, Message: "changed"}} + })).To(BeFalse()) + }) + + It("passes a spec change", func() { + Expect(update(func(bp *bindplanev1alpha1.Bindplane) { bp.Generation = 2 })).To(BeTrue()) + }) + + It("passes an annotation change", func() { + Expect(update(func(bp *bindplanev1alpha1.Bindplane) { + bp.Annotations[annotationPauseReconciliation] = "true" + })).To(BeTrue()) + }) + + It("passes a label change", func() { + Expect(update(func(bp *bindplanev1alpha1.Bindplane) { + bp.Labels = map[string]string{"x": "y"} + })).To(BeTrue()) + }) + + It("passes a finalizer change", func() { + Expect(update(func(bp *bindplanev1alpha1.Bindplane) { bp.Finalizers = nil })).To(BeTrue()) + }) + + It("passes a deletion", func() { + Expect(update(func(bp *bindplanev1alpha1.Bindplane) { + now := metav1.Now() + bp.DeletionTimestamp = &now + })).To(BeTrue()) + }) + + It("passes create and delete events", func() { + p := ignoreStatusOnlyUpdates() + Expect(p.Create(event.CreateEvent{Object: base()})).To(BeTrue()) + Expect(p.Delete(event.DeleteEvent{Object: base()})).To(BeTrue()) + }) +})