From f21799f31e8c0159e55d3e5a4b72a267c6e79c49 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 3 Oct 2026 15:39:49 +0000 Subject: [PATCH 1/3] fix: return an empty namespaced LIST where no Coder backend serves the namespace (#209) Change-Id: I2df6cd4bc4b1e5c1362a422beba4c663183f189c Signed-off-by: Thomas Kosiewski --- docs/how-to/troubleshooting.md | 2 + docs/reference/aggregated-api-behavior.md | 21 ++ hack/e2e-workspace-lifecycle.sh | 23 +- hack/e2e-workspace-lifecycle_test.sh | 11 +- .../aggregated/coder/controlplane_provider.go | 6 +- .../coder/namespace_not_served_test.go | 141 +++++++++++ internal/aggregated/coder/provider.go | 30 ++- internal/aggregated/storage/storage_test.go | 10 +- internal/aggregated/storage/template.go | 9 + .../aggregated/storage/templateversion.go | 3 + .../storage/unserved_namespace_test.go | 230 ++++++++++++++++++ internal/aggregated/storage/workspace.go | 17 ++ 12 files changed, 482 insertions(+), 21 deletions(-) create mode 100644 internal/aggregated/coder/namespace_not_served_test.go create mode 100644 internal/aggregated/storage/unserved_namespace_test.go diff --git a/docs/how-to/troubleshooting.md b/docs/how-to/troubleshooting.md index b09d6468a..814f8d1ab 100644 --- a/docs/how-to/troubleshooting.md +++ b/docs/how-to/troubleshooting.md @@ -166,6 +166,8 @@ This section is about `ServiceUnavailable` errors from the aggregated API server - `all` mode: no eligible `CoderControlPlane` exists yet. A control plane is eligible when its operator access is enabled and ready, its status has an operator token reference and a URL, and its name does not contain a `.` character. - Standalone mode (`--app=aggregated-apiserver`): set all three flags `--coder-url`, `--coder-session-token`, and `--coder-namespace`. +A `list` in one namespace that has no eligible control plane does not return this error. It returns an empty list. See [Namespaces without a Coder backend](../reference/aggregated-api-behavior.md#namespaces-without-a-coder-backend). + The logs show which provider configuration the server used. ## Aggregated reads return `multiple eligible CoderControlPlane ...` diff --git a/docs/reference/aggregated-api-behavior.md b/docs/reference/aggregated-api-behavior.md index f65480a37..635420335 100644 --- a/docs/reference/aggregated-api-behavior.md +++ b/docs/reference/aggregated-api-behavior.md @@ -10,6 +10,7 @@ The aggregated API server serves `coderworkspaces`, `codertemplates` and `codert | [Delete preconditions](#delete-preconditions) | The server checks `uid` and `resourceVersion`. A mismatch returns `409` and does not change Coder. | | [Workspace `resourceVersion`](#workspace-resourceversion) | An opaque fingerprint. Compare it only for equality. Workspace activity alone can cause `409`. | | [Watch](#watch) | Shows only writes made through this server. No replay, no initial events. | +| [Namespaces without a Coder backend](#namespaces-without-a-coder-backend) | A `list` in one namespace returns an empty list. Other requests return an error. Thus such a namespace can be deleted. | | [Server-side apply](#server-side-apply) | Create-on-update works. The server does not keep field ownership. | | [Server-side dry-run](#server-side-dry-run) | Not supported for writes to `coderworkspaces` and `codertemplates`. `kubectl diff` and `--dry-run=server` return `400` and do not change Coder. A promotion with `dryRun=All` is a read-only preview. Start and stop accept dry-run too. | | [Template versions](#template-versions) | Read-only: `get` and `list`, no watch. Reads never download template source. | @@ -137,6 +138,26 @@ The server rejects these requests: | `resourceVersionMatch` set | Rejected | | `sendInitialEvents=false` without a matching option | `422 Invalid` (rejected upstream) | +## Namespaces without a Coder backend + +A namespace has no Coder backend in these cases: + +- `all` mode: the namespace has no eligible `CoderControlPlane`. For the eligibility rules, see [Aggregated reads return `ServiceUnavailable`](../how-to/troubleshooting.md#aggregated-reads-return-serviceunavailable). +- Standalone mode: the namespace is not the one that `--coder-namespace` names. + +In such a namespace, the server answers requests as follows: + +| Request | Result | +| --- | --- | +| `list` of `coderworkspaces`, `codertemplates`, or `codertemplateversions` in that namespace | `200` with an empty list. The server does not call Coder. | +| `get`, `create`, `update`, `delete`, and the subresources | An error: `503` in `all` mode, `400` in standalone mode | +| `watch` | Works as described in [Watch](#watch). It shows no events, because no write can succeed in that namespace. | +| `list` in all namespaces (`-A`) | Unchanged. In `all` mode, it returns `503` when no `CoderControlPlane` is eligible in any namespace. | + +The namespace controller lists every resource type in a namespace before it removes the namespace. An error from a list keeps the namespace in `Terminating` forever, so the empty list lets the deletion finish. + +An empty list means that no Coder backend serves the namespace now. It does not mean that Coder has no objects. If a control plane stops being eligible for a short time, for example while its operator access is not ready, a list in its namespace is empty during that time. A client that caches a list, such as an informer, then sees all objects as deleted, and sees them again when the control plane is eligible again. + ## Server-side apply `kubectl apply --server-side` can create a resource that does not exist yet. For a missing workspace or template, the update path creates the object instead (`forceAllowCreate=true`). diff --git a/hack/e2e-workspace-lifecycle.sh b/hack/e2e-workspace-lifecycle.sh index 02fa224b1..cf43b44ec 100755 --- a/hack/e2e-workspace-lifecycle.sh +++ b/hack/e2e-workspace-lifecycle.sh @@ -575,19 +575,22 @@ wait_until "template test e2e-ns-delete to report CoderUnavailable" tt_unavailab T0=$SECONDS k delete namespace "$NS" --wait=false >/dev/null || fail "cannot delete namespace $NS" gone_obj() { ! k -n "$NS" get "$1" "$2" -o name >/dev/null 2>"$WORK/obj.err" && grep -q NotFound "$WORK/obj.err"; } -ns_released() { # the test and the control plane are gone, and no namespace content or finalizer remains +NS_SEEN="" +ns_gone() { # the test and the control plane go first, then the namespace itself disappears (#209) tt_state e2e-ns-delete || true # logs the ControlPlaneGone release while the object still exists gone_obj codertemplatetest e2e-ns-delete && gone_obj codercontrolplane coder || return 1 - k get namespace "$NS" -o json >"$WORK/ns.json" 2>"$WORK/ns.err" || { grep -q NotFound "$WORK/ns.err"; return; } - jq -e '[.status.conditions[]? | select(.type == "NamespaceContentRemaining" or .type == "NamespaceFinalizersRemaining")] | - length == 2 and all(.status == "False")' "$WORK/ns.json" >/dev/null + if k get namespace "$NS" -o json >"$WORK/ns.json" 2>"$WORK/ns.err"; then + local seen # diagnostics: the phase and every True condition, logged when they change + seen=$(jq -r '[.status.phase // "unknown"] + [.status.conditions[]? | select(.status == "True") | + "\(.type): \(.message)"] | join("; ")' "$WORK/ns.json") || seen="unparsable namespace JSON" + [[ $seen == "$NS_SEEN" ]] || log "namespace $NS still exists: $seen" + NS_SEEN=$seen + return 1 + fi + grep -q NotFound "$WORK/ns.err" } -TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, control plane, and content of namespace $NS deleted (ControlPlaneGone release)" ns_released +TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, control plane, and namespace $NS deleted (ControlPlaneGone release)" ns_gone NS_DELETE_SECONDS=$((SECONDS - T0)) -# Known issue #209: a namespaced aggregated LIST without an eligible control plane answers 503, so the namespace -# stays Terminating. Once #209 is fixed, this check becomes "the namespace disappears". -[[ ! -s $WORK/ns.json ]] || log "namespace $NS is $(jq -r '.status.phase' "$WORK/ns.json"), known issue #209: $(jq -r '[.status.conditions[]? | - select(.type == "NamespaceDeletionContentFailure" and .status == "True") | .message] | join("; ")' "$WORK/ns.json")" -log "namespace $NS: test, control plane, and content deleted in ${NS_DELETE_SECONDS}s" +log "namespace $NS deleted in ${NS_DELETE_SECONDS}s, after its test and control plane" CASES+=("case: $CURRENT = passed") && CURRENT="" && RESULT=PASS log "PASS: workspace lifecycle" diff --git a/hack/e2e-workspace-lifecycle_test.sh b/hack/e2e-workspace-lifecycle_test.sh index 14d42bf68..7ac0f5caa 100755 --- a/hack/e2e-workspace-lifecycle_test.sh +++ b/hack/e2e-workspace-lifecycle_test.sh @@ -93,7 +93,9 @@ case "${pos[0]}:${pos[1]:-}" in tt_json Failed AgentStartError '[{"type":"WorkspaceDeleted","status":"True","reason":"Deleted"}]' else tt_json Succeeded Succeeded '[{"type":"Ready","status":"True","reason":"Succeeded"},{"type":"WorkspaceDeleted","status":"True","reason":"Deleted"}]' fi ;; - get:namespace) # content gone, but the namespace stays Terminating (the aggregated API LIST answers 503) + get:namespace) # Terminating on the first read, then gone; ns-stays-terminating: Terminating forever (#209) + n=$(($(cat "$S/ns-reads" 2>/dev/null || echo 0) + 1)) && echo "$n" >"$S/ns-reads" + [[ $SCENARIO == ns-stays-terminating || $n -lt 2 ]] || err NotFound 'namespaces "coder" not found' jq -n '{status: {phase: "Terminating", conditions: [{type: "NamespaceDeletionContentFailure", status: "True", message: "no eligible CoderControlPlane"}, {type: "NamespaceContentRemaining", status: "False"}, {type: "NamespaceFinalizersRemaining", status: "False"}]}}' ;; delete:namespace) touch "$S/ns-deleted"; echo 'namespace "coder" deleted' ;; @@ -381,9 +383,9 @@ check "psql only counts rows: operator and tester api_keys, the first pass test' check "owner patch names the tester; the namespace test waits 120 s for its agent" eval 'grep -qF "\"ownerUserID\":\"user-tester\"" "$S/calls.log" && jq -e ".spec.parameters == [{name: \"startup_delay\", value: \"120\"}] and .spec.template == \"coder.e2e-agent\"" "$T/work/tt-e2e-ns-delete.json" >/dev/null && jq -e ".spec.version.active and (.spec | has(\"parameters\") | not)" "$T/work/tt-e2e-pass-1.json" >/dev/null' -check "namespace deleted only after CoderUnavailable; the release went through ControlPlaneGone" eval '[[ $(grep -n "e2e-ns-delete: Running CoderUnavailable" "$T/out" | cut -d: -f1) -lt $(grep -n "content deleted in" "$T/out" | cut -d: -f1) ]] && +check "namespace deleted only after CoderUnavailable; the release went through ControlPlaneGone; the namespace disappeared" eval '[[ $(grep -n "e2e-ns-delete: Running CoderUnavailable" "$T/out" | cut -d: -f1) -lt $(grep -n "namespace coder deleted in" "$T/out" | cut -d: -f1) ]] && out_has "e2e-ns-delete: Failed ControlPlaneGone deleted=Unknown/ControlPlaneGone" && grep -q "replicas.:0" "$S/calls.log" && - out_has "namespace coder is Terminating, known issue #209: no eligible CoderControlPlane"' + out_has "namespace coder still exists: Terminating; NamespaceDeletionContentFailure: no eligible CoderControlPlane" && [[ $(<"$S/ns-reads") == 2 ]]' check "tester: password user in the default organization; receipt records its username and id" eval 'jq -e ".login_type == \"password\" and .organization_ids == [\"org-1\"]" "$S/tester-body.json" >/dev/null && grep -qx "tester=e2e-tester/user-tester" "$T/work/receipt.txt"' # shellcheck disable=SC2034 # pw is used by the eval'd check below @@ -586,7 +588,8 @@ ws-rows-2|expected exactly one workspaces row named ktt-e2e-pass-1 (deleted rows coder-rolls|setting templateTests.ownerUserID rolled deploy/coder|$TESTS keys-after-fails|cannot count the api_keys rows of the tester after the tests|$PHASED coder-stays-up|timed out after 3s waiting for: deploy/coder without pods|$PHASED,kubectl create,kubectl patch -ns-delete-stuck|timed out after 2s waiting for: test, control plane, and content of namespace coder deleted|$PHASED,kubectl create,kubectl patch,kubectl delete +ns-delete-stuck|timed out after 2s waiting for: test, control plane, and namespace coder deleted|$PHASED,kubectl create,kubectl patch,kubectl delete +ns-stays-terminating|timed out after 2s waiting for: test, control plane, and namespace coder deleted|$PHASED,kubectl create,kubectl patch,kubectl delete fail-ignored|template test e2e-agent-fail ended Succeeded Succeeded deleted=True/Deleted, want Failed AgentStartError|$TESTS,kubectl create badparam-created|template test e2e-bad-param: expected 0 workspaces rows named ktt-e2e-bad-param (deleted rows included), found 1|$TESTS,kubectl create,kubectl create restart-failed|template test e2e-restart failed: Failed AgentStartError|$TESTS,kubectl create,kubectl create,kubectl create,kubectl delete diff --git a/internal/aggregated/coder/controlplane_provider.go b/internal/aggregated/coder/controlplane_provider.go index acf066501..a3b322dc3 100644 --- a/internal/aggregated/coder/controlplane_provider.go +++ b/internal/aggregated/coder/controlplane_provider.go @@ -80,7 +80,11 @@ func (p *ControlPlaneClientProvider) ClientForNamespace(ctx context.Context, nam switch len(eligible) { case 0: - return nil, apierrors.NewServiceUnavailable(noEligibleControlPlaneMessage(namespace)) + unavailable := apierrors.NewServiceUnavailable(noEligibleControlPlaneMessage(namespace)) + if namespace == "" { + return nil, unavailable + } + return nil, newNamespaceNotServedError(unavailable) case 1: // handled below default: diff --git a/internal/aggregated/coder/namespace_not_served_test.go b/internal/aggregated/coder/namespace_not_served_test.go new file mode 100644 index 000000000..ebcfc7b38 --- /dev/null +++ b/internal/aggregated/coder/namespace_not_served_test.go @@ -0,0 +1,141 @@ +package coder + +import ( + "context" + "errors" + "fmt" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" +) + +// Issue #209: a namespaced LIST in a namespace that no Coder backend serves answers with an empty +// list. The provider marks exactly those errors, and the marked errors keep their status for all +// other verbs. +func TestControlPlaneClientProviderMarksNamespaceNotServed(t *testing.T) { + t.Parallel() + + notReady := eligibleControlPlane("team-a", "coder") + notReady.Status.OperatorAccessReady = false + + tests := []struct { + name string + controlPlanes []coderv1alpha1.CoderControlPlane + secrets []corev1.Secret + namespace string + wantNotServed bool + wantStatus func(error) bool + }{ + { + name: "no control plane anywhere", + namespace: "team-a", + wantNotServed: true, + wantStatus: apierrors.IsServiceUnavailable, + }, + { + name: "eligible control plane only in another namespace", + controlPlanes: []coderv1alpha1.CoderControlPlane{eligibleControlPlane("team-b", "coder")}, + namespace: "team-a", + wantNotServed: true, + wantStatus: apierrors.IsServiceUnavailable, + }, + { + name: "control plane in namespace is not eligible", + controlPlanes: []coderv1alpha1.CoderControlPlane{notReady}, + namespace: "team-a", + wantNotServed: true, + wantStatus: apierrors.IsServiceUnavailable, + }, + { + name: "all namespaces without eligible control plane", + namespace: "", + wantStatus: apierrors.IsServiceUnavailable, + }, + { + name: "multiple eligible control planes in namespace", + controlPlanes: []coderv1alpha1.CoderControlPlane{ + eligibleControlPlane("team-a", "coder-a"), + eligibleControlPlane("team-a", "coder-b"), + }, + namespace: "team-a", + wantStatus: apierrors.IsBadRequest, + }, + { + name: "eligible control plane with unusable token secret", + controlPlanes: []coderv1alpha1.CoderControlPlane{eligibleControlPlane("team-a", "coder")}, + secrets: []corev1.Secret{ + secretWithStringData("team-a", "operator-token", map[string]string{"other": "value"}), + }, + namespace: "team-a", + wantStatus: apierrors.IsServiceUnavailable, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + provider, _ := newControlPlaneProviderForTest(t, tt.controlPlanes, tt.secrets) + + _, err := provider.ClientForNamespace(context.Background(), tt.namespace) + if err == nil { + t.Fatal("expected error") + } + if !tt.wantStatus(err) { + t.Fatalf("unexpected status for error %v", err) + } + if got := IsNamespaceNotServed(err); got != tt.wantNotServed { + t.Fatalf("IsNamespaceNotServed() = %t, want %t (error %v)", got, tt.wantNotServed, err) + } + // Callers wrap provider errors; the mark and the status must survive wrapping. + wrapped := fmt.Errorf("resolve client: %w", err) + if got := IsNamespaceNotServed(wrapped); got != tt.wantNotServed { + t.Fatalf("IsNamespaceNotServed(wrapped) = %t, want %t", got, tt.wantNotServed) + } + var statusErr *apierrors.StatusError + if !errors.As(wrapped, &statusErr) || !tt.wantStatus(statusErr) { + t.Fatalf("expected wrapped error to unwrap to its status error, got %v", wrapped) + } + }) + } +} + +func TestStaticClientProviderMarksNamespaceNotServed(t *testing.T) { + t.Parallel() + + client, err := NewSDKClient(Config{ + CoderURL: mustParseURL(t, "https://coder.example.com"), + SessionToken: "session-token", + }) + if err != nil { + t.Fatalf("create SDK client: %v", err) + } + + pinned := &StaticClientProvider{Client: client, Namespace: "control-plane"} + _, err = pinned.ClientForNamespace(context.Background(), "other-namespace") + if !apierrors.IsBadRequest(err) { + t.Fatalf("expected BadRequest for a namespace mismatch, got %v", err) + } + if !IsNamespaceNotServed(err) { + t.Fatalf("expected a namespace mismatch to be marked as not served, got %v", err) + } + + unpinned := &StaticClientProvider{Client: client} + _, err = unpinned.ClientForNamespace(context.Background(), "other-namespace") + if !apierrors.IsServiceUnavailable(err) { + t.Fatalf("expected ServiceUnavailable for an unpinned provider, got %v", err) + } + if IsNamespaceNotServed(err) { + t.Fatalf("expected missing configuration not to be marked as not served, got %v", err) + } + + if IsNamespaceNotServed(nil) { + t.Fatal("expected nil error not to be marked as not served") + } + if IsNamespaceNotServed(apierrors.NewServiceUnavailable("unrelated")) { + t.Fatal("expected an unmarked status error not to be marked as not served") + } +} diff --git a/internal/aggregated/coder/provider.go b/internal/aggregated/coder/provider.go index 58a5397b1..5f4d29948 100644 --- a/internal/aggregated/coder/provider.go +++ b/internal/aggregated/coder/provider.go @@ -2,6 +2,7 @@ package coder import ( "context" + "errors" "fmt" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -28,6 +29,31 @@ type NamespaceLister interface { EligibleNamespaces(ctx context.Context) ([]string, error) } +// namespaceNotServedError marks a ClientForNamespace error that means no Coder backend serves the +// request namespace. It unwraps to the status error that GET, CREATE and other verbs return. +type namespaceNotServedError struct { + status *apierrors.StatusError +} + +func (e *namespaceNotServedError) Error() string { return e.status.Error() } + +func (e *namespaceNotServedError) Unwrap() error { return e.status } + +func newNamespaceNotServedError(status *apierrors.StatusError) error { + if status == nil { + panic("assertion failed: namespace-not-served status error must not be nil") + } + return &namespaceNotServedError{status: status} +} + +// IsNamespaceNotServed reports whether err from ClientForNamespace means that no Coder backend +// serves the named namespace. Nothing can exist there, so a namespaced LIST returns an empty list +// instead of the error. Otherwise the namespace controller cannot delete the namespace (#209). +func IsNamespaceNotServed(err error) bool { + var target *namespaceNotServedError + return errors.As(err, &target) +} + // StaticClientProvider returns one static client, optionally restricted to one namespace. type StaticClientProvider struct { Client *codersdk.Client @@ -60,13 +86,13 @@ func (p *StaticClientProvider) ClientForNamespace(ctx context.Context, namespace namespace = p.Namespace } if namespace != p.Namespace { - return nil, apierrors.NewBadRequest( + return nil, newNamespaceNotServedError(apierrors.NewBadRequest( fmt.Sprintf( "namespace %q is not served by this aggregated API server (configured for %q)", namespace, p.Namespace, ), - ) + )) } return p.Client, nil diff --git a/internal/aggregated/storage/storage_test.go b/internal/aggregated/storage/storage_test.go index dd7327ca0..40506280f 100644 --- a/internal/aggregated/storage/storage_test.go +++ b/internal/aggregated/storage/storage_test.go @@ -1197,7 +1197,7 @@ func TestTemplateStorageListNamespacedRequestBypassesFanOut(t *testing.T) { } } -func TestTemplateStorageListPreservesProviderStatusErrors(t *testing.T) { +func TestTemplateStorageGetPreservesProviderStatusErrors(t *testing.T) { t.Parallel() server, _ := newMockCoderServer(t) @@ -1215,7 +1215,8 @@ func TestTemplateStorageListPreservesProviderStatusErrors(t *testing.T) { Namespace: "control-plane", }) - _, err = templateStorage.List(namespacedContext("other-namespace"), nil) + // A namespaced LIST in an unserved namespace is empty (#209); GET keeps the provider error. + _, err = templateStorage.Get(namespacedContext("other-namespace"), "acme.starter-template", nil) if !apierrors.IsBadRequest(err) { t.Fatalf("expected BadRequest from provider namespace restriction, got %v", err) } @@ -2626,7 +2627,7 @@ func TestWorkspaceStorageListAggregatesAcrossNamespaces(t *testing.T) { } } -func TestWorkspaceStorageListPreservesProviderStatusErrors(t *testing.T) { +func TestWorkspaceStorageGetPreservesProviderStatusErrors(t *testing.T) { t.Parallel() server, _ := newMockCoderServer(t) @@ -2644,7 +2645,8 @@ func TestWorkspaceStorageListPreservesProviderStatusErrors(t *testing.T) { Namespace: "control-plane", }) - _, err = workspaceStorage.List(namespacedContext("other-namespace"), nil) + // A namespaced LIST in an unserved namespace is empty (#209); GET keeps the provider error. + _, err = workspaceStorage.Get(namespacedContext("other-namespace"), "acme.alice.dev-workspace", nil) if !apierrors.IsBadRequest(err) { t.Fatalf("expected BadRequest from provider namespace restriction, got %v", err) } diff --git a/internal/aggregated/storage/template.go b/internal/aggregated/storage/template.go index 5a91620da..ca2deb322 100644 --- a/internal/aggregated/storage/template.go +++ b/internal/aggregated/storage/template.go @@ -243,6 +243,15 @@ func (s *TemplateStorage) List(ctx context.Context, _ *metainternalversion.ListO } sdk, err := s.clientForNamespace(ctx, namespace) + if listsUnservedNamespace(ctx, err) { + return &aggregationv1alpha1.CoderTemplateList{ + TypeMeta: metav1.TypeMeta{ + Kind: "CoderTemplateList", + APIVersion: aggregationv1alpha1.SchemeGroupVersion.String(), + }, + Items: make([]aggregationv1alpha1.CoderTemplate, 0), + }, nil + } if err != nil { return nil, wrapClientError(err) } diff --git a/internal/aggregated/storage/templateversion.go b/internal/aggregated/storage/templateversion.go index fe0f9ef3c..0dfa602d4 100644 --- a/internal/aggregated/storage/templateversion.go +++ b/internal/aggregated/storage/templateversion.go @@ -180,6 +180,9 @@ func (s *TemplateVersionStorage) list(ctx context.Context, opts *metainternalver func (s *TemplateVersionStorage) listNamespace(ctx context.Context, namespace string) ([]aggregationv1alpha1.CoderTemplateVersion, error) { resource := aggregationv1alpha1.Resource("codertemplateversions") sdk, err := s.clientForNamespace(ctx, namespace) + if listsUnservedNamespace(ctx, err) { + return nil, nil + } if err != nil { return nil, wrapClientError(err) } diff --git a/internal/aggregated/storage/unserved_namespace_test.go b/internal/aggregated/storage/unserved_namespace_test.go new file mode 100644 index 000000000..2744d4a60 --- /dev/null +++ b/internal/aggregated/storage/unserved_namespace_test.go @@ -0,0 +1,230 @@ +package storage + +import ( + "context" + "reflect" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/registry/rest" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + aggregationv1alpha1 "github.com/coder/coder-k8s/api/aggregation/v1alpha1" + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" + "github.com/coder/coder-k8s/internal/aggregated/coder" +) + +// newControlPlaneProviderForStorageTest builds the dynamic provider that --app=all uses, backed by a +// fake client that holds controlPlanes. +func newControlPlaneProviderForStorageTest(t *testing.T, controlPlanes ...coderv1alpha1.CoderControlPlane) coder.ClientProvider { + t.Helper() + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(coderv1alpha1.AddToScheme(scheme)) + reader := fake.NewClientBuilder(). + WithScheme(scheme). + WithLists(&coderv1alpha1.CoderControlPlaneList{Items: controlPlanes}). + Build() + + provider, err := coder.NewControlPlaneClientProvider(reader, reader, 10*time.Second) + if err != nil { + t.Fatalf("new control plane client provider: %v", err) + } + return provider +} + +func storageTestControlPlane(namespace, name string, ready bool) coderv1alpha1.CoderControlPlane { + return coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Status: coderv1alpha1.CoderControlPlaneStatus{ + URL: "https://coder.example.com", + OperatorAccessReady: ready, + OperatorTokenSecretRef: &coderv1alpha1.SecretKeySelector{Name: "operator-token", Key: "token"}, + }, + } +} + +type listStorage interface { + List(ctx context.Context, opts *metainternalversion.ListOptions) (runtime.Object, error) +} + +// listStoragesForTest returns the three LIST implementations of the aggregated API, keyed by resource. +func listStoragesForTest(provider coder.ClientProvider) map[string]listStorage { + return map[string]listStorage{ + "coderworkspaces": NewWorkspaceStorage(provider), + "codertemplates": NewTemplateStorage(provider), + "codertemplateversions": NewTemplateVersionStorage(provider), + } +} + +func assertEmptyTypedList(t *testing.T, resource string, obj runtime.Object) { + t.Helper() + + var items any + switch list := obj.(type) { + case *aggregationv1alpha1.CoderWorkspaceList: + if resource != "coderworkspaces" || list.Kind != "CoderWorkspaceList" { + t.Fatalf("%s: unexpected list %T kind %q", resource, obj, list.Kind) + } + items = list.Items + case *aggregationv1alpha1.CoderTemplateList: + if resource != "codertemplates" || list.Kind != "CoderTemplateList" { + t.Fatalf("%s: unexpected list %T kind %q", resource, obj, list.Kind) + } + items = list.Items + case *aggregationv1alpha1.CoderTemplateVersionList: + if resource != "codertemplateversions" || list.Kind != "CoderTemplateVersionList" { + t.Fatalf("%s: unexpected list %T kind %q", resource, obj, list.Kind) + } + items = list.Items + default: + t.Fatalf("%s: unexpected list type %T", resource, obj) + } + // items must be an empty, non-nil slice so that the JSON response holds "items": []. + value := reflect.ValueOf(items) + if value.IsNil() || value.Len() != 0 { + t.Fatalf("%s: expected empty non-nil items, got %#v", resource, items) + } +} + +// Issue #209: the namespace controller lists every deletable resource in a namespace before it +// removes the namespace. A namespaced LIST in a namespace that no Coder backend serves must answer +// with an empty list, or the namespace never finishes deleting. +func TestNamespacedListInUnservedNamespaceReturnsEmptyList(t *testing.T) { + t.Parallel() + + server, state := newMockCoderServer(t) + defer server.Close() + + providers := map[string]coder.ClientProvider{ + "standalone provider pinned to another namespace": &coder.StaticClientProvider{ + Client: newTestSDKClient(t, server.URL), Namespace: "control-plane", + }, + "no control plane": newControlPlaneProviderForStorageTest(t), + "control plane only in another namespace": newControlPlaneProviderForStorageTest(t, + storageTestControlPlane("control-plane", "coder", true)), + "control plane in namespace is not ready": newControlPlaneProviderForStorageTest(t, + storageTestControlPlane("empty-namespace", "coder", false)), + } + + for providerName, provider := range providers { + for resource, storage := range listStoragesForTest(provider) { + obj, err := storage.List(namespacedContext("empty-namespace"), nil) + if err != nil { + t.Fatalf("%s: %s LIST: expected empty list, got error %v", providerName, resource, err) + } + assertEmptyTypedList(t, resource, obj) + } + } + if requests := state.requests(); len(requests) != 0 { + t.Fatalf("expected no Coder requests for unserved namespaces, got %v", requests) + } +} + +// LIST keeps every error that does not mean "this namespace is not served". +func TestListKeepsErrorsOtherThanUnservedNamespace(t *testing.T) { + t.Parallel() + + noControlPlane := newControlPlaneProviderForStorageTest(t) + for resource, storage := range listStoragesForTest(noControlPlane) { + _, err := storage.List(namespacedContext(""), nil) + if !apierrors.IsServiceUnavailable(err) { + t.Fatalf("%s: all-namespaces LIST without a control plane: expected ServiceUnavailable, got %v", resource, err) + } + } + + duplicate := newControlPlaneProviderForStorageTest(t, + storageTestControlPlane("team-a", "coder-a", true), + storageTestControlPlane("team-a", "coder-b", true)) + for resource, storage := range listStoragesForTest(duplicate) { + _, err := storage.List(namespacedContext("team-a"), nil) + if !apierrors.IsBadRequest(err) { + t.Fatalf("%s: LIST with two eligible control planes: expected BadRequest, got %v", resource, err) + } + } + + // The eligible control plane has no token Secret, so resolving its client fails. + missingSecret := newControlPlaneProviderForStorageTest(t, storageTestControlPlane("team-a", "coder", true)) + for resource, storage := range listStoragesForTest(missingSecret) { + obj, err := storage.List(namespacedContext("team-a"), nil) + if err == nil { + t.Fatalf("%s: LIST with an unreadable token Secret: expected error, got %#v", resource, obj) + } + } + + server, _ := newMockCoderServer(t) + defer server.Close() + unpinned := &coder.StaticClientProvider{Client: newTestSDKClient(t, server.URL)} + for resource, storage := range listStoragesForTest(unpinned) { + _, err := storage.List(namespacedContext("team-a"), nil) + if !apierrors.IsServiceUnavailable(err) { + t.Fatalf("%s: LIST with an unpinned standalone provider: expected ServiceUnavailable, got %v", resource, err) + } + } +} + +// GET, CREATE and DELETE in a namespace without an eligible control plane keep their 503. +func TestOtherVerbsInNamespaceWithoutControlPlaneKeepServiceUnavailable(t *testing.T) { + t.Parallel() + + provider := newControlPlaneProviderForStorageTest(t) + ctx := namespacedContext("empty-namespace") + workspaceStorage := NewWorkspaceStorage(provider) + templateStorage := NewTemplateStorage(provider) + + assertServiceUnavailable := func(verb string, err error) { + t.Helper() + if !apierrors.IsServiceUnavailable(err) { + t.Fatalf("%s: expected ServiceUnavailable, got %v", verb, err) + } + assertTopLevelStatusError(t, err) + } + + _, err := workspaceStorage.Get(ctx, "acme.alice.dev", nil) + assertServiceUnavailable("workspace GET", err) + _, err = workspaceStorage.Create(ctx, &aggregationv1alpha1.CoderWorkspace{ + ObjectMeta: metav1.ObjectMeta{Name: "acme.alice.dev"}, + Spec: aggregationv1alpha1.CoderWorkspaceSpec{Organization: "acme", TemplateName: "starter-template"}, + }, rest.ValidateAllObjectFunc, nil) + assertServiceUnavailable("workspace CREATE", err) + _, _, err = workspaceStorage.Delete(ctx, "acme.alice.dev", rest.ValidateAllObjectFunc, nil) + assertServiceUnavailable("workspace DELETE", err) + + _, err = templateStorage.Get(ctx, "acme.starter-template", nil) + assertServiceUnavailable("template GET", err) + _, err = templateStorage.Create(ctx, &aggregationv1alpha1.CoderTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: "acme.starter-template"}, + Spec: aggregationv1alpha1.CoderTemplateSpec{Organization: "acme"}, + }, rest.ValidateAllObjectFunc, nil) + assertServiceUnavailable("template CREATE", err) + _, _, err = templateStorage.Delete(ctx, "acme.starter-template", rest.ValidateAllObjectFunc, nil) + assertServiceUnavailable("template DELETE", err) +} + +// WATCH never asks the provider: it serves local events, so it already works in a namespace +// without a control plane and across all namespaces. +func TestWatchInNamespaceWithoutControlPlaneSucceeds(t *testing.T) { + t.Parallel() + + provider := newControlPlaneProviderForStorageTest(t) + for _, namespace := range []string{"empty-namespace", ""} { + workspaceWatch, err := NewWorkspaceStorage(provider).Watch(namespacedContext(namespace), nil) + if err != nil { + t.Fatalf("workspace WATCH in namespace %q: %v", namespace, err) + } + workspaceWatch.Stop() + + templateWatch, err := NewTemplateStorage(provider).Watch(namespacedContext(namespace), nil) + if err != nil { + t.Fatalf("template WATCH in namespace %q: %v", namespace, err) + } + templateWatch.Stop() + } +} diff --git a/internal/aggregated/storage/workspace.go b/internal/aggregated/storage/workspace.go index 326f7a8b3..e2e96ed57 100644 --- a/internal/aggregated/storage/workspace.go +++ b/internal/aggregated/storage/workspace.go @@ -228,6 +228,15 @@ func (s *WorkspaceStorage) List(ctx context.Context, _ *metainternalversion.List } sdk, err := s.clientForNamespace(ctx, namespace) + if listsUnservedNamespace(ctx, err) { + return &aggregationv1alpha1.CoderWorkspaceList{ + TypeMeta: metav1.TypeMeta{ + Kind: "CoderWorkspaceList", + APIVersion: aggregationv1alpha1.SchemeGroupVersion.String(), + }, + Items: make([]aggregationv1alpha1.CoderWorkspace, 0), + }, nil + } if err != nil { return nil, wrapClientError(err) } @@ -839,6 +848,14 @@ func namespaceFromRequestContext(ctx context.Context) (string, error) { return genericapirequest.NamespaceValue(ctx), nil } +// listsUnservedNamespace reports whether a LIST must answer err from clientForNamespace with an +// empty list: the request names one namespace, and no Coder backend serves that namespace. Nothing +// can exist there. The namespace controller lists every resource before it deletes a namespace, +// and it never finishes on an error (#209). An all-namespaces LIST keeps the error. +func listsUnservedNamespace(ctx context.Context, err error) bool { + return genericapirequest.NamespaceValue(ctx) != "" && coder.IsNamespaceNotServed(err) +} + func requiredNamespaceFromRequestContext(ctx context.Context) (string, error) { namespace, err := namespaceFromRequestContext(ctx) if err != nil { From 845e3e405962089b86832cfc6cb2bb4f5f28b958 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 3 Oct 2026 16:14:50 +0000 Subject: [PATCH 2/3] fix: keep the 503 for LIST while a CoderControlPlane exists but is not eligible Change-Id: Ia4dc887f37a8b6516dd035c24649d98aae485e37 Signed-off-by: Thomas Kosiewski --- docs/how-to/troubleshooting.md | 2 +- docs/reference/aggregated-api-behavior.md | 8 +++-- .../aggregated/coder/controlplane_provider.go | 31 +++++++++++-------- .../coder/namespace_not_served_test.go | 3 +- .../storage/unserved_namespace_test.go | 13 ++++++-- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/docs/how-to/troubleshooting.md b/docs/how-to/troubleshooting.md index 814f8d1ab..1ad0f8ddd 100644 --- a/docs/how-to/troubleshooting.md +++ b/docs/how-to/troubleshooting.md @@ -166,7 +166,7 @@ This section is about `ServiceUnavailable` errors from the aggregated API server - `all` mode: no eligible `CoderControlPlane` exists yet. A control plane is eligible when its operator access is enabled and ready, its status has an operator token reference and a URL, and its name does not contain a `.` character. - Standalone mode (`--app=aggregated-apiserver`): set all three flags `--coder-url`, `--coder-session-token`, and `--coder-namespace`. -A `list` in one namespace that has no eligible control plane does not return this error. It returns an empty list. See [Namespaces without a Coder backend](../reference/aggregated-api-behavior.md#namespaces-without-a-coder-backend). +A `list` in one namespace that contains no `CoderControlPlane` at all does not return this error. It returns an empty list. See [Namespaces without a Coder backend](../reference/aggregated-api-behavior.md#namespaces-without-a-coder-backend). The logs show which provider configuration the server used. diff --git a/docs/reference/aggregated-api-behavior.md b/docs/reference/aggregated-api-behavior.md index 635420335..0ce592953 100644 --- a/docs/reference/aggregated-api-behavior.md +++ b/docs/reference/aggregated-api-behavior.md @@ -142,7 +142,7 @@ The server rejects these requests: A namespace has no Coder backend in these cases: -- `all` mode: the namespace has no eligible `CoderControlPlane`. For the eligibility rules, see [Aggregated reads return `ServiceUnavailable`](../how-to/troubleshooting.md#aggregated-reads-return-serviceunavailable). +- `all` mode: the namespace contains no `CoderControlPlane` at all. - Standalone mode: the namespace is not the one that `--coder-namespace` names. In such a namespace, the server answers requests as follows: @@ -154,9 +154,11 @@ In such a namespace, the server answers requests as follows: | `watch` | Works as described in [Watch](#watch). It shows no events, because no write can succeed in that namespace. | | `list` in all namespaces (`-A`) | Unchanged. In `all` mode, it returns `503` when no `CoderControlPlane` is eligible in any namespace. | -The namespace controller lists every resource type in a namespace before it removes the namespace. An error from a list keeps the namespace in `Terminating` forever, so the empty list lets the deletion finish. +A namespace that contains a `CoderControlPlane` that is not eligible is not in this group. For example, operator access is not ready, or the name contains a `.` character. For the eligibility rules, see [Aggregated reads return `ServiceUnavailable`](../how-to/troubleshooting.md#aggregated-reads-return-serviceunavailable). In such a namespace, every request returns `503`, `list` included. -An empty list means that no Coder backend serves the namespace now. It does not mean that Coder has no objects. If a control plane stops being eligible for a short time, for example while its operator access is not ready, a list in its namespace is empty during that time. A client that caches a list, such as an informer, then sees all objects as deleted, and sees them again when the control plane is eligible again. +The namespace controller lists every resource type in a namespace before it removes the namespace. An error from a list keeps the namespace in `Terminating` forever, so the empty list lets the deletion finish. If the namespace still contains a `CoderControlPlane`, the namespace controller deletes it in the same pass. The lists return `503` until the control plane is gone, and then they return empty lists. + +A short outage of a control plane does not empty a list. For example, the operator sets `operatorAccessReady=false` after a Postgres error. During that time, a list in its namespace returns `503`, so a client that caches a list, such as an informer, does not see objects as deleted. ## Server-side apply diff --git a/internal/aggregated/coder/controlplane_provider.go b/internal/aggregated/coder/controlplane_provider.go index a3b322dc3..62d51a9c2 100644 --- a/internal/aggregated/coder/controlplane_provider.go +++ b/internal/aggregated/coder/controlplane_provider.go @@ -73,7 +73,7 @@ func (p *ControlPlaneClientProvider) ClientForNamespace(ctx context.Context, nam return nil, fmt.Errorf("assertion failed: secret reader must not be nil") } - eligible, err := p.findEligibleControlPlanes(ctx, namespace) + eligible, listed, err := p.findEligibleControlPlanes(ctx, namespace) if err != nil { return nil, err } @@ -81,7 +81,10 @@ func (p *ControlPlaneClientProvider) ClientForNamespace(ctx context.Context, nam switch len(eligible) { case 0: unavailable := apierrors.NewServiceUnavailable(noEligibleControlPlaneMessage(namespace)) - if namespace == "" { + // Only a namespace without any CoderControlPlane is "not served". A control plane that exists + // but is not eligible can be in a short outage (operator access not ready after a Postgres + // error), so a namespaced LIST must keep the 503 instead of reporting every object as gone. + if namespace == "" || listed > 0 { return nil, unavailable } return nil, newNamespaceNotServedError(unavailable) @@ -193,7 +196,7 @@ func (p *ControlPlaneClientProvider) ClientForNamespace(ctx context.Context, nam // DefaultNamespace resolves the namespace for all-namespaces LIST requests. func (p *ControlPlaneClientProvider) DefaultNamespace(ctx context.Context) (string, error) { - eligible, err := p.findEligibleControlPlanes(ctx, "") + eligible, _, err := p.findEligibleControlPlanes(ctx, "") if err != nil { return "", err } @@ -221,7 +224,7 @@ func (p *ControlPlaneClientProvider) EligibleNamespaces(ctx context.Context) ([] return nil, fmt.Errorf("assertion failed: context must not be nil") } - eligible, err := p.findEligibleControlPlanes(ctx, "") + eligible, _, err := p.findEligibleControlPlanes(ctx, "") if err != nil { return nil, err } @@ -251,18 +254,20 @@ func (p *ControlPlaneClientProvider) EligibleNamespaces(ctx context.Context) ([] return namespaces, nil } +// findEligibleControlPlanes returns the eligible CoderControlPlanes in namespace (all namespaces when +// empty) and the number of CoderControlPlanes listed there, eligible or not. func (p *ControlPlaneClientProvider) findEligibleControlPlanes( ctx context.Context, namespace string, -) ([]coderv1alpha1.CoderControlPlane, error) { +) (eligible []coderv1alpha1.CoderControlPlane, listed int, err error) { if p == nil { - return nil, fmt.Errorf("assertion failed: control plane client provider must not be nil") + return nil, 0, fmt.Errorf("assertion failed: control plane client provider must not be nil") } if ctx == nil { - return nil, fmt.Errorf("assertion failed: context must not be nil") + return nil, 0, fmt.Errorf("assertion failed: context must not be nil") } if p.cpReader == nil { - return nil, fmt.Errorf("assertion failed: control plane reader must not be nil") + return nil, 0, fmt.Errorf("assertion failed: control plane reader must not be nil") } controlPlaneList := &coderv1alpha1.CoderControlPlaneList{} @@ -270,15 +275,15 @@ func (p *ControlPlaneClientProvider) findEligibleControlPlanes( if namespace != "" { listOptions = append(listOptions, client.InNamespace(namespace)) } - if err := p.cpReader.List(ctx, controlPlaneList, listOptions...); err != nil { + if err = p.cpReader.List(ctx, controlPlaneList, listOptions...); err != nil { if namespace == "" { - return nil, fmt.Errorf("list CoderControlPlane resources across all namespaces: %w", err) + return nil, 0, fmt.Errorf("list CoderControlPlane resources across all namespaces: %w", err) } - return nil, fmt.Errorf("list CoderControlPlane resources in namespace %q: %w", namespace, err) + return nil, 0, fmt.Errorf("list CoderControlPlane resources in namespace %q: %w", namespace, err) } - eligible := make([]coderv1alpha1.CoderControlPlane, 0, 1) + eligible = make([]coderv1alpha1.CoderControlPlane, 0, 1) for i := range controlPlaneList.Items { controlPlane := controlPlaneList.Items[i] if strings.Contains(controlPlane.Name, ".") { @@ -305,7 +310,7 @@ func (p *ControlPlaneClientProvider) findEligibleControlPlanes( eligible = append(eligible, controlPlane) } - return eligible, nil + return eligible, len(controlPlaneList.Items), nil } func noEligibleControlPlaneMessage(namespace string) string { diff --git a/internal/aggregated/coder/namespace_not_served_test.go b/internal/aggregated/coder/namespace_not_served_test.go index ebcfc7b38..839272597 100644 --- a/internal/aggregated/coder/namespace_not_served_test.go +++ b/internal/aggregated/coder/namespace_not_served_test.go @@ -43,10 +43,11 @@ func TestControlPlaneClientProviderMarksNamespaceNotServed(t *testing.T) { wantStatus: apierrors.IsServiceUnavailable, }, { + // A control plane that exists but is not ready can be in a short outage (for example a + // Postgres error during operator access setup). LIST must keep the 503 then. name: "control plane in namespace is not eligible", controlPlanes: []coderv1alpha1.CoderControlPlane{notReady}, namespace: "team-a", - wantNotServed: true, wantStatus: apierrors.IsServiceUnavailable, }, { diff --git a/internal/aggregated/storage/unserved_namespace_test.go b/internal/aggregated/storage/unserved_namespace_test.go index 2744d4a60..7c44d4892 100644 --- a/internal/aggregated/storage/unserved_namespace_test.go +++ b/internal/aggregated/storage/unserved_namespace_test.go @@ -110,8 +110,6 @@ func TestNamespacedListInUnservedNamespaceReturnsEmptyList(t *testing.T) { "no control plane": newControlPlaneProviderForStorageTest(t), "control plane only in another namespace": newControlPlaneProviderForStorageTest(t, storageTestControlPlane("control-plane", "coder", true)), - "control plane in namespace is not ready": newControlPlaneProviderForStorageTest(t, - storageTestControlPlane("empty-namespace", "coder", false)), } for providerName, provider := range providers { @@ -140,6 +138,17 @@ func TestListKeepsErrorsOtherThanUnservedNamespace(t *testing.T) { } } + // A control plane that exists but is not ready (for example during a short Postgres outage) still + // serves the namespace: an empty list would tell watch clients that every object was deleted. + notReady := newControlPlaneProviderForStorageTest(t, storageTestControlPlane("team-a", "coder", false)) + for resource, storage := range listStoragesForTest(notReady) { + _, err := storage.List(namespacedContext("team-a"), nil) + if !apierrors.IsServiceUnavailable(err) { + t.Fatalf("%s: LIST with a control plane that is not ready: expected ServiceUnavailable, got %v", resource, err) + } + assertTopLevelStatusError(t, err) + } + duplicate := newControlPlaneProviderForStorageTest(t, storageTestControlPlane("team-a", "coder-a", true), storageTestControlPlane("team-a", "coder-b", true)) From 13405d04a7a1bb9cecb09a552060110b4be4ebf2 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 3 Oct 2026 16:37:57 +0000 Subject: [PATCH 3/3] test: log namespace conditions from the first deletion poll Change-Id: I3515ac9d98be4a5416aff96cf74e7d2dc1193d1b Signed-off-by: Thomas Kosiewski --- hack/e2e-workspace-lifecycle.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hack/e2e-workspace-lifecycle.sh b/hack/e2e-workspace-lifecycle.sh index cf43b44ec..d8d35ea65 100755 --- a/hack/e2e-workspace-lifecycle.sh +++ b/hack/e2e-workspace-lifecycle.sh @@ -576,18 +576,19 @@ T0=$SECONDS k delete namespace "$NS" --wait=false >/dev/null || fail "cannot delete namespace $NS" gone_obj() { ! k -n "$NS" get "$1" "$2" -o name >/dev/null 2>"$WORK/obj.err" && grep -q NotFound "$WORK/obj.err"; } NS_SEEN="" -ns_gone() { # the test and the control plane go first, then the namespace itself disappears (#209) +ns_gone() { # the namespace disappears (#209), and with it the test and the control plane tt_state e2e-ns-delete || true # logs the ControlPlaneGone release while the object still exists - gone_obj codertemplatetest e2e-ns-delete && gone_obj codercontrolplane coder || return 1 if k get namespace "$NS" -o json >"$WORK/ns.json" 2>"$WORK/ns.err"; then - local seen # diagnostics: the phase and every True condition, logged when they change + # Diagnostics from the first poll on, so a run shows NamespaceDeletionContentFailure (the aggregated + # LIST errors while the control plane still exists): the phase and every True condition, on change. + local seen seen=$(jq -r '[.status.phase // "unknown"] + [.status.conditions[]? | select(.status == "True") | "\(.type): \(.message)"] | join("; ")' "$WORK/ns.json") || seen="unparsable namespace JSON" [[ $seen == "$NS_SEEN" ]] || log "namespace $NS still exists: $seen" NS_SEEN=$seen return 1 fi - grep -q NotFound "$WORK/ns.err" + grep -q NotFound "$WORK/ns.err" && gone_obj codertemplatetest e2e-ns-delete && gone_obj codercontrolplane coder } TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, control plane, and namespace $NS deleted (ControlPlaneGone release)" ns_gone NS_DELETE_SECONDS=$((SECONDS - T0))