so-l2l: clean orphan PVs when namespace is already gone
down() used to early-return when read_namespace returned 404, skipping all cleanup. That left cluster-scoped PVs orphaned after a prior 'stop --delete-namespace' (namespace cascades delete PVCs, but PVs with Retain reclaim policy survive). Split _delete_labeled_resources into namespaced and cluster- scoped phases via a namespace_present flag. When the namespace is missing, jump straight to _delete_labeled_pvs (for --delete-volumes) and the cluster-scoped half of the wait. _wait_for_labeled_deletions now builds its lister set based on whether the namespace still exists. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>pull/743/head
parent
3d83c6ad27
commit
2f99e6f7c9
|
|
@ -926,23 +926,31 @@ class K8sDeployer(Deployer):
|
||||||
label_selector = f"app={self.cluster_info.app_name}"
|
label_selector = f"app={self.cluster_info.app_name}"
|
||||||
|
|
||||||
ns = self.k8s_namespace
|
ns = self.k8s_namespace
|
||||||
# Namespace may not exist yet on first-time deployments.
|
# Check whether the namespace exists. If it's already gone, skip the
|
||||||
|
# namespaced cleanup (nothing to do, list/delete calls would 404),
|
||||||
|
# but cluster-scoped PVs may still be labeled with this stack — so
|
||||||
|
# fall through to the cluster-scoped half of _delete_labeled_resources.
|
||||||
try:
|
try:
|
||||||
self.core_api.read_namespace(name=ns)
|
self.core_api.read_namespace(name=ns)
|
||||||
|
namespace_present = True
|
||||||
except ApiException as e:
|
except ApiException as e:
|
||||||
if e.status == 404:
|
if e.status == 404:
|
||||||
|
namespace_present = False
|
||||||
if opts.o.debug:
|
if opts.o.debug:
|
||||||
print(f"Namespace {ns} not found; nothing to delete")
|
print(f"Namespace {ns} not found; cleaning cluster-scoped only")
|
||||||
if self.is_kind() and not self.skip_cluster_management:
|
else:
|
||||||
destroy_cluster(self.kind_cluster_name)
|
raise
|
||||||
return
|
|
||||||
raise
|
|
||||||
|
|
||||||
self._delete_labeled_resources(ns, label_selector, delete_volumes=volumes)
|
self._delete_labeled_resources(
|
||||||
|
ns,
|
||||||
|
label_selector,
|
||||||
|
delete_volumes=volumes,
|
||||||
|
namespace_present=namespace_present,
|
||||||
|
)
|
||||||
|
|
||||||
# Full teardown: nuke the namespace and wait for termination so that a
|
# Full teardown: nuke the namespace and wait for termination so that a
|
||||||
# subsequent up() can recreate it cleanly.
|
# subsequent up() can recreate it cleanly. No-op if already missing.
|
||||||
if delete_namespace:
|
if delete_namespace and namespace_present:
|
||||||
self._delete_namespace()
|
self._delete_namespace()
|
||||||
self._wait_for_namespace_gone()
|
self._wait_for_namespace_gone()
|
||||||
|
|
||||||
|
|
@ -950,9 +958,19 @@ class K8sDeployer(Deployer):
|
||||||
destroy_cluster(self.kind_cluster_name)
|
destroy_cluster(self.kind_cluster_name)
|
||||||
|
|
||||||
def _delete_labeled_resources(
|
def _delete_labeled_resources(
|
||||||
self, namespace: str, label_selector: str, delete_volumes: bool
|
self,
|
||||||
|
namespace: str,
|
||||||
|
label_selector: str,
|
||||||
|
delete_volumes: bool,
|
||||||
|
namespace_present: bool = True,
|
||||||
):
|
):
|
||||||
"""Delete all stack-labeled resources in the namespace.
|
"""Delete all stack-labeled resources.
|
||||||
|
|
||||||
|
Namespaced resources (Deployments, Services, ConfigMaps, Secrets,
|
||||||
|
PVCs, Pods, Endpoints, Ingresses, Jobs) are only touched when the
|
||||||
|
namespace still exists. Cluster-scoped PVs are always candidates
|
||||||
|
for deletion if delete_volumes is set — they can outlive a deleted
|
||||||
|
namespace (e.g. after an earlier stop --delete-namespace).
|
||||||
|
|
||||||
Keeps the namespace Active so that a subsequent up() can recreate
|
Keeps the namespace Active so that a subsequent up() can recreate
|
||||||
resources without racing against k8s namespace termination.
|
resources without racing against k8s namespace termination.
|
||||||
|
|
@ -971,6 +989,15 @@ class K8sDeployer(Deployer):
|
||||||
if e.status not in (404, 405):
|
if e.status not in (404, 405):
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
if not namespace_present:
|
||||||
|
# Jump straight to cluster-scoped cleanup.
|
||||||
|
if delete_volumes:
|
||||||
|
self._delete_labeled_pvs(label_selector)
|
||||||
|
self._wait_for_labeled_deletions(
|
||||||
|
namespace, label_selector, delete_volumes=delete_volumes
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
# Ingresses first so external traffic stops before pods disappear.
|
# Ingresses first so external traffic stops before pods disappear.
|
||||||
_swallow_404(
|
_swallow_404(
|
||||||
lambda: self.networking_api.delete_collection_namespaced_ingress(
|
lambda: self.networking_api.delete_collection_namespaced_ingress(
|
||||||
|
|
@ -1047,31 +1074,37 @@ class K8sDeployer(Deployer):
|
||||||
namespace=namespace, label_selector=label_selector
|
namespace=namespace, label_selector=label_selector
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
# Cluster-scoped PVs.
|
self._delete_labeled_pvs(label_selector)
|
||||||
try:
|
|
||||||
pvs = self.core_api.list_persistent_volume(
|
|
||||||
label_selector=label_selector
|
|
||||||
)
|
|
||||||
for pv in pvs.items:
|
|
||||||
if opts.o.debug:
|
|
||||||
print(f"Deleting PV: {pv.metadata.name}")
|
|
||||||
try:
|
|
||||||
self.core_api.delete_persistent_volume(name=pv.metadata.name)
|
|
||||||
except ApiException as e:
|
|
||||||
_check_delete_exception(e)
|
|
||||||
except ApiException as e:
|
|
||||||
if opts.o.debug:
|
|
||||||
print(f"Error listing PVs: {e}")
|
|
||||||
|
|
||||||
self._wait_for_labeled_deletions(
|
self._wait_for_labeled_deletions(
|
||||||
namespace, label_selector, delete_volumes=delete_volumes
|
namespace,
|
||||||
|
label_selector,
|
||||||
|
delete_volumes=delete_volumes,
|
||||||
|
namespace_present=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _delete_labeled_pvs(self, label_selector: str):
|
||||||
|
"""Delete cluster-scoped PVs matching the stack label."""
|
||||||
|
try:
|
||||||
|
pvs = self.core_api.list_persistent_volume(label_selector=label_selector)
|
||||||
|
except ApiException as e:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"Error listing PVs: {e}")
|
||||||
|
return
|
||||||
|
for pv in pvs.items:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"Deleting PV: {pv.metadata.name}")
|
||||||
|
try:
|
||||||
|
self.core_api.delete_persistent_volume(name=pv.metadata.name)
|
||||||
|
except ApiException as e:
|
||||||
|
_check_delete_exception(e)
|
||||||
|
|
||||||
def _wait_for_labeled_deletions(
|
def _wait_for_labeled_deletions(
|
||||||
self,
|
self,
|
||||||
namespace: str,
|
namespace: str,
|
||||||
label_selector: str,
|
label_selector: str,
|
||||||
delete_volumes: bool,
|
delete_volumes: bool,
|
||||||
|
namespace_present: bool = True,
|
||||||
timeout_seconds: int = 120,
|
timeout_seconds: int = 120,
|
||||||
):
|
):
|
||||||
"""Block until stack-labeled resources finish terminating.
|
"""Block until stack-labeled resources finish terminating.
|
||||||
|
|
@ -1084,8 +1117,10 @@ class K8sDeployer(Deployer):
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
|
|
||||||
# (kind name, lister callable) — lister returns an object with .items
|
# (kind name, lister callable) — lister returns an object with .items.
|
||||||
listers = [
|
# Namespaced kinds are skipped when the namespace is already gone
|
||||||
|
# (there's nothing to list).
|
||||||
|
listers = [] if not namespace_present else [
|
||||||
(
|
(
|
||||||
"deployment",
|
"deployment",
|
||||||
lambda: self.apps_api.list_namespaced_deployment(
|
lambda: self.apps_api.list_namespaced_deployment(
|
||||||
|
|
@ -1130,14 +1165,17 @@ class K8sDeployer(Deployer):
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
if delete_volumes:
|
if delete_volumes:
|
||||||
listers.append(
|
if namespace_present:
|
||||||
(
|
listers.append(
|
||||||
"persistentvolumeclaim",
|
(
|
||||||
lambda: self.core_api.list_namespaced_persistent_volume_claim(
|
"persistentvolumeclaim",
|
||||||
namespace=namespace, label_selector=label_selector
|
lambda: self.core_api.list_namespaced_persistent_volume_claim(
|
||||||
),
|
namespace=namespace, label_selector=label_selector
|
||||||
|
),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
# PVs are cluster-scoped — wait for them even when the namespace
|
||||||
|
# is already gone (orphaned from a prior --delete-namespace).
|
||||||
listers.append(
|
listers.append(
|
||||||
(
|
(
|
||||||
"persistentvolume",
|
"persistentvolume",
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue