Fix failing k8s and external-stack CI test scripts (#739)

- Add `--perform-cluster-management` to container-registry, k8s-deployment-control, and database test scripts (`--skip-cluster-management` is now the default)
- Fix `wait_for_log_output()` in all k8s tests - "No logs available" is non-empty, so the check was passing prematurely
- Use HTTPS for container-registry catalog check (Caddy redirects HTTP->HTTPS)
- Fix external-stack sync test: sed pattern used `=` but spec is YAML (`: `), so the substitution never matched
- Workaround hyphenated env var name (`test-variable-1`) from upstream test-external-stack repo - docker compose v2 rejects hyphens
- Quote `echo $log_output` vars to prevent glob expansion in error output
- Use stack name (instead of cluster-id) derived namespace in k8s-deployment-control test

tests/container-registry/run-test.sh

@@ -34,7 +34,7 @@ wait_for_log_output () {
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 
-        if [[ ! -z "$log_output" ]]; then
+        if [[ ! -z "$log_output" ]] && [[ "$log_output" != *"No logs available"* ]] && [[ "$log_output" != *"Pods not running"* ]]; then
             # if ready, return
             return
         else
@@ -49,7 +49,7 @@ wait_for_log_output () {
 
 
 delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --perform-cluster-management
     exit 1
 }
 
@@ -111,7 +111,7 @@ echo "deploy create test: passed"
 docker pull registry:2.8
 
 # Try to start the deployment
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start --perform-cluster-management
 wait_for_pods_started
 # Check logs command works
 wait_for_log_output
@@ -121,7 +121,7 @@ if [[ "$log_output_3" == *"listening on"* ]]; then
     echo "deployment logs test: passed"
 else
     echo "deployment logs test: FAILED"
-    echo $log_output_3
+    echo "$log_output_3"
     delete_cluster_exit
 fi
 
@@ -132,15 +132,25 @@ docker tag hello-world localhost:80/hello-world
 docker push localhost:80/hello-world
 # Then do a quick check that we actually pushed something there
 # See: https://stackoverflow.com/questions/31251356/how-to-get-a-list-of-images-on-docker-registry-v2
-registry_response=$(curl -s -X GET http://localhost:80/v2/_catalog)
+# Wait for the catalog to reflect the pushed image
-if [[ "$registry_response" == *"{\"repositories\":[\"hello-world\"]}"* ]]; then
+registry_ok=false
+for i in {1..10}; do
+    registry_response=$(curl -s -k -X GET https://localhost/v2/_catalog)
+    if [[ "$registry_response" == *"hello-world"* ]]; then
+        registry_ok=true
+        break
+    fi
+    echo "Waiting for registry catalog (attempt $i)..."
+    sleep 3
+done
+if $registry_ok; then
     echo "registry content test: passed"
 else
     echo "registry content test: FAILED"
-    echo $registry_response
+    echo "Response: $registry_response"
     delete_cluster_exit
 fi
 
 # Stop and clean up
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --perform-cluster-management
 echo "Test passed"

tests/database/run-test.sh

@@ -57,7 +57,7 @@ wait_for_test_complete () {
 
 
 delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --perform-cluster-management
     exit 1
 }
 
@@ -98,7 +98,7 @@ fi
 echo "deploy create test: passed"
 
 # Try to start the deployment
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start --perform-cluster-management
 wait_for_pods_started
 # Check logs command works
 wait_for_test_complete
@@ -111,10 +111,10 @@ else
 fi
 
 # Stop then start again and check the volume was preserved
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --skip-cluster-management
 # Sleep a bit just in case
 sleep 20
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start --skip-cluster-management
 wait_for_pods_started
 wait_for_test_complete
 
@@ -127,5 +127,5 @@ else
 fi
 
 # Stop and clean up
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --perform-cluster-management
 echo "Test passed"

tests/external-stack/run-test.sh

@@ -31,6 +31,11 @@ rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
 # Clone the external test stack
 $TEST_TARGET_SO fetch-stack git.vdb.to/cerc-io/test-external-stack
+# Workaround: fix hyphenated variable name in external stack's init() defaults
+# (docker compose v2 rejects hyphens in env var names)
+# TODO: remove once upstream test-external-stack repo is fixed
+stack_commands="$CERC_REPO_BASE_DIR/test-external-stack/stack-orchestrator/stacks/test-external-stack/deploy/commands.py"
+sed -i 's/test-variable-1/test_variable_1/g' "$stack_commands"
 stack_name="$CERC_REPO_BASE_DIR/test-external-stack/stack-orchestrator/stacks/test-external-stack"
 TEST_TARGET_SO_STACK="$TEST_TARGET_SO --stack ${stack_name}"
 # Test bringing the test container up and down
@@ -135,7 +140,7 @@ original_marker_content=$(<$test_data_marker)
 # Verify deployment file exists and preserve its cluster ID
 original_cluster_id=$(grep "cluster-id:" "$test_deployment_dir/deployment.yml" 2>/dev/null || echo "")
 # Modify spec file to simulate an update
-sed -i.bak 's/CERC_TEST_PARAM_1=PASSED/CERC_TEST_PARAM_1=UPDATED/' $test_deployment_spec
+sed -i.bak 's/CERC_TEST_PARAM_1: PASSED/CERC_TEST_PARAM_1: UPDATED/' $test_deployment_spec
 # Run sync to update deployment files without destroying data
 $TEST_TARGET_SO_STACK deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir --update
 # Verify the spec file was updated in deployment dir
@@ -179,7 +184,7 @@ else
     exit 1
 fi
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
-if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
+if [[ "$log_output_3" == *"Test-param-1: UPDATED"* ]]; then
     echo "deployment config test: passed"
 else
     echo "deployment config test: FAILED"

tests/k8s-deploy/run-deploy-test.sh

@@ -32,7 +32,7 @@ wait_for_log_output () {
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 
-        if [[ ! -z "$log_output" ]]; then
+        if [[ ! -z "$log_output" ]] && [[ "$log_output" != *"No logs available"* ]] && [[ "$log_output" != *"Pods not running"* ]]; then
             # if ready, return
             return
         else
@@ -129,7 +129,7 @@ if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
     echo "deployment logs test: passed"
 else
     echo "deployment logs test: FAILED"
-    echo $log_output_3
+    echo "$log_output_3"
     delete_cluster_exit
 fi
 
@@ -164,7 +164,7 @@ if [[ "$log_output_5" == *"/data: MOUNTED"* ]]; then
     echo "deployment bind volumes test: passed"
 else
     echo "deployment bind volumes test: FAILED"
-    echo $log_output_5
+    echo "$log_output_5"
     delete_cluster_exit
 fi
 
@@ -174,7 +174,7 @@ if [[ "$log_output_6" == *"/data2: MOUNTED"* ]]; then
     echo "deployment provisioner volumes test: passed"
 else
     echo "deployment provisioner volumes test: FAILED"
-    echo $log_output_6
+    echo "$log_output_6"
     delete_cluster_exit
 fi
 

tests/k8s-deployment-control/run-test.sh

@@ -38,7 +38,7 @@ wait_for_log_output () {
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 
-        if [[ ! -z "$log_output" ]]; then
+        if [[ ! -z "$log_output" ]] && [[ "$log_output" != *"No logs available"* ]] && [[ "$log_output" != *"Pods not running"* ]]; then
             # if ready, return
             return
         else
@@ -52,7 +52,7 @@ wait_for_log_output () {
 }
 
 delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --perform-cluster-management
     exit 1
 }
 
@@ -189,7 +189,7 @@ EOF
 deployment_id=$(cat ${test_deployment_dir}/deployment.yml | cut -d ' ' -f 2)
 
 # Try to start the deployment
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start --perform-cluster-management
 wait_for_pods_started
 # Check logs command works
 wait_for_log_output
@@ -199,14 +199,15 @@ if [[ "$log_output_1" == *"filesystem is fresh"* ]]; then
     echo "deployment of pod test: passed"
 else
     echo "deployment pod test: FAILED"
-    echo $log_output_1
+    echo "$log_output_1"
     delete_cluster_exit
 fi
 
 # The deployment's pod should be scheduled onto node: worker3
 # Check that's what happened
 # Get get the node onto which the stack pod has been deployed
-deployment_node=$(kubectl get pods -n laconic-${deployment_id} -l app=${deployment_id} -o=jsonpath='{.items..spec.nodeName}')
+# Namespace is now derived from stack name, not cluster-id
+deployment_node=$(kubectl get pods -n laconic-test -l app=${deployment_id} -o=jsonpath='{.items..spec.nodeName}')
 expected_node=${deployment_id}-worker3
 echo "Stack pod deployed to node: ${deployment_node}"
 if [[ ${deployment_node} == ${expected_node} ]]; then
@@ -218,5 +219,5 @@ else
 fi
 
 # Stop and clean up
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --perform-cluster-management
 echo "Test passed"