#!/usr/bin/env bash set -e if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x echo "Environment variables:" env fi # Helper functions: TODO move into a separate file (mirrors run-deploy-test.sh:10). wait_for_pods_started () { local dir=$1 for i in {1..50} do local ps_output=$( $TEST_TARGET_SO deployment --dir $dir ps ) if [[ "$ps_output" == *"Running containers:"* ]]; then return else sleep 5 fi done echo "waiting for pods to start: FAILED" cleanup_and_exit } # Multi-pod stacks aren't visible to 'deployment ps' (deploy_k8s.py:1366 # filters by app_name-deployment substring, which doesn't match # laconic---deployment- names). Wait via kubectl. wait_for_k8s_pods_ready () { local ns=$1 local timeout=240 local waited=0 # First wait for at least one pod to appear in the namespace. while [ $waited -lt $timeout ]; do local count=$(kubectl get pods -n "$ns" --no-headers 2>/dev/null | wc -l) if [ "$count" -gt 0 ]; then break fi sleep 2 waited=$((waited + 2)) done if ! kubectl wait --for=condition=Ready pod --all \ -n "$ns" --timeout=$((timeout - waited))s 2>&1; then echo "kubectl wait pods ready: FAILED (ns=$ns)" kubectl get pods -n "$ns" 2>&1 || true kubectl describe pods -n "$ns" 2>&1 | tail -80 || true cleanup_and_exit fi } # Best-effort full teardown so CI runners don't leak namespaces/PVs/clusters # between runs. Variables may be unset depending on which phase tripped. cleanup_and_exit () { if [ -n "$DEP1" ] && [ -d "$DEP1" ]; then $TEST_TARGET_SO deployment --dir $DEP1 \ stop --delete-volumes --delete-namespace --skip-cluster-management || true fi if [ -n "$DEP2" ] && [ -d "$DEP2" ]; then $TEST_TARGET_SO deployment --dir $DEP2 \ stop --delete-volumes --delete-namespace --perform-cluster-management || true fi exit 1 } # Make a clone usable for `git commit` without touching the runner's global config. configure_git_identity () { local repo_dir=$1 git -C $repo_dir config user.email "test@stack-orchestrator.test" git -C $repo_dir config user.name "test" } TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) echo "Testing this package: $TEST_TARGET_SO" WORK_DIR=~/stack-orchestrator-test/restart # Multi-repo pod working clones land here; resolved by get_plugin_code_paths. export CERC_REPO_BASE_DIR=$WORK_DIR/repo-base rm -rf $WORK_DIR mkdir -p $WORK_DIR $CERC_REPO_BASE_DIR # Source location of the test stacks shipped in this checkout. The test stages # them into a temp git repo so 'deployment restart' (which runs 'git pull' on # the stack source) has a real repo to pull from. DATA_DIR=stack_orchestrator/data # ============================================================================ # Phase 1 — single-repo restart cycle. Verifies that: # * deploy create copies commands.py into /hooks/ # * deployment start runs the copied start() hook # * mutating the stack-source commands.py and running 'deployment restart' # re-copies the new file into hooks/ and re-executes the new start() # ============================================================================ echo "=== Phase 1: single-repo restart cycle ===" BARE1=$WORK_DIR/stack-single.git CLONE1=$WORK_DIR/stack-single git init -b main --bare $BARE1 git clone $BARE1 $CLONE1 configure_git_identity $CLONE1 # External-stack layout: /stack-orchestrator/{stacks,compose}/... mkdir -p $CLONE1/stack-orchestrator/stacks $CLONE1/stack-orchestrator/compose cp -r $DATA_DIR/stacks/test-restart $CLONE1/stack-orchestrator/stacks/ cp $DATA_DIR/compose/docker-compose-test-restart.yml $CLONE1/stack-orchestrator/compose/ git -C $CLONE1 add . git -C $CLONE1 commit -m "test-restart v1" git -C $CLONE1 push -u origin main STACK_PATH_SINGLE=$CLONE1/stack-orchestrator/stacks/test-restart SPEC1=$WORK_DIR/spec-single.yml DEP1=$WORK_DIR/dep-single $TEST_TARGET_SO --stack $STACK_PATH_SINGLE deploy --deploy-to k8s-kind init --output $SPEC1 $TEST_TARGET_SO --stack $STACK_PATH_SINGLE deploy create --spec-file $SPEC1 --deployment-dir $DEP1 if [ ! -f "$DEP1/hooks/commands.py" ]; then echo "single-repo deploy create test: FAILED (hooks/commands.py missing)" cleanup_and_exit fi if ! grep -q '"v1"' "$DEP1/hooks/commands.py"; then echo "single-repo deploy create test: FAILED (hooks/commands.py does not contain v1 marker)" cleanup_and_exit fi echo "single-repo deploy create test: passed" $TEST_TARGET_SO deployment --dir $DEP1 start --perform-cluster-management wait_for_pods_started $DEP1 # call_stack_deploy_start runs synchronously inside the start command # (deploy_k8s.py:1026), so the marker is on disk before 'start' returns. if [ ! -f "$DEP1/marker" ]; then echo "single-repo start v1 test: FAILED (marker file missing)" cleanup_and_exit fi marker_v1=$(cat $DEP1/marker) if [ "$marker_v1" != "v1" ]; then echo "single-repo start v1 test: FAILED (got: $marker_v1)" cleanup_and_exit fi echo "single-repo start v1 test: passed" # Mutate the stack-source working tree v1 -> v2. No commit needed: 'deployment # restart' runs 'git pull' against the bare which is a no-op, and _copy_hooks # reads the working tree directly via get_plugin_code_paths. sed -i 's/"v1"/"v2"/' $STACK_PATH_SINGLE/deploy/commands.py $TEST_TARGET_SO deployment --dir $DEP1 restart --stack-path $STACK_PATH_SINGLE if ! grep -q '"v2"' "$DEP1/hooks/commands.py"; then echo "single-repo restart re-copy test: FAILED (hooks/commands.py still v1)" cleanup_and_exit fi echo "single-repo restart re-copy test: passed" marker_v2=$(cat $DEP1/marker) if [ "$marker_v2" != "v2" ]; then echo "single-repo restart re-execute test: FAILED (got: $marker_v2)" cleanup_and_exit fi echo "single-repo restart re-execute test: passed" # Stop phase 1 deployment but keep the cluster for phase 2. $TEST_TARGET_SO deployment --dir $DEP1 \ stop --delete-volumes --delete-namespace --skip-cluster-management # ============================================================================ # Phase 2 — multi-repo create + start. Verifies that a stack with N pods, each # from a separate repo, produces hooks/commands_0.py ... commands_{N-1}.py and # that call_stack_deploy_start invokes every module's start(). # ============================================================================ echo "=== Phase 2: multi-repo create + start ===" # Pod repos: stack.yml's pods[].repository = 'cerc-io/test-restart-pod-X' # resolves (via get_plugin_code_paths) to # $CERC_REPO_BASE_DIR/test-restart-pod-X//stack/... for label in a b; do POD_BARE=$WORK_DIR/pod-$label.git POD_CLONE=$CERC_REPO_BASE_DIR/test-restart-pod-$label git init -b main --bare $POD_BARE git clone $POD_BARE $POD_CLONE configure_git_identity $POD_CLONE mkdir -p $POD_CLONE/stack/deploy # For dict-form pods, get_pod_file_path resolves the compose file at # //docker-compose.yml — owned by the pod repo, not # the stack repo. get_plugin_code_paths adds the trailing 'stack/', so # commands.py lives at //stack/deploy/commands.py. cat > $POD_CLONE/docker-compose.yml < $POD_CLONE/stack/deploy/commands.py <