Implement docker-compose variable expansion for K8s

- Implement _expand_shell_vars to handle ${VAR}, ${VAR:-default}, ${VAR-default}
- Pass config file values to envs_from_compose_file for variable substitution
- Variables from config file still override expanded defaults

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

infra_pyproject.toml

@@ -0,0 +1,115 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "downpore-infra"
+version = "0.1.0"
+description = "Infrastructure execution engine for the downpore distributed download system"
+readme = "README.md"
+license = {file = "LICENSE"}
+requires-python = ">=3.8"
+authors = [
+    {name = "Downpore Project"},
+]
+keywords = ["infrastructure", "ansible", "digitalocean", "vm-management"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: System :: Systems Administration",
+    "Topic :: Internet :: WWW/HTTP",
+]
+
+dependencies = [
+    "ansible>=6.0.0",
+    "python-digitalocean>=1.17.0",
+    "paramiko>=2.11.0",
+    "pyyaml>=6.0",
+    "jinja2>=3.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "black>=22.0.0",
+    "flake8>=5.0.0",
+    "pyright>=1.1.0",
+    "ansible-lint>=6.0.0",
+    "yamllint>=1.28.0",
+    "pre-commit>=3.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/downpore/downpore-infra"
+Repository = "https://github.com/downpore/downpore-infra"
+Issues = "https://github.com/downpore/downpore-infra/issues"
+
+[project.scripts]
+cleanup-vms = "downpore_infra.scripts.cleanup_vms:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/downpore_infra"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.black]
+line-length = 88
+target-version = ['py38']
+
+[tool.flake8]
+max-line-length = 88
+extend-ignore = ["E203", "W503"]
+
+[tool.pyright]
+pythonVersion = "3.9"
+typeCheckingMode = "basic"
+reportMissingImports = "none"
+reportMissingModuleSource = "none"
+reportUnusedImport = "error"
+include = ["src/**/*.py", "tests/**/*.py"]
+exclude = ["**/build/**", "**/__pycache__/**"]
+pythonPath = "src"
+
+[tool.mypy]
+python_version = "3.8"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks tests as end-to-end (requires real infrastructure)",
+]
+addopts = [
+    "--cov",
+    "--cov-report=term-missing",
+    "--cov-report=html",
+    "--strict-markers",
+]
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.coverage.run]
+source = ["src/downpore_infra"]
+disable_warnings = ["couldnt-parse"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+]

stack_orchestrator/data/container-build/cerc-optimism-contracts/old_Dockerfile

@@ -0,0 +1,22 @@
+FROM cerc/foundry:local
+
+# Install node (local foundry is a debian based image)
+RUN apt-get update \
+  && apt-get install -y curl wget \
+  && curl --silent --location https://deb.nodesource.com/setup_16.x | bash - \
+  && apt-get update \
+  && apt-get install -y nodejs git busybox jq \
+  && node -v
+
+RUN corepack enable \
+  && yarn --version
+
+WORKDIR /app
+
+# Copy optimism repo contents
+COPY . .
+
+RUN echo "Building optimism" && \
+  pnpm install && pnpm build
+
+WORKDIR /app/packages/contracts-bedrock

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -354,7 +354,7 @@ class ClusterInfo:
                         print(f"service ports: {container_ports}")
                 merged_envs = merge_envs(
                     envs_from_compose_file(
-                        service_info["environment"]), self.environment_variables.map
+                        service_info["environment"], self.environment_variables.map), self.environment_variables.map
                 ) if "environment" in service_info else self.environment_variables.map
                 envs = envs_from_environment_variables_map(merged_envs)
                 if opts.o.debug:

stack_orchestrator/deploy/k8s/helpers.py

@@ -334,23 +334,34 @@ def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]:
     return result
 
 
-def _expand_shell_vars(raw_val: str) -> str:
+def _expand_shell_vars(raw_val: str, env_map: Mapping[str, str] = None) -> str:
-    # could be: <string> or ${<env-var-name>} or ${<env-var-name>:-<default-value>}
+    # Expand docker-compose style variable substitution:
-    # TODO: implement support for variable substitution and default values
+    # ${VAR} - use VAR value or empty string
-    # if raw_val is like ${<something>} print a warning and substitute an empty string
+    # ${VAR:-default} - use VAR value or default if unset/empty
-    # otherwise return raw_val
+    # ${VAR-default} - use VAR value or default if unset
-    match = re.search(r"^\$\{(.*)\}$", raw_val)
+    if env_map is None:
+        env_map = {}
+    if raw_val is None:
+        return ""
+    match = re.search(r"^\$\{([^}]+)\}$", raw_val)
     if match:
-        print(f"WARNING: found unimplemented environment variable substitution: {raw_val}")
+        inner = match.group(1)
-    else:
+        # Check for default value syntax
-        return raw_val
+        if ":-" in inner:
+            var_name, default_val = inner.split(":-", 1)
+            return env_map.get(var_name, "") or default_val
+        elif "-" in inner:
+            var_name, default_val = inner.split("-", 1)
+            return env_map.get(var_name, default_val)
+        else:
+            return env_map.get(inner, "")
+    return raw_val
 
 
-# TODO: handle the case where the same env var is defined in multiple places
+def envs_from_compose_file(compose_file_envs: Mapping[str, str], env_map: Mapping[str, str] = None) -> Mapping[str, str]:
-def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]:
     result = {}
     for env_var, env_val in compose_file_envs.items():
-        expanded_env_val = _expand_shell_vars(env_val)
+        expanded_env_val = _expand_shell_vars(env_val, env_map)
         result.update({env_var: expanded_env_val})
     return result