From bce898ec35a5bba49977015ec3f2dc20d99834c5 Mon Sep 17 00:00:00 2001 From: "A. F. Dudley" Date: Tue, 20 Jan 2026 05:46:37 -0500 Subject: [PATCH] Implement docker-compose variable expansion for K8s - Implement _expand_shell_vars to handle ${VAR}, ${VAR:-default}, ${VAR-default} - Pass config file values to envs_from_compose_file for variable substitution - Variables from config file still override expanded defaults Co-Authored-By: Claude Opus 4.5 --- .DS_Store | Bin 0 -> 8196 bytes infra_pyproject.toml | 115 ++++++++++++++++++ .../cerc-optimism-contracts/old_Dockerfile | 22 ++++ stack_orchestrator/deploy/k8s/cluster_info.py | 2 +- stack_orchestrator/deploy/k8s/helpers.py | 35 ++++-- 5 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 .DS_Store create mode 100644 infra_pyproject.toml create mode 100644 stack_orchestrator/data/container-build/cerc-optimism-contracts/old_Dockerfile diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e0b29474a9139b91d808cd474a4a1b1347bf1f9f GIT binary patch literal 8196 zcmeHM&ubGw7=5GB20R!G(jIyk@G3&_E^CN*v`_@mgKZLN#HJh4M9|az2_E_fc=sYb zdk|E_qkn-%!LwJz_svYUGuf=fi&*?Vn0edp$9`{Kmz~ZQfN4+ro4`7NCWm0_0;dHg zaTygm6FeJ4!+PxRZEpAS;i!_S4pl%EPz6*0RX`Q^R}|o$E!M2r_kC1nRX`Q^FBOpY zgXIv+0u~8@bGVo zm6FE*Q*=`%Yw|*w%ZRyr|5|dXWSy|7&r0~AQkMb{+Fc7-2Hk9oJW@88L^peR?H46La^8nbc<`eDG5Etdh?JyX5o) z52?tQR|7sP;vLgJ5nrN_{q$qmx|sXrSH(=frFzCI`qSEEoi}!5o#(g`zpC^%j-v-@ zH0FHgxYRqUkC`pbNczuqOJPwp*=j9^U>30OC@%5q{x3qPsRE}^fwP|Tx}5)S)qnp# zg)39as(>o6tOBOd+3vKN?RNDARkF)ljvWp$$tyg%5>Ck<0{prBhas;mQ>7*gSa_s| PrXK>F3_7U-e^r5BPKX2Q literal 0 HcmV?d00001 diff --git a/infra_pyproject.toml b/infra_pyproject.toml new file mode 100644 index 00000000..1137eee0 --- /dev/null +++ b/infra_pyproject.toml @@ -0,0 +1,115 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "downpore-infra" +version = "0.1.0" +description = "Infrastructure execution engine for the downpore distributed download system" +readme = "README.md" +license = {file = "LICENSE"} +requires-python = ">=3.8" +authors = [ + {name = "Downpore Project"}, +] +keywords = ["infrastructure", "ansible", "digitalocean", "vm-management"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: System :: Systems Administration", + "Topic :: Internet :: WWW/HTTP", +] + +dependencies = [ + "ansible>=6.0.0", + "python-digitalocean>=1.17.0", + "paramiko>=2.11.0", + "pyyaml>=6.0", + "jinja2>=3.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=22.0.0", + "flake8>=5.0.0", + "pyright>=1.1.0", + "ansible-lint>=6.0.0", + "yamllint>=1.28.0", + "pre-commit>=3.0.0", +] + +[project.urls] +Homepage = "https://github.com/downpore/downpore-infra" +Repository = "https://github.com/downpore/downpore-infra" +Issues = "https://github.com/downpore/downpore-infra/issues" + +[project.scripts] +cleanup-vms = "downpore_infra.scripts.cleanup_vms:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/downpore_infra"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.black] +line-length = 88 +target-version = ['py38'] + +[tool.flake8] +max-line-length = 88 +extend-ignore = ["E203", "W503"] + +[tool.pyright] +pythonVersion = "3.9" +typeCheckingMode = "basic" +reportMissingImports = "none" +reportMissingModuleSource = "none" +reportUnusedImport = "error" +include = ["src/**/*.py", "tests/**/*.py"] +exclude = ["**/build/**", "**/__pycache__/**"] +pythonPath = "src" + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "e2e: marks tests as end-to-end (requires real infrastructure)", +] +addopts = [ + "--cov", + "--cov-report=term-missing", + "--cov-report=html", + "--strict-markers", +] +asyncio_default_fixture_loop_scope = "function" + +[tool.coverage.run] +source = ["src/downpore_infra"] +disable_warnings = ["couldnt-parse"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", +] diff --git a/stack_orchestrator/data/container-build/cerc-optimism-contracts/old_Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-contracts/old_Dockerfile new file mode 100644 index 00000000..2499df0a --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-optimism-contracts/old_Dockerfile @@ -0,0 +1,22 @@ +FROM cerc/foundry:local + +# Install node (local foundry is a debian based image) +RUN apt-get update \ + && apt-get install -y curl wget \ + && curl --silent --location https://deb.nodesource.com/setup_16.x | bash - \ + && apt-get update \ + && apt-get install -y nodejs git busybox jq \ + && node -v + +RUN corepack enable \ + && yarn --version + +WORKDIR /app + +# Copy optimism repo contents +COPY . . + +RUN echo "Building optimism" && \ + pnpm install && pnpm build + +WORKDIR /app/packages/contracts-bedrock diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index d0b71300..088ac748 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -354,7 +354,7 @@ class ClusterInfo: print(f"service ports: {container_ports}") merged_envs = merge_envs( envs_from_compose_file( - service_info["environment"]), self.environment_variables.map + service_info["environment"], self.environment_variables.map), self.environment_variables.map ) if "environment" in service_info else self.environment_variables.map envs = envs_from_environment_variables_map(merged_envs) if opts.o.debug: diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 2fa5b048..f07eeacc 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -334,23 +334,34 @@ def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]: return result -def _expand_shell_vars(raw_val: str) -> str: - # could be: or ${} or ${:-} - # TODO: implement support for variable substitution and default values - # if raw_val is like ${} print a warning and substitute an empty string - # otherwise return raw_val - match = re.search(r"^\$\{(.*)\}$", raw_val) +def _expand_shell_vars(raw_val: str, env_map: Mapping[str, str] = None) -> str: + # Expand docker-compose style variable substitution: + # ${VAR} - use VAR value or empty string + # ${VAR:-default} - use VAR value or default if unset/empty + # ${VAR-default} - use VAR value or default if unset + if env_map is None: + env_map = {} + if raw_val is None: + return "" + match = re.search(r"^\$\{([^}]+)\}$", raw_val) if match: - print(f"WARNING: found unimplemented environment variable substitution: {raw_val}") - else: - return raw_val + inner = match.group(1) + # Check for default value syntax + if ":-" in inner: + var_name, default_val = inner.split(":-", 1) + return env_map.get(var_name, "") or default_val + elif "-" in inner: + var_name, default_val = inner.split("-", 1) + return env_map.get(var_name, default_val) + else: + return env_map.get(inner, "") + return raw_val -# TODO: handle the case where the same env var is defined in multiple places -def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]: +def envs_from_compose_file(compose_file_envs: Mapping[str, str], env_map: Mapping[str, str] = None) -> Mapping[str, str]: result = {} for env_var, env_val in compose_file_envs.items(): - expanded_env_val = _expand_shell_vars(env_val) + expanded_env_val = _expand_shell_vars(env_val, env_map) result.update({env_var: expanded_env_val}) return result