From 6d620ba9c2969d363acfd197b8e8ccfe23a9ea52 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 May 2023 09:49:26 +0800 Subject: [PATCH 1/3] git branch in stack and on command line (#409) * Support @branch notation in stack.yml * Refactor and support branches directive --- app/data/stacks/test/stack.yml | 2 +- app/setup_repositories.py | 190 +++++++++++++++++++++------------ 2 files changed, 121 insertions(+), 71 deletions(-) diff --git a/app/data/stacks/test/stack.yml b/app/data/stacks/test/stack.yml index b564cca..a62da19 100644 --- a/app/data/stacks/test/stack.yml +++ b/app/data/stacks/test/stack.yml @@ -3,7 +3,7 @@ name: test description: "A test stack" repos: - github.com/cerc-io/laconicd - - git.vdb.to/cerc-io/test-project + - git.vdb.to/cerc-io/test-project@test-branch containers: - cerc/test-container pods: diff --git a/app/setup_repositories.py b/app/setup_repositories.py index 770af6d..f6ee1e0 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -52,15 +52,112 @@ def is_git_repo(path): # ) +def branch_strip(s): + return s.split('@')[0] + + +def host_and_path_for_repo(fully_qualified_repo): + repo_branch_split = fully_qualified_repo.split("@") + repo_branch = repo_branch_split[-1] if len(repo_branch_split) > 1 else None + repo_host_split = repo_branch_split[0].split("/") + # Legacy unqualified repo means github + if len(repo_host_split) == 2: + return "github.com", "/".join(repo_host_split), repo_branch + else: + if len(repo_host_split) == 3: + # First part is the host + return repo_host_split[0], "/".join(repo_host_split[1:]), repo_branch + + +# TODO: fix the messy arg list here +def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo): + repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo) + git_ssh_prefix = f"git@{repo_host}:" + git_http_prefix = f"https://{repo_host}/" + full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" + repoName = repo_path.split("/")[-1] + full_filesystem_repo_path = os.path.join(dev_root_path, repoName) + is_present = os.path.isdir(full_filesystem_repo_path) + current_repo_branch = git.Repo(full_filesystem_repo_path).active_branch.name if is_present else None + if not quiet: + present_text = f"already exists active branch: {current_repo_branch}" if is_present \ + else 'Needs to be fetched' + print(f"Checking: {full_filesystem_repo_path}: {present_text}") + # Quick check that it's actually a repo + if is_present: + if not is_git_repo(full_filesystem_repo_path): + print(f"Error: {full_filesystem_repo_path} does not contain a valid git repository") + sys.exit(1) + else: + if pull: + if verbose: + print(f"Running git pull for {full_filesystem_repo_path}") + if not check_only: + git_repo = git.Repo(full_filesystem_repo_path) + origin = git_repo.remotes.origin + origin.pull(progress=None if quiet else GitProgress()) + else: + print("(git pull skipped)") + if not is_present: + # Clone + if verbose: + print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') + if not dry_run: + git.Repo.clone_from(full_github_repo_path, + full_filesystem_repo_path, + progress=None if quiet else GitProgress()) + else: + print("(git clone skipped)") + # Checkout the requested branch, if one was specified + branch_to_checkout = None + if branches_array: + # Find the current repo in the branches list + print("Checking") + for repo_branch in branches_array: + repo_branch_tuple = repo_branch.split(" ") + if repo_branch_tuple[0] == branch_strip(fully_qualified_repo): + # checkout specified branch + branch_to_checkout = repo_branch_tuple[1] + else: + branch_to_checkout = repo_branch + + print(f"branch_to_checkout: {branch_to_checkout}") + if branch_to_checkout: + if current_repo_branch and (current_repo_branch != branch_to_checkout): + if not quiet: + print(f"switching to branch {branch_to_checkout} in repo {repo_path}") + git_repo = git.Repo(full_filesystem_repo_path) + git_repo.git.checkout(branch_to_checkout) + else: + if verbose: + print(f"repo {repo_path} is already switched to branch {branch_to_checkout}") + + +def parse_branches(branches_string): + if branches_string: + result_array = [] + branches_directives = branches_string.split(",") + for branch_directive in branches_directives: + split_directive = branch_directive.split("@") + if len(split_directive) != 2: + print(f"Error: branch specified is not valid: {branch_directive}") + sys.exit(1) + result_array.append(f"{split_directive[0]} {split_directive[1]}") + return result_array + else: + return None + + @click.command() @click.option("--include", help="only clone these repositories") @click.option("--exclude", help="don\'t clone these repositories") @click.option('--git-ssh', is_flag=True, default=False) @click.option('--check-only', is_flag=True, default=False) @click.option('--pull', is_flag=True, default=False) +@click.option("--branches", help="override branches for repositories") @click.option('--branches-file', help="checkout branches specified in this file") @click.pass_context -def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): +def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches_file): '''git clone the set of repositories required to build the complete system from source''' quiet = ctx.obj.quiet @@ -68,16 +165,29 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): dry_run = ctx.obj.dry_run stack = ctx.obj.stack - branches = [] + branches_array = [] # TODO: branches file needs to be re-worked in the context of stacks if branches_file: - if verbose: - print(f"loading branches from: {branches_file}") - with open(branches_file) as branches_file_open: - branches = branches_file_open.read().splitlines() - if verbose: - print(f"Branches are: {branches}") + if branches: + print("Error: can't specify both --branches and --branches-file") + sys.exit(1) + else: + if verbose: + print(f"loading branches from: {branches_file}") + with open(branches_file) as branches_file_open: + branches_array = branches_file_open.read().splitlines() + + print(f"branches: {branches}") + if branches: + if branches_file: + print("Error: can't specify both --branches and --branches-file") + sys.exit(1) + else: + branches_array = parse_branches(branches) + + if branches_array and verbose: + print(f"Branches are: {branches_array}") local_stack = ctx.obj.local_stack @@ -119,75 +229,15 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file): repos = [] for repo in repos_in_scope: - if include_exclude_check(repo, include, exclude): + if include_exclude_check(branch_strip(repo), include, exclude): repos.append(repo) else: if verbose: print(f"Excluding: {repo}") - def host_and_path_for_repo(fully_qualified_repo): - repo_split = fully_qualified_repo.split("/") - # Legacy unqualified repo means github - if len(repo_split) == 2: - return "github.com", "/".join(repo_split) - else: - if len(repo_split) == 3: - # First part is the host - return repo_split[0], "/".join(repo_split[1:]) - - def process_repo(fully_qualified_repo): - repo_host, repo_path = host_and_path_for_repo(fully_qualified_repo) - git_ssh_prefix = f"git@{repo_host}:" - git_http_prefix = f"https://{repo_host}/" - full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" - repoName = repo_path.split("/")[-1] - full_filesystem_repo_path = os.path.join(dev_root_path, repoName) - is_present = os.path.isdir(full_filesystem_repo_path) - if not quiet: - present_text = f"already exists active branch: {git.Repo(full_filesystem_repo_path).active_branch}" if is_present \ - else 'Needs to be fetched' - print(f"Checking: {full_filesystem_repo_path}: {present_text}") - # Quick check that it's actually a repo - if is_present: - if not is_git_repo(full_filesystem_repo_path): - print(f"Error: {full_filesystem_repo_path} does not contain a valid git repository") - sys.exit(1) - else: - if pull: - if verbose: - print(f"Running git pull for {full_filesystem_repo_path}") - if not check_only: - git_repo = git.Repo(full_filesystem_repo_path) - origin = git_repo.remotes.origin - origin.pull(progress=None if quiet else GitProgress()) - else: - print("(git pull skipped)") - if not is_present: - # Clone - if verbose: - print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') - if not dry_run: - git.Repo.clone_from(full_github_repo_path, - full_filesystem_repo_path, - progress=None if quiet else GitProgress()) - else: - print("(git clone skipped)") - # Checkout the requested branch, if one was specified - if branches: - # Find the current repo in the branches list - for repo_branch in branches: - repo_branch_tuple = repo_branch.split(" ") - if repo_branch_tuple[0] == repo: - # checkout specified branch - branch_to_checkout = repo_branch_tuple[1] - if verbose: - print(f"checking out branch {branch_to_checkout} in repo {repo}") - git_repo = git.Repo(full_filesystem_repo_path) - git_repo.git.checkout(branch_to_checkout) - for repo in repos: try: - process_repo(repo) + process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, repo) except git.exc.GitCommandError as error: print(f"\n******* git command returned error exit status:\n{error}") sys.exit(1) From 938f51ef8cf67873d55ba4ac20f2d137a281a33e Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 May 2023 10:00:42 +0800 Subject: [PATCH 2/3] Specify chunker stack branches (#410) * Specify v5 branches * Fix logic for branch switch --- app/data/stacks/chain-chunker/stack.yml | 8 ++++---- app/setup_repositories.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/data/stacks/chain-chunker/stack.yml b/app/data/stacks/chain-chunker/stack.yml index 671859c..2d36330 100644 --- a/app/data/stacks/chain-chunker/stack.yml +++ b/app/data/stacks/chain-chunker/stack.yml @@ -2,10 +2,10 @@ version: "1.0" name: chain-chunker decription: "Stack to build containers for chain-chunker" repos: - - github.com/cerc-io/ipld-eth-state-snapshot - - github.com/cerc-io/eth-statediff-service - - github.com/cerc-io/ipld-eth-db - - github.com/cerc-io/ipld-eth-server + - github.com/cerc-io/ipld-eth-state-snapshot@v5 + - github.com/cerc-io/eth-statediff-service@v5 + - github.com/cerc-io/ipld-eth-db@v5 + - github.com/cerc-io/ipld-eth-server@v5 containers: - cerc/ipld-eth-state-snapshot - cerc/eth-statediff-service diff --git a/app/setup_repositories.py b/app/setup_repositories.py index f6ee1e0..bfcbe0e 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -121,9 +121,8 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa else: branch_to_checkout = repo_branch - print(f"branch_to_checkout: {branch_to_checkout}") if branch_to_checkout: - if current_repo_branch and (current_repo_branch != branch_to_checkout): + if current_repo_branch is None or (current_repo_branch and (current_repo_branch != branch_to_checkout)): if not quiet: print(f"switching to branch {branch_to_checkout} in repo {repo_path}") git_repo = git.Repo(full_filesystem_repo_path) From 6ec55ba46018d0f424bf70a417501e728dc6c642 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 25 May 2023 11:21:08 -0500 Subject: [PATCH 3/3] Add a plugeth-based version of the fixturenet (#411) * plugeth version of the fixturenet * Use pre-built plugeth. --- .../docker-compose-fixturenet-plugeth.yml | 129 ++++++++++++++++++ .../fixturenet-plugeth/plugins/README.md | 1 + .../cerc-fixturenet-eth-plugeth/Dockerfile | 27 ++++ .../cerc-fixturenet-eth-plugeth/build.sh | 17 +++ .../stacks/fixturenet-plugeth-tx/README.md | 19 +++ .../stacks/fixturenet-plugeth-tx/stack.yml | 13 ++ 6 files changed, 206 insertions(+) create mode 100644 app/data/compose/docker-compose-fixturenet-plugeth.yml create mode 100644 app/data/config/fixturenet-plugeth/plugins/README.md create mode 100644 app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile create mode 100755 app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh create mode 100644 app/data/stacks/fixturenet-plugeth-tx/README.md create mode 100644 app/data/stacks/fixturenet-plugeth-tx/stack.yml diff --git a/app/data/compose/docker-compose-fixturenet-plugeth.yml b/app/data/compose/docker-compose-fixturenet-plugeth.yml new file mode 100644 index 0000000..9011c8e --- /dev/null +++ b/app/data/compose/docker-compose-fixturenet-plugeth.yml @@ -0,0 +1,129 @@ +services: + fixturenet-eth-bootnode-geth: + restart: always + hostname: fixturenet-eth-bootnode-geth + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + environment: + RUN_BOOTNODE: "true" + image: cerc/fixturenet-eth-plugeth:local + volumes: + - fixturenet_plugeth_bootnode_geth_data:/root/ethdata + - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins + ports: + - "9898" + - "30303" + + fixturenet-eth-geth-1: + restart: always + hostname: fixturenet-eth-geth-1 + cap_add: + - SYS_PTRACE + environment: + CERC_REMOTE_DEBUG: "true" + CERC_RUN_STATEDIFF: "detect" + CERC_STATEDIFF_DB_NODE_ID: 1 + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + image: cerc/fixturenet-eth-plugeth:local + volumes: + - fixturenet_plugeth_geth_1_data:/root/ethdata + - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8545/"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + depends_on: + - fixturenet-eth-bootnode-geth + ports: + - "8545" + - "40000" + - "6060" + + fixturenet-eth-geth-2: + restart: always + hostname: fixturenet-eth-geth-2 + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8545/"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + environment: + CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true" + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + image: cerc/fixturenet-eth-plugeth:local + depends_on: + - fixturenet-eth-bootnode-geth + volumes: + - fixturenet_plugeth_geth_2_data:/root/ethdata + - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins + + fixturenet-eth-bootnode-lighthouse: + restart: always + hostname: fixturenet-eth-bootnode-lighthouse + environment: + RUN_BOOTNODE: "true" + image: cerc/fixturenet-eth-lighthouse:local + + fixturenet-eth-lighthouse-1: + restart: always + hostname: fixturenet-eth-lighthouse-1 + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8001/eth/v2/beacon/blocks/head"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 30s + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + environment: + NODE_NUMBER: "1" + ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545" + EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-1:8551" + image: cerc/fixturenet-eth-lighthouse:local + volumes: + - fixturenet_plugeth_lighthouse_1_data:/opt/testnet/build/cl + depends_on: + fixturenet-eth-bootnode-lighthouse: + condition: service_started + fixturenet-eth-geth-1: + condition: service_healthy + ports: + - "8001" + + fixturenet-eth-lighthouse-2: + restart: always + hostname: fixturenet-eth-lighthouse-2 + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8001/eth/v2/beacon/blocks/head"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 30s + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + environment: + NODE_NUMBER: "2" + ETH1_ENDPOINT: "http://fixturenet-eth-geth-2:8545" + EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-2:8551" + LIGHTHOUSE_GENESIS_STATE_URL: "http://fixturenet-eth-lighthouse-1:8001/eth/v2/debug/beacon/states/0" + image: cerc/fixturenet-eth-lighthouse:local + volumes: + - fixturenet_plugeth_lighthouse_2_data:/opt/testnet/build/cl + depends_on: + fixturenet-eth-bootnode-lighthouse: + condition: service_started + fixturenet-eth-geth-2: + condition: service_healthy + +volumes: + fixturenet_plugeth_bootnode_geth_data: + fixturenet_plugeth_geth_1_data: + fixturenet_plugeth_geth_2_data: + fixturenet_plugeth_lighthouse_1_data: + fixturenet_plugeth_lighthouse_2_data: diff --git a/app/data/config/fixturenet-plugeth/plugins/README.md b/app/data/config/fixturenet-plugeth/plugins/README.md new file mode 100644 index 0000000..aa5aac1 --- /dev/null +++ b/app/data/config/fixturenet-plugeth/plugins/README.md @@ -0,0 +1 @@ +See: https://docs.plugeth.org/ diff --git a/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile b/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile new file mode 100644 index 0000000..90cd38d --- /dev/null +++ b/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile @@ -0,0 +1,27 @@ +FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen + +FROM golang:1.19.4-bullseye AS delve +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM ubuntu:22.04 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=delve /go/bin/dlv /usr/local/bin/ +COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/ +COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/ +COPY --from=ethgen /apps /apps + +RUN wget -O /usr/local/bin/geth https://github.com/openrelayxyz/plugeth/releases/download/v1.11.6.1.0/geth-linux-amd64-v1.1.0-v1.11.6.1.0 && chmod a+x /usr/local/bin/geth +RUN cd /apps/el-gen && pip3 install -r requirements.txt + +COPY genesis /opt/testnet +COPY run-el.sh /opt/testnet/run.sh + +RUN cd /opt/testnet && make genesis-el + +RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey + +ENTRYPOINT ["/opt/testnet/run.sh"] diff --git a/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh b/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh new file mode 100755 index 0000000..e81bc93 --- /dev/null +++ b/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Build cerc/fixturenet-eth-plugeth +set -x + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if [ ! -d "${SCRIPT_DIR}/genesis" ]; then + cp -frp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/genesis ${SCRIPT_DIR}/genesis +fi + +if [ ! -d "${SCRIPT_DIR}/run-el.sh" ]; then + cp -fp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/run-el.sh ${SCRIPT_DIR}/ +fi + +docker build -t cerc/fixturenet-eth-plugeth:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR diff --git a/app/data/stacks/fixturenet-plugeth-tx/README.md b/app/data/stacks/fixturenet-plugeth-tx/README.md new file mode 100644 index 0000000..832ed74 --- /dev/null +++ b/app/data/stacks/fixturenet-plugeth-tx/README.md @@ -0,0 +1,19 @@ +# fixturenet-plugeth-tx + +A variation of `fixturenet-eth` that uses `plugeth` instead of `go-ethereum`. + +See `stacks/fixturenet-eth/README.md` for more information. + +## Containers + +* cerc/lighthouse +* cerc/fixturenet-eth-plugeth +* cerc/fixturenet-eth-lighthouse +* cerc/tx-spammer + +## Deploy the stack +``` +$ laconic-so --stack fixturenet-plugeth-tx setup-repositories +$ laconic-so --stack fixturenet-plugeth-tx build-containers +$ laconic-so --stack fixturenet-plugeth-tx deploy up +``` diff --git a/app/data/stacks/fixturenet-plugeth-tx/stack.yml b/app/data/stacks/fixturenet-plugeth-tx/stack.yml new file mode 100644 index 0000000..14637da --- /dev/null +++ b/app/data/stacks/fixturenet-plugeth-tx/stack.yml @@ -0,0 +1,13 @@ +version: "1.2" +name: fixturenet-plugeth-tx +decription: "plugeth Ethereum Fixturenet w/ tx-spammer" +repos: + - github.com/cerc-io/tx-spammer +containers: + - cerc/lighthouse + - cerc/fixturenet-eth-plugeth + - cerc/fixturenet-eth-lighthouse + - cerc/tx-spammer +pods: + - fixturenet-plugeth + - tx-spammer