Update README.md

* cloud init install

* add dev mode script + description

* instructions

README.md

@@ -1,5 +1,7 @@
 # Stack Orchestrator
 
+
+
 Stack Orchestrator allows building and deployment of a Laconic Stack on a single machine with minimial prerequisites. It is a Python3 CLI tool that runs on any OS with Python3 and Docker. The following diagram summarizes the relevant repositories in the Laconic Stack - and the relationship to Stack Orchestrator.
 
 ![The Stack](/docs/images/laconic-stack.png)

app/build_containers.py

@@ -22,13 +22,11 @@
 
 import os
 import sys
+from decouple import config
 import subprocess
+import click
 import importlib.resources
 from pathlib import Path
-
-import click
-from decouple import config
-
 from .util import include_exclude_check, get_parsed_stack_config
 
 # TODO: find a place for this

app/data/compose/docker-compose-peer-test-app.yml

@@ -1,8 +1,9 @@
 version: '3.2'
 
 services:
+  # Builds and serves the peer-test react-app
   peer-test-app:
-    # Builds and serves the peer-test react-app
+    restart: unless-stopped
     image: cerc/react-peer:local
     working_dir: /scripts
     env_file:

app/data/config/fixturenet-eth/fixturenet-eth.env

@@ -17,7 +17,8 @@ CERC_STATEDIFF_DB_PORT=5432
 CERC_STATEDIFF_DB_NAME="cerc_testing"
 CERC_STATEDIFF_DB_USER="vdbm"
 CERC_STATEDIFF_DB_PASSWORD="password"
-CERC_STATEDIFF_DB_GOOSE_MIN_VER=18
+CERC_STATEDIFF_DB_GOOSE_MIN_VER=${CERC_STATEDIFF_DB_GOOSE_MIN_VER:-18}
 CERC_STATEDIFF_DB_LOG_STATEMENTS="false"
+CERC_STATEDIFF_WORKERS=2
 
 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"

app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile

@@ -22,6 +22,18 @@ COPY run-el.sh /opt/testnet/run.sh
 RUN cd /opt/testnet && make genesis-el
 
 COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
+
+# Snag the genesis block info.
 RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
+RUN cp -rp ~/ethdata ~/tmpeth && \
+    geth --datadir ~/tmpeth init /opt/testnet/build/el/geth.json && \
+    geth --datadir ~/tmpeth --http & \
+    sleep 5 && \
+    curl -q --location 'localhost:8545' \
+        --header 'Content-Type: application/json' \
+        --data '{ "jsonrpc": "2.0", "id": 14, "method": "eth_getBlockByNumber", "params": ["0x0", false] }' \
+        -o /opt/testnet/build/el/genesis_block.json && \
+    killall -9 geth && \
+    rm -rf ~/tmpeth
 
 ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -92,6 +92,7 @@ else
       --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \
       --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \
       --statediff.waitforsync=true \
+      --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \
       --statediff.writing=true"
     fi
 

app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile

@@ -1,4 +1,4 @@
-FROM sigp/lcli:local AS lcli
+FROM cerc/lighthouse-cli:local AS lcli
 FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-eth-geth:local AS fnetgeth
 

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh

@@ -27,12 +27,14 @@ lcli \
   --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --testnet-dir $TESTNET_DIR \
   --min-genesis-active-validator-count $GENESIS_VALIDATOR_COUNT \
+  --validator-count $VALIDATOR_COUNT \
   --min-genesis-time $GENESIS_TIME \
   --genesis-delay $GENESIS_DELAY \
   --genesis-fork-version $GENESIS_FORK_VERSION \
   --altair-fork-epoch $ALTAIR_FORK_EPOCH \
-  --merge-fork-epoch $MERGE_FORK_EPOCH \
+  --bellatrix-fork-epoch $MERGE_FORK_EPOCH \
   --eth1-id $ETH1_CHAIN_ID \
+  --eth1-block-hash $ETH1_BLOCK_HASH \
   --eth1-follow-distance 1 \
   --seconds-per-slot $SECONDS_PER_SLOT \
   --seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env

@@ -42,7 +42,9 @@ VC_ARGS=${VC_ARGS:-""}
 EXECUTION_ENDPOINT=${EXECUTION_ENDPOINT:-http://localhost:8551}
 
 ETH1_GENESIS_JSON=${ETH1_GENESIS_JSON:-"../build/el/geth.json"}
+ETH1_GENESIS_BLOCK_JSON=${ETH1_GENESIS_BLOCK_JSON:-"../build/el/genesis_block.json"}
 ETH1_CONFIG_YAML=${ETH1_CONFIG_YAML:-"../el/el-config.yaml"}
+ETH1_BLOCK_HASH=${ETH1_BLOCK_HASH:-`cat $ETH1_GENESIS_BLOCK_JSON | jq -r '.result.hash' | cut -d'x' -f2`}
 
 ETH1_CHAIN_ID=${ETH1_CHAIN_ID:-`cat $ETH1_GENESIS_JSON | jq -r '.config.chainId'`}
 ETH1_TTD=${ETH1_TTD:-`cat $ETH1_GENESIS_JSON | jq -r '.config.terminalTotalDifficulty'`}

app/data/container-build/cerc-fixturenet-plugeth-lighthouse/Dockerfile

@@ -1,4 +1,4 @@
-FROM sigp/lcli:v4.1.0 AS lcli
+FROM cerc/lighthouse-cli:local AS lcli
 FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-plugeth-plugeth:local AS fnetgeth
 

app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile

@@ -6,7 +6,7 @@ RUN go install github.com/go-delve/delve/cmd/dlv@latest
 FROM ubuntu:22.04
 RUN apt-get update && \
       apt-get install -y --no-install-recommends \
-        python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat && \
+        python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat psmisc && \
       rm -rf /var/lib/apt/lists/*
 
 COPY --from=delve  /go/bin/dlv /usr/local/bin/
@@ -24,4 +24,17 @@ RUN cd /opt/testnet && make genesis-el
 
 RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
 
+# Snag the genesis block info.
+RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
+RUN cp -rp ~/ethdata ~/tmpeth && \
+    geth --datadir ~/tmpeth init /opt/testnet/build/el/geth.json && \
+    geth --datadir ~/tmpeth --http & \
+    sleep 5 && \
+    curl -q --location 'localhost:8545' \
+        --header 'Content-Type: application/json' \
+        --data '{ "jsonrpc": "2.0", "id": 14, "method": "eth_getBlockByNumber", "params": ["0x0", false] }' \
+        -o /opt/testnet/build/el/genesis_block.json && \
+    killall -9 geth && \
+    rm -rf ~/tmpeth
+
 ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-lighthouse-cli/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/lighthouse-cli
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+project_dir=${CERC_REPO_BASE_DIR}/lighthouse
+docker build -t cerc/lighthouse-cli:local --build-arg PORTABLE=true -f ${project_dir}/lcli/Dockerfile ${build_command_args} ${project_dir}

app/data/container-build/cerc-lighthouse/Dockerfile

@@ -1,4 +1,5 @@
-FROM sigp/lighthouse:v4.1.0-modern
+ARG TAG_SUFFIX="-modern"
+FROM sigp/lighthouse:v4.1.0${TAG_SUFFIX}
 
 RUN apt-get update; apt-get install bash netcat curl less jq -y;
 

app/data/stacks/fixturenet-eth-loaded/stack.yml

@@ -6,9 +6,11 @@ repos:
   - github.com/cerc-io/tx-spammer
   - github.com/cerc-io/ipld-eth-server
   - github.com/cerc-io/ipld-eth-db
+  - github.com/cerc-io/lighthouse
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/ipld-eth-server

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -4,10 +4,12 @@ decription: "Ethereum Fixturenet w/ tx-spammer"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer
-  - dboreham/foundry
+  - github.com/dboreham/foundry
+  - github.com/cerc-io/lighthouse
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/tx-spammer

app/data/stacks/fixturenet-eth/stack.yml

@@ -3,10 +3,12 @@ name: fixturenet-eth
 decription: "Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
+  - github.com/cerc-io/lighthouse
   - github.com/dboreham/foundry
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/foundry

app/data/stacks/fixturenet-optimism/README.md

@@ -14,14 +14,6 @@ laconic-so --stack fixturenet-optimism setup-repositories
 # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
 ```
 
-Checkout to the required versions and branches in repos:
-
-```bash
-# Optimism
-cd ~/cerc/optimism
-git checkout v1.0.4
-```
-
 Build the container images:
 
 ```bash

app/data/stacks/fixturenet-optimism/l2-only.md

@@ -14,14 +14,6 @@ laconic-so --stack fixturenet-optimism setup-repositories --exclude github.com/c
 # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
 ```
 
-Checkout to the required versions and branches in repos:
-
-```bash
-# Optimism
-cd ~/cerc/optimism
-git checkout v1.0.4
-```
-
 Build the container images:
 
 ```bash

app/data/stacks/fixturenet-optimism/stack.yml

@@ -3,12 +3,14 @@ name: fixturenet-optimism
 decription: "Optimism Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
+  - github.com/cerc-io/lighthouse
   - github.com/dboreham/foundry
-  - github.com/ethereum-optimism/optimism
+  - github.com/ethereum-optimism/optimism@v1.0.4
-  - github.com/ethereum-optimism/op-geth
+  - github.com/ethereum-optimism/op-geth@v1.101105.2
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/foundry

app/data/stacks/fixturenet-plugeth-tx/stack.yml

@@ -3,11 +3,16 @@ name: fixturenet-plugeth-tx
 decription: "plugeth Ethereum Fixturenet w/ tx-spammer"
 repos:
   - github.com/cerc-io/tx-spammer
+  - github.com/dboreham/foundry
+  - github.com/cerc-io/lighthouse
 containers:
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-plugeth-plugeth
   - cerc/fixturenet-plugeth-lighthouse
   - cerc/tx-spammer
+  - cerc/foundry
 pods:
   - fixturenet-plugeth
+  - foundry
   - tx-spammer

app/data/stacks/fixturenet-pocket/stack.yml

@@ -3,11 +3,13 @@ name: fixturenet-pocket
 description: "A single node pocket chain that can serve relays from the geth-1 node in eth-fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
+  - github.com/cerc-io/lighthouse
   - github.com/pokt-network/pocket-core
   - github.com/pokt-network/pocket-core-deployments # contains the dockerfile
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/pocket

app/data/stacks/mobymask-v2/README.md

@@ -18,26 +18,6 @@ laconic-so --stack mobymask-v2 setup-repositories
 NOTE: If repositories already exist and are checked out to different versions, `setup-repositories` command will throw an error.
 For getting around this, the repositories mentioned below can be removed and then run the command.
 
-Checkout to the required versions and branches in repos
-
-```bash
-# watcher-ts
-cd ~/cerc/watcher-ts
-git checkout v0.2.41
-
-# mobymask-v2-watcher-ts
-cd ~/cerc/mobymask-v2-watcher-ts
-git checkout v0.1.1
-
-# MobyMask
-cd ~/cerc/MobyMask
-git checkout v0.1.2
-
-# Optimism
-cd ~/cerc/optimism
-git checkout v1.0.4
-```
-
 Build the container images:
 
 ```bash

app/data/stacks/mobymask-v2/mobymask-only.md

@@ -14,22 +14,6 @@ laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/M
 # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
 ```
 
-Checkout to the required versions and branches in repos:
-
-```bash
-# watcher-ts
-cd ~/cerc/watcher-ts
-git checkout v0.2.41
-
-# mobymask-v2-watcher-ts
-cd ~/cerc/mobymask-v2-watcher-ts
-git checkout v0.1.1
-
-# MobyMask
-cd ~/cerc/MobyMask
-git checkout v0.1.2
-```
-
 Build the container images:
 
 ```bash

app/data/stacks/mobymask-v2/stack.yml

@@ -2,21 +2,24 @@ version: "1.0"
 name: mobymask-v2
 repos:
   - github.com/cerc-io/go-ethereum
+  - github.com/cerc-io/lighthouse
   - github.com/dboreham/foundry
-  - github.com/ethereum-optimism/optimism
+  - github.com/ethereum-optimism/optimism@v1.0.4
-  - github.com/ethereum-optimism/op-geth
+  - github.com/ethereum-optimism/op-geth@v1.101105.2
-  - github.com/cerc-io/watcher-ts
+  - github.com/cerc-io/watcher-ts@v0.2.43
-  - github.com/cerc-io/mobymask-v2-watcher-ts
+  - github.com/cerc-io/mobymask-v2-watcher-ts@v0.1.1
-  - github.com/cerc-io/MobyMask
+  - github.com/cerc-io/MobyMask@v0.1.2
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
+  - cerc/lighthouse-cli
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/foundry
   - cerc/optimism-contracts
   - cerc/optimism-l2geth
   - cerc/optimism-op-batcher
+  - cerc/optimism-op-proposer
   - cerc/optimism-op-node
   - cerc/watcher-ts
   - cerc/watcher-mobymask-v2

app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md

@@ -30,22 +30,6 @@ Clone required repositories:
   # 100%|##############################################################################################################################################| 1.41k/1.41k [00:18<00:00, 76.4B/s]
   ```
 
-Checkout to the required versions and branches in repos:
-
-  ```bash
-  # watcher-ts
-  cd ~/cerc/watcher-ts
-  git checkout v0.2.41
-
-  # mobymask-v2-watcher-ts
-  cd ~/cerc/mobymask-v2-watcher-ts
-  git checkout v0.1.1
-
-  # MobyMask
-  cd ~/cerc/MobyMask
-  git checkout v0.1.2
-  ```
-
 Build the container images:
 
   ```bash

scripts/cloud-init-dev-mode-install.yaml

@@ -0,0 +1,44 @@
+#cloud-config
+
+# Used for easily testing stacks-in-development on cloud platforms
+# Assumes Ubuntu, edit the last line if targeting a different OS
+
+# Once SSH'd into the server, run:
+# `$ cd stack-orchestrator`
+# `$ git checkout <branch>
+# `$ ./scripts/developer-mode-setup.sh`
+# `$ source ./venv/bin/activate`
+
+# Followed by the stack instructions.
+
+package_update: true
+package_upgrade: true
+
+groups:
+  - docker
+
+system_info:
+  default_user:
+    groups: [ docker ]
+
+packages:
+  - apt-transport-https
+  - ca-certificates
+  - curl
+  - jq
+  - git
+  - gnupg
+  - lsb-release
+  - unattended-upgrades
+  - python3.10-venv
+  - pip
+
+runcmd:
+  - mkdir -p /etc/apt/keyrings
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+  - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+  - apt-get update
+  - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+  - systemctl enable docker
+  - systemctl start docker
+  - git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator

scripts/cloud-init-user-mode-install.yaml

@@ -0,0 +1,35 @@
+#cloud-config
+
+# Used for installing Stack Orchestrator on platforms that support `cloud-init`
+# Tested on Ubuntu
+
+package_update: true
+package_upgrade: true
+
+groups:
+  - docker
+
+system_info:
+  default_user:
+    groups: [ docker ]
+
+packages:
+  - apt-transport-https
+  - ca-certificates
+  - curl
+  - jq
+  - git
+  - gnupg
+  - lsb-release
+  - unattended-upgrades
+
+runcmd:
+  - mkdir -p /etc/apt/keyrings
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+  - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+  - apt-get update
+  - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+  - systemctl enable docker
+  - systemctl start docker
+  - curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+  - chmod +x /usr/local/bin/laconic-so