Pass extra args to custom create command (#972)

This is needed to allow custom deploy commands to handle arbitrary args.

* Adds a `DeploymentContext.modify_yaml` helper
* Removes `laconicd` from test stack to simplify it

Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/972
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -26,8 +26,14 @@ fi
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
+if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
+  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
 cd "${WORK_DIR}" || exit 1
 
+  ./build-webapp.sh || exit 1
+  exit 0
+fi
+
 if [ -f "next.config.mjs" ]; then
   NEXT_CONFIG_JS="next.config.mjs"
   IMPORT_OR_REQUIRE="import"

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -30,6 +30,13 @@ fi
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
 
+if [ -f "./run-webapp.sh" ]; then
+  echo "Running webapp with run-webapp.sh ..."
+  cd "${WORK_DIR}" || exit 1
+  ./run-webapp.sh &
+  tpid=$!
+  wait $tpid
+else 
   "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
   mv .next .next.old
   mv .next-r/.next .
@@ -63,3 +70,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
   fi
 
   $CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}
+fi

stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
-from ruamel.yaml import YAML
 
 
 def create(context: DeploymentContext, extra_args):
@@ -23,17 +22,12 @@ def create(context: DeploymentContext, extra_args):
     # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment
     fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml')
 
-    with open(fixturenet_eth_compose_file, 'r') as yaml_file:
-        yaml = YAML()
-        yaml_data = yaml.load(yaml_file)
-
     new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh'
 
+    def add_geth_volume(yaml_data):
         if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']:
             yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script)
 
-    with open(fixturenet_eth_compose_file, 'w') as yaml_file:
-        yaml = YAML()
-        yaml.dump(yaml_data, yaml_file)
+    context.modify_yaml(fixturenet_eth_compose_file, add_geth_volume)
 
     return None

stack_orchestrator/data/stacks/test/stack.yml

@@ -2,7 +2,6 @@ version: "1.0"
 name: test
 description: "A test stack"
 repos:
-  - git.vdb.to/cerc-io/laconicd
   - git.vdb.to/cerc-io/test-project@test-branch
 containers:
   - cerc/test-container

stack_orchestrator/deploy/deployment_context.py

@@ -45,18 +45,20 @@ class DeploymentContext:
     def get_compose_dir(self):
         return self.deployment_dir.joinpath(constants.compose_dir_name)
 
+    def get_compose_file(self, name: str):
+        return self.get_compose_dir() / f"docker-compose-{name}.yml"
+
     def get_cluster_id(self):
         return self.id
 
-    def init(self, dir):
-        self.deployment_dir = dir
+    def init(self, dir: Path):
+        self.deployment_dir = dir.absolute()
         self.spec = Spec()
         self.spec.init_from_file(self.get_spec_file())
         self.stack = Stack(self.spec.obj["stack"])
         self.stack.init_from_file(self.get_stack_file())
         deployment_file_path = self.get_deployment_file()
         if deployment_file_path.exists():
-            with deployment_file_path:
             obj = get_yaml().load(open(deployment_file_path, "r"))
             self.id = obj[constants.cluster_id_key]
         # Handle the case of a legacy deployment with no file
@@ -67,3 +69,19 @@ class DeploymentContext:
             unique_cluster_descriptor = f"{path},{self.get_stack_file()},None,None"
             hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16]
             self.id = f"{constants.cluster_name_prefix}{hash}"
+
+    def modify_yaml(self, file_path: Path, modifier_func):
+        """
+        Load a YAML from the deployment, apply a modification function, and write it back.
+        """
+        if not file_path.absolute().is_relative_to(self.deployment_dir):
+            raise ValueError(f"File is not inside deployment directory: {file_path}")
+
+        yaml = get_yaml()
+        with open(file_path, 'r') as f:
+            yaml_data = yaml.load(f)
+
+        modifier_func(yaml_data)
+
+        with open(file_path, 'w') as f:
+            yaml.dump(yaml_data, f)

stack_orchestrator/deploy/deployment_create.py

@@ -443,18 +443,16 @@ def _check_volume_definitions(spec):
 @click.command()
 @click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
-# TODO: Hack
-@click.option("--network-dir", help="Network configuration supplied in this directory")
-@click.option("--initial-peers", help="Initial set of persistent peers")
+@click.argument('extra_args', nargs=-1, type=click.UNPROCESSED)
 @click.pass_context
-def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
+def create(ctx, spec_file, deployment_dir, extra_args):
     deployment_command_context = ctx.obj
-    return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers)
+    return create_operation(deployment_command_context, spec_file, deployment_dir, extra_args)
 
 
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
-def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
+def create_operation(deployment_command_context, spec_file, deployment_dir, extra_args):
     parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
     _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
@@ -541,7 +539,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context)
     # TODO: make deployment_dir_path a Path above
     deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])
+    call_stack_deploy_create(deployment_context, extra_args)
 
 
 # TODO: this code should be in the stack .py files but

stack_orchestrator/deploy/spec.py

@@ -92,7 +92,6 @@ class Spec:
         return self.obj.get(item, default)
 
     def init_from_file(self, file_path: Path):
-        with file_path:
         self.obj = get_yaml().load(open(file_path, "r"))
         self.file_path = file_path
 

stack_orchestrator/deploy/stack.py

@@ -27,5 +27,4 @@ class Stack:
         self.name = name
 
     def init_from_file(self, file_path: Path):
-        with file_path:
         self.obj = get_yaml().load(open(file_path, "r"))

stack_orchestrator/deploy/webapp/deploy_webapp.py

@@ -92,7 +92,6 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist
         spec_file_name,
         deployment_dir,
         None,
-        None
     )
     # Fix up the container tag inside the deployment compose file
     _fixup_container_tag(deployment_dir, image)

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -342,17 +342,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="Requests must have a minimum payment to be processed (in alnt)",
     default=0,
 )
-@click.option(
-    "--atom-payment-address",
-    help="Cosmos ATOM address to receive payments",
-    default=None,
-)
-@click.option(
-    "--min-atom-payment",
-    help="Minimum required ATOM payment amount",
-    default=1,
-    type=float,
-)
 @click.option("--lrn", help="The LRN of this deployer.", required=True)
 @click.option(
     "--all-requests",
@@ -405,8 +394,6 @@ def command(  # noqa: C901
     recreate_on_deploy,
     log_dir,
     min_required_payment,
-    atom_payment_address,
-    min_atom_payment,
     lrn,
     config_upload_dir,
     private_key_file,
@@ -643,8 +630,6 @@ def command(  # noqa: C901
                     payment_address,
                     min_required_payment,
                     main_logger,
-                    atom_payment_address,
-                    min_atom_payment,
                 ):
                     main_logger.log(f"{r.id}: Payment confirmed.")
                     requests_to_execute.append(r)

stack_orchestrator/deploy/webapp/publish_webapp_deployer.py

@@ -46,17 +46,6 @@ from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient
     help="List the minimum required payment (in alnt) to process a deployment request.",
     default=0,
 )
-@click.option(
-    "--atom-payment-address",
-    help="The Cosmos ATOM address to which payments should be made.",
-    default=None,
-)
-@click.option(
-    "--min-atom-payment",
-    help="List the minimum required payment (in ATOM) to process a deployment request.",
-    default=1,
-    type=float,
-)
 @click.option(
     "--dry-run",
     help="Don't publish anything, just report what would be done.",
@@ -71,8 +60,6 @@ def command(  # noqa: C901
     lrn,
     payment_address,
     min_required_payment,
-    atom_payment_address,
-    min_atom_payment,
     dry_run,
 ):
     laconic = LaconicRegistryClient(laconic_config)
@@ -97,10 +84,6 @@ def command(  # noqa: C901
             "minimumPayment"
         ] = f"{min_required_payment}alnt"
 
-    if atom_payment_address:
-        webapp_deployer_record["record"]["atomPaymentAddress"] = atom_payment_address
-        webapp_deployer_record["record"]["minimumAtomPayment"] = min_atom_payment
-
     if dry_run:
         yaml.dump(webapp_deployer_record, sys.stdout)
         return

stack_orchestrator/deploy/webapp/util.py

@@ -801,7 +801,7 @@ def skip_by_tag(r, include_tags, exclude_tags):
     return False
 
 
-def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger, atom_payment_address=None, atom_min_amount=None):
+def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger):
     req_owner = laconic.get_owner(record)
     if req_owner == payment_address:
         # No need to confirm payment if the sender and recipient are the same account.
@@ -811,9 +811,11 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min
         logger.log(f"{record.id}: no payment tx info")
         return False
 
-    # Try to verify as a laconic payment first
     tx = laconic.get_tx(record.attributes.payment)
-    if tx:
+    if not tx:
+        logger.log(f"{record.id}: cannot locate payment tx")
+        return False
+
     if tx.code != 0:
         logger.log(
             f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}"
@@ -871,55 +873,6 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min
 
     return True
 
-    # If we get here, the transaction hash wasn't found in the laconic testnet
-    # Let's check if it's a valid Cosmos ATOM payment if configuration is available
-    if atom_payment_address:
-        logger.log(f"{record.id}: checking if payment is a valid Cosmos ATOM transaction")
-        
-        try:
-            import requests
-            
-            # Use the webapp-deployment-status-api to verify the ATOM payment
-            deployer_record = laconic.get_record(record.attributes.deployer)
-            if not deployer_record or not deployer_record.attributes.apiUrl:
-                logger.log(f"{record.id}: cannot find deployer API URL to verify ATOM payment")
-                return False
-                
-            api_url = deployer_record.attributes.apiUrl
-            verify_url = f"{api_url}/verify/atom-payment"
-            
-            # Make a request to the API to verify the ATOM payment
-            # Pass markAsUsed=true to prevent this transaction from being used again
-            response = requests.post(
-                verify_url,
-                json={
-                    "txHash": record.attributes.payment,
-                    "minAmount": atom_min_amount,
-                    "markAsUsed": True
-                },
-                timeout=10
-            )
-            
-            if response.status_code != 200:
-                logger.log(f"{record.id}: ATOM payment verification API request failed with status {response.status_code}")
-                return False
-                
-            result = response.json()
-            if not result.get("valid", False):
-                logger.log(f"{record.id}: ATOM payment verification failed: {result.get('reason', 'unknown reason')}")
-                return False
-                
-            # Payment is valid
-            logger.log(f"{record.id}: ATOM payment verified successfully, amount: {result.get('amount')} ATOM")
-            return True
-            
-        except Exception as e:
-            logger.log(f"{record.id}: error verifying ATOM payment: {str(e)}")
-            return False
-    
-    logger.log(f"{record.id}: payment tx {record.attributes.payment} not found in laconic testnet and ATOM payment verification not configured")
-    return False
-
 
 def confirm_auction(laconic: LaconicRegistryClient, record, deployer_lrn, payment_address, logger):
     auction_id = record.attributes.auction

stack_orchestrator/util.py

@@ -180,9 +180,7 @@ def get_k8s_dir():
 def get_parsed_deployment_spec(spec_file):
     spec_file_path = Path(spec_file)
     try:
-        with spec_file_path:
-            deploy_spec = get_yaml().load(open(spec_file_path, "r"))
-            return deploy_spec
+        return get_yaml().load(open(spec_file_path, "r"))
     except FileNotFoundError as error:
         # We try here to generate a useful diagnostic error
         print(f"Error: spec file: {spec_file_path} does not exist")

tests/deploy/run-deploy-test.sh

@@ -14,8 +14,13 @@ delete_cluster_exit () {
 
 # Test basic stack-orchestrator deploy
 echo "Running stack-orchestrator deploy test"
-# Bit of a hack, test the most recent package
+
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
     TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
+
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"