From 503a34bdc5c77c266afd54195744e848bfc129a7 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 14:44:38 +0100
Subject: [PATCH 01/40] testing

---
 honda-japan.yaml                              | 419 ++++++++++++++----
 main.sh                                       |  15 +-
 .../activate_monitoring.sh                    |   6 -
 .../plot_monitoring.sh                        |   1 -
 .../transfer_outputs.sh                       |  22 -
 scripts/activate_monitoring.sh                |   6 +
 .../adapt_wtx_paths.py                        |   0
 scripts/cpu_and_memory_usage.py               | 101 +++++
 .../cpu_and_memory_usage_requirements.yaml    | 153 +++++++
 .../dry_run.sh                                |   0
 .../estimate_end_time.py                      |   0
 .../get_group_allocation_balance.py           |   0
 .../montecarlo.sh                             |   0
 scripts/plot_monitoring.sh                    |   1 +
 .../run_dcs.sh                                |   7 +-
 .../sensitivity.sh                            |   0
 .../transfer_and_process_inputs.sh            |   3 +-
 scripts/transfer_outputs.sh                   |   5 +
 18 files changed, 601 insertions(+), 138 deletions(-)
 delete mode 100644 resources/001_simulation_executor/activate_monitoring.sh
 delete mode 100644 resources/001_simulation_executor/plot_monitoring.sh
 delete mode 100644 resources/001_simulation_executor/transfer_outputs.sh
 create mode 100644 scripts/activate_monitoring.sh
 rename {resources/001_simulation_executor => scripts}/adapt_wtx_paths.py (100%)
 create mode 100644 scripts/cpu_and_memory_usage.py
 create mode 100644 scripts/cpu_and_memory_usage_requirements.yaml
 rename {resources/001_simulation_executor => scripts}/dry_run.sh (100%)
 rename {resources/001_simulation_executor => scripts}/estimate_end_time.py (100%)
 rename get_group_allocation_balance.py => scripts/get_group_allocation_balance.py (100%)
 rename {resources/001_simulation_executor => scripts}/montecarlo.sh (100%)
 create mode 100644 scripts/plot_monitoring.sh
 rename {resources/001_simulation_executor => scripts}/run_dcs.sh (58%)
 rename {resources/001_simulation_executor => scripts}/sensitivity.sh (100%)
 rename resources/001_simulation_executor/transfer_inputs.sh => scripts/transfer_and_process_inputs.sh (94%)
 create mode 100644 scripts/transfer_outputs.sh

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 208bd28..950c4d1 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -1,30 +1,304 @@
 permissions:
   - '*'
+env:
+  MAX_JOBS: '2'
 jobs:
-  main:
+  local_preprocessing:
     steps:
-      - name: Main
-        run: bash main.sh > logs/main/step_0/logs.out 2>&1 
-        cleanup: |
-          touch COMPLETED
-          source resources/001_simulation_executor/inputs.sh
-          timeout 180 bash cancel.sh  || true
-          exit 0
-  stream:
+
+      - name: Sanity Checks
+        run: |
+          source /etc/profile.d/parallelworks.sh
+          source /etc/profile.d/parallelworks-env.sh
+          source /pw/.miniconda3/etc/profile.d/conda.sh
+
+          python3 scripts/get_group_allocation_balance.py ${{ inputs.run_hours_3dcs_group }} ${{ inputs.org_name }}
+          if [ $? -ne 0 ]; then
+              echo "$(date) ERROR: No 3DCS balance is available. Exiting workflow." >&2
+              exit 1
+          fi
+
+  remote_preprocessing:
+    working-directory: ${{ inputs.rundir }}
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
     steps:
-      - name: Stream Logs
-        early-cancel: any-job-failed
-        run: bash stream.sh > logs/stream/step_0/logs.out 2>&1 
+      - name: Checkout
+        uses: parallelworks/checkout
+        with:
+          repo: https://github.com/parallelworks/dcs-workflow.git
+          branch: main
+
+      - name: Sanity Checks
+        run: |
+          if [[ "${{ inputs.dcs.output_directory }}" == "${{ inputs.dcs.model_directory }}" || "${{ inputs.dcs.output_directory }}" == "${{ inputs.dcs.model_directory }}/"* ]]; then
+              echo "$(date) ERROR: Output directory ${{ inputs.dcs.output_directory }} is a subdirectory of model directory ${{ inputs.dcs.model_directory }}. Exiting workflow" >&2
+              exit 1
+          fi
+
+          # Check if file is provided as an argument
+          if ! [ -f "dcs_environment/${{ inputs.dcs.version }}.sh" ]; then
+              echo "$(date) ERROR: Missing file dcs_environment/${{ inputs.dcs.version }}.sh required to load and run 3DCS. Exiting workflow." >&2
+              exit 1
+          fi
+
+      - name: Transfer and Process Input Files
+        run: |
+
+          mkdir tmp-data-transfer
+          cd tmp-data-transfer
+
+          pw buckets cp -r ${{ inputs.dcs.bucket.uri }}/${{ inputs.dcs.model_directory }} .
+
+          # Find all files ending in ".wtx" in the current directory, excluding subdirectories
+          export dcs_model_file=$(find . -type f -name "*.wtx")
+
+          # Check if dcs_model_file is empty
+          if [ -z "$dcs_model_file" ]; then
+              echo "$(date) ERROR: No '.wtx' files found. Exiting workflow" >&2
+              exit 1
+          fi
+
+          echo "dcs_model_file=${dcs_model_file}" >> $OUTPUTS
+
+          # Count the number of files found
+          file_count=$(echo "$dcs_model_file" | wc -l)
+
+          # Check if only one file ending in ".wtx" is found
+          if [ "$file_count" -eq 1 ]; then
+              echo "Found file ${dcs_model_file}"
+          else
+              echo "$(date) ERROR: Found $file_count '.wtx' files. Expected only one. Exiting workflow" >&2
+              exit 1
+          fi
+
+          # Get the count of directories in the current directory
+          dir_count=$(ls -d */ | wc -l)
+
+          # If there's only one directory
+          if [ "$dir_count" -eq 1 ]; then
+              # Get the name of the directory
+              fea_dir=$(ls -d */ | head -n 1)
+              export fea_dir=${fea_dir%/}  # Remove trailing slash
+              echo "Directory found: $fea_dir"
+          else
+              # If no directory or multiple directories exist
+              echo "Error: Either no FEA directory found or multiple directories exist." >&2
+              exit 1
+          fi
+
+          # Process WTX file to adapt the paths to the files
+          retries=5
+          while true; do
+              python3 ${{ inputs.rundir }}/scripts/adapt_wtx_paths.py ${dcs_model_file} ${fea_dir}
+              exit_code=$?
+              if [ ${exit_code} -ne 0 ]; then
+                retries=$((retries-1))
+                if [ ${retries} -gt 0 ]; then
+                    sleep 10
+                else
+                    echo
+                    echo "$(date) ERROR: Failed to process WTX file with adapt_wtx_paths.py"  >&2
+                    exit 1
+                fi
+              else
+                break
+              fi
+          done
+
+          # List all downloaded file
+          find . -mindepth 1 > downloaded_files.txt
+
+          mv tmp-data-transfer/*  .
+          rmdir tmp-data-transfer
+
+      - name: General Preprocessing
+        run: |
+
+          source dcs_environment/${dcs_version}.sh 
+          sudo chown -R ${USER}:pwuser ${WINEPREFIX}
+
+          mkdir ./usage
+          mkdir ./usage_completed
+
+          cat <<'EOF' >> inputs.sh
+          export PATH=$HOME/pw:$PATH
+          export dcs_model_file="${{ needs.remote_preprocessing.outputs.dcs_model_file }}"
+          export dcs_dry_run="${{ inputs.dcs.dry_run }}"
+          export dcs_version="${{ inputs.dcs.version }}"
+          export dcs_analysis_type="${{ inputs.dcs.analysis_type }}"
+          export dcs_bucket_uri="${{ inputs.dcs.bucket.uri }}"
+          export dcs_bucket_uri="${{ inputs.dcs.bucket.uri }}"
+          export dcs_model_directory="${{ inputs.dcs.model_directory }}"
+          export dcs_output_directory="${{ inputs.dcs.output_directory }}"
+          export dcs_num_seeds="${{ inputs.dcs.num_seeds }}"
+          export dcs_concurrency="${{ inputs.dcs.concurrency }}"
+          export dcs_thread="${{ inputs.dcs.thread }}"
+          export metering_user="${{ inputs.metering_user }}"
+          export metering_ip="${{ inputs.metering_ip }}"
+          export run_hours_3dcs_group="${{ inputs.run_hours_3dcs_group }}"
+          export org_name="${{ inputs.org_name }}"
+          export monitoring_conda_dir="${{ inputs.monitoring_conda_dir }}"
+          export monitoring_conda_env="${{ inputs.monitoring_conda_env }}"
+          EOF
+          echo "export PW_PARENT_JOB_DIR=${PW_PARENT_JOB_DIR}"
+
+          echo "$(date) INFO: Inputs file"
+          cat inputs.sh
+
+      - name: Installing Python Dependencies for CPU and Memory Monitoring
+        run: |
+          source inputs.sh
+
+          install_miniconda() {
+              local install_dir=$1
+              local conda_repo="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+              local ID="${RANDOM}-$(date +%s)" # This script may run at the same time!
+              echo "Installing Miniconda to ${install_dir}"
+              wget --no-check-certificate "${conda_repo}" -O "/tmp/miniconda-${ID}.sh" > "/tmp/miniconda_wget-${ID}.out" 2>&1
+              rm -rf "${install_dir}"
+              mkdir -p "$(dirname "${install_dir}")"
+              bash "/tmp/miniconda-${ID}.sh" -b -p "${install_dir}" > "/tmp/miniconda_sh-${ID}.out" 2>&1
+          }
+
+          create_conda_env_from_yaml() {
+              local CONDA_DIR=$1
+              local CONDA_ENV=$2
+              local CONDA_YAML=$3
+              local CONDA_SH="${CONDA_DIR}/etc/profile.d/conda.sh"
+
+              # Remove name/prefix fields and empty lines so conda env update accepts the file
+              sed -i -e 's/name.*$//' -e 's/prefix.*$//' -e '/^$/d' "${CONDA_YAML}"
+
+              if [ ! -d "${CONDA_DIR}" ]; then
+                  echo "Conda directory <${CONDA_DIR}> not found. Installing Conda..."
+                  install_miniconda "${CONDA_DIR}"
+              fi
+
+              echo "Sourcing Conda: ${CONDA_SH}"
+              source "${CONDA_SH}"
+
+              if conda env list | grep -q "^${CONDA_ENV} "; then
+                  echo "Conda environment <${CONDA_ENV}> already exists. Skipping installation."
+              else
+                  echo "Conda environment <${CONDA_ENV}> not found. Creating from <${CONDA_YAML}>..."
+                  conda env update -n "${CONDA_ENV}" -q -f "${CONDA_YAML}" || {
+                      echo "ERROR: Failed to create conda environment <${CONDA_ENV}>. Exiting."
+                      exit 1
+                  }
+              fi
+
+              conda activate "${CONDA_ENV}"
+          }
+
+
+          create_conda_env_from_yaml ${monitoring_conda_dir} ${monitoring_conda_env} scripts/cpu_and_memory_usage_requirements.yaml
+
+      - name: Dry Run
+        if: ${{ inputs.dcs.dry_run == true }}
+        run: |
+          echo "$(date) INFO: Running the workflow in dry run mode"
+          echo > scripts/activate_monitoring.sh
+          echo > scripts/plot_monitoring.sh
+          echo > scripts/activate_monitoring.sh
+          mv scripts/dry_run.sh scripts/run_dcs.sh
+
+
+      - name: Create Job Scripts
+        run: |
+          set -x
+
+          source inputs.sh
+
+          if [ ${dcs_concurrency} -gt ${MAX_JOBS} ]; then
+            echo "$(date) ERROR: Selected number of jobs is greater than maximum number of jobs. Exiting workflow."
+            exit 1
+          fi
 
-  estimate_end_time:
+          for nj in $(seq 1 ${MAX_JOBS}); do
+            if [ ${nj} -le ${dcs_concurrency} ]; then
+              echo "$(date) INFO: Preparing directory for job ${nj}"
+              job_dir=job_dir_${nj}
+              mkdir -p ${job_dir}
+              echo "${job_dir}=${PWD}/${job_dir}" >> $OUTPUTS
+
+              echo '#!/bin/bash' > ${job_dir}/cancel.sh
+              chmod +x ${job_dir}/cancel.sh
+
+              cat inputs.sh >> ${job_dir}/run_case.sh
+              echo "export case_index=${nj}" >> ${job_dir}/run_case.sh
+
+              cat dcs_environment/${dcs_version}.sh >> ${job_dir}/run_case.sh
+              echo >> ${job_dir}/run_case.sh
+
+              cat scripts/${dcs_analysis_type}.sh >> ${job_dir}/run_case.sh
+              cat scripts/activate_monitoring.sh >> ${job_dir}/run_case.sh
+              cat scrtips/run_dcs.sh >> ${job_dir}/run_case.sh
+              cat scripts/plot_monitoring.sh >> ${job_dir}/run_case.sh
+              cat scripts/transfer_outputs.sh >> ${job_dir}/run_case.sh
+
+              echo "run_job_${nj}=true" | tee -a $OUTPUTS
+            else
+              echo "run_job_${nj}=false" | tee -a $OUTPUTS
+            fi
+          done
+
+
+  job_1:
+    if: ${{ needs.preprocessing.outputs.run_job_1 == "true" }}
+    working-directory: ${{ needs.preprocessing.outputs.job_dir_1 }}
+    needs:
+      - preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
     steps:
-      - name: Estimate End Time
+      - uses: marketplace/script_submitter/v3.5
         early-cancel: any-job-failed
-        run: bash estimate_end_time.sh > logs/estimate_end_time/step_0/logs.out 2>&1 
-        cleanup: |
-          bash cancel_stream.sh || true
-          exit 0
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PWD}
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
 
+  job_2:
+    if: ${{ needs.preprocessing.outputs.run_job_2 == "true" }}
+    working-directory: ${{ needs.preprocessing.outputs.job_dir_2 }}
+    needs:
+      - preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PWD}
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
 
 'on':
   execute:
@@ -49,6 +323,21 @@ jobs:
         type: string
         default: honda
         hidden: true
+      monitoring_conda_dir:
+        label: PW Conda Directory
+        type: string
+        default: /dcs/pw/miniconda
+        hidden: true
+      monitoring_conda_env:
+        label: PW Conda Environment
+        type: string
+        default: psutil
+        hidden: true
+      resource:
+        type: compute-clusters
+        label: Service host
+        include-workspace: false
+        tooltip: Resource to host the service
       dcs:
         type: group
         label: 3DCS Options
@@ -80,10 +369,10 @@ jobs:
                 label: Monte Carlo Simulation
               - value: sensitivity
                 label: Contributor Analysis
-          bucket_id:
-            label: Bucket ID or namespace
-            type: string
-            tooltip: Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name]
+          bucket:
+            label: Bucket
+            type: bucket
+            tooltip: Select the storage bucket used to download input files and upload output files.
           model_directory:
             label: Model Path
             type: string
@@ -115,84 +404,30 @@ jobs:
             max: 64
             default: 1
             tooltip: Number of threads used to run the simulation. Be aware that N threads require more than N times the memory needed for a single thread.
-
-      pwrl_001_simulation_executor:
+      slurm:
         type: group
-        label: Simulation Executor
+        label: SLURM Directives
         items:
-          monitoring_conda_dir:
-            label: PW Conda Directory
-            type: string
-            default: /dcs/pw/miniconda
-            hidden: true
-          monitoring_conda_env:
-            label: PW Conda Environment
-            type: string
-            default: psutil
-            hidden: true
-          resource:
-            label: Resource
-            type: compute-clusters
-            tooltip: Resource to run the simulation task
-            include-workspace: false
-          jobschedulertype:
-            label: Select Controller, SLURM Partition or PBS Queue
-            type: string
-            default: SLURM
+          is_disabled:
+            type: boolean
             hidden: true
-          _sch__dd_partition_e_:
+            default: false
+            label: Is SLURM disabled?
+          partition:
             type: slurm-partitions
             label: SLURM partition
             optional: true
-            tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option.
-            resource: ${{ inputs.pwrl_001_simulation_executor.resource }}
-          _sch__dd_time_e_:
+            resource: ${{ inputs.resource }}
+            tooltip: Select a partition from the drop down menu.
+          time:
             label: Walltime
             type: string
             default: '999:00:00'
-            tooltip: Maximum walltime per job
+            tooltip: '--time= SLURM directive to set the maximum wall-clock time limit for the job'
           scheduler_directives:
-            label: Scheduler directives
-            type: string
-            default: '--exclusive'
-            tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.
-      pwrl_002_merge_executor:
-        type: group
-        label: Merge Executor
-        items:
-          monitoring_conda_dir:
-            label: PW Conda Directory
-            type: string
-            default: /dcs/pw/miniconda
-            hidden: true
-          monitoring_conda_env:
-            label: PW Conda Environment
-            type: string
-            default: psutil
-            hidden: true
-          resource:
-            label: Resource
-            type: compute-clusters
-            tooltip: Resource to run the simulation task
-            include-workspace: false
-          jobschedulertype:
-            label: Select Controller, SLURM Partition or PBS Queue
-            type: string
-            default: SLURM
-            hidden: true
-          _sch__dd_partition_e_:
-            type: slurm-partitions
-            label: SLURM partition
+            type: editor
             optional: true
-            tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option.
-            resource: ${{ inputs.pwrl_002_merge_executor.resource }}
-          _sch__dd_time_e_:
-            label: Walltime
-            type: string
-            default: '999:00:00'
-            tooltip: Maximum walltime per job
-          scheduler_directives:
-            label: Scheduler directives
-            type: string
-            default: '--exclusive'
-            tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.
+            tooltip: |
+              Type in additional scheduler directives.
+            default: |
+              #SBATCH --exclusive
diff --git a/main.sh b/main.sh
index 1f1d348..650b5f5 100755
--- a/main.sh
+++ b/main.sh
@@ -2,17 +2,6 @@
 source inputs.sh
 chmod +x cancel.sh
 
-if [[ "${dcs_output_directory}" == "${dcs_model_directory}" || "${dcs_output_directory}" == "${dcs_model_directory}/"* ]]; then
-    echo "Error: Output directory is a subdirectory of model directory." >&2
-    exit 1
-fi
-
-# Check if file is provided as an argument
-if ! [ -f "dcs_environment/${dcs_version}.sh" ]; then
-    echo "Error: Missing file dcs_environment/${dcs_version}.sh required to load and run 3DCS. Exiting workflow."
-    exit 1
-fi
-
 
 # Use the resource wrapper
 source /etc/profile.d/parallelworks.sh
@@ -38,7 +27,7 @@ fi
 
 # Check balance
 echo; echo "3DCS allocation balance"
-python3 get_group_allocation_balance.py ${run_hours_3dcs_group} ${org_name}
+python3 utils/get_group_allocation_balance.py ${run_hours_3dcs_group} ${org_name}
 
 
 if [ $? -ne 0 ]; then
@@ -102,6 +91,8 @@ scp bucket_credentials ${resource_publicIp}:${resource_jobdir}/bucket_credential
 reload_bucket_credential_pid=$!
 echo "kill ${reload_bucket_credential_pid} || true # bucket credentials" >> cancel.sh
 
+# HERE
+
 echo; echo; echo "PREPARING AND SUBMITTING 3DCS RUN JOBS"
 single_cluster_rsync_exec resources/001_simulation_executor/cluster_rsync_exec.sh
 return_code=$?
diff --git a/resources/001_simulation_executor/activate_monitoring.sh b/resources/001_simulation_executor/activate_monitoring.sh
deleted file mode 100644
index 5d809a2..0000000
--- a/resources/001_simulation_executor/activate_monitoring.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-source ${monitoring_conda_dir}/etc/profile.d/conda.sh 
-conda activate ${monitoring_conda_env}
-monitoring_txt="case-${case_index}-${HOSTNAME}-jobid-${SLURM_JOB_ID}.txt"
-python ${resource_jobdir}/${resource_label}/cpu_and_memory_usage.py --write-usage --txt ${monitoring_txt} &
-monitoring_pid=$!
-echo "kill ${monitoring_pid}" >> ${resource_jobdir}/${resource_label}/cancel.sh
diff --git a/resources/001_simulation_executor/plot_monitoring.sh b/resources/001_simulation_executor/plot_monitoring.sh
deleted file mode 100644
index 7327892..0000000
--- a/resources/001_simulation_executor/plot_monitoring.sh
+++ /dev/null
@@ -1 +0,0 @@
-python ${resource_jobdir}/${resource_label}/cpu_and_memory_usage.py --plot-usage --txt ${monitoring_txt}
diff --git a/resources/001_simulation_executor/transfer_outputs.sh b/resources/001_simulation_executor/transfer_outputs.sh
deleted file mode 100644
index 7ff846d..0000000
--- a/resources/001_simulation_executor/transfer_outputs.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-# HARDCODED TO AWS
-# A dynamicStorage parameter type would be very helpful for this
-unset BUCKET_URI
-unset AWS_ACCESS_KEY_ID
-unset AWS_SECRET_ACCESS_KEY
-unset AWS_SESSION_TOKEN
-
-source ${resource_jobdir}/bucket_credentials
-
-# Loop until all variables are non-empty
-while [[ -z "$BUCKET_URI" \
-     || -z "$AWS_ACCESS_KEY_ID" \
-     || -z "$AWS_SECRET_ACCESS_KEY" \
-     || -z "$AWS_SESSION_TOKEN" ]]
-do
-    echo "$(date) Waiting for required environment variables to be set..."
-    sleep 5
-    source ${resource_jobdir}/bucket_credentials
-done
-
-# Copy path/to/worker_<i> to bucket
-aws s3 cp --recursive ${PWD} ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/$(basename ${PWD})
diff --git a/scripts/activate_monitoring.sh b/scripts/activate_monitoring.sh
new file mode 100644
index 0000000..2624db3
--- /dev/null
+++ b/scripts/activate_monitoring.sh
@@ -0,0 +1,6 @@
+source ${monitoring_conda_dir}/etc/profile.d/conda.sh 
+conda activate ${monitoring_conda_env}
+monitoring_txt="case-${case_index}-${HOSTNAME}-jobid-${SLURM_JOB_ID}.txt"
+python ${PW_PARENT_JOB_DIR}/scripts/cpu_and_memory_usage.py --write-usage --txt ${monitoring_txt} &
+monitoring_pid=$!
+echo "kill ${monitoring_pid} | true # kill monitoring" >> cancel.sh
diff --git a/resources/001_simulation_executor/adapt_wtx_paths.py b/scripts/adapt_wtx_paths.py
similarity index 100%
rename from resources/001_simulation_executor/adapt_wtx_paths.py
rename to scripts/adapt_wtx_paths.py
diff --git a/scripts/cpu_and_memory_usage.py b/scripts/cpu_and_memory_usage.py
new file mode 100644
index 0000000..1586bfe
--- /dev/null
+++ b/scripts/cpu_and_memory_usage.py
@@ -0,0 +1,101 @@
+import argparse
+import psutil
+import matplotlib.pyplot as plt
+from datetime import datetime
+import time
+import os
+import subprocess
+
+"""
+This script monitors CPU and memory usage over time and provides options to either write the data to a text file or plot it. 
+If '--write-usage' option is provided, it continuously monitors CPU and memory usage and writes the data to the specified text file.
+If '--plot-usage' option is provided, it reads CPU and memory usage data from the specified text file and plots it over time.
+"""
+
+def kill_job():
+    try:
+        with open("job_id.submitted", "r") as file:
+            job_id = file.readline().strip()
+            if job_id:
+                subprocess.run(["scancel", job_id], check=True)
+                print(f"Job {job_id} cancelled successfully.")
+            else:
+                print("No job ID found in job_id.submitted.")
+    except FileNotFoundError:
+        print("File job_id.submitted not found.")
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to cancel job {job_id}. Error: {e}")
+
+def get_usage():
+    cpu_percent = psutil.cpu_percent()
+    memory_percent = psutil.virtual_memory().percent
+    return cpu_percent, memory_percent
+
+def write_usage_data(txt_file):
+    with open(txt_file, "w") as data_file:
+        print("Monitoring CPU and memory usage...")
+        while True:
+            timestamp = datetime.now()
+            cpu_usage, memory_usage = get_usage()
+            if memory_usage > 98:
+                print('Memory exceeded 98%. Killing job.', flush = True)
+                kill_job()
+
+            # Write data to file
+            data_file.write(f"{timestamp},{cpu_usage},{memory_usage}\n")
+            data_file.flush()
+
+            # Wait for 1 second before collecting next data point
+            time.sleep(1)
+
+
+def plot_usage_data(txt_file):
+    timestamps = []
+    cpu_usages = []
+    memory_usages = []
+
+    try:
+        with open(txt_file, "r") as data_file:
+            print("Reading CPU and memory usage data...")
+            for line in data_file:
+                parts = line.strip().split(",")
+                timestamp = datetime.fromisoformat(parts[0])
+                cpu_usage = float(parts[1])
+                memory_usage = float(parts[2])
+                timestamps.append(timestamp)
+                cpu_usages.append(cpu_usage)
+                memory_usages.append(memory_usage)
+
+        # Plot CPU and memory usage over time
+        plt.figure(figsize=(10, 6))
+        plt.plot(timestamps, cpu_usages, label='CPU Usage (%)')
+        plt.plot(timestamps, memory_usages, label='Memory Usage (%)')
+        plt.xlabel('Time')
+        plt.ylabel('Usage (%)')
+        plt.title('CPU and Memory Usage Over Time')
+        plt.legend()
+        plt.grid(True)
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+
+        # Save plot image
+        img_path = os.path.splitext(txt_file)[0] + ".png"
+        plt.savefig(img_path)
+        print(f"Plot image saved as {img_path}")
+    except FileNotFoundError:
+        print(f"File '{txt_file}' not found.")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="CPU and Memory Usage Monitor and Plotter")
+    parser.add_argument("--write-usage", action="store_true", help="Write CPU and memory usage data to a text file")
+    parser.add_argument("--plot-usage", action="store_true", help="Plot CPU and memory usage data from a text file")
+    parser.add_argument("--txt", type=str, help="Specify the text file to read/write usage data")
+
+    args = parser.parse_args()
+
+    if args.write_usage and args.txt:
+        write_usage_data(args.txt)
+    elif args.plot_usage and args.txt:
+        plot_usage_data(args.txt)
+    else:
+        print("Invalid arguments. Please specify '--write-usage' or '--plot-usage' along with '--txt' option.")
diff --git a/scripts/cpu_and_memory_usage_requirements.yaml b/scripts/cpu_and_memory_usage_requirements.yaml
new file mode 100644
index 0000000..e73c7d8
--- /dev/null
+++ b/scripts/cpu_and_memory_usage_requirements.yaml
@@ -0,0 +1,153 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - alsa-lib=1.2.11=hd590300_1
+  - attr=2.5.1=h166bdaf_1
+  - brotli=1.1.0=hd590300_1
+  - brotli-bin=1.1.0=hd590300_1
+  - bzip2=1.0.8=hd590300_5
+  - ca-certificates=2024.2.2=hbcca054_0
+  - cairo=1.18.0=h3faef2a_0
+  - certifi=2024.2.2=pyhd8ed1ab_0
+  - contourpy=1.2.1=py312h8572e83_0
+  - cycler=0.12.1=pyhd8ed1ab_0
+  - dbus=1.13.6=h5008d03_3
+  - expat=2.6.2=h59595ed_0
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_1
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.51.0=py312h98912ed_0
+  - freetype=2.12.1=h267a509_2
+  - gettext=0.22.5=h59595ed_2
+  - gettext-tools=0.22.5=h59595ed_2
+  - glib=2.80.0=hf2295e7_5
+  - glib-tools=2.80.0=hde27a5a_5
+  - graphite2=1.3.13=h59595ed_1003
+  - gst-plugins-base=1.24.1=hfa15dee_1
+  - gstreamer=1.24.1=h98fc4e7_1
+  - harfbuzz=8.3.0=h3d44ed6_0
+  - icu=73.2=h59595ed_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.5=py312h8572e83_1
+  - krb5=1.21.2=h659d440_0
+  - lame=3.100=h166bdaf_1003
+  - lcms2=2.16=hb7c19ff_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libasprintf=0.22.5=h661eb56_2
+  - libasprintf-devel=0.22.5=h661eb56_2
+  - libblas=3.9.0=22_linux64_openblas
+  - libbrotlicommon=1.1.0=hd590300_1
+  - libbrotlidec=1.1.0=hd590300_1
+  - libbrotlienc=1.1.0=hd590300_1
+  - libcap=2.69=h0f662aa_0
+  - libcblas=3.9.0=22_linux64_openblas
+  - libclang-cpp15=15.0.7=default_h127d8a8_5
+  - libclang13=18.1.3=default_h5d6823c_0
+  - libcups=2.3.3=h4637d8d_4
+  - libdeflate=1.20=hd590300_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libevent=2.1.12=hf998b51_1
+  - libexpat=2.6.2=h59595ed_0
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.3=h59595ed_0
+  - libgcc-ng=13.2.0=h807b86a_5
+  - libgcrypt=1.10.3=hd590300_0
+  - libgettextpo=0.22.5=h59595ed_2
+  - libgettextpo-devel=0.22.5=h59595ed_2
+  - libgfortran-ng=13.2.0=h69a702a_5
+  - libgfortran5=13.2.0=ha4646dd_5
+  - libglib=2.80.0=hf2295e7_5
+  - libgomp=13.2.0=h807b86a_5
+  - libgpg-error=1.48=h71f35ed_0
+  - libiconv=1.17=hd590300_2
+  - libjpeg-turbo=3.0.0=hd590300_1
+  - liblapack=3.9.0=22_linux64_openblas
+  - libllvm15=15.0.7=hb3ce162_4
+  - libllvm18=18.1.3=h2448989_0
+  - libnsl=2.0.1=hd590300_0
+  - libogg=1.3.4=h7f98852_1
+  - libopenblas=0.3.27=pthreads_h413a1c8_0
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.43=h2797004_0
+  - libpq=16.2=h33b98f1_1
+  - libsndfile=1.2.2=hc60ed4a_1
+  - libsqlite=3.45.2=h2797004_0
+  - libstdcxx-ng=13.2.0=h7e041cc_5
+  - libsystemd0=255=h3516f8a_1
+  - libtiff=4.6.0=h1dd3fc0_3
+  - libuuid=2.38.1=h0b41bf4_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libwebp-base=1.4.0=hd590300_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxkbcommon=1.7.0=h662e7e4_0
+  - libxml2=2.12.6=h232c23b_2
+  - libzlib=1.2.13=hd590300_5
+  - lz4-c=1.9.4=hcb278e6_0
+  - matplotlib=3.8.4=py312h7900ff3_0
+  - matplotlib-base=3.8.4=py312he5832f3_0
+  - mpg123=1.32.6=h59595ed_0
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - mysql-common=8.3.0=hf1915f5_4
+  - mysql-libs=8.3.0=hca2cd23_4
+  - ncurses=6.4.20240210=h59595ed_0
+  - nspr=4.35=h27087fc_0
+  - nss=3.98=h1d7d5a4_0
+  - numpy=1.26.4=py312heda63a1_0
+  - openjpeg=2.5.2=h488ebb8_0
+  - openssl=3.2.1=hd590300_1
+  - packaging=24.0=pyhd8ed1ab_0
+  - pcre2=10.43=hcad00b1_0
+  - pillow=10.3.0=py312hdcec9eb_0
+  - pip=24.0=pyhd8ed1ab_0
+  - pixman=0.43.2=h59595ed_0
+  - ply=3.11=pyhd8ed1ab_2
+  - psutil=5.9.8=py312h98912ed_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - pulseaudio-client=17.0=hb77b528_0
+  - pyparsing=3.1.2=pyhd8ed1ab_0
+  - pyqt=5.15.9=py312h949fe66_5
+  - pyqt5-sip=12.12.2=py312h30efb56_5
+  - python=3.12.3=hab00c5b_0_cpython
+  - python-dateutil=2.9.0=pyhd8ed1ab_0
+  - python_abi=3.12=4_cp312
+  - qt-main=5.15.8=hc9dc06e_21
+  - readline=8.2=h8228510_1
+  - setuptools=69.5.1=pyhd8ed1ab_0
+  - sip=6.7.12=py312h30efb56_0
+  - six=1.16.0=pyh6c4a22f_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - toml=0.10.2=pyhd8ed1ab_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - tornado=6.4=py312h98912ed_0
+  - tzdata=2024a=h0c530f3_0
+  - wheel=0.43.0=pyhd8ed1ab_1
+  - xcb-util=0.4.0=hd590300_1
+  - xcb-util-image=0.4.0=h8ee46fc_1
+  - xcb-util-keysyms=0.4.0=h8ee46fc_1
+  - xcb-util-renderutil=0.3.9=hd590300_1
+  - xcb-util-wm=0.4.1=h8ee46fc_1
+  - xkeyboard-config=2.41=hd590300_0
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.2.6=h166bdaf_0
+  - zlib=1.2.13=hd590300_5
+  - zstd=1.5.5=hfc55251_0
\ No newline at end of file
diff --git a/resources/001_simulation_executor/dry_run.sh b/scripts/dry_run.sh
similarity index 100%
rename from resources/001_simulation_executor/dry_run.sh
rename to scripts/dry_run.sh
diff --git a/resources/001_simulation_executor/estimate_end_time.py b/scripts/estimate_end_time.py
similarity index 100%
rename from resources/001_simulation_executor/estimate_end_time.py
rename to scripts/estimate_end_time.py
diff --git a/get_group_allocation_balance.py b/scripts/get_group_allocation_balance.py
similarity index 100%
rename from get_group_allocation_balance.py
rename to scripts/get_group_allocation_balance.py
diff --git a/resources/001_simulation_executor/montecarlo.sh b/scripts/montecarlo.sh
similarity index 100%
rename from resources/001_simulation_executor/montecarlo.sh
rename to scripts/montecarlo.sh
diff --git a/scripts/plot_monitoring.sh b/scripts/plot_monitoring.sh
new file mode 100644
index 0000000..06cda51
--- /dev/null
+++ b/scripts/plot_monitoring.sh
@@ -0,0 +1 @@
+python ${PW_PARENT_JOB_DIR}/scripts/cpu_and_memory_usage.py --plot-usage --txt ${monitoring_txt}
diff --git a/resources/001_simulation_executor/run_dcs.sh b/scripts/run_dcs.sh
similarity index 58%
rename from resources/001_simulation_executor/run_dcs.sh
rename to scripts/run_dcs.sh
index b3aaff3..d429d8a 100644
--- a/resources/001_simulation_executor/run_dcs.sh
+++ b/scripts/run_dcs.sh
@@ -5,7 +5,7 @@
 cat >> metering.sh <<HERE
 #!/bin/bash 
 while true; do
-    date >> ${resource_jobdir}/usage/$(hostname)-${job_number}
+    date >> ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER}
     sleep \$((RANDOM % 30 + 30))
 done
 HERE
@@ -13,14 +13,15 @@ HERE
 chmod +x metering.sh
 ./metering.sh &
 metering_pid=$!
+echo "kill ${metering_pid} || true # Metering" >> cancel.sh
 
 # Run 3dcs
 SECONDS=0
 eval "${dcs_run}"  macroScript.txt
 kill ${metering_pid}
-mv ${resource_jobdir}/usage/$(hostname)-${job_number} ${resource_jobdir}/usage_completed/$(hostname)-${job_number}
+mv ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER} ${PW_PARENT_JOB_DIR}/usage_completed/$(hostname)-${PW_JOB_NUMBER}
 
 # Results is own by root
 sudo chmod 777 Results/ -R
-echo ${SECONDS} > Results/dcs-runtime_${case_index}.txt
+echo ${SECONDS} > Results/dcs-runtime-${case_index}.txt
 
diff --git a/resources/001_simulation_executor/sensitivity.sh b/scripts/sensitivity.sh
similarity index 100%
rename from resources/001_simulation_executor/sensitivity.sh
rename to scripts/sensitivity.sh
diff --git a/resources/001_simulation_executor/transfer_inputs.sh b/scripts/transfer_and_process_inputs.sh
similarity index 94%
rename from resources/001_simulation_executor/transfer_inputs.sh
rename to scripts/transfer_and_process_inputs.sh
index 9022ae0..2428225 100644
--- a/resources/001_simulation_executor/transfer_inputs.sh
+++ b/scripts/transfer_and_process_inputs.sh
@@ -4,8 +4,7 @@
 mkdir tmp-data-transfer
 cd tmp-data-transfer
 
-# dcs_model_directory can end with or without /
-aws s3 sync ${BUCKET_URI}/${dcs_model_directory} . --cli-read-timeout 60
+pw buckets cp -r ${BUCKET_URI}/${dcs_model_directory} .
 
 # User aws s3 cp --recursive ../test ${BUCKET_URI}/path/to/dir/test to transfer to the bucket
 
diff --git a/scripts/transfer_outputs.sh b/scripts/transfer_outputs.sh
new file mode 100644
index 0000000..d41ba6c
--- /dev/null
+++ b/scripts/transfer_outputs.sh
@@ -0,0 +1,5 @@
+
+# Copy path/to/worker_<i> to bucket
+pw buckets cp -r ${PWD} ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/$(basename ${PWD})
+
+

From 81810436c9494a6265b21174908e71665143a6f3 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 14:54:25 +0100
Subject: [PATCH 02/40] testing

---
 honda-japan.yaml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 950c4d1..8888d02 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -44,6 +44,7 @@ jobs:
 
       - name: Transfer and Process Input Files
         run: |
+          set -x
 
           mkdir tmp-data-transfer
           cd tmp-data-transfer
@@ -90,7 +91,7 @@ jobs:
           # Process WTX file to adapt the paths to the files
           retries=5
           while true; do
-              python3 ${{ inputs.rundir }}/scripts/adapt_wtx_paths.py ${dcs_model_file} ${fea_dir}
+              python3 ./scripts/adapt_wtx_paths.py ${dcs_model_file} ${fea_dir}
               exit_code=$?
               if [ ${exit_code} -ne 0 ]; then
                 retries=$((retries-1))
@@ -245,10 +246,10 @@ jobs:
 
 
   job_1:
-    if: ${{ needs.preprocessing.outputs.run_job_1 == "true" }}
-    working-directory: ${{ needs.preprocessing.outputs.job_dir_1 }}
+    if: ${{ needs.remote_preprocessing.outputs.run_job_1 == "true" }}
+    working-directory: ${{ needs.remote_preprocessing.outputs.job_dir_1 }}
     needs:
-      - preprocessing
+      - remote_preprocessing
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
@@ -273,10 +274,10 @@ jobs:
             is_disabled: true
 
   job_2:
-    if: ${{ needs.preprocessing.outputs.run_job_2 == "true" }}
-    working-directory: ${{ needs.preprocessing.outputs.job_dir_2 }}
+    if: ${{ needs.remote_preprocessing.outputs.run_job_2 == "true" }}
+    working-directory: ${{ needs.remote_preprocessing.outputs.job_dir_2 }}
     needs:
-      - preprocessing
+      - remote_preprocessing
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:

From 21e18d9bfaa05a3e1583ae3a72c36e7d5f20b1c9 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:00:10 +0100
Subject: [PATCH 03/40] testing

---
 honda-japan.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 8888d02..283a2a5 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -91,7 +91,7 @@ jobs:
           # Process WTX file to adapt the paths to the files
           retries=5
           while true; do
-              python3 ./scripts/adapt_wtx_paths.py ${dcs_model_file} ${fea_dir}
+              python3 ${PW_PARENT_JOB_DIR}/scripts/adapt_wtx_paths.py ${dcs_model_file} ${fea_dir}
               exit_code=$?
               if [ ${exit_code} -ne 0 ]; then
                 retries=$((retries-1))

From 8e08fc46611e1cd14bb9b432cde244f08faf0503 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:02:55 +0100
Subject: [PATCH 04/40] testing

---
 honda-japan.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 283a2a5..4376ce0 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -27,7 +27,7 @@ jobs:
         uses: parallelworks/checkout
         with:
           repo: https://github.com/parallelworks/dcs-workflow.git
-          branch: main
+          branch: v4
 
       - name: Sanity Checks
         run: |

From 8fa8819a0d718abc39a1530cd559677dc969c363 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:08:35 +0100
Subject: [PATCH 05/40] testing

---
 honda-japan.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 4376ce0..e9c6f43 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -110,6 +110,7 @@ jobs:
           # List all downloaded file
           find . -mindepth 1 > downloaded_files.txt
 
+          cd ..
           mv tmp-data-transfer/*  .
           rmdir tmp-data-transfer
 

From a233af9d25f0ea68937b9d3b4d4b5a9ce8598097 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:15:33 +0100
Subject: [PATCH 06/40] testing

---
 honda-japan.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index e9c6f43..35e19dc 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -130,7 +130,6 @@ jobs:
           export dcs_version="${{ inputs.dcs.version }}"
           export dcs_analysis_type="${{ inputs.dcs.analysis_type }}"
           export dcs_bucket_uri="${{ inputs.dcs.bucket.uri }}"
-          export dcs_bucket_uri="${{ inputs.dcs.bucket.uri }}"
           export dcs_model_directory="${{ inputs.dcs.model_directory }}"
           export dcs_output_directory="${{ inputs.dcs.output_directory }}"
           export dcs_num_seeds="${{ inputs.dcs.num_seeds }}"
@@ -222,7 +221,7 @@ jobs:
               echo "$(date) INFO: Preparing directory for job ${nj}"
               job_dir=job_dir_${nj}
               mkdir -p ${job_dir}
-              echo "${job_dir}=${PWD}/${job_dir}" >> $OUTPUTS
+              echo "${job_dir}=${PWD}/${job_dir}" | tee -a $OUTPUTS
 
               echo '#!/bin/bash' > ${job_dir}/cancel.sh
               chmod +x ${job_dir}/cancel.sh
@@ -235,7 +234,7 @@ jobs:
 
               cat scripts/${dcs_analysis_type}.sh >> ${job_dir}/run_case.sh
               cat scripts/activate_monitoring.sh >> ${job_dir}/run_case.sh
-              cat scrtips/run_dcs.sh >> ${job_dir}/run_case.sh
+              cat scripts/run_dcs.sh >> ${job_dir}/run_case.sh
               cat scripts/plot_monitoring.sh >> ${job_dir}/run_case.sh
               cat scripts/transfer_outputs.sh >> ${job_dir}/run_case.sh
 

From 06a6755d052f05ffcfa1b747f66cb489ad213e8b Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:19:39 +0100
Subject: [PATCH 07/40] testing

---
 honda-japan.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 35e19dc..a78a0ec 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -221,7 +221,6 @@ jobs:
               echo "$(date) INFO: Preparing directory for job ${nj}"
               job_dir=job_dir_${nj}
               mkdir -p ${job_dir}
-              echo "${job_dir}=${PWD}/${job_dir}" | tee -a $OUTPUTS
 
               echo '#!/bin/bash' > ${job_dir}/cancel.sh
               chmod +x ${job_dir}/cancel.sh
@@ -247,7 +246,7 @@ jobs:
 
   job_1:
     if: ${{ needs.remote_preprocessing.outputs.run_job_1 == "true" }}
-    working-directory: ${{ needs.remote_preprocessing.outputs.job_dir_1 }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_1
     needs:
       - remote_preprocessing
     ssh:
@@ -275,7 +274,7 @@ jobs:
 
   job_2:
     if: ${{ needs.remote_preprocessing.outputs.run_job_2 == "true" }}
-    working-directory: ${{ needs.remote_preprocessing.outputs.job_dir_2 }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_2
     needs:
       - remote_preprocessing
     ssh:

From b995e026aa0671dd92d96630f31ff492cf25dec4 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:24:18 +0100
Subject: [PATCH 08/40] testing

---
 honda-japan.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index a78a0ec..f25ecab 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -257,7 +257,7 @@ jobs:
         with:
           resource: ${{ inputs.resource }}
           shebang: '#!/bin/bash'
-          rundir: ${PWD}
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_1
           use_existing_script: true
           script_path: ./run_case.sh
           define_cleanup_script: true
@@ -285,7 +285,7 @@ jobs:
         with:
           resource: ${{ inputs.resource }}
           shebang: '#!/bin/bash'
-          rundir: ${PWD}
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_2
           use_existing_script: true
           script_path: ./run_case.sh
           define_cleanup_script: true

From 0eb563b94b07b621ea7e08c746bc714031bb152c Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:38:56 +0100
Subject: [PATCH 09/40] testing

---
 honda-japan.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index f25ecab..af922a5 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -60,7 +60,7 @@ jobs:
               exit 1
           fi
 
-          echo "dcs_model_file=${dcs_model_file}" >> $OUTPUTS
+          echo "dcs_model_file=${PW_PARENT_JOB_DIR}/${dcs_model_file}" >> $OUTPUTS
 
           # Count the number of files found
           file_count=$(echo "$dcs_model_file" | wc -l)

From fd879ef56d72b520821b57648c76d32f2a6d2251 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 15:41:58 +0100
Subject: [PATCH 10/40] testing

---
 scripts/activate_monitoring.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/activate_monitoring.sh b/scripts/activate_monitoring.sh
index 2624db3..1a4d243 100644
--- a/scripts/activate_monitoring.sh
+++ b/scripts/activate_monitoring.sh
@@ -3,4 +3,4 @@ conda activate ${monitoring_conda_env}
 monitoring_txt="case-${case_index}-${HOSTNAME}-jobid-${SLURM_JOB_ID}.txt"
 python ${PW_PARENT_JOB_DIR}/scripts/cpu_and_memory_usage.py --write-usage --txt ${monitoring_txt} &
 monitoring_pid=$!
-echo "kill ${monitoring_pid} | true # kill monitoring" >> cancel.sh
+echo "kill ${monitoring_pid} || true # kill monitoring" >> cancel.sh

From efa8386ce7b35908348909d30fa52f07291a200e Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 16:05:25 +0100
Subject: [PATCH 11/40] testing

---
 scripts/transfer_outputs.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/transfer_outputs.sh b/scripts/transfer_outputs.sh
index d41ba6c..81d9e2d 100644
--- a/scripts/transfer_outputs.sh
+++ b/scripts/transfer_outputs.sh
@@ -1,5 +1,6 @@
 
 # Copy path/to/worker_<i> to bucket
-pw buckets cp -r ${PWD} ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/$(basename ${PWD})
+set -x
+pw buckets cp -r ${PWD} ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/$(basename ${PWD}) > bucket_upload.log 2>&1
 
 

From 44726ca6384fc0377fefed5a839df698e2ac5d7b Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Fri, 6 Mar 2026 16:27:07 +0100
Subject: [PATCH 12/40] testing

---
 honda-japan.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index af922a5..3858486 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -142,7 +142,9 @@ jobs:
           export monitoring_conda_dir="${{ inputs.monitoring_conda_dir }}"
           export monitoring_conda_env="${{ inputs.monitoring_conda_env }}"
           EOF
-          echo "export PW_PARENT_JOB_DIR=${PW_PARENT_JOB_DIR}"
+          echo "export PW_PARENT_JOB_DIR=${PW_PARENT_JOB_DIR}" >> inputs.sh
+          echo "export PW_WORKFLOW_NAME=${PW_WORKFLOW_NAME}" >> inputs.sh
+          echo "export PW_JOB_NUMBER=${PW_JOB_NUMBER}" >> inputs.sh
 
           echo "$(date) INFO: Inputs file"
           cat inputs.sh

From fbcff8338b05a420d6dbe853afef262d2aea2640 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 11:15:08 +0100
Subject: [PATCH 13/40] Add merge step

---
 honda-japan.yaml                              |  51 ++++++++
 .../cluster_rsync_exec.sh                     | 122 ------------------
 .../002_merge_executor/cluster_rsync_exec.sh  |  62 ---------
 .../002_merge_executor/transfer_inputs.sh     |  27 ----
 .../002_merge_executor/transfer_outputs.sh    |   6 -
 .../merge}/clean_job_directory.sh             |   0
 .../merge}/dry_run.sh                         |   0
 .../merge}/montecarlo.sh                      |   0
 .../merge}/run_dcs.sh                         |   5 +-
 .../merge}/sensitivity.sh                     |   0
 scripts/merge/transfer_inputs.sh              |   3 +
 scripts/merge/transfer_outputs.sh             |   7 +
 12 files changed, 64 insertions(+), 219 deletions(-)
 delete mode 100644 resources/001_simulation_executor/cluster_rsync_exec.sh
 delete mode 100644 resources/002_merge_executor/cluster_rsync_exec.sh
 delete mode 100644 resources/002_merge_executor/transfer_inputs.sh
 delete mode 100644 resources/002_merge_executor/transfer_outputs.sh
 rename {resources/002_merge_executor => scripts/merge}/clean_job_directory.sh (100%)
 rename {resources/002_merge_executor => scripts/merge}/dry_run.sh (100%)
 rename {resources/002_merge_executor => scripts/merge}/montecarlo.sh (100%)
 rename {resources/002_merge_executor => scripts/merge}/run_dcs.sh (64%)
 rename {resources/002_merge_executor => scripts/merge}/sensitivity.sh (100%)
 create mode 100644 scripts/merge/transfer_inputs.sh
 create mode 100644 scripts/merge/transfer_outputs.sh

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 3858486..3e156d0 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -205,6 +205,7 @@ jobs:
           echo > scripts/plot_monitoring.sh
           echo > scripts/activate_monitoring.sh
           mv scripts/dry_run.sh scripts/run_dcs.sh
+          mv scripts/merge/dry_run.sh scripts/merge/dry_run.sh
 
 
       - name: Create Job Scripts
@@ -302,6 +303,56 @@ jobs:
           pbs:
             is_disabled: true
 
+  merge:
+    if: ${{ always }}
+    needs:
+      - job_1
+      - job_2
+    working-directory: ${{ inputs.rundir }}
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - name: Create Merge Script
+        early-cancel: any-job-failed
+        run: |
+          set -x 
+          source inputs.sh
+          echo "$(date) INFO: Preparing merge job"
+          mkdir -p merge
+
+          echo '#!/bin/bash' > merge/cancel.sh
+          chmod +x merge/cancel.sh
+
+          cat inputs.sh > merge/merge.sh
+          cat dcs_environment/${dcs_version}.sh >> merge/merge.sh
+          cat scripts/merge/transfer_inputs.sh >> merge/merge.sh
+          cat scripts/merge/${dcs_analysis_type}.sh >> merge/merge.sh
+          cat scripts/merge/run_dcs.sh >> merge/merge.sh
+          cat scripts/merge/clean_job_directory.sh >> merge/merge.sh
+          cat scripts/merge/transfer_outputs.sh >> merge/merge.sh
+ 
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/merge
+          use_existing_script: true
+          script_path: ./merge.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+
 'on':
   execute:
     inputs:
diff --git a/resources/001_simulation_executor/cluster_rsync_exec.sh b/resources/001_simulation_executor/cluster_rsync_exec.sh
deleted file mode 100644
index 87f82f7..0000000
--- a/resources/001_simulation_executor/cluster_rsync_exec.sh
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/bin/bash
-cd $(dirname $0)
-
-source inputs.sh
-source workflow-libs.sh
-
-source dcs_environment/${dcs_version}.sh 
-sudo chown -R ${USER}:pwuser ${WINEPREFIX}
-
-# Prepare for metering
-mkdir ${resource_jobdir}/usage
-mkdir ${resource_jobdir}/usage_completed
-
-echo '#!/bin/bash' > cancel.sh
-chmod +x cancel.sh
-
-create_case(){
-    # The merge tasks syncs the results from an S3 bucket. To simplify the path to the
-    # results in the S3 bucket ww use the resource job dir
-    case_dir=${resource_jobdir}/worker_${case_index}
-    mkdir -p ${case_dir}
-
-    echo "    Writing job script"
-    cp batch_header.sh  ${case_dir}/run_case.sh
-
-    if [[ ${jobschedulertype} == "SLURM" ]]; then 
-        echo "#SBATCH -o ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh
-        echo "#SBATCH -e ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh
-    elif [[ ${jobschedulertype} == "PBS" ]]; then
-        echo "#PBS -o ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh
-        echo "#PBS -e ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh
-    fi
-    
-    # Main script
-    echo "mkdir -p ${case_dir}" >> ${case_dir}/run_case.sh
-    echo "cd ${case_dir}" >> ${case_dir}/run_case.sh
-    
-    # FIXME: This is needed because run directory is not shared between controller and compute nodes
-    #echo "rsync -avzq ${resource_privateIp}:${case_dir}/ ."  >> ${case_dir}/run_case.sh
-
-    # Main script
-    echo >> ${case_dir}/run_case.sh
-    cat inputs.sh >> ${case_dir}/run_case.sh
-    cat dcs_environment/${dcs_version}.sh >> ${case_dir}/run_case.sh
-    echo "export case_index=${case_index}" >> ${case_dir}/run_case.sh
-    echo "export dcs_model_file=${dcs_model_file}" >> ${case_dir}/run_case.sh
-    echo >> ${case_dir}/run_case.sh
-
-    cat ${dcs_analysis_type}.sh >> ${case_dir}/run_case.sh
-    cat activate_monitoring.sh >> ${case_dir}/run_case.sh
-    cat run_dcs.sh >> ${case_dir}/run_case.sh
-    cat plot_monitoring.sh >> ${case_dir}/run_case.sh
-    echo "source ${resource_jobdir}/bucket_credentials" >> ${case_dir}/run_case.sh
-    cat transfer_outputs.sh >> ${case_dir}/run_case.sh
-
-}
-
-cat_slurm_logs() {
-    for f in  $(find ${resource_jobdir} -name logs_*.out); do
-	    echo; echo "Contents of ${f}:"
-	    cat ${f}
-    done
-	      
-}
-
-
-echo; echo; echo "STARTING INPUT DATA TRANSFER"
-source ${resource_jobdir}/bucket_credentials
-source transfer_inputs.sh
-
-if [[ ${dcs_dry_run} == "true" ]]; then
-    echo "RUNNING THE WORKFLOW IN DRY RUN MODE"
-    unset monitoring_conda_dir monitoring_conda_env
-    echo > activate_monitoring.sh
-    echo > plot_monitoring.sh
-    echo > activate_monitoring.sh
-    mv dry_run.sh run_dcs.sh
-else
-    rm dry_run.sh
-fi
-
-# If no conda environment is specified for the CPU and Mem python monitoring utility
-# the workflow assumes monitoring is disabled
-if [ -z "${monitoring_conda_dir}" ] || [ -z "${monitoring_conda_env}" ]; then
-    echo "CPU and Memory monitoring are disabled"
-    echo > activate_monitoring.sh
-    echo > plot_monitoring.sh
-else
-    echo; echo; echo "INSTALLING PYTHON DEPENDENCIES FOR CPU AND MEMORY MONITORING"
-    create_conda_env_from_yaml ${monitoring_conda_dir} ${monitoring_conda_env} ./cpu_and_memory_usage_requirements.yaml
-fi
-
-echo; echo; echo "CREATING JOB SCRIPTS"
-for case_index in $(seq 1 ${dcs_concurrency}); do
-    echo; echo "  Case ${case_index}"
-    create_case
-done
-
-echo; echo; echo "SUBMITTING JOB SCRIPTS"
-for case_index in $(seq 1 ${dcs_concurrency}); do
-    case_dir=${resource_jobdir}/worker_${case_index}
-    echo; echo "  Case ${case_index}"
-    cp ${resource_jobdir}/${dcs_model_file} ${case_dir}
-
-    submit_job_sh=${case_dir}/run_case.sh
-    echo "  Job script ${submit_job_sh}"
-
-    if [[ ${jobschedulertype} == "SLURM" ]]; then 
-        job_id=$(${submit_cmd} ${submit_job_sh} | tail -1 | awk -F ' ' '{print $4}')
-    elif [[ ${jobschedulertype} == "PBS" ]]; then
-        job_id=$(${submit_cmd} ${submit_job_sh} | tail -1)
-    fi
-    
-    if [ -z "${job_id}" ]; then
-        echo "  ERROR: ${submit_cmd} ${submit_job_sh} failed"
-        exit 1
-    else
-        echo "  Submitted job ${job_id}"
-        echo "${cancel_cmd} ${job_id}" >> ${resource_jobdir}/${resource_label}/cancel.sh
-        echo ${job_id} > ${case_dir}/job_id.submitted
-    fi
-done
\ No newline at end of file
diff --git a/resources/002_merge_executor/cluster_rsync_exec.sh b/resources/002_merge_executor/cluster_rsync_exec.sh
deleted file mode 100644
index 038d050..0000000
--- a/resources/002_merge_executor/cluster_rsync_exec.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-cd $(dirname $0)
-
-source inputs.sh
-source workflow-libs.sh
-
-if [[ ${dcs_dry_run} == "true" ]]; then
-    echo "RUNNING THE WORKFLOW IN DRY RUN MODE"
-    mv dry_run.sh run_dcs.sh 
-else
-    rm dry_run.sh
-fi
-
-echo; echo; echo "MERGING RESULTS"
-pwd
-echo "    Writing job script"
-cp batch_header.sh merge.sh
-
-if [[ ${jobschedulertype} == "SLURM" ]]; then 
-    echo "#SBATCH -o ${PWD}/${case_dir}/logs_merge.out" >> merge.sh
-    echo "#SBATCH -e ${PWD}/${case_dir}/logs_merge.out" >> merge.sh
-elif [[ ${jobschedulertype} == "PBS" ]]; then
-    echo "#PBS -o ${PWD}/${case_dir}/logs_merge.out" >> merge.sh
-    echo "#PBS -e ${PWD}/${case_dir}/logs_merge.out" >> merge.sh
-fi
-    
-# FIXME: This is needed because run directory is not shared between controller and compute nodes
-#echo "rsync -avzq ${resource_privateIp}:${PWD}/ ."  >> merge.sh
-
-
-# Main script
-cat inputs.sh >> merge.sh
-cat dcs_environment/${dcs_version}.sh >> merge.sh
-echo "source ${resource_jobdir}/bucket_credentials" >> merge.sh
-cat transfer_inputs.sh >> merge.sh
-cat ${dcs_analysis_type}.sh >> merge.sh
-cat run_dcs.sh >> merge.sh
-echo "source ${resource_jobdir}/bucket_credentials" >> merge.sh
-cat clean_job_directory.sh >> merge.sh
-cat transfer_outputs.sh >> merge.sh
-
-# FIXME: This is needed because run directory is not shared between controller and compute nodes
-#echo "rsync -avzq . ${resource_privateIp}:${PWD}/"  >> merge.sh
-
-echo; echo; echo "SUBMITTING MERGE JOB"
-submit_job_sh=./merge.sh
-echo "  Job script ${submit_job_sh}"
-
-if [[ ${jobschedulertype} == "SLURM" ]]; then 
-    jobid=$(${submit_cmd} ${submit_job_sh} | tail -1 | awk -F ' ' '{print $4}')
-elif [[ ${jobschedulertype} == "PBS" ]]; then
-    jobid=$(${submit_cmd} ${submit_job_sh} | tail -1)
-fi
-    
-if [ -z "${jobid}" ]; then
-    echo "  ERROR: ${submit_cmd} ${submit_job_sh} failed"
-    cat ${PWD}/${case_dir}/logs_merge.out
-    exit 1
-fi
-
-echo ${jobid} > ${resource_jobdir}/job_id.submitted
-echo "${cancel_cmd} ${jobid}" >> ${resource_jobdir}/${resource_label}/cancel.sh
\ No newline at end of file
diff --git a/resources/002_merge_executor/transfer_inputs.sh b/resources/002_merge_executor/transfer_inputs.sh
deleted file mode 100644
index 5e59b4c..0000000
--- a/resources/002_merge_executor/transfer_inputs.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-# HARDCODED TO AWS
-# A dynamicStorage parameter type would be very helpful for this
-
-# Transfer model
-# dcs_model_directory can end with or without /
-aws s3 sync ${BUCKET_URI}/${dcs_model_directory} .
-aws s3 sync ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number} .
-
-# Find all files ending in ".wtx" in the current directory, excluding subdirectories
-dcs_model_file=$(find . -maxdepth 1 -type f -name "*.wtx")
-
-# Check if dcs_model_file is empty
-if [ -z "$dcs_model_file" ]; then
-    echo "Error: No '.wtx' files found."
-    exit 1
-fi
-
-# Count the number of files found
-file_count=$(echo "$dcs_model_file" | wc -l)
-
-# Check if only one file ending in ".wtx" is found
-if [ "$file_count" -eq 1 ]; then
-    echo "Found file ${dcs_model_file}"
-else
-    echo "Error: Found $file_count '.wtx' files. Expected only one."
-    exit 1
-fi
diff --git a/resources/002_merge_executor/transfer_outputs.sh b/resources/002_merge_executor/transfer_outputs.sh
deleted file mode 100644
index 4ab73ab..0000000
--- a/resources/002_merge_executor/transfer_outputs.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-# HARDCODED TO AWS
-# A dynamicStorage parameter type would be very helpful for this
-
-aws s3 cp ${resource_label}/merge.sh ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/merge.sh
-aws s3 cp --recursive Results ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/Results
-aws s3 cp --recursive TempData ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/TempData
diff --git a/resources/002_merge_executor/clean_job_directory.sh b/scripts/merge/clean_job_directory.sh
similarity index 100%
rename from resources/002_merge_executor/clean_job_directory.sh
rename to scripts/merge/clean_job_directory.sh
diff --git a/resources/002_merge_executor/dry_run.sh b/scripts/merge/dry_run.sh
similarity index 100%
rename from resources/002_merge_executor/dry_run.sh
rename to scripts/merge/dry_run.sh
diff --git a/resources/002_merge_executor/montecarlo.sh b/scripts/merge/montecarlo.sh
similarity index 100%
rename from resources/002_merge_executor/montecarlo.sh
rename to scripts/merge/montecarlo.sh
diff --git a/resources/002_merge_executor/run_dcs.sh b/scripts/merge/run_dcs.sh
similarity index 64%
rename from resources/002_merge_executor/run_dcs.sh
rename to scripts/merge/run_dcs.sh
index d23ebb6..7e27552 100644
--- a/resources/002_merge_executor/run_dcs.sh
+++ b/scripts/merge/run_dcs.sh
@@ -6,7 +6,7 @@
 cat >> metering.sh <<HERE
 #!/bin/bash
 while true; do
-    date >> ${resource_jobdir}/usage/$(hostname)-${job_number}-merge
+    date >> ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER}-merge
     sleep \$((RANDOM % 30 + 30))
 done
 HERE
@@ -14,12 +14,13 @@ HERE
 chmod +x metering.sh
 ./metering.sh &
 metering_pid=$!
+echo "kill ${metering_pid} || true # Metering" >> cancel.sh
 
 # Run 3dcs
 SECONDS=0
 eval "${dcs_run}"  macroScript.txt
 kill ${metering_pid}
-mv ${resource_jobdir}/usage/$(hostname)-${job_number}-merge ${resource_jobdir}/usage_completed/$(hostname)-${job_number}-merge
+mv ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER}-merge ${PW_PARENT_JOB_DIR}/usage_completed/$(hostname)-${PW_JOB_NUMBER}-merge
 
 
 # Results is own by root
diff --git a/resources/002_merge_executor/sensitivity.sh b/scripts/merge/sensitivity.sh
similarity index 100%
rename from resources/002_merge_executor/sensitivity.sh
rename to scripts/merge/sensitivity.sh
diff --git a/scripts/merge/transfer_inputs.sh b/scripts/merge/transfer_inputs.sh
new file mode 100644
index 0000000..9f75e0e
--- /dev/null
+++ b/scripts/merge/transfer_inputs.sh
@@ -0,0 +1,3 @@
+
+
+pw buckets cp -r ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER} .  > bucket_download.log 2>&1
diff --git a/scripts/merge/transfer_outputs.sh b/scripts/merge/transfer_outputs.sh
new file mode 100644
index 0000000..2df1a8f
--- /dev/null
+++ b/scripts/merge/transfer_outputs.sh
@@ -0,0 +1,7 @@
+
+pw buckets cp merge.sh ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/merge.sh > bucket_upload.log 2>&1
+echo >> bucket_upload.log 
+pw buckets cp -r Results ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/Results >> bucket_upload.log 2>&1
+echo >> bucket_upload.log
+pw buckets cp -r TempData ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/TempData >> bucket_upload.log 2>&1
+

From f069d4afc6727344edbd278a0a4e64eeb793126d Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 11:35:46 +0100
Subject: [PATCH 14/40] Add merge step

---
 honda-japan.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 3e156d0..58481be 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -331,7 +331,6 @@ jobs:
           cat scripts/merge/clean_job_directory.sh >> merge/merge.sh
           cat scripts/merge/transfer_outputs.sh >> merge/merge.sh
  
-    steps:
       - uses: marketplace/script_submitter/v3.5
         early-cancel: any-job-failed
         with:

From 679f73028d7d143c845b895097c27d8d313b04a9 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 11:52:16 +0100
Subject: [PATCH 15/40] Add streaming

---
 honda-japan.yaml         | 17 +++++++++++++++++
 scripts/merge/run_dcs.sh |  7 +++++++
 scripts/run_dcs.sh       |  7 +++++++
 3 files changed, 31 insertions(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 58481be..15175d7 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -3,6 +3,23 @@ permissions:
 env:
   MAX_JOBS: '2'
 jobs:
+
+  stream_logs:
+    steps:
+
+      - name: Sanity Checks
+        run: |
+          source /etc/profile.d/parallelworks.sh
+          source /etc/profile.d/parallelworks-env.sh
+          source /pw/.miniconda3/etc/profile.d/conda.sh
+
+          python3 scripts/get_group_allocation_balance.py ${{ inputs.run_hours_3dcs_group }} ${{ inputs.org_name }}
+          if [ $? -ne 0 ]; then
+              echo "$(date) ERROR: No 3DCS balance is available. Exiting workflow." >&2
+              exit 1
+          fi
+
+
   local_preprocessing:
     steps:
 
diff --git a/scripts/merge/run_dcs.sh b/scripts/merge/run_dcs.sh
index 7e27552..630ab45 100644
--- a/scripts/merge/run_dcs.sh
+++ b/scripts/merge/run_dcs.sh
@@ -16,6 +16,13 @@ chmod +x metering.sh
 metering_pid=$!
 echo "kill ${metering_pid} || true # Metering" >> cancel.sh
 
+# Stream logs
+log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt"
+touch ${log_path}
+tail -f ${log_path} &
+tail_pid=$!
+echop "kill ${tail_pid} || true # Tail logs" >> cancel.sh
+
 # Run 3dcs
 SECONDS=0
 eval "${dcs_run}"  macroScript.txt
diff --git a/scripts/run_dcs.sh b/scripts/run_dcs.sh
index d429d8a..28f9657 100644
--- a/scripts/run_dcs.sh
+++ b/scripts/run_dcs.sh
@@ -15,6 +15,13 @@ chmod +x metering.sh
 metering_pid=$!
 echo "kill ${metering_pid} || true # Metering" >> cancel.sh
 
+# Stream logs
+log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt"
+touch ${log_path}
+tail -f ${log_path} &
+tail_pid=$!
+echop "kill ${tail_pid} || true # Tail logs" >> cancel.sh
+
 # Run 3dcs
 SECONDS=0
 eval "${dcs_run}"  macroScript.txt

From 1e8ebfc678ff6c4eb1fe8d026165ca29092f1aab Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 12:11:53 +0100
Subject: [PATCH 16/40] Add endtime estimation

---
 honda-japan.yaml | 77 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 61 insertions(+), 16 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 15175d7..4fbae8c 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -4,22 +4,6 @@ env:
   MAX_JOBS: '2'
 jobs:
 
-  stream_logs:
-    steps:
-
-      - name: Sanity Checks
-        run: |
-          source /etc/profile.d/parallelworks.sh
-          source /etc/profile.d/parallelworks-env.sh
-          source /pw/.miniconda3/etc/profile.d/conda.sh
-
-          python3 scripts/get_group_allocation_balance.py ${{ inputs.run_hours_3dcs_group }} ${{ inputs.org_name }}
-          if [ $? -ne 0 ]; then
-              echo "$(date) ERROR: No 3DCS balance is available. Exiting workflow." >&2
-              exit 1
-          fi
-
-
   local_preprocessing:
     steps:
 
@@ -250,6 +234,8 @@ jobs:
 
               cat dcs_environment/${dcs_version}.sh >> ${job_dir}/run_case.sh
               echo >> ${job_dir}/run_case.sh
+              echo "touch job.started" >> ${job_dir}/run_case.sh
+              echo >> ${job_dir}/run_case.sh
 
               cat scripts/${dcs_analysis_type}.sh >> ${job_dir}/run_case.sh
               cat scripts/activate_monitoring.sh >> ${job_dir}/run_case.sh
@@ -264,6 +250,60 @@ jobs:
           done
 
 
+  estimate_end_time:
+    needs:
+      - remote_preprocessing
+    working-directory: ${{ inputs.rundir }}
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - name: Estimate End Time
+        run: |
+
+          # Source the inputs file
+          source inputs.sh
+
+          if [ ${dcs_thread} -eq 1 ]; then
+              echo "$(date) INFO: Number of threads must be larger than 1 to enable estimates"
+              echo "$(date) INFO: Exiting job..."
+          fi 
+
+          log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt"
+
+          wait_for_all_simulations_to_start() {
+              while true; do
+                  n_running_workers=$(ls -d job_dir_*/job.started | wc -l)
+                  if [ $? -ne 0 ]; then
+                      n_running_workers=0
+                  fi
+                  if [ "${n_running_workers}" -lt "${dcs_concurrency}" ]; then
+                      echo "$(date) INFO: ${n_running_workers}/${dcs_concurrency} simulations started"
+                      echo "$(date) INFO: Waiting for all simulations to start..."
+                      sleep 15
+                  else
+                      break
+                  fi
+              done
+          }
+
+          wait_for_all_simulations_to_start
+
+          echo "$(date) INFO: All simulations are started!"
+          echo; echo
+          echo "$(date) INFO: Calculating completion time estimates"
+
+          log_files=$(ls -d ./job_dir_*/${log_path})
+          log_files=$(echo ${log_files} | tr ' ' ',')
+
+          echo "$(date) INFO: Log files:"
+          echo ${log_files} | tr ',' '\n'
+
+          while true; do
+              echo; echo; echo "$(date)"
+              python3 scripts/estimate_end_time.py ${log_files}"
+              sleep 120
+          done
+
   job_1:
     if: ${{ needs.remote_preprocessing.outputs.run_job_1 == "true" }}
     working-directory: ${PW_PARENT_JOB_DIR}/job_dir_1
@@ -329,6 +369,11 @@ jobs:
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
+      - name: Cancel End Time Estimation
+        uses: parallelworks/cancel-jobs
+        with:
+          jobs:
+            - estimate_end_time
       - name: Create Merge Script
         early-cancel: any-job-failed
         run: |

From ed17911ca7bb52d79030a3c2b92a41a6daef045e Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 12:25:23 +0100
Subject: [PATCH 17/40] Add endtime estimation

---
 honda-japan.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 4fbae8c..eb96b17 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -300,9 +300,10 @@ jobs:
 
           while true; do
               echo; echo; echo "$(date)"
-              python3 scripts/estimate_end_time.py ${log_files}"
+              python3 scripts/estimate_end_time.py ${log_files}
               sleep 120
           done
+          exit 0
 
   job_1:
     if: ${{ needs.remote_preprocessing.outputs.run_job_1 == "true" }}

From d507210a93e6135fcf4bd3e5879feb578d988192 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 12:42:44 +0100
Subject: [PATCH 18/40] Only run merge if more than one job was submitted

---
 honda-japan.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index eb96b17..f7c03f9 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -380,6 +380,15 @@ jobs:
         run: |
           set -x 
           source inputs.sh
+
+          # Only run merge if more than one job was submitted
+          if [ ${dcs_concurrency} == 1 ]; then
+            echo "$(date) INFO: 3DCS concurrency is 1. Exiting workflow."
+            echo "run_merge=false" | tee -a $OUTPUTS
+            exit 0
+          fi
+          echo "run_merge=true" | tee -a $OUTPUTS
+
           echo "$(date) INFO: Preparing merge job"
           mkdir -p merge
 
@@ -395,6 +404,7 @@ jobs:
           cat scripts/merge/transfer_outputs.sh >> merge/merge.sh
  
       - uses: marketplace/script_submitter/v3.5
+        if: ${{ needs.merge.outputs.run_merge == "true" }}
         early-cancel: any-job-failed
         with:
           resource: ${{ inputs.resource }}

From 000f82d1a21a9402722f6811442b6593147a9e82 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 13:02:36 +0100
Subject: [PATCH 19/40] testing

---
 scripts/merge/run_dcs.sh | 2 +-
 scripts/run_dcs.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/merge/run_dcs.sh b/scripts/merge/run_dcs.sh
index 630ab45..c609dd7 100644
--- a/scripts/merge/run_dcs.sh
+++ b/scripts/merge/run_dcs.sh
@@ -21,7 +21,7 @@ log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').t
 touch ${log_path}
 tail -f ${log_path} &
 tail_pid=$!
-echop "kill ${tail_pid} || true # Tail logs" >> cancel.sh
+echo "kill ${tail_pid} || true # Tail logs" >> cancel.sh
 
 # Run 3dcs
 SECONDS=0
diff --git a/scripts/run_dcs.sh b/scripts/run_dcs.sh
index 28f9657..2704946 100644
--- a/scripts/run_dcs.sh
+++ b/scripts/run_dcs.sh
@@ -20,7 +20,7 @@ log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').t
 touch ${log_path}
 tail -f ${log_path} &
 tail_pid=$!
-echop "kill ${tail_pid} || true # Tail logs" >> cancel.sh
+echo "kill ${tail_pid} || true # Tail logs" >> cancel.sh
 
 # Run 3dcs
 SECONDS=0

From 38a90ec161f1cfb1df69da2e237451434cc34516 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 14:35:35 +0100
Subject: [PATCH 20/40] Add usage metering

---
 honda-japan.yaml             | 30 ++++++++----
 metering.sh                  |  7 ---
 reload_bucket_credentials.sh | 15 ------
 stream.sh                    | 88 ------------------------------------
 windows-aws-credentials.yaml | 18 --------
 5 files changed, 22 insertions(+), 136 deletions(-)
 delete mode 100755 metering.sh
 delete mode 100755 reload_bucket_credentials.sh
 delete mode 100755 stream.sh
 delete mode 100644 windows-aws-credentials.yaml

diff --git a/honda-japan.yaml b/honda-japan.yaml
index f7c03f9..256bd3d 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -19,7 +19,22 @@ jobs:
               exit 1
           fi
 
+  usage_metering:
+    needs:
+      - local_preprocessing
+    steps:
+      #!/bin/bash
+      mkdir -p usage
+      while true; do
+          sleep 60
+          echo "$(date) INFO: Transferring usage data" 
+          rsync -avz --delete ${{ inputs.resource.ip }}:${{ inputs.rundir }}/usage/ usage
+          rsync -avz ${PWD}/usage/ ${{ inputs.metering_user }}@${{ inputs.metering_ip }}:~/.3dcs/usage-pending
+      done
+
   remote_preprocessing:
+    needs:
+      - local_preprocessing
     working-directory: ${{ inputs.rundir }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
@@ -292,16 +307,15 @@ jobs:
           echo; echo
           echo "$(date) INFO: Calculating completion time estimates"
 
-          log_files=$(ls -d ./job_dir_*/${log_path})
-          log_files=$(echo ${log_files} | tr ' ' ',')
+          while true; do
+            log_files=$(ls -d ./job_dir_*/${log_path})
+            log_files=$(echo ${log_files} | tr ' ' ',')
 
-          echo "$(date) INFO: Log files:"
-          echo ${log_files} | tr ',' '\n'
+            echo "$(date) INFO: Log files:"
+            echo ${log_files} | tr ',' '\n'
 
-          while true; do
-              echo; echo; echo "$(date)"
-              python3 scripts/estimate_end_time.py ${log_files}
-              sleep 120
+            python3 scripts/estimate_end_time.py ${log_files}
+            sleep 120
           done
           exit 0
 
diff --git a/metering.sh b/metering.sh
deleted file mode 100755
index aa26f66..0000000
--- a/metering.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-mkdir -p usage
-while true; do
-    sleep 60
-    rsync -avz --delete ${resource_publicIp}:${resource_jobdir}/usage/ usage >> metering.out 2>&1
-    rsync -avz ${resource_jobdir}/usage/ ${metering_user}@${metering_ip}:~/.3dcs/usage-pending >> metering.out 2>&1
-done
\ No newline at end of file
diff --git a/reload_bucket_credentials.sh b/reload_bucket_credentials.sh
deleted file mode 100755
index 265f945..0000000
--- a/reload_bucket_credentials.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-source resources/001_simulation_executor/inputs.sh
-set -x
-while true; do
-    sleep 300
-    date
-    pw buckets get-token pw://${dcs_bucket_id} > bucket_credentials
-    source bucket_credentials
-    # Check if BUCKET_NAME is empty
-    if ! [ -n "${BUCKET_URI}" ]; then
-        echo "ERROR: Unable to load bucket credentials!"
-        exit 1
-    fi
-    scp bucket_credentials ${resource_publicIp}:${resource_jobdir}/bucket_credentials
-done
diff --git a/stream.sh b/stream.sh
deleted file mode 100755
index 8e87a10..0000000
--- a/stream.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-while [ ! -f "SUBMITTED" ]; do
-    echo "$(date '+%Y-%m-%d %H:%M:%S') - Waiting for simulations to be submitted"
-    sleep 10
-done
-
-# Source the inputs file
-source resources/001_simulation_executor/inputs.sh
-
-export sshcmd="ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=3 ${resource_publicIp}"
-
-log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt"
-
-wait_for_all_simulations_to_start() {
-    while true; do
-        n_running_workers=$(${sshcmd} ls -d ${resource_jobdir}/worker_*/${log_path} | wc -l)
-        if [ $? -ne 0 ]; then
-            n_running_workers=0
-        fi
-        if [ "${n_running_workers}" -lt "${dcs_concurrency}" ]; then
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - ${n_running_workers}/${dcs_concurrency} simulations started"
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - Waiting for all simulations to start..."
-            sleep 15
-        else
-            break
-        fi
-        if [ -f "COMPLETED" ]; then
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - Simulations are completed"
-            exit 0
-        fi
-    done
-}
-
-wait_for_all_simulations_to_start
-
-echo "$(date '+%Y-%m-%d %H:%M:%S') - All simulations are started!"
-echo; echo
-echo "$(date '+%Y-%m-%d %H:%M:%S') - Initiating streaming"
-
-stream_logs() {
-    local max_retries=5
-    local retry_delay=10
-    local attempt=1
-
-    while [ $attempt -le $max_retries ]; do
-        if [ -f "COMPLETED" ]; then
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - COMPLETED file detected, stopping log streaming"
-            return 0
-        fi
-        echo "$(date '+%Y-%m-%d %H:%M:%S') - Attempt $attempt of $max_retries to stream logs"
-        ${sshcmd} "tail -f ${resource_jobdir}/worker_*/${log_path}"
-        local exit_code=$?
-        if [ $exit_code -eq 0 ]; then
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - Streaming completed successfully"
-            return 0
-        else
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - SSH tail failed with exit code $exit_code. Retrying in $retry_delay seconds..."
-            sleep $retry_delay
-            attempt=$((attempt + 1))
-        fi
-    done
-
-    if [ $attempt -gt $max_retries ]; then
-        if [ -f "COMPLETED" ]; then
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - COMPLETED file detected, ignoring retry limit"
-            return 0
-        else
-            echo "$(date '+%Y-%m-%d %H:%M:%S') - Failed to stream logs after $max_retries attempts"
-            return 1
-        fi
-    fi
-}
-
-# Start log streaming in the background
-echo "$(date '+%Y-%m-%d %H:%M:%S') - Starting background log streaming"
-stream_logs &
-stream_pid=$!
-echo "kill ${stream_pid}" > cancel_stream.sh
-
-# Wait for either COMPLETED file or stream_logs to exit
-while [ ! -f "COMPLETED" ]; do
-    # Check if the stream_logs process is still running
-    if ! ps -p $stream_pid > /dev/null; then
-        echo "$(date '+%Y-%m-%d %H:%M:%S') - Stream logs process (PID: $stream_pid) has exited"
-        break
-    fi
-    sleep 10
-done
\ No newline at end of file
diff --git a/windows-aws-credentials.yaml b/windows-aws-credentials.yaml
deleted file mode 100644
index c5525b9..0000000
--- a/windows-aws-credentials.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-permissions:
-  - '*'
-jobs:
-  main:
-    steps:
-      - name: Generate Windows S3 Credentials
-        run: |
-          source /etc/profile.d/parallelworks.sh
-          source /etc/profile.d/parallelworks-env.sh
-          source /pw/.miniconda3/etc/profile.d/conda.sh
-          python3 bucket_token_generator.py --bucket_id ${{ inputs.bucket_id }} --token_format text --platform windows
-'on':
-  execute:
-    inputs:
-      bucket_id:
-        label: Bucket ID or namespace
-        type: string
-        tooltip: Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name]

From 4d610d9b45aeac8e6b185b3353530c97053d1612 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 14:49:38 +0100
Subject: [PATCH 21/40] Add usage metering

---
 honda-japan.yaml | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 256bd3d..9ba729c 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -23,14 +23,16 @@ jobs:
     needs:
       - local_preprocessing
     steps:
-      #!/bin/bash
-      mkdir -p usage
-      while true; do
-          sleep 60
-          echo "$(date) INFO: Transferring usage data" 
-          rsync -avz --delete ${{ inputs.resource.ip }}:${{ inputs.rundir }}/usage/ usage
-          rsync -avz ${PWD}/usage/ ${{ inputs.metering_user }}@${{ inputs.metering_ip }}:~/.3dcs/usage-pending
-      done
+      - name: Usage Metering
+        run: |
+          #!/bin/bash
+          mkdir -p usage
+          while true; do
+              sleep 60
+              echo "$(date) INFO: Transferring usage data" 
+              rsync -avz --delete ${{ inputs.resource.ip }}:${{ inputs.rundir }}/usage/ usage
+              rsync -avz ${PWD}/usage/ ${{ inputs.metering_user }}@${{ inputs.metering_ip }}:~/.3dcs/usage-pending
+          done
 
   remote_preprocessing:
     needs:

From 3a3ca0ecca16d49437c65199d77783723d0bf227 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 15:19:34 +0100
Subject: [PATCH 22/40] Add usage metering

---
 honda-japan.yaml | 515 ++++++++++++++++++++++++++++++++++++++++++++++-
 workflow.xml     | 201 ------------------
 workflow.yaml    | 172 ----------------
 3 files changed, 513 insertions(+), 375 deletions(-)
 delete mode 100644 workflow.xml
 delete mode 100644 workflow.yaml

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 9ba729c..7c70e93 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -1,7 +1,7 @@
 permissions:
   - '*'
 env:
-  MAX_JOBS: '2'
+  MAX_JOBS: '20'
 jobs:
 
   local_preprocessing:
@@ -377,6 +377,511 @@ jobs:
           pbs:
             is_disabled: true
 
+  job_3:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_3 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_3
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_3
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_4:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_4 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_4
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_4
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_5:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_5 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_5
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_5
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_6:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_6 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_6
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_6
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_7:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_7 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_7
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_7
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_8:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_8 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_8
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_8
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_9:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_9 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_9
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_9
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_10:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_10 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_10
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_10
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_11:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_11 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_11
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_11
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_12:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_12 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_12
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_12
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_13:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_13 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_13
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_13
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_14:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_14 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_14
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_14
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_15:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_15 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_15
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_15
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_16:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_16 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_16
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_16
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_17:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_17 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_17
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_17
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_18:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_18 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_18
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_18
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_19:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_19 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_19
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_19
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+  job_20:
+    if: ${{ needs.remote_preprocessing.outputs.run_job_20 == "true" }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_20
+    needs:
+      - remote_preprocessing
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - uses: marketplace/script_submitter/v3.5
+        early-cancel: any-job-failed
+        with:
+          resource: ${{ inputs.resource }}
+          shebang: '#!/bin/bash'
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_20
+          use_existing_script: true
+          script_path: ./run_case.sh
+          define_cleanup_script: true
+          cleanup_script_path: ./cancel.sh
+          scheduler: true
+          use_scheduler_agent: false
+          slurm:
+            is_disabled: false
+            partition: ${{ inputs.slurm.partition }}
+            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
+            time: ${{ inputs.slurm.time }}
+          pbs:
+            is_disabled: true
+
+
   merge:
     if: ${{ always }}
     needs:
@@ -440,6 +945,12 @@ jobs:
           pbs:
             is_disabled: true
 
+      - name: Cancel Usage Metering
+        uses: parallelworks/cancel-jobs
+        with:
+          jobs:
+            - usage_metering
+
 
 'on':
   execute:
@@ -535,7 +1046,7 @@ jobs:
             label: Number of Workers
             type: number
             min: 1
-            max: 50
+            max: 20
             default: 1
             tooltip: Number of workers used to run the simulations. A SLURM job is submitted for each worker.
           thread:
diff --git a/workflow.xml b/workflow.xml
deleted file mode 100644
index a44814a..0000000
--- a/workflow.xml
+++ /dev/null
@@ -1,201 +0,0 @@
-<tool id='NA' name='NA'>
-  <command interpreter='bash'>main.sh</command>
-  <cancel interpreter='bash'>cancel.sh</cancel>
-  <inputs>
-    <param 
-        name='metering_user' 
-        label='metering_user' 
-        type='hidden' 
-        value='__metering_user__'  
-    ></param>
-    <param 
-        name='metering_ip' 
-        label='metering_ip' 
-        type='hidden' 
-        value='34.132.102.65'  
-    ></param>
-<section name='dcs' type='section' title='3DCS Options' expanded='true'>
-        <param 
-            name="dry_run" 
-            type="boolean" 
-            truevalue="Yes" 
-            falsevalue="No" 
-            checked="False" 
-            value="false"
-            label="Dry run?" 
-            help='Dry run generates the macroScript.txt file and executes every step of the workflow except running 3DCS'  
-        ></param>
-        <param 
-            name='version' 
-            type='select' 
-            label='3DCS Version' 
-            help='Select the 3DCS version' 
-            multiple='false'
-        >
-            <option value="8.0.0.2"  selected="true">8.0.0.2</option>
-            <option value="7.10.0.2" >7.10.0.2</option>
-        </param>
-        <param 
-            name='analysis_type' 
-            type='select' 
-            label='Analysis Type' 
-            help='Select Montecarlo simulation or contributor analysis (sensitivity)' 
-            multiple='false'
-        >
-            <option value="montecarlo"  selected="true">Monte Carlo Simulation</option>
-            <option value="sensitivity" >Contributor Analysis</option>
-        </param>
-        <param name='bucket_id' 
-            label='Bucket ID or namespace' 
-            type='text' 
-            help='Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name]' 
-        ></param>
-        <param 
-            name='model_directory' 
-            label='Model Path' 
-            type='text' 
-            help='Directory within the bucket with the model and all the required files' 
-        ></param>
-        <param 
-            name='output_directory' 
-            label='Output Path' 
-            type='text' 
-            help='Output directory within the bucket to store outputs and logs. It cannot be inside the Model Directory!' 
-        ></param>
-        <param 
-            name='num_seeds' 
-            type='integer' 
-            label='Number of Monte Carlo Simulations' 
-            min='1' 
-            max='100000' 
-            value='2000'
-            depends_on='dcs.analysis_type'
-            show_if="montecarlo"
-        ></param>
-        <param 
-            name='concurrency' 
-            type='integer' 
-            label='Number of Workers' 
-            min='1' 
-            max='100' 
-            value='2' 
-            help='Number of workers used to run the simulations. A SLURM job is submitted for each worker.' 
-        ></param>
-        <param 
-            name='thread' 
-            type='integer' 
-            label='Number of Threads per Worker' 
-            min='1' 
-            max='64' 
-            value='1' 
-            help='Number of threads used to run the simulation. Be aware that N threads require more than N times the memory needed for a single thread.'
-        ></param>
-    </section>
-    <section name='pwrl_001_simulation_executor' type='section' title='Simulation Executor' expanded='true'>
-        <param 
-            name='monitoring_conda_dir' 
-            label='PW Conda Directory' 
-            type='hidden' 
-            value='/dcs/pw/miniconda'
-        ></param>
-        <param 
-            name='monitoring_conda_env' 
-            label='PW Conda Directory' 
-            type='hidden' 
-            value='psutil'
-        ></param>
-        <param 
-            name='resource' 
-            type='computeResource' 
-            label='Resource' 
-            hideUserWorkspace='true' 
-            help='Resource to run the simulation task'>
-        </param>
-        <param 
-            name='jobschedulertype' 
-            label='Select Controller, SLURM Partition or PBS Queue' 
-            type='hidden' 
-            value='SLURM' 
-        >
-        </param>
-       <param
-           name='_sch__dd_partition_e_'
-           label='SLURM partition'
-           type='dynamicPartitionDropdown'
-           resource='pwrl_001_simulation_executor.resource'
-           help='Partition to submit the interactive job. Leave empty to let SLURM pick the optimal option.'
-           optional='true'  
-           dependent='false'
-       ></param>
-        <param 
-            name='_sch__dd_time_e_' 
-            label='Walltime' 
-            type='text' 
-            help='Maximum walltime per job' 
-            value='999:00:00'
-            optional='true' 
-        ></param>
-        <param 
-            name='scheduler_directives' 
-            label='Scheduler directives' 
-            type='text' 
-            help='e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.' 
-            value='--exclusive'
-            optional='true'  
-        ></param>
-    </section>
-    <section name='pwrl_002_merge_executor' type='section' title='Merge Executor' expanded='true'>
-        <param 
-            name='monitoring_conda_dir' 
-            label='PW Conda Directory' 
-            type='hidden' 
-            value='/dcs/pw/miniconda'
-        ></param>
-        <param 
-            name='monitoring_conda_env' 
-            label='PW Conda Directory' 
-            type='hidden' 
-            value='psutil'
-        ></param>
-        <param 
-            name='resource' 
-            type='computeResource' 
-            label='Resource' 
-            hideUserWorkspace='true' 
-            help='Resource to run the simulation task'>
-        </param>
-        <param 
-            name='jobschedulertype' 
-            label='Select Controller, SLURM Partition or PBS Queue' 
-            type='hidden' 
-            value='SLURM' 
-        >
-        </param>
-        <param
-           name='_sch__dd_partition_e_'
-           label='SLURM partition'
-           type='dynamicPartitionDropdown'
-           resource='pwrl_002_merge_executor.resource'
-           help='Partition to submit the interactive job. Leave empty to let SLURM pick the optimal option.'
-           optional='true'  
-           dependent='false'
-        ></param>
-        <param 
-            name='_sch__dd_time_e_' 
-            label='Walltime' 
-            type='text' 
-            help='Maximum walltime per job' 
-            value='999:00:00'
-            optional='true' 
-        ></param>
-        <param 
-            name='scheduler_directives' 
-            label='Scheduler directives' 
-            type='text' 
-            help='e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.' 
-            value='--exclusive'
-            optional='true'  
-        ></param>
-    </section>
-    </inputs>
-</tool>
diff --git a/workflow.yaml b/workflow.yaml
deleted file mode 100644
index 55434b5..0000000
--- a/workflow.yaml
+++ /dev/null
@@ -1,172 +0,0 @@
-permissions:
-  - '*'
-jobs:
-  main:
-    steps:
-      - name: Main
-        run: bash main.sh
-        cleanup: |
-          bash cancel.sh  || true
-          exit 0
-'on':
-  execute:
-    inputs:
-      metering_user:
-        label: Metering User
-        type: string
-        default: __metering_user__
-        hidden: true
-      metering_ip:
-        label: Metering IP
-        type: string
-        default: 34.132.102.65
-        hidden: true
-      dcs:
-        type: group
-        label: 3DCS Options
-        items:
-          dry_run:
-            label: Dry run?
-            type: boolean
-            default: false
-            tooltip: Dry run generates the macroScript.txt file and executes every step of the workflow except running 3DCS
-          version:
-            label: 3DCS Version
-            type: dropdown
-            default: 8.0.0.2
-            tooltip: Select the 3DCS version
-            options:
-              - value: 8.0.0.2
-                label: 8.0.0.2
-              - value: 7.10.0.2
-                label: 7.10.0.2
-          analysis_type:
-            label: Analysis Type
-            type: dropdown
-            tooltip: Select Montecarlo simulation or contributor analysis (sensitivity)
-            default: montecarlo
-            options:
-              - value: montecarlo
-                label: Monte Carlo Simulation
-              - value: sensitivity
-                label: Contributor Analysis
-          bucket_id:
-            label: Bucket ID or namespace
-            type: string
-            tooltip: Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name]
-          model_directory:
-            label: Model Path
-            type: string
-            tooltip: Directory within the bucket with the model and all the required files
-          output_directory:
-            label: Output Path
-            type: string
-            tooltip: Output directory within the bucket to store outputs and logs. It cannot be inside the Model Directory!
-          num_seeds:
-            label: Number of Monte Carlo Simulations
-            type: number
-            min: 1
-            max: 100000
-            default: 2000
-            hidden: ${{ 'montecarlo' !=inputs.dcs.analysis_type }}
-            ignore: ${{ .hidden }}
-            optional: ${{ .hidden }}
-          concurrency:
-            label: Number of Workers
-            type: number
-            min: 1
-            max: 50
-            default: 1
-            tooltip: Number of workers used to run the simulations. A SLURM job is submitted for each worker.
-          thread:
-            label: Number of Threads per Worker
-            type: number
-            min: 1
-            max: 64
-            default: 1
-            tooltip: Number of threads used to run the simulation. Be aware that N threads require more than N times the memory needed for a single thread.
-
-      pwrl_001_simulation_executor:
-        type: group
-        label: Simulation Executor
-        items:
-          monitoring_conda_dir:
-            label: PW Conda Directory
-            type: string
-            default: /dcs/pw/miniconda
-            hidden: true
-          monitoring_conda_env:
-            label: PW Conda Environment
-            type: string
-            default: psutil
-            hidden: true
-          resource:
-            label: Resource
-            type: compute-clusters
-            tooltip: Resource to run the simulation task
-            include-workspace: false
-          jobschedulertype:
-            label: Select Controller, SLURM Partition or PBS Queue
-            type: string
-            default: SLURM
-            hidden: true
-          _sch__dd_partition_e_:
-            type: slurm-partitions
-            label: SLURM partition
-            hidden: ${{ 'SLURM' !=inputs.pwrl_001_simulation_executor.jobschedulertype }}
-            ignore: ${{ .hidden }}
-            optional: true
-            tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option.
-            resource: ${{ inputs.pwrl_001_simulation_executor.resource }}
-          _sch__dd_time_e_:
-            label: Walltime
-            type: string
-            default: '999:00:00'
-            tooltip: Maximum walltime per job
-          scheduler_directives:
-            label: Scheduler directives
-            type: string
-            default: '--exclusive'
-            tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.
-      pwrl_002_merge_executor:
-        type: group
-        label: Merge Executor
-        items:
-          monitoring_conda_dir:
-            label: PW Conda Directory
-            type: string
-            default: /dcs/pw/miniconda
-            hidden: true
-          monitoring_conda_env:
-            label: PW Conda Environment
-            type: string
-            default: psutil
-            hidden: true
-          resource:
-            label: Resource
-            type: compute-clusters
-            tooltip: Resource to run the simulation task
-            include-workspace: false
-          jobschedulertype:
-            label: Select Controller, SLURM Partition or PBS Queue
-            type: string
-            default: SLURM
-            hidden: true
-          _sch__dd_partition_e_:
-            type: slurm-partitions
-            label: SLURM partition
-            hidden: ${{ 'SLURM' !=inputs.pwrl_002_merge_executor.jobschedulertype }}
-            ignore: ${{ .hidden }}
-            optional: true
-            tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option.
-            resource: ${{ inputs.pwrl_002_merge_executor.resource }}
-          _sch__dd_time_e_:
-            label: Walltime
-            type: string
-            default: '999:00:00'
-            tooltip: Maximum walltime per job
-          scheduler_directives:
-            label: Scheduler directives
-            type: string
-            default: '--exclusive'
-            tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.

From b207a1d06ca9f2e9f304427bde2abc58802700f8 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 9 Mar 2026 15:21:54 +0100
Subject: [PATCH 23/40] Add usage metering

---
 honda-japan.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 7c70e93..286a487 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -24,6 +24,7 @@ jobs:
       - local_preprocessing
     steps:
       - name: Usage Metering
+        early-cancel: any-job-failed
         run: |
           #!/bin/bash
           mkdir -p usage

From d2113c1f5e75371a4a82cac902cd1c1009f8048f Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 10 Mar 2026 14:35:31 +0100
Subject: [PATCH 24/40] Increase SSH Session Limit

---
 honda-japan.yaml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 286a487..7416c40 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -48,6 +48,29 @@ jobs:
           repo: https://github.com/parallelworks/dcs-workflow.git
           branch: v4
 
+      - name: Increase SSH Session Limit
+        run: |
+          #!/bin/bash
+          set -euo pipefail
+
+          SSHD_CONFIG="/etc/ssh/sshd_config"
+          MAX_SESSIONS=50
+          MAX_STARTUPS="50:30:200"
+
+          sudo cp "$SSHD_CONFIG" "${SSHD_CONFIG}.bak.$(date +%Y%m%d%H%M%S)"
+          sudo sed -i "s/^#\?MaxSessions.*/MaxSessions ${MAX_SESSIONS}/" "$SSHD_CONFIG"
+          sudo sed -i "s/^#\?MaxStartups.*/MaxStartups ${MAX_STARTUPS}/" "$SSHD_CONFIG"
+
+          sudo sshd -t || {
+              echo "sshd config validation failed. Restoring backup." >&2
+              sudo cp "${SSHD_CONFIG}.bak."* "$SSHD_CONFIG"
+              exit 1
+          }
+          sudo systemctl reload sshd
+
+          echo "SSH limits updated:"
+          sudo grep -E "^MaxSessions|^MaxStartups" "$SSHD_CONFIG"
+
       - name: Sanity Checks
         run: |
           if [[ "${{ inputs.dcs.output_directory }}" == "${{ inputs.dcs.model_directory }}" || "${{ inputs.dcs.output_directory }}" == "${{ inputs.dcs.model_directory }}/"* ]]; then

From 476d8b60270efaea655112f187a74c8e9896c597 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 10 Mar 2026 14:42:09 +0100
Subject: [PATCH 25/40] Add 20 jobs to merge dependency

---
 honda-japan.yaml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 7416c40..5b241fa 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -911,6 +911,25 @@ jobs:
     needs:
       - job_1
       - job_2
+      - job_3
+      - job_4
+      - job_5
+      - job_6
+      - job_7
+      - job_8
+      - job_9
+      - job_10
+      - job_11
+      - job_12
+      - job_13
+      - job_14
+      - job_15
+      - job_16
+      - job_17
+      - job_18
+      - job_19
+      - job_20
+
     working-directory: ${{ inputs.rundir }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}

From 962705b20902a037193a31b848c33b6f7ddd65b2 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 10 Mar 2026 15:17:44 +0100
Subject: [PATCH 26/40] Add 20 jobs to merge dependency

---
 honda-japan.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 5b241fa..d5cd19e 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -53,6 +53,13 @@ jobs:
           #!/bin/bash
           set -euo pipefail
 
+          if [ -f "${HOME}/.ssh/sessions_increased" ]; then
+            echo "$(date) INFO: SSH session limit already increased"
+            exit 0
+          fi
+          touch "${HOME}/.ssh/sessions_increased"
+          echo "$(date) INFO: Increasing SSH session limit"
+
           SSHD_CONFIG="/etc/ssh/sshd_config"
           MAX_SESSIONS=50
           MAX_STARTUPS="50:30:200"

From ba67c2660e549019ee359a8cd66a4487387916f6 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 16 Mar 2026 10:26:54 +0100
Subject: [PATCH 27/40] Use matrix feature

---
 honda-japan.yaml | 667 ++++-------------------------------------------
 1 file changed, 52 insertions(+), 615 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index d5cd19e..66e47b9 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -48,35 +48,6 @@ jobs:
           repo: https://github.com/parallelworks/dcs-workflow.git
           branch: v4
 
-      - name: Increase SSH Session Limit
-        run: |
-          #!/bin/bash
-          set -euo pipefail
-
-          if [ -f "${HOME}/.ssh/sessions_increased" ]; then
-            echo "$(date) INFO: SSH session limit already increased"
-            exit 0
-          fi
-          touch "${HOME}/.ssh/sessions_increased"
-          echo "$(date) INFO: Increasing SSH session limit"
-
-          SSHD_CONFIG="/etc/ssh/sshd_config"
-          MAX_SESSIONS=50
-          MAX_STARTUPS="50:30:200"
-
-          sudo cp "$SSHD_CONFIG" "${SSHD_CONFIG}.bak.$(date +%Y%m%d%H%M%S)"
-          sudo sed -i "s/^#\?MaxSessions.*/MaxSessions ${MAX_SESSIONS}/" "$SSHD_CONFIG"
-          sudo sed -i "s/^#\?MaxStartups.*/MaxStartups ${MAX_STARTUPS}/" "$SSHD_CONFIG"
-
-          sudo sshd -t || {
-              echo "sshd config validation failed. Restoring backup." >&2
-              sudo cp "${SSHD_CONFIG}.bak."* "$SSHD_CONFIG"
-              exit 1
-          }
-          sudo systemctl reload sshd
-
-          echo "SSH limits updated:"
-          sudo grep -E "^MaxSessions|^MaxStartups" "$SSHD_CONFIG"
 
       - name: Sanity Checks
         run: |
@@ -268,33 +239,27 @@ jobs:
             exit 1
           fi
 
-          for nj in $(seq 1 ${MAX_JOBS}); do
-            if [ ${nj} -le ${dcs_concurrency} ]; then
-              echo "$(date) INFO: Preparing directory for job ${nj}"
-              job_dir=job_dir_${nj}
-              mkdir -p ${job_dir}
-
-              echo '#!/bin/bash' > ${job_dir}/cancel.sh
-              chmod +x ${job_dir}/cancel.sh
-
-              cat inputs.sh >> ${job_dir}/run_case.sh
-              echo "export case_index=${nj}" >> ${job_dir}/run_case.sh
-
-              cat dcs_environment/${dcs_version}.sh >> ${job_dir}/run_case.sh
-              echo >> ${job_dir}/run_case.sh
-              echo "touch job.started" >> ${job_dir}/run_case.sh
-              echo >> ${job_dir}/run_case.sh
-
-              cat scripts/${dcs_analysis_type}.sh >> ${job_dir}/run_case.sh
-              cat scripts/activate_monitoring.sh >> ${job_dir}/run_case.sh
-              cat scripts/run_dcs.sh >> ${job_dir}/run_case.sh
-              cat scripts/plot_monitoring.sh >> ${job_dir}/run_case.sh
-              cat scripts/transfer_outputs.sh >> ${job_dir}/run_case.sh
-
-              echo "run_job_${nj}=true" | tee -a $OUTPUTS
-            else
-              echo "run_job_${nj}=false" | tee -a $OUTPUTS
-            fi
+          for nj in $(seq 1 ${dcs_concurrency}); do
+            echo "$(date) INFO: Preparing directory for job ${nj}"
+            job_dir=job_dir_${nj}
+            mkdir -p ${job_dir}
+
+            echo '#!/bin/bash' > ${job_dir}/cancel.sh
+            chmod +x ${job_dir}/cancel.sh
+
+            cat inputs.sh >> ${job_dir}/run_case.sh
+            echo "export case_index=${nj}" >> ${job_dir}/run_case.sh
+
+            cat dcs_environment/${dcs_version}.sh >> ${job_dir}/run_case.sh
+            echo >> ${job_dir}/run_case.sh
+            echo "touch job.started" >> ${job_dir}/run_case.sh
+            echo >> ${job_dir}/run_case.sh
+
+            cat scripts/${dcs_analysis_type}.sh >> ${job_dir}/run_case.sh
+            cat scripts/activate_monitoring.sh >> ${job_dir}/run_case.sh
+            cat scripts/run_dcs.sh >> ${job_dir}/run_case.sh
+            cat scripts/plot_monitoring.sh >> ${job_dir}/run_case.sh
+            cat scripts/transfer_outputs.sh >> ${job_dir}/run_case.sh
           done
 
 
@@ -352,39 +317,36 @@ jobs:
           done
           exit 0
 
-  job_1:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_1 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_1
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_1
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_2:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_2 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_2
+  workers:
     needs:
       - remote_preprocessing
+    strategy:
+      fail-fast: true
+      max-parallel: ${{ inputs.dcs.concurrency }}
+      matrix:
+        job_id:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          - 11
+          - 12
+          - 13
+          - 14
+          - 15
+          - 16
+          - 17
+          - 18
+          - 19
+          - 20
+    if: ${{ matrix.job_id <= inputs.dcs.concurrency }}
+    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_${{ matrix.job_id }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
@@ -393,7 +355,7 @@ jobs:
         with:
           resource: ${{ inputs.resource }}
           shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_2
+          rundir: ${PW_PARENT_JOB_DIR}/job_dir_${{ matrix.job_id }}
           use_existing_script: true
           script_path: ./run_case.sh
           define_cleanup_script: true
@@ -408,535 +370,10 @@ jobs:
           pbs:
             is_disabled: true
 
-  job_3:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_3 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_3
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_3
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_4:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_4 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_4
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_4
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_5:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_5 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_5
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_5
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_6:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_6 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_6
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_6
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_7:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_7 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_7
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_7
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_8:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_8 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_8
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_8
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_9:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_9 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_9
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_9
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_10:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_10 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_10
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_10
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_11:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_11 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_11
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_11
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_12:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_12 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_12
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_12
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_13:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_13 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_13
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_13
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_14:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_14 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_14
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_14
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_15:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_15 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_15
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_15
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_16:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_16 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_16
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_16
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_17:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_17 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_17
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_17
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_18:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_18 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_18
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_18
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_19:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_19 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_19
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_19
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-  job_20:
-    if: ${{ needs.remote_preprocessing.outputs.run_job_20 == "true" }}
-    working-directory: ${PW_PARENT_JOB_DIR}/job_dir_20
-    needs:
-      - remote_preprocessing
-    ssh:
-      remoteHost: ${{ inputs.resource.ip }}
-    steps:
-      - uses: marketplace/script_submitter/v3.5
-        early-cancel: any-job-failed
-        with:
-          resource: ${{ inputs.resource }}
-          shebang: '#!/bin/bash'
-          rundir: ${PW_PARENT_JOB_DIR}/job_dir_20
-          use_existing_script: true
-          script_path: ./run_case.sh
-          define_cleanup_script: true
-          cleanup_script_path: ./cancel.sh
-          scheduler: true
-          use_scheduler_agent: false
-          slurm:
-            is_disabled: false
-            partition: ${{ inputs.slurm.partition }}
-            scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
-            time: ${{ inputs.slurm.time }}
-          pbs:
-            is_disabled: true
-
-
   merge:
     if: ${{ always }}
     needs:
-      - job_1
-      - job_2
-      - job_3
-      - job_4
-      - job_5
-      - job_6
-      - job_7
-      - job_8
-      - job_9
-      - job_10
-      - job_11
-      - job_12
-      - job_13
-      - job_14
-      - job_15
-      - job_16
-      - job_17
-      - job_18
-      - job_19
-      - job_20
-
+      - workers
     working-directory: ${{ inputs.rundir }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
@@ -1132,4 +569,4 @@ jobs:
             tooltip: |
               Type in additional scheduler directives.
             default: |
-              #SBATCH --exclusive
+              #SBATCH --exclusive
\ No newline at end of file

From 197518427d1526bf2f8c9d81baea2cc006561925 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 16 Mar 2026 11:44:38 +0100
Subject: [PATCH 28/40] Fix issue with paths to FEA directory

---
 README.md                              | 16 ++++++++--------
 scripts/adapt_wtx_paths.py             | 24 ++++++++----------------
 scripts/transfer_and_process_inputs.sh | 17 +----------------
 3 files changed, 17 insertions(+), 40 deletions(-)

diff --git a/README.md b/README.md
index b3d42a9..8001d6d 100644
--- a/README.md
+++ b/README.md
@@ -1,23 +1,22 @@
 ## 3DCS Parallel Workflow
-This workflow enables running a 3DCS Monte Carlo or contributor analysis job using multiple workers on a SLURM cluster in the cloud. Here’s how to use it:
+This workflow enables running a 3DCS Monte Carlo or contributor analysis job using multiple workers on a SLURM cluster in the cloud. Here's how to use it:
 
 ### 1. Upload Files
-- Upload your model’s files to a cloud bucket on the platform (see this [link](https://parallelworks.com/docs/storage/transferring-data/obtaining-credentials)).
+- Upload your model's files to a cloud bucket on the platform (see this [link](https://parallelworks.com/docs/storage/transferring-data/obtaining-credentials)).
 
 ### 2. Run the Job
-- Fill in the workflow’s input form.
+- Fill in the workflow's input form.
 - Click the execute button.
 - For parameter descriptions, hover over the help (?) icon next to each parameter's name.
 
 ### 3. Job Execution
-- The job is divided into sub-jobs based on the selected number of workers.
-- Each worker uses the specified number of threads.
+- Before running, the workflow checks that your group has sufficient 3DCS allocation balance.
+- The job is divided into sub-jobs based on the selected number of workers (up to 20).
+- Each worker is submitted as an individual SLURM job and uses the specified number of threads.
 - The simulation runs in the resource defined in the simulation executor section of the input form.
 
 ### 4. Node Utilization
-- The "max workers per node" parameter sets the maximum number of workers per node.
-- To reduce compute costs, estimate the required memory for a single worker and maximize memory utilization by fitting as many workers as possible on a single node.
-- Required memory for N workers is N times the memory for a single worker. However, the required memory for N threads is more than N times the memory for a single thread.
+- The `#SBATCH --exclusive` flag is set by default in the SLURM directives due to issues running multiple 3DCS workers on a single node. This means each worker occupies a full node.
 
 ![Sample Configuration](https://raw.githubusercontent.com/parallelworks/dcs-workflow/main/2-Nodes_4-Workers_2-Threads_Per_Worker_Configuration.png)
 
@@ -33,3 +32,4 @@ This workflow enables running a 3DCS Monte Carlo or contributor analysis job usi
 - 3DCS usage is calculated based on the total number of hours a node uses any number of 3DCS workers.
 - For example, if 2 nodes run a 3DCS job for 10 hours with 3 workers per node and 4 threads per worker, the usage is 2 nodes x 10 hours = 20 hours.
 - To minimize 3DCS license usage, fit as many workers as possible on a single node.
+- Usage data is continuously transferred to the metering server during the run.
diff --git a/scripts/adapt_wtx_paths.py b/scripts/adapt_wtx_paths.py
index b686555..f7c6a75 100644
--- a/scripts/adapt_wtx_paths.py
+++ b/scripts/adapt_wtx_paths.py
@@ -2,7 +2,6 @@
 import sys
 
 wtx_file_path = sys.argv[1]
-fea_dir = sys.argv[2].replace('/', '')
 
 def replace_between_angle_brackets(input_string, replacement):
     """
@@ -27,14 +26,6 @@ def write_lines_to_file(lines, file_path, encoding='utf-8'):
     with open(file_path, 'w', encoding=encoding) as file:
         file.writelines(lines)
 
-def get_search_file_paths(file_paths):
-    """
-    Create search patterns for the file paths.
-
-    :param file_paths: List of file paths.
-    :return: List of search patterns for the file paths.
-    """
-    return [os.path.basename(file_path) for file_path in file_paths]
 
 def read_file_with_encodings(file_path, encodings=['utf-8', 'shift_jis', 'euc-jp', 'iso-2022-jp', 'latin-1']):
     """
@@ -53,20 +44,22 @@ def read_file_with_encodings(file_path, encodings=['utf-8', 'shift_jis', 'euc-jp
             continue
     raise UnicodeDecodeError(f"Unable to decode file {file_path} with any of the provided encodings.")
 
-def process_wtx_file(wtx_file_path, search_file_paths):
+def process_wtx_file(wtx_file_path, file_paths):
     """
     Process the WTX file, replacing text between angle brackets.
 
     :param wtx_file_path: Path to the WTX file.
-    :param search_file_paths: List of search patterns for file paths.
+    :param file_paths: List of relative file paths found in the current directory.
     :return: List of processed lines from the WTX file.
     """
     new_wtx_file_lines = []
     lines, encoding = read_file_with_encodings(wtx_file_path)
     for line in lines:
-        for file_path in search_file_paths:
-            if file_path in line:
-                line = replace_between_angle_brackets(line, '..\\' + fea_dir + '\\' + file_path)
+        for file_path in file_paths:
+            basename = os.path.basename(file_path)
+            if basename in line:
+                win_path = '..\\'  + file_path.replace('/', '\\')
+                line = replace_between_angle_brackets(line, win_path)
         new_wtx_file_lines.append(line)
     return new_wtx_file_lines, encoding
 
@@ -84,9 +77,8 @@ def main():
     wtx_file_path = sys.argv[1]
 
     file_paths = find_all_files_os()
-    search_file_paths = get_search_file_paths(file_paths)
 
-    new_wtx_file_lines, encoding = process_wtx_file(wtx_file_path, search_file_paths)
+    new_wtx_file_lines, encoding = process_wtx_file(wtx_file_path, file_paths)
     write_lines_to_file(new_wtx_file_lines, wtx_file_path, encoding=encoding)
 
 if __name__ == '__main__':
diff --git a/scripts/transfer_and_process_inputs.sh b/scripts/transfer_and_process_inputs.sh
index 2428225..627ac6f 100644
--- a/scripts/transfer_and_process_inputs.sh
+++ b/scripts/transfer_and_process_inputs.sh
@@ -29,25 +29,10 @@ else
 fi
 
 
-# Get the count of directories in the current directory
-dir_count=$(ls -d */ | wc -l)
-
-# If there's only one directory
-if [ "$dir_count" -eq 1 ]; then
-    # Get the name of the directory
-    fea_dir=$(ls -d */ | head -n 1)
-    export fea_dir=${fea_dir%/}  # Remove trailing slash
-    echo "Directory found: $fea_dir"
-else
-    # If no directory or multiple directories exist
-    echo "Error: Either no FEA directory found or multiple directories exist." >&2
-    exit 1
-fi
-
 # Process WTX file to adapt the paths to the files
 retries=5
 while true; do
-    python3 ../adapt_wtx_paths.py ${dcs_model_file} ${fea_dir}
+    python3 ../adapt_wtx_paths.py ${dcs_model_file}
     exit_code=$?
     if [ ${exit_code} -ne 0 ]; then
 	    retries=$((retries-1))

From c29736e379f12dcdc7ced2fba6861f05a3ebe1ea Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Thu, 19 Mar 2026 11:01:03 +0100
Subject: [PATCH 29/40] Testing

---
 honda-japan.yaml | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 66e47b9..e48a51e 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -136,9 +136,10 @@ jobs:
 
       - name: General Preprocessing
         run: |
-
+          set -x
           source dcs_environment/${dcs_version}.sh 
           sudo chown -R ${USER}:pwuser ${WINEPREFIX}
+          ls -lat ${WINEPREFIX}
 
           mkdir ./usage
           mkdir ./usage_completed
@@ -350,7 +351,7 @@ jobs:
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
-      - uses: marketplace/script_submitter/v3.5
+      - uses: marketplace/script_submitter/v3.6
         early-cancel: any-job-failed
         with:
           resource: ${{ inputs.resource }}
@@ -363,12 +364,12 @@ jobs:
           scheduler: true
           use_scheduler_agent: false
           slurm:
-            is_disabled: false
+            is_enabled: true
             partition: ${{ inputs.slurm.partition }}
             scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
             time: ${{ inputs.slurm.time }}
           pbs:
-            is_disabled: true
+            is_enabled: false
 
   merge:
     if: ${{ always }}
@@ -425,12 +426,12 @@ jobs:
           scheduler: true
           use_scheduler_agent: false
           slurm:
-            is_disabled: false
+            is_enabled: true
             partition: ${{ inputs.slurm.partition }}
             scheduler_directives: ${{ inputs.slurm.scheduler_directives }}
             time: ${{ inputs.slurm.time }}
           pbs:
-            is_disabled: true
+            is_enabled: false
 
       - name: Cancel Usage Metering
         uses: parallelworks/cancel-jobs
@@ -547,11 +548,6 @@ jobs:
         type: group
         label: SLURM Directives
         items:
-          is_disabled:
-            type: boolean
-            hidden: true
-            default: false
-            label: Is SLURM disabled?
           partition:
             type: slurm-partitions
             label: SLURM partition

From a65be41fd7ffbba510e396c3ae8869bd99b8314c Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Thu, 19 Mar 2026 11:10:59 +0100
Subject: [PATCH 30/40] Testing

---
 honda-japan.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index e48a51e..0466a07 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -138,8 +138,9 @@ jobs:
         run: |
           set -x
           source dcs_environment/${dcs_version}.sh 
+          ls -latd ${WINEPREFIX}
           sudo chown -R ${USER}:pwuser ${WINEPREFIX}
-          ls -lat ${WINEPREFIX}
+          ls -latd ${WINEPREFIX}
 
           mkdir ./usage
           mkdir ./usage_completed

From c661334102e5953918162f3ff6d7b5a86b95197b Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Thu, 19 Mar 2026 11:14:50 +0100
Subject: [PATCH 31/40] Testing

---
 honda-japan.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 0466a07..2a92d8f 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -137,7 +137,7 @@ jobs:
       - name: General Preprocessing
         run: |
           set -x
-          source dcs_environment/${dcs_version}.sh 
+          source dcs_environment/${{ inputs.dcs.version }}.sh 
           ls -latd ${WINEPREFIX}
           sudo chown -R ${USER}:pwuser ${WINEPREFIX}
           ls -latd ${WINEPREFIX}

From 8f23a8710b4d160967e360d6a075b77802ce523e Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Thu, 19 Mar 2026 11:40:09 +0100
Subject: [PATCH 32/40] Testing metering

---
 honda-japan.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 2a92d8f..a6eaef4 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -31,8 +31,10 @@ jobs:
           while true; do
               sleep 60
               echo "$(date) INFO: Transferring usage data" 
+              set -x
               rsync -avz --delete ${{ inputs.resource.ip }}:${{ inputs.rundir }}/usage/ usage
               rsync -avz ${PWD}/usage/ ${{ inputs.metering_user }}@${{ inputs.metering_ip }}:~/.3dcs/usage-pending
+              set +x
           done
 
   remote_preprocessing:

From b015858f16f0ff3cc14a42b649dd70126971eadd Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Thu, 19 Mar 2026 11:44:51 +0100
Subject: [PATCH 33/40] Testing metering

---
 honda-japan.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index a6eaef4..978773d 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -32,7 +32,7 @@ jobs:
               sleep 60
               echo "$(date) INFO: Transferring usage data" 
               set -x
-              rsync -avz --delete ${{ inputs.resource.ip }}:${{ inputs.rundir }}/usage/ usage
+              rsync -avz --delete ${{ inputs.resource.ip }}:${PW_PARENT_JOB_DIR}/usage/ usage
               rsync -avz ${PWD}/usage/ ${{ inputs.metering_user }}@${{ inputs.metering_ip }}:~/.3dcs/usage-pending
               set +x
           done
@@ -40,7 +40,6 @@ jobs:
   remote_preprocessing:
     needs:
       - local_preprocessing
-    working-directory: ${{ inputs.rundir }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
@@ -270,7 +269,6 @@ jobs:
   estimate_end_time:
     needs:
       - remote_preprocessing
-    working-directory: ${{ inputs.rundir }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
@@ -378,7 +376,6 @@ jobs:
     if: ${{ always }}
     needs:
       - workers
-    working-directory: ${{ inputs.rundir }}
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:

From 122eb0405e05bc69bee9996ef4f6a6301153ec2b Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 24 Mar 2026 10:51:02 +0100
Subject: [PATCH 34/40] Support 40 workers

---
 honda-japan.yaml | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 978773d..4fcab22 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -1,7 +1,7 @@
 permissions:
   - '*'
 env:
-  MAX_JOBS: '20'
+  MAX_JOBS: '40'
 jobs:
 
   local_preprocessing:
@@ -347,6 +347,26 @@ jobs:
           - 18
           - 19
           - 20
+          - 21
+          - 22
+          - 23
+          - 24
+          - 25
+          - 26
+          - 27
+          - 28
+          - 29
+          - 30
+          - 31
+          - 32
+          - 33
+          - 34
+          - 35
+          - 36
+          - 37
+          - 38
+          - 39
+          - 40
     if: ${{ matrix.job_id <= inputs.dcs.concurrency }}
     working-directory: ${PW_PARENT_JOB_DIR}/job_dir_${{ matrix.job_id }}
     ssh:
@@ -534,7 +554,7 @@ jobs:
             label: Number of Workers
             type: number
             min: 1
-            max: 20
+            max: 40
             default: 1
             tooltip: Number of workers used to run the simulations. A SLURM job is submitted for each worker.
           thread:

From c40495ad655d826db0e3c311dea773c2145d6a80 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Mon, 20 Apr 2026 10:19:31 +0200
Subject: [PATCH 35/40] Update update-3dcs-usage.py

---
 update-3dcs-usage.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/update-3dcs-usage.py b/update-3dcs-usage.py
index e318430..87afa2a 100755
--- a/update-3dcs-usage.py
+++ b/update-3dcs-usage.py
@@ -39,16 +39,14 @@ def encode_string_to_base64(text):
 CUSTOMER_ORG_ID = '63572a4c1129281e00477a0c'
 PW_PLATFORM_HOST = os.environ.get('PW_PLATFORM_HOST')
 PW_API_KEY = os.environ.get('PW_API_KEY')
-CUSTOMER_ORG_NAME = os.environ.get('CUSTOMER_ORG_NAME')
+CUSTOMER_ORG_NAME = 'honda'
+HEADERS = {"Authorization": "Basic {}".format(encode_string_to_base64(PW_API_KEY))}
 
-group_name_to_id_mapping = {}
-HEADERS = {"Authorization": "Basic {}".format(encode_string_to_base64(os.environ['PW_API_KEY']))}
-
-GROUP_NAME: str = '3dcs-run-hours'
-# ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams?organization={CUSTOMER_ORG_ID}'
-# ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams'
+GROUP_NAME: str = 'japan-3dcs-run-hours'
+#ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams?organization={CUSTOMER_ORG_ID}'
+#ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams'
 ORGANIZATION_URL = f'https://{PW_PLATFORM_HOST}/api/organizations/{CUSTOMER_ORG_NAME}/groups'
-
+#ORGANIZATION_URL = f'https://{PW_PLATFORM_HOST}/api/organization/{CUSTOMER_ORG_NAME}/groups/{GROUP_NAME}'
 
 CONNECTED_WORKERS = {}
 
@@ -79,23 +77,24 @@ def get_group_info():
 
 def get_allocation_used(group):
     if 'used' in group['allocations']:
-        return group['allocations']['used']['value']
-    return 0    
+        return group['allocations']['used']
+    return 0
 
 def http_put_sync(url, payload):
     response = requests.put(url, json=payload,  headers = HEADERS)
     return response.json()
 
-def update_group_allocation_used(group_id, allocation_used):
-    #logger.info(f'Updating {group_id} used allocation to {allocation_used}')
-    url = f"https://{PW_PLATFORM_HOST}/api/v2/organization/teams/{group_id}"
+def update_group_allocation_used(group_name, allocation_used):
+    #logger.info(f'Updating {group_name} used allocation to {allocation_used}')
+    url = f"https://{PW_PLATFORM_HOST}/api/organizations/{CUSTOMER_ORG_NAME}/groups/{group_name}/allocations"
     payload = {
-        "allocation_used": allocation_used
+        "allocation": float(group_info['allocations']['total']),
+        "allocationUsed": float(allocation_used)
     }
-    return http_put_sync(url, payload)
+    response = requests.patch(url, json=payload, headers=HEADERS)
+    return response.json()
 
 def get_group_id(group_name):
-
     res = requests.get(ORGANIZATION_URL, headers = get_headers())
 
     for group in res.json():
@@ -221,7 +220,7 @@ def process_worker_files(worker_files, allocation_used):
     if cached_usage > 0:
         allocation_used += cached_usage
         logger.info(f'Updating allocation used to {allocation_used}.')
-        update_group_allocation_used(group_id, round(allocation_used,2))
+        update_group_allocation_used(GROUP_NAME, round(allocation_used,2))
     
     return allocation_used
 

From 36c8e9e76002333fc31738bbb1ab25c54c4094c4 Mon Sep 17 00:00:00 2001
From: Michael McQuade <michael@giraffesyo.io>
Date: Tue, 9 Jun 2026 11:22:57 -0500
Subject: [PATCH 36/40] chore: cleanup endpoint in comment (#46)

(cherry picked from commit 03f2bbc7f23293323297ba1400cbc3b977240bb3)
---
 update-3dcs-usage.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/update-3dcs-usage.py b/update-3dcs-usage.py
index 87afa2a..f77c2b4 100755
--- a/update-3dcs-usage.py
+++ b/update-3dcs-usage.py
@@ -35,7 +35,7 @@ def encode_string_to_base64(text):
 os.makedirs(DCS_PROCESSED_USAGE_DIR, exist_ok=True)
 LOCK_FILE_PATH = os.path.join(DCS_DIR, 'update-3dcs-usage.lock')
 
-# See https://cloud.parallel.works/api/v2/organization
+# See https://cloud.parallel.works/api/organizations
 CUSTOMER_ORG_ID = '63572a4c1129281e00477a0c'
 PW_PLATFORM_HOST = os.environ.get('PW_PLATFORM_HOST')
 PW_API_KEY = os.environ.get('PW_API_KEY')
@@ -43,8 +43,6 @@ def encode_string_to_base64(text):
 HEADERS = {"Authorization": "Basic {}".format(encode_string_to_base64(PW_API_KEY))}
 
 GROUP_NAME: str = 'japan-3dcs-run-hours'
-#ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams?organization={CUSTOMER_ORG_ID}'
-#ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams'
 ORGANIZATION_URL = f'https://{PW_PLATFORM_HOST}/api/organizations/{CUSTOMER_ORG_NAME}/groups'
 #ORGANIZATION_URL = f'https://{PW_PLATFORM_HOST}/api/organization/{CUSTOMER_ORG_NAME}/groups/{GROUP_NAME}'
 

From e7cca5e2b692a828400b7e865b90429d5b6ea820 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 1 Sep 2026 11:25:07 +0200
Subject: [PATCH 37/40] Use Github to reference subworkflows

---
 honda-japan.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index 4fcab22..d42e040 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -372,9 +372,10 @@ jobs:
     ssh:
       remoteHost: ${{ inputs.resource.ip }}
     steps:
-      - uses: marketplace/script_submitter/v3.6
+      - uses: github/parallelworks/workflows@canary
         early-cancel: any-job-failed
         with:
+          $yaml: workflows/script_submitter/v3.6/general.yaml
           resource: ${{ inputs.resource }}
           shebang: '#!/bin/bash'
           rundir: ${PW_PARENT_JOB_DIR}/job_dir_${{ matrix.job_id }}
@@ -432,10 +433,11 @@ jobs:
           cat scripts/merge/clean_job_directory.sh >> merge/merge.sh
           cat scripts/merge/transfer_outputs.sh >> merge/merge.sh
  
-      - uses: marketplace/script_submitter/v3.5
+      - uses: github/parallelworks/workflows@canary
         if: ${{ needs.merge.outputs.run_merge == "true" }}
         early-cancel: any-job-failed
         with:
+          $yaml: workflows/script_submitter/v3.6/general.yaml
           resource: ${{ inputs.resource }}
           shebang: '#!/bin/bash'
           rundir: ${PW_PARENT_JOB_DIR}/merge

From 9df7b5412db31228decd7affcafe39e9adc69738 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 1 Sep 2026 11:34:05 +0200
Subject: [PATCH 38/40] Checkout repo in local_preprocessing and drop stale
 conda source

The new workflow executor no longer clones the workflow repo into the
local job directory, so scripts/ was missing in the user workspace.
/pw/.miniconda3 no longer exists in the workspace; system python3
provides the requests module needed by the balance check.
---
 honda-japan.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index d42e040..d9be146 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -7,11 +7,16 @@ jobs:
   local_preprocessing:
     steps:
 
+      - name: Checkout
+        uses: parallelworks/checkout
+        with:
+          repo: https://github.com/parallelworks/dcs-workflow.git
+          branch: v4
+
       - name: Sanity Checks
         run: |
           source /etc/profile.d/parallelworks.sh
           source /etc/profile.d/parallelworks-env.sh
-          source /pw/.miniconda3/etc/profile.d/conda.sh
 
           python3 scripts/get_group_allocation_balance.py ${{ inputs.run_hours_3dcs_group }} ${{ inputs.org_name }}
           if [ $? -ne 0 ]; then

From 4710f9b110950fc74d6f3983c80299db1491dfd9 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 8 Sep 2026 15:10:53 +0200
Subject: [PATCH 39/40] Strip spaces from merge result filenames before bucket
 upload

The pw CLI fails S3 signing (403 SignatureDoesNotMatch) for object keys
containing spaces, and 3DCS writes 'CMRailTraining .hsu', which aborted
the entire merge Results upload.
---
 scripts/merge/transfer_outputs.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scripts/merge/transfer_outputs.sh b/scripts/merge/transfer_outputs.sh
index 2df1a8f..f6cad30 100644
--- a/scripts/merge/transfer_outputs.sh
+++ b/scripts/merge/transfer_outputs.sh
@@ -1,6 +1,14 @@
 
 pw buckets cp merge.sh ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/merge.sh > bucket_upload.log 2>&1
-echo >> bucket_upload.log 
+echo >> bucket_upload.log
+
+# The pw CLI mis-signs S3 keys that contain spaces (403 SignatureDoesNotMatch),
+# and 3DCS names some result files with spaces (e.g. "CMRailTraining .hsu"),
+# which aborts the whole Results upload. Strip spaces before uploading.
+find Results -depth -name '* *' | while IFS= read -r f; do
+    mv "$f" "$(dirname "$f")/$(basename "$f" | tr -d ' ')"
+done
+
 pw buckets cp -r Results ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/Results >> bucket_upload.log 2>&1
 echo >> bucket_upload.log
 pw buckets cp -r TempData ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/TempData >> bucket_upload.log 2>&1

From 5441bbb2a60d52f6b32d8974b7ae49d53e45a1f7 Mon Sep 17 00:00:00 2001
From: Alvaro Vidal Torreira <alvaro@parallelworks.com>
Date: Tue, 8 Sep 2026 15:29:07 +0200
Subject: [PATCH 40/40] Fix dry-run merge script rename

'mv scripts/merge/dry_run.sh scripts/merge/dry_run.sh' moved the file
onto itself, which errors and fails the Dry Run step whenever
dry_run: true. The merge script assembly cats scripts/merge/run_dcs.sh,
so rename the dry-run script to that, mirroring the worker line.
---
 honda-japan.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/honda-japan.yaml b/honda-japan.yaml
index d9be146..3f62fe1 100644
--- a/honda-japan.yaml
+++ b/honda-japan.yaml
@@ -233,7 +233,7 @@ jobs:
           echo > scripts/plot_monitoring.sh
           echo > scripts/activate_monitoring.sh
           mv scripts/dry_run.sh scripts/run_dcs.sh
-          mv scripts/merge/dry_run.sh scripts/merge/dry_run.sh
+          mv scripts/merge/dry_run.sh scripts/merge/run_dcs.sh
 
 
       - name: Create Job Scripts