diff --git a/README.md b/README.md index b3d42a9..8001d6d 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,22 @@ ## 3DCS Parallel Workflow -This workflow enables running a 3DCS Monte Carlo or contributor analysis job using multiple workers on a SLURM cluster in the cloud. Here’s how to use it: +This workflow enables running a 3DCS Monte Carlo or contributor analysis job using multiple workers on a SLURM cluster in the cloud. Here's how to use it: ### 1. Upload Files -- Upload your model’s files to a cloud bucket on the platform (see this [link](https://parallelworks.com/docs/storage/transferring-data/obtaining-credentials)). +- Upload your model's files to a cloud bucket on the platform (see this [link](https://parallelworks.com/docs/storage/transferring-data/obtaining-credentials)). ### 2. Run the Job -- Fill in the workflow’s input form. +- Fill in the workflow's input form. - Click the execute button. - For parameter descriptions, hover over the help (?) icon next to each parameter's name. ### 3. Job Execution -- The job is divided into sub-jobs based on the selected number of workers. -- Each worker uses the specified number of threads. +- Before running, the workflow checks that your group has sufficient 3DCS allocation balance. +- The job is divided into sub-jobs based on the selected number of workers (up to 20). +- Each worker is submitted as an individual SLURM job and uses the specified number of threads. - The simulation runs in the resource defined in the simulation executor section of the input form. ### 4. Node Utilization -- The "max workers per node" parameter sets the maximum number of workers per node. -- To reduce compute costs, estimate the required memory for a single worker and maximize memory utilization by fitting as many workers as possible on a single node. -- Required memory for N workers is N times the memory for a single worker. However, the required memory for N threads is more than N times the memory for a single thread. +- The `#SBATCH --exclusive` flag is set by default in the SLURM directives due to issues running multiple 3DCS workers on a single node. This means each worker occupies a full node. ![Sample Configuration](https://raw.githubusercontent.com/parallelworks/dcs-workflow/main/2-Nodes_4-Workers_2-Threads_Per_Worker_Configuration.png) @@ -33,3 +32,4 @@ This workflow enables running a 3DCS Monte Carlo or contributor analysis job usi - 3DCS usage is calculated based on the total number of hours a node uses any number of 3DCS workers. - For example, if 2 nodes run a 3DCS job for 10 hours with 3 workers per node and 4 threads per worker, the usage is 2 nodes x 10 hours = 20 hours. - To minimize 3DCS license usage, fit as many workers as possible on a single node. +- Usage data is continuously transferred to the metering server during the run. diff --git a/honda-japan.yaml b/honda-japan.yaml index 208bd28..3f62fe1 100644 --- a/honda-japan.yaml +++ b/honda-japan.yaml @@ -1,30 +1,471 @@ permissions: - '*' +env: + MAX_JOBS: '40' jobs: - main: + + local_preprocessing: steps: - - name: Main - run: bash main.sh > logs/main/step_0/logs.out 2>&1 - cleanup: | - touch COMPLETED - source resources/001_simulation_executor/inputs.sh - timeout 180 bash cancel.sh || true - exit 0 - stream: + + - name: Checkout + uses: parallelworks/checkout + with: + repo: https://github.com/parallelworks/dcs-workflow.git + branch: v4 + + - name: Sanity Checks + run: | + source /etc/profile.d/parallelworks.sh + source /etc/profile.d/parallelworks-env.sh + + python3 scripts/get_group_allocation_balance.py ${{ inputs.run_hours_3dcs_group }} ${{ inputs.org_name }} + if [ $? -ne 0 ]; then + echo "$(date) ERROR: No 3DCS balance is available. Exiting workflow." >&2 + exit 1 + fi + + usage_metering: + needs: + - local_preprocessing steps: - - name: Stream Logs + - name: Usage Metering early-cancel: any-job-failed - run: bash stream.sh > logs/stream/step_0/logs.out 2>&1 + run: | + #!/bin/bash + mkdir -p usage + while true; do + sleep 60 + echo "$(date) INFO: Transferring usage data" + set -x + rsync -avz --delete ${{ inputs.resource.ip }}:${PW_PARENT_JOB_DIR}/usage/ usage + rsync -avz ${PWD}/usage/ ${{ inputs.metering_user }}@${{ inputs.metering_ip }}:~/.3dcs/usage-pending + set +x + done + + remote_preprocessing: + needs: + - local_preprocessing + ssh: + remoteHost: ${{ inputs.resource.ip }} + steps: + - name: Checkout + uses: parallelworks/checkout + with: + repo: https://github.com/parallelworks/dcs-workflow.git + branch: v4 + + + - name: Sanity Checks + run: | + if [[ "${{ inputs.dcs.output_directory }}" == "${{ inputs.dcs.model_directory }}" || "${{ inputs.dcs.output_directory }}" == "${{ inputs.dcs.model_directory }}/"* ]]; then + echo "$(date) ERROR: Output directory ${{ inputs.dcs.output_directory }} is a subdirectory of model directory ${{ inputs.dcs.model_directory }}. Exiting workflow" >&2 + exit 1 + fi + + # Check if file is provided as an argument + if ! [ -f "dcs_environment/${{ inputs.dcs.version }}.sh" ]; then + echo "$(date) ERROR: Missing file dcs_environment/${{ inputs.dcs.version }}.sh required to load and run 3DCS. Exiting workflow." >&2 + exit 1 + fi + + - name: Transfer and Process Input Files + run: | + set -x + + mkdir tmp-data-transfer + cd tmp-data-transfer + + pw buckets cp -r ${{ inputs.dcs.bucket.uri }}/${{ inputs.dcs.model_directory }} . + + # Find all files ending in ".wtx" in the current directory, excluding subdirectories + export dcs_model_file=$(find . -type f -name "*.wtx") + + # Check if dcs_model_file is empty + if [ -z "$dcs_model_file" ]; then + echo "$(date) ERROR: No '.wtx' files found. Exiting workflow" >&2 + exit 1 + fi + + echo "dcs_model_file=${PW_PARENT_JOB_DIR}/${dcs_model_file}" >> $OUTPUTS + + # Count the number of files found + file_count=$(echo "$dcs_model_file" | wc -l) + + # Check if only one file ending in ".wtx" is found + if [ "$file_count" -eq 1 ]; then + echo "Found file ${dcs_model_file}" + else + echo "$(date) ERROR: Found $file_count '.wtx' files. Expected only one. Exiting workflow" >&2 + exit 1 + fi + + # Get the count of directories in the current directory + dir_count=$(ls -d */ | wc -l) + + # If there's only one directory + if [ "$dir_count" -eq 1 ]; then + # Get the name of the directory + fea_dir=$(ls -d */ | head -n 1) + export fea_dir=${fea_dir%/} # Remove trailing slash + echo "Directory found: $fea_dir" + else + # If no directory or multiple directories exist + echo "Error: Either no FEA directory found or multiple directories exist." >&2 + exit 1 + fi + + # Process WTX file to adapt the paths to the files + retries=5 + while true; do + python3 ${PW_PARENT_JOB_DIR}/scripts/adapt_wtx_paths.py ${dcs_model_file} ${fea_dir} + exit_code=$? + if [ ${exit_code} -ne 0 ]; then + retries=$((retries-1)) + if [ ${retries} -gt 0 ]; then + sleep 10 + else + echo + echo "$(date) ERROR: Failed to process WTX file with adapt_wtx_paths.py" >&2 + exit 1 + fi + else + break + fi + done + + # List all downloaded file + find . -mindepth 1 > downloaded_files.txt + + cd .. + mv tmp-data-transfer/* . + rmdir tmp-data-transfer + + - name: General Preprocessing + run: | + set -x + source dcs_environment/${{ inputs.dcs.version }}.sh + ls -latd ${WINEPREFIX} + sudo chown -R ${USER}:pwuser ${WINEPREFIX} + ls -latd ${WINEPREFIX} + + mkdir ./usage + mkdir ./usage_completed + + cat <<'EOF' >> inputs.sh + export PATH=$HOME/pw:$PATH + export dcs_model_file="${{ needs.remote_preprocessing.outputs.dcs_model_file }}" + export dcs_dry_run="${{ inputs.dcs.dry_run }}" + export dcs_version="${{ inputs.dcs.version }}" + export dcs_analysis_type="${{ inputs.dcs.analysis_type }}" + export dcs_bucket_uri="${{ inputs.dcs.bucket.uri }}" + export dcs_model_directory="${{ inputs.dcs.model_directory }}" + export dcs_output_directory="${{ inputs.dcs.output_directory }}" + export dcs_num_seeds="${{ inputs.dcs.num_seeds }}" + export dcs_concurrency="${{ inputs.dcs.concurrency }}" + export dcs_thread="${{ inputs.dcs.thread }}" + export metering_user="${{ inputs.metering_user }}" + export metering_ip="${{ inputs.metering_ip }}" + export run_hours_3dcs_group="${{ inputs.run_hours_3dcs_group }}" + export org_name="${{ inputs.org_name }}" + export monitoring_conda_dir="${{ inputs.monitoring_conda_dir }}" + export monitoring_conda_env="${{ inputs.monitoring_conda_env }}" + EOF + echo "export PW_PARENT_JOB_DIR=${PW_PARENT_JOB_DIR}" >> inputs.sh + echo "export PW_WORKFLOW_NAME=${PW_WORKFLOW_NAME}" >> inputs.sh + echo "export PW_JOB_NUMBER=${PW_JOB_NUMBER}" >> inputs.sh + + echo "$(date) INFO: Inputs file" + cat inputs.sh + + - name: Installing Python Dependencies for CPU and Memory Monitoring + run: | + source inputs.sh + + install_miniconda() { + local install_dir=$1 + local conda_repo="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" + local ID="${RANDOM}-$(date +%s)" # This script may run at the same time! + echo "Installing Miniconda to ${install_dir}" + wget --no-check-certificate "${conda_repo}" -O "/tmp/miniconda-${ID}.sh" > "/tmp/miniconda_wget-${ID}.out" 2>&1 + rm -rf "${install_dir}" + mkdir -p "$(dirname "${install_dir}")" + bash "/tmp/miniconda-${ID}.sh" -b -p "${install_dir}" > "/tmp/miniconda_sh-${ID}.out" 2>&1 + } + + create_conda_env_from_yaml() { + local CONDA_DIR=$1 + local CONDA_ENV=$2 + local CONDA_YAML=$3 + local CONDA_SH="${CONDA_DIR}/etc/profile.d/conda.sh" + + # Remove name/prefix fields and empty lines so conda env update accepts the file + sed -i -e 's/name.*$//' -e 's/prefix.*$//' -e '/^$/d' "${CONDA_YAML}" + + if [ ! -d "${CONDA_DIR}" ]; then + echo "Conda directory <${CONDA_DIR}> not found. Installing Conda..." + install_miniconda "${CONDA_DIR}" + fi + + echo "Sourcing Conda: ${CONDA_SH}" + source "${CONDA_SH}" + + if conda env list | grep -q "^${CONDA_ENV} "; then + echo "Conda environment <${CONDA_ENV}> already exists. Skipping installation." + else + echo "Conda environment <${CONDA_ENV}> not found. Creating from <${CONDA_YAML}>..." + conda env update -n "${CONDA_ENV}" -q -f "${CONDA_YAML}" || { + echo "ERROR: Failed to create conda environment <${CONDA_ENV}>. Exiting." + exit 1 + } + fi + + conda activate "${CONDA_ENV}" + } + + + create_conda_env_from_yaml ${monitoring_conda_dir} ${monitoring_conda_env} scripts/cpu_and_memory_usage_requirements.yaml + + - name: Dry Run + if: ${{ inputs.dcs.dry_run == true }} + run: | + echo "$(date) INFO: Running the workflow in dry run mode" + echo > scripts/activate_monitoring.sh + echo > scripts/plot_monitoring.sh + echo > scripts/activate_monitoring.sh + mv scripts/dry_run.sh scripts/run_dcs.sh + mv scripts/merge/dry_run.sh scripts/merge/run_dcs.sh + + + - name: Create Job Scripts + run: | + set -x + + source inputs.sh + + if [ ${dcs_concurrency} -gt ${MAX_JOBS} ]; then + echo "$(date) ERROR: Selected number of jobs is greater than maximum number of jobs. Exiting workflow." + exit 1 + fi + + for nj in $(seq 1 ${dcs_concurrency}); do + echo "$(date) INFO: Preparing directory for job ${nj}" + job_dir=job_dir_${nj} + mkdir -p ${job_dir} + + echo '#!/bin/bash' > ${job_dir}/cancel.sh + chmod +x ${job_dir}/cancel.sh + + cat inputs.sh >> ${job_dir}/run_case.sh + echo "export case_index=${nj}" >> ${job_dir}/run_case.sh + + cat dcs_environment/${dcs_version}.sh >> ${job_dir}/run_case.sh + echo >> ${job_dir}/run_case.sh + echo "touch job.started" >> ${job_dir}/run_case.sh + echo >> ${job_dir}/run_case.sh + + cat scripts/${dcs_analysis_type}.sh >> ${job_dir}/run_case.sh + cat scripts/activate_monitoring.sh >> ${job_dir}/run_case.sh + cat scripts/run_dcs.sh >> ${job_dir}/run_case.sh + cat scripts/plot_monitoring.sh >> ${job_dir}/run_case.sh + cat scripts/transfer_outputs.sh >> ${job_dir}/run_case.sh + done + estimate_end_time: + needs: + - remote_preprocessing + ssh: + remoteHost: ${{ inputs.resource.ip }} steps: - name: Estimate End Time - early-cancel: any-job-failed - run: bash estimate_end_time.sh > logs/estimate_end_time/step_0/logs.out 2>&1 - cleanup: | - bash cancel_stream.sh || true + run: | + + # Source the inputs file + source inputs.sh + + if [ ${dcs_thread} -eq 1 ]; then + echo "$(date) INFO: Number of threads must be larger than 1 to enable estimates" + echo "$(date) INFO: Exiting job..." + fi + + log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt" + + wait_for_all_simulations_to_start() { + while true; do + n_running_workers=$(ls -d job_dir_*/job.started | wc -l) + if [ $? -ne 0 ]; then + n_running_workers=0 + fi + if [ "${n_running_workers}" -lt "${dcs_concurrency}" ]; then + echo "$(date) INFO: ${n_running_workers}/${dcs_concurrency} simulations started" + echo "$(date) INFO: Waiting for all simulations to start..." + sleep 15 + else + break + fi + done + } + + wait_for_all_simulations_to_start + + echo "$(date) INFO: All simulations are started!" + echo; echo + echo "$(date) INFO: Calculating completion time estimates" + + while true; do + log_files=$(ls -d ./job_dir_*/${log_path}) + log_files=$(echo ${log_files} | tr ' ' ',') + + echo "$(date) INFO: Log files:" + echo ${log_files} | tr ',' '\n' + + python3 scripts/estimate_end_time.py ${log_files} + sleep 120 + done exit 0 + workers: + needs: + - remote_preprocessing + strategy: + fail-fast: true + max-parallel: ${{ inputs.dcs.concurrency }} + matrix: + job_id: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + if: ${{ matrix.job_id <= inputs.dcs.concurrency }} + working-directory: ${PW_PARENT_JOB_DIR}/job_dir_${{ matrix.job_id }} + ssh: + remoteHost: ${{ inputs.resource.ip }} + steps: + - uses: github/parallelworks/workflows@canary + early-cancel: any-job-failed + with: + $yaml: workflows/script_submitter/v3.6/general.yaml + resource: ${{ inputs.resource }} + shebang: '#!/bin/bash' + rundir: ${PW_PARENT_JOB_DIR}/job_dir_${{ matrix.job_id }} + use_existing_script: true + script_path: ./run_case.sh + define_cleanup_script: true + cleanup_script_path: ./cancel.sh + scheduler: true + use_scheduler_agent: false + slurm: + is_enabled: true + partition: ${{ inputs.slurm.partition }} + scheduler_directives: ${{ inputs.slurm.scheduler_directives }} + time: ${{ inputs.slurm.time }} + pbs: + is_enabled: false + + merge: + if: ${{ always }} + needs: + - workers + ssh: + remoteHost: ${{ inputs.resource.ip }} + steps: + - name: Cancel End Time Estimation + uses: parallelworks/cancel-jobs + with: + jobs: + - estimate_end_time + - name: Create Merge Script + early-cancel: any-job-failed + run: | + set -x + source inputs.sh + + # Only run merge if more than one job was submitted + if [ ${dcs_concurrency} == 1 ]; then + echo "$(date) INFO: 3DCS concurrency is 1. Exiting workflow." + echo "run_merge=false" | tee -a $OUTPUTS + exit 0 + fi + echo "run_merge=true" | tee -a $OUTPUTS + + echo "$(date) INFO: Preparing merge job" + mkdir -p merge + + echo '#!/bin/bash' > merge/cancel.sh + chmod +x merge/cancel.sh + + cat inputs.sh > merge/merge.sh + cat dcs_environment/${dcs_version}.sh >> merge/merge.sh + cat scripts/merge/transfer_inputs.sh >> merge/merge.sh + cat scripts/merge/${dcs_analysis_type}.sh >> merge/merge.sh + cat scripts/merge/run_dcs.sh >> merge/merge.sh + cat scripts/merge/clean_job_directory.sh >> merge/merge.sh + cat scripts/merge/transfer_outputs.sh >> merge/merge.sh + + - uses: github/parallelworks/workflows@canary + if: ${{ needs.merge.outputs.run_merge == "true" }} + early-cancel: any-job-failed + with: + $yaml: workflows/script_submitter/v3.6/general.yaml + resource: ${{ inputs.resource }} + shebang: '#!/bin/bash' + rundir: ${PW_PARENT_JOB_DIR}/merge + use_existing_script: true + script_path: ./merge.sh + define_cleanup_script: true + cleanup_script_path: ./cancel.sh + scheduler: true + use_scheduler_agent: false + slurm: + is_enabled: true + partition: ${{ inputs.slurm.partition }} + scheduler_directives: ${{ inputs.slurm.scheduler_directives }} + time: ${{ inputs.slurm.time }} + pbs: + is_enabled: false + + - name: Cancel Usage Metering + uses: parallelworks/cancel-jobs + with: + jobs: + - usage_metering + 'on': execute: @@ -49,6 +490,21 @@ jobs: type: string default: honda hidden: true + monitoring_conda_dir: + label: PW Conda Directory + type: string + default: /dcs/pw/miniconda + hidden: true + monitoring_conda_env: + label: PW Conda Environment + type: string + default: psutil + hidden: true + resource: + type: compute-clusters + label: Service host + include-workspace: false + tooltip: Resource to host the service dcs: type: group label: 3DCS Options @@ -80,10 +536,10 @@ jobs: label: Monte Carlo Simulation - value: sensitivity label: Contributor Analysis - bucket_id: - label: Bucket ID or namespace - type: string - tooltip: Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name] + bucket: + label: Bucket + type: bucket + tooltip: Select the storage bucket used to download input files and upload output files. model_directory: label: Model Path type: string @@ -105,7 +561,7 @@ jobs: label: Number of Workers type: number min: 1 - max: 50 + max: 40 default: 1 tooltip: Number of workers used to run the simulations. A SLURM job is submitted for each worker. thread: @@ -115,84 +571,25 @@ jobs: max: 64 default: 1 tooltip: Number of threads used to run the simulation. Be aware that N threads require more than N times the memory needed for a single thread. - - pwrl_001_simulation_executor: + slurm: type: group - label: Simulation Executor + label: SLURM Directives items: - monitoring_conda_dir: - label: PW Conda Directory - type: string - default: /dcs/pw/miniconda - hidden: true - monitoring_conda_env: - label: PW Conda Environment - type: string - default: psutil - hidden: true - resource: - label: Resource - type: compute-clusters - tooltip: Resource to run the simulation task - include-workspace: false - jobschedulertype: - label: Select Controller, SLURM Partition or PBS Queue - type: string - default: SLURM - hidden: true - _sch__dd_partition_e_: + partition: type: slurm-partitions label: SLURM partition optional: true - tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option. - resource: ${{ inputs.pwrl_001_simulation_executor.resource }} - _sch__dd_time_e_: + resource: ${{ inputs.resource }} + tooltip: Select a partition from the drop down menu. + time: label: Walltime type: string default: '999:00:00' - tooltip: Maximum walltime per job + tooltip: '--time= SLURM directive to set the maximum wall-clock time limit for the job' scheduler_directives: - label: Scheduler directives - type: string - default: '--exclusive' - tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword. - pwrl_002_merge_executor: - type: group - label: Merge Executor - items: - monitoring_conda_dir: - label: PW Conda Directory - type: string - default: /dcs/pw/miniconda - hidden: true - monitoring_conda_env: - label: PW Conda Environment - type: string - default: psutil - hidden: true - resource: - label: Resource - type: compute-clusters - tooltip: Resource to run the simulation task - include-workspace: false - jobschedulertype: - label: Select Controller, SLURM Partition or PBS Queue - type: string - default: SLURM - hidden: true - _sch__dd_partition_e_: - type: slurm-partitions - label: SLURM partition + type: editor optional: true - tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option. - resource: ${{ inputs.pwrl_002_merge_executor.resource }} - _sch__dd_time_e_: - label: Walltime - type: string - default: '999:00:00' - tooltip: Maximum walltime per job - scheduler_directives: - label: Scheduler directives - type: string - default: '--exclusive' - tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword. + tooltip: | + Type in additional scheduler directives. + default: | + #SBATCH --exclusive \ No newline at end of file diff --git a/main.sh b/main.sh index 1f1d348..650b5f5 100755 --- a/main.sh +++ b/main.sh @@ -2,17 +2,6 @@ source inputs.sh chmod +x cancel.sh -if [[ "${dcs_output_directory}" == "${dcs_model_directory}" || "${dcs_output_directory}" == "${dcs_model_directory}/"* ]]; then - echo "Error: Output directory is a subdirectory of model directory." >&2 - exit 1 -fi - -# Check if file is provided as an argument -if ! [ -f "dcs_environment/${dcs_version}.sh" ]; then - echo "Error: Missing file dcs_environment/${dcs_version}.sh required to load and run 3DCS. Exiting workflow." - exit 1 -fi - # Use the resource wrapper source /etc/profile.d/parallelworks.sh @@ -38,7 +27,7 @@ fi # Check balance echo; echo "3DCS allocation balance" -python3 get_group_allocation_balance.py ${run_hours_3dcs_group} ${org_name} +python3 utils/get_group_allocation_balance.py ${run_hours_3dcs_group} ${org_name} if [ $? -ne 0 ]; then @@ -102,6 +91,8 @@ scp bucket_credentials ${resource_publicIp}:${resource_jobdir}/bucket_credential reload_bucket_credential_pid=$! echo "kill ${reload_bucket_credential_pid} || true # bucket credentials" >> cancel.sh +# HERE + echo; echo; echo "PREPARING AND SUBMITTING 3DCS RUN JOBS" single_cluster_rsync_exec resources/001_simulation_executor/cluster_rsync_exec.sh return_code=$? diff --git a/metering.sh b/metering.sh deleted file mode 100755 index aa26f66..0000000 --- a/metering.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -mkdir -p usage -while true; do - sleep 60 - rsync -avz --delete ${resource_publicIp}:${resource_jobdir}/usage/ usage >> metering.out 2>&1 - rsync -avz ${resource_jobdir}/usage/ ${metering_user}@${metering_ip}:~/.3dcs/usage-pending >> metering.out 2>&1 -done \ No newline at end of file diff --git a/reload_bucket_credentials.sh b/reload_bucket_credentials.sh deleted file mode 100755 index 265f945..0000000 --- a/reload_bucket_credentials.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -source resources/001_simulation_executor/inputs.sh -set -x -while true; do - sleep 300 - date - pw buckets get-token pw://${dcs_bucket_id} > bucket_credentials - source bucket_credentials - # Check if BUCKET_NAME is empty - if ! [ -n "${BUCKET_URI}" ]; then - echo "ERROR: Unable to load bucket credentials!" - exit 1 - fi - scp bucket_credentials ${resource_publicIp}:${resource_jobdir}/bucket_credentials -done diff --git a/resources/001_simulation_executor/activate_monitoring.sh b/resources/001_simulation_executor/activate_monitoring.sh deleted file mode 100644 index 5d809a2..0000000 --- a/resources/001_simulation_executor/activate_monitoring.sh +++ /dev/null @@ -1,6 +0,0 @@ -source ${monitoring_conda_dir}/etc/profile.d/conda.sh -conda activate ${monitoring_conda_env} -monitoring_txt="case-${case_index}-${HOSTNAME}-jobid-${SLURM_JOB_ID}.txt" -python ${resource_jobdir}/${resource_label}/cpu_and_memory_usage.py --write-usage --txt ${monitoring_txt} & -monitoring_pid=$! -echo "kill ${monitoring_pid}" >> ${resource_jobdir}/${resource_label}/cancel.sh diff --git a/resources/001_simulation_executor/cluster_rsync_exec.sh b/resources/001_simulation_executor/cluster_rsync_exec.sh deleted file mode 100644 index 87f82f7..0000000 --- a/resources/001_simulation_executor/cluster_rsync_exec.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -cd $(dirname $0) - -source inputs.sh -source workflow-libs.sh - -source dcs_environment/${dcs_version}.sh -sudo chown -R ${USER}:pwuser ${WINEPREFIX} - -# Prepare for metering -mkdir ${resource_jobdir}/usage -mkdir ${resource_jobdir}/usage_completed - -echo '#!/bin/bash' > cancel.sh -chmod +x cancel.sh - -create_case(){ - # The merge tasks syncs the results from an S3 bucket. To simplify the path to the - # results in the S3 bucket ww use the resource job dir - case_dir=${resource_jobdir}/worker_${case_index} - mkdir -p ${case_dir} - - echo " Writing job script" - cp batch_header.sh ${case_dir}/run_case.sh - - if [[ ${jobschedulertype} == "SLURM" ]]; then - echo "#SBATCH -o ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh - echo "#SBATCH -e ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh - elif [[ ${jobschedulertype} == "PBS" ]]; then - echo "#PBS -o ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh - echo "#PBS -e ${case_dir}/logs_${case_index}.out" >> ${case_dir}/run_case.sh - fi - - # Main script - echo "mkdir -p ${case_dir}" >> ${case_dir}/run_case.sh - echo "cd ${case_dir}" >> ${case_dir}/run_case.sh - - # FIXME: This is needed because run directory is not shared between controller and compute nodes - #echo "rsync -avzq ${resource_privateIp}:${case_dir}/ ." >> ${case_dir}/run_case.sh - - # Main script - echo >> ${case_dir}/run_case.sh - cat inputs.sh >> ${case_dir}/run_case.sh - cat dcs_environment/${dcs_version}.sh >> ${case_dir}/run_case.sh - echo "export case_index=${case_index}" >> ${case_dir}/run_case.sh - echo "export dcs_model_file=${dcs_model_file}" >> ${case_dir}/run_case.sh - echo >> ${case_dir}/run_case.sh - - cat ${dcs_analysis_type}.sh >> ${case_dir}/run_case.sh - cat activate_monitoring.sh >> ${case_dir}/run_case.sh - cat run_dcs.sh >> ${case_dir}/run_case.sh - cat plot_monitoring.sh >> ${case_dir}/run_case.sh - echo "source ${resource_jobdir}/bucket_credentials" >> ${case_dir}/run_case.sh - cat transfer_outputs.sh >> ${case_dir}/run_case.sh - -} - -cat_slurm_logs() { - for f in $(find ${resource_jobdir} -name logs_*.out); do - echo; echo "Contents of ${f}:" - cat ${f} - done - -} - - -echo; echo; echo "STARTING INPUT DATA TRANSFER" -source ${resource_jobdir}/bucket_credentials -source transfer_inputs.sh - -if [[ ${dcs_dry_run} == "true" ]]; then - echo "RUNNING THE WORKFLOW IN DRY RUN MODE" - unset monitoring_conda_dir monitoring_conda_env - echo > activate_monitoring.sh - echo > plot_monitoring.sh - echo > activate_monitoring.sh - mv dry_run.sh run_dcs.sh -else - rm dry_run.sh -fi - -# If no conda environment is specified for the CPU and Mem python monitoring utility -# the workflow assumes monitoring is disabled -if [ -z "${monitoring_conda_dir}" ] || [ -z "${monitoring_conda_env}" ]; then - echo "CPU and Memory monitoring are disabled" - echo > activate_monitoring.sh - echo > plot_monitoring.sh -else - echo; echo; echo "INSTALLING PYTHON DEPENDENCIES FOR CPU AND MEMORY MONITORING" - create_conda_env_from_yaml ${monitoring_conda_dir} ${monitoring_conda_env} ./cpu_and_memory_usage_requirements.yaml -fi - -echo; echo; echo "CREATING JOB SCRIPTS" -for case_index in $(seq 1 ${dcs_concurrency}); do - echo; echo " Case ${case_index}" - create_case -done - -echo; echo; echo "SUBMITTING JOB SCRIPTS" -for case_index in $(seq 1 ${dcs_concurrency}); do - case_dir=${resource_jobdir}/worker_${case_index} - echo; echo " Case ${case_index}" - cp ${resource_jobdir}/${dcs_model_file} ${case_dir} - - submit_job_sh=${case_dir}/run_case.sh - echo " Job script ${submit_job_sh}" - - if [[ ${jobschedulertype} == "SLURM" ]]; then - job_id=$(${submit_cmd} ${submit_job_sh} | tail -1 | awk -F ' ' '{print $4}') - elif [[ ${jobschedulertype} == "PBS" ]]; then - job_id=$(${submit_cmd} ${submit_job_sh} | tail -1) - fi - - if [ -z "${job_id}" ]; then - echo " ERROR: ${submit_cmd} ${submit_job_sh} failed" - exit 1 - else - echo " Submitted job ${job_id}" - echo "${cancel_cmd} ${job_id}" >> ${resource_jobdir}/${resource_label}/cancel.sh - echo ${job_id} > ${case_dir}/job_id.submitted - fi -done \ No newline at end of file diff --git a/resources/001_simulation_executor/plot_monitoring.sh b/resources/001_simulation_executor/plot_monitoring.sh deleted file mode 100644 index 7327892..0000000 --- a/resources/001_simulation_executor/plot_monitoring.sh +++ /dev/null @@ -1 +0,0 @@ -python ${resource_jobdir}/${resource_label}/cpu_and_memory_usage.py --plot-usage --txt ${monitoring_txt} diff --git a/resources/001_simulation_executor/run_dcs.sh b/resources/001_simulation_executor/run_dcs.sh deleted file mode 100644 index b3aaff3..0000000 --- a/resources/001_simulation_executor/run_dcs.sh +++ /dev/null @@ -1,26 +0,0 @@ -# run in bat file to get correct exit code from the software -#echo set DCS2FLMD_LICENSE_FILE="$DCS2FLMD_LICENSE_FILE" > run.bat - -# Create metering script -cat >> metering.sh <> ${resource_jobdir}/usage/$(hostname)-${job_number} - sleep \$((RANDOM % 30 + 30)) -done -HERE - -chmod +x metering.sh -./metering.sh & -metering_pid=$! - -# Run 3dcs -SECONDS=0 -eval "${dcs_run}" macroScript.txt -kill ${metering_pid} -mv ${resource_jobdir}/usage/$(hostname)-${job_number} ${resource_jobdir}/usage_completed/$(hostname)-${job_number} - -# Results is own by root -sudo chmod 777 Results/ -R -echo ${SECONDS} > Results/dcs-runtime_${case_index}.txt - diff --git a/resources/001_simulation_executor/transfer_outputs.sh b/resources/001_simulation_executor/transfer_outputs.sh deleted file mode 100644 index 7ff846d..0000000 --- a/resources/001_simulation_executor/transfer_outputs.sh +++ /dev/null @@ -1,22 +0,0 @@ -# HARDCODED TO AWS -# A dynamicStorage parameter type would be very helpful for this -unset BUCKET_URI -unset AWS_ACCESS_KEY_ID -unset AWS_SECRET_ACCESS_KEY -unset AWS_SESSION_TOKEN - -source ${resource_jobdir}/bucket_credentials - -# Loop until all variables are non-empty -while [[ -z "$BUCKET_URI" \ - || -z "$AWS_ACCESS_KEY_ID" \ - || -z "$AWS_SECRET_ACCESS_KEY" \ - || -z "$AWS_SESSION_TOKEN" ]] -do - echo "$(date) Waiting for required environment variables to be set..." - sleep 5 - source ${resource_jobdir}/bucket_credentials -done - -# Copy path/to/worker_ to bucket -aws s3 cp --recursive ${PWD} ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/$(basename ${PWD}) diff --git a/resources/002_merge_executor/cluster_rsync_exec.sh b/resources/002_merge_executor/cluster_rsync_exec.sh deleted file mode 100644 index 038d050..0000000 --- a/resources/002_merge_executor/cluster_rsync_exec.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -cd $(dirname $0) - -source inputs.sh -source workflow-libs.sh - -if [[ ${dcs_dry_run} == "true" ]]; then - echo "RUNNING THE WORKFLOW IN DRY RUN MODE" - mv dry_run.sh run_dcs.sh -else - rm dry_run.sh -fi - -echo; echo; echo "MERGING RESULTS" -pwd -echo " Writing job script" -cp batch_header.sh merge.sh - -if [[ ${jobschedulertype} == "SLURM" ]]; then - echo "#SBATCH -o ${PWD}/${case_dir}/logs_merge.out" >> merge.sh - echo "#SBATCH -e ${PWD}/${case_dir}/logs_merge.out" >> merge.sh -elif [[ ${jobschedulertype} == "PBS" ]]; then - echo "#PBS -o ${PWD}/${case_dir}/logs_merge.out" >> merge.sh - echo "#PBS -e ${PWD}/${case_dir}/logs_merge.out" >> merge.sh -fi - -# FIXME: This is needed because run directory is not shared between controller and compute nodes -#echo "rsync -avzq ${resource_privateIp}:${PWD}/ ." >> merge.sh - - -# Main script -cat inputs.sh >> merge.sh -cat dcs_environment/${dcs_version}.sh >> merge.sh -echo "source ${resource_jobdir}/bucket_credentials" >> merge.sh -cat transfer_inputs.sh >> merge.sh -cat ${dcs_analysis_type}.sh >> merge.sh -cat run_dcs.sh >> merge.sh -echo "source ${resource_jobdir}/bucket_credentials" >> merge.sh -cat clean_job_directory.sh >> merge.sh -cat transfer_outputs.sh >> merge.sh - -# FIXME: This is needed because run directory is not shared between controller and compute nodes -#echo "rsync -avzq . ${resource_privateIp}:${PWD}/" >> merge.sh - -echo; echo; echo "SUBMITTING MERGE JOB" -submit_job_sh=./merge.sh -echo " Job script ${submit_job_sh}" - -if [[ ${jobschedulertype} == "SLURM" ]]; then - jobid=$(${submit_cmd} ${submit_job_sh} | tail -1 | awk -F ' ' '{print $4}') -elif [[ ${jobschedulertype} == "PBS" ]]; then - jobid=$(${submit_cmd} ${submit_job_sh} | tail -1) -fi - -if [ -z "${jobid}" ]; then - echo " ERROR: ${submit_cmd} ${submit_job_sh} failed" - cat ${PWD}/${case_dir}/logs_merge.out - exit 1 -fi - -echo ${jobid} > ${resource_jobdir}/job_id.submitted -echo "${cancel_cmd} ${jobid}" >> ${resource_jobdir}/${resource_label}/cancel.sh \ No newline at end of file diff --git a/resources/002_merge_executor/transfer_inputs.sh b/resources/002_merge_executor/transfer_inputs.sh deleted file mode 100644 index 5e59b4c..0000000 --- a/resources/002_merge_executor/transfer_inputs.sh +++ /dev/null @@ -1,27 +0,0 @@ -# HARDCODED TO AWS -# A dynamicStorage parameter type would be very helpful for this - -# Transfer model -# dcs_model_directory can end with or without / -aws s3 sync ${BUCKET_URI}/${dcs_model_directory} . -aws s3 sync ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number} . - -# Find all files ending in ".wtx" in the current directory, excluding subdirectories -dcs_model_file=$(find . -maxdepth 1 -type f -name "*.wtx") - -# Check if dcs_model_file is empty -if [ -z "$dcs_model_file" ]; then - echo "Error: No '.wtx' files found." - exit 1 -fi - -# Count the number of files found -file_count=$(echo "$dcs_model_file" | wc -l) - -# Check if only one file ending in ".wtx" is found -if [ "$file_count" -eq 1 ]; then - echo "Found file ${dcs_model_file}" -else - echo "Error: Found $file_count '.wtx' files. Expected only one." - exit 1 -fi diff --git a/resources/002_merge_executor/transfer_outputs.sh b/resources/002_merge_executor/transfer_outputs.sh deleted file mode 100644 index 4ab73ab..0000000 --- a/resources/002_merge_executor/transfer_outputs.sh +++ /dev/null @@ -1,6 +0,0 @@ -# HARDCODED TO AWS -# A dynamicStorage parameter type would be very helpful for this - -aws s3 cp ${resource_label}/merge.sh ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/merge.sh -aws s3 cp --recursive Results ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/Results -aws s3 cp --recursive TempData ${BUCKET_URI}/${dcs_output_directory}/${USER}/${workflow_name}/${job_number}/TempData diff --git a/scripts/activate_monitoring.sh b/scripts/activate_monitoring.sh new file mode 100644 index 0000000..1a4d243 --- /dev/null +++ b/scripts/activate_monitoring.sh @@ -0,0 +1,6 @@ +source ${monitoring_conda_dir}/etc/profile.d/conda.sh +conda activate ${monitoring_conda_env} +monitoring_txt="case-${case_index}-${HOSTNAME}-jobid-${SLURM_JOB_ID}.txt" +python ${PW_PARENT_JOB_DIR}/scripts/cpu_and_memory_usage.py --write-usage --txt ${monitoring_txt} & +monitoring_pid=$! +echo "kill ${monitoring_pid} || true # kill monitoring" >> cancel.sh diff --git a/resources/001_simulation_executor/adapt_wtx_paths.py b/scripts/adapt_wtx_paths.py similarity index 79% rename from resources/001_simulation_executor/adapt_wtx_paths.py rename to scripts/adapt_wtx_paths.py index b686555..f7c6a75 100644 --- a/resources/001_simulation_executor/adapt_wtx_paths.py +++ b/scripts/adapt_wtx_paths.py @@ -2,7 +2,6 @@ import sys wtx_file_path = sys.argv[1] -fea_dir = sys.argv[2].replace('/', '') def replace_between_angle_brackets(input_string, replacement): """ @@ -27,14 +26,6 @@ def write_lines_to_file(lines, file_path, encoding='utf-8'): with open(file_path, 'w', encoding=encoding) as file: file.writelines(lines) -def get_search_file_paths(file_paths): - """ - Create search patterns for the file paths. - - :param file_paths: List of file paths. - :return: List of search patterns for the file paths. - """ - return [os.path.basename(file_path) for file_path in file_paths] def read_file_with_encodings(file_path, encodings=['utf-8', 'shift_jis', 'euc-jp', 'iso-2022-jp', 'latin-1']): """ @@ -53,20 +44,22 @@ def read_file_with_encodings(file_path, encodings=['utf-8', 'shift_jis', 'euc-jp continue raise UnicodeDecodeError(f"Unable to decode file {file_path} with any of the provided encodings.") -def process_wtx_file(wtx_file_path, search_file_paths): +def process_wtx_file(wtx_file_path, file_paths): """ Process the WTX file, replacing text between angle brackets. :param wtx_file_path: Path to the WTX file. - :param search_file_paths: List of search patterns for file paths. + :param file_paths: List of relative file paths found in the current directory. :return: List of processed lines from the WTX file. """ new_wtx_file_lines = [] lines, encoding = read_file_with_encodings(wtx_file_path) for line in lines: - for file_path in search_file_paths: - if file_path in line: - line = replace_between_angle_brackets(line, '..\\' + fea_dir + '\\' + file_path) + for file_path in file_paths: + basename = os.path.basename(file_path) + if basename in line: + win_path = '..\\' + file_path.replace('/', '\\') + line = replace_between_angle_brackets(line, win_path) new_wtx_file_lines.append(line) return new_wtx_file_lines, encoding @@ -84,9 +77,8 @@ def main(): wtx_file_path = sys.argv[1] file_paths = find_all_files_os() - search_file_paths = get_search_file_paths(file_paths) - new_wtx_file_lines, encoding = process_wtx_file(wtx_file_path, search_file_paths) + new_wtx_file_lines, encoding = process_wtx_file(wtx_file_path, file_paths) write_lines_to_file(new_wtx_file_lines, wtx_file_path, encoding=encoding) if __name__ == '__main__': diff --git a/scripts/cpu_and_memory_usage.py b/scripts/cpu_and_memory_usage.py new file mode 100644 index 0000000..1586bfe --- /dev/null +++ b/scripts/cpu_and_memory_usage.py @@ -0,0 +1,101 @@ +import argparse +import psutil +import matplotlib.pyplot as plt +from datetime import datetime +import time +import os +import subprocess + +""" +This script monitors CPU and memory usage over time and provides options to either write the data to a text file or plot it. +If '--write-usage' option is provided, it continuously monitors CPU and memory usage and writes the data to the specified text file. +If '--plot-usage' option is provided, it reads CPU and memory usage data from the specified text file and plots it over time. +""" + +def kill_job(): + try: + with open("job_id.submitted", "r") as file: + job_id = file.readline().strip() + if job_id: + subprocess.run(["scancel", job_id], check=True) + print(f"Job {job_id} cancelled successfully.") + else: + print("No job ID found in job_id.submitted.") + except FileNotFoundError: + print("File job_id.submitted not found.") + except subprocess.CalledProcessError as e: + print(f"Failed to cancel job {job_id}. Error: {e}") + +def get_usage(): + cpu_percent = psutil.cpu_percent() + memory_percent = psutil.virtual_memory().percent + return cpu_percent, memory_percent + +def write_usage_data(txt_file): + with open(txt_file, "w") as data_file: + print("Monitoring CPU and memory usage...") + while True: + timestamp = datetime.now() + cpu_usage, memory_usage = get_usage() + if memory_usage > 98: + print('Memory exceeded 98%. Killing job.', flush = True) + kill_job() + + # Write data to file + data_file.write(f"{timestamp},{cpu_usage},{memory_usage}\n") + data_file.flush() + + # Wait for 1 second before collecting next data point + time.sleep(1) + + +def plot_usage_data(txt_file): + timestamps = [] + cpu_usages = [] + memory_usages = [] + + try: + with open(txt_file, "r") as data_file: + print("Reading CPU and memory usage data...") + for line in data_file: + parts = line.strip().split(",") + timestamp = datetime.fromisoformat(parts[0]) + cpu_usage = float(parts[1]) + memory_usage = float(parts[2]) + timestamps.append(timestamp) + cpu_usages.append(cpu_usage) + memory_usages.append(memory_usage) + + # Plot CPU and memory usage over time + plt.figure(figsize=(10, 6)) + plt.plot(timestamps, cpu_usages, label='CPU Usage (%)') + plt.plot(timestamps, memory_usages, label='Memory Usage (%)') + plt.xlabel('Time') + plt.ylabel('Usage (%)') + plt.title('CPU and Memory Usage Over Time') + plt.legend() + plt.grid(True) + plt.xticks(rotation=45) + plt.tight_layout() + + # Save plot image + img_path = os.path.splitext(txt_file)[0] + ".png" + plt.savefig(img_path) + print(f"Plot image saved as {img_path}") + except FileNotFoundError: + print(f"File '{txt_file}' not found.") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="CPU and Memory Usage Monitor and Plotter") + parser.add_argument("--write-usage", action="store_true", help="Write CPU and memory usage data to a text file") + parser.add_argument("--plot-usage", action="store_true", help="Plot CPU and memory usage data from a text file") + parser.add_argument("--txt", type=str, help="Specify the text file to read/write usage data") + + args = parser.parse_args() + + if args.write_usage and args.txt: + write_usage_data(args.txt) + elif args.plot_usage and args.txt: + plot_usage_data(args.txt) + else: + print("Invalid arguments. Please specify '--write-usage' or '--plot-usage' along with '--txt' option.") diff --git a/scripts/cpu_and_memory_usage_requirements.yaml b/scripts/cpu_and_memory_usage_requirements.yaml new file mode 100644 index 0000000..e73c7d8 --- /dev/null +++ b/scripts/cpu_and_memory_usage_requirements.yaml @@ -0,0 +1,153 @@ +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - alsa-lib=1.2.11=hd590300_1 + - attr=2.5.1=h166bdaf_1 + - brotli=1.1.0=hd590300_1 + - brotli-bin=1.1.0=hd590300_1 + - bzip2=1.0.8=hd590300_5 + - ca-certificates=2024.2.2=hbcca054_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - contourpy=1.2.1=py312h8572e83_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - dbus=1.13.6=h5008d03_3 + - expat=2.6.2=h59595ed_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_1 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.51.0=py312h98912ed_0 + - freetype=2.12.1=h267a509_2 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - glib=2.80.0=hf2295e7_5 + - glib-tools=2.80.0=hde27a5a_5 + - graphite2=1.3.13=h59595ed_1003 + - gst-plugins-base=1.24.1=hfa15dee_1 + - gstreamer=1.24.1=h98fc4e7_1 + - harfbuzz=8.3.0=h3d44ed6_0 + - icu=73.2=h59595ed_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.5=py312h8572e83_1 + - krb5=1.21.2=h659d440_0 + - lame=3.100=h166bdaf_1003 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libblas=3.9.0=22_linux64_openblas + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcap=2.69=h0f662aa_0 + - libcblas=3.9.0=22_linux64_openblas + - libclang-cpp15=15.0.7=default_h127d8a8_5 + - libclang13=18.1.3=default_h5d6823c_0 + - libcups=2.3.3=h4637d8d_4 + - libdeflate=1.20=hd590300_0 + - libedit=3.1.20191231=he28a2e2_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.2=h59595ed_0 + - libffi=3.4.2=h7f98852_5 + - libflac=1.4.3=h59595ed_0 + - libgcc-ng=13.2.0=h807b86a_5 + - libgcrypt=1.10.3=hd590300_0 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=13.2.0=h69a702a_5 + - libgfortran5=13.2.0=ha4646dd_5 + - libglib=2.80.0=hf2295e7_5 + - libgomp=13.2.0=h807b86a_5 + - libgpg-error=1.48=h71f35ed_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - liblapack=3.9.0=22_linux64_openblas + - libllvm15=15.0.7=hb3ce162_4 + - libllvm18=18.1.3=h2448989_0 + - libnsl=2.0.1=hd590300_0 + - libogg=1.3.4=h7f98852_1 + - libopenblas=0.3.27=pthreads_h413a1c8_0 + - libopus=1.3.1=h7f98852_1 + - libpng=1.6.43=h2797004_0 + - libpq=16.2=h33b98f1_1 + - libsndfile=1.2.2=hc60ed4a_1 + - libsqlite=3.45.2=h2797004_0 + - libstdcxx-ng=13.2.0=h7e041cc_5 + - libsystemd0=255=h3516f8a_1 + - libtiff=4.6.0=h1dd3fc0_3 + - libuuid=2.38.1=h0b41bf4_0 + - libvorbis=1.3.7=h9c3ff4c_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.15=h0b41bf4_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h662e7e4_0 + - libxml2=2.12.6=h232c23b_2 + - libzlib=1.2.13=hd590300_5 + - lz4-c=1.9.4=hcb278e6_0 + - matplotlib=3.8.4=py312h7900ff3_0 + - matplotlib-base=3.8.4=py312he5832f3_0 + - mpg123=1.32.6=h59595ed_0 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=8.3.0=hf1915f5_4 + - mysql-libs=8.3.0=hca2cd23_4 + - ncurses=6.4.20240210=h59595ed_0 + - nspr=4.35=h27087fc_0 + - nss=3.98=h1d7d5a4_0 + - numpy=1.26.4=py312heda63a1_0 + - openjpeg=2.5.2=h488ebb8_0 + - openssl=3.2.1=hd590300_1 + - packaging=24.0=pyhd8ed1ab_0 + - pcre2=10.43=hcad00b1_0 + - pillow=10.3.0=py312hdcec9eb_0 + - pip=24.0=pyhd8ed1ab_0 + - pixman=0.43.2=h59595ed_0 + - ply=3.11=pyhd8ed1ab_2 + - psutil=5.9.8=py312h98912ed_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - pulseaudio-client=17.0=hb77b528_0 + - pyparsing=3.1.2=pyhd8ed1ab_0 + - pyqt=5.15.9=py312h949fe66_5 + - pyqt5-sip=12.12.2=py312h30efb56_5 + - python=3.12.3=hab00c5b_0_cpython + - python-dateutil=2.9.0=pyhd8ed1ab_0 + - python_abi=3.12=4_cp312 + - qt-main=5.15.8=hc9dc06e_21 + - readline=8.2=h8228510_1 + - setuptools=69.5.1=pyhd8ed1ab_0 + - sip=6.7.12=py312h30efb56_0 + - six=1.16.0=pyh6c4a22f_0 + - tk=8.6.13=noxft_h4845f30_101 + - toml=0.10.2=pyhd8ed1ab_0 + - tomli=2.0.1=pyhd8ed1ab_0 + - tornado=6.4=py312h98912ed_0 + - tzdata=2024a=h0c530f3_0 + - wheel=0.43.0=pyhd8ed1ab_1 + - xcb-util=0.4.0=hd590300_1 + - xcb-util-image=0.4.0=h8ee46fc_1 + - xcb-util-keysyms=0.4.0=h8ee46fc_1 + - xcb-util-renderutil=0.3.9=hd590300_1 + - xcb-util-wm=0.4.1=h8ee46fc_1 + - xkeyboard-config=2.41=hd590300_0 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - zlib=1.2.13=hd590300_5 + - zstd=1.5.5=hfc55251_0 \ No newline at end of file diff --git a/resources/001_simulation_executor/dry_run.sh b/scripts/dry_run.sh similarity index 100% rename from resources/001_simulation_executor/dry_run.sh rename to scripts/dry_run.sh diff --git a/resources/001_simulation_executor/estimate_end_time.py b/scripts/estimate_end_time.py similarity index 100% rename from resources/001_simulation_executor/estimate_end_time.py rename to scripts/estimate_end_time.py diff --git a/get_group_allocation_balance.py b/scripts/get_group_allocation_balance.py similarity index 100% rename from get_group_allocation_balance.py rename to scripts/get_group_allocation_balance.py diff --git a/resources/002_merge_executor/clean_job_directory.sh b/scripts/merge/clean_job_directory.sh similarity index 100% rename from resources/002_merge_executor/clean_job_directory.sh rename to scripts/merge/clean_job_directory.sh diff --git a/resources/002_merge_executor/dry_run.sh b/scripts/merge/dry_run.sh similarity index 100% rename from resources/002_merge_executor/dry_run.sh rename to scripts/merge/dry_run.sh diff --git a/resources/002_merge_executor/montecarlo.sh b/scripts/merge/montecarlo.sh similarity index 100% rename from resources/002_merge_executor/montecarlo.sh rename to scripts/merge/montecarlo.sh diff --git a/resources/002_merge_executor/run_dcs.sh b/scripts/merge/run_dcs.sh similarity index 50% rename from resources/002_merge_executor/run_dcs.sh rename to scripts/merge/run_dcs.sh index d23ebb6..c609dd7 100644 --- a/resources/002_merge_executor/run_dcs.sh +++ b/scripts/merge/run_dcs.sh @@ -6,7 +6,7 @@ cat >> metering.sh <> ${resource_jobdir}/usage/$(hostname)-${job_number}-merge + date >> ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER}-merge sleep \$((RANDOM % 30 + 30)) done HERE @@ -14,12 +14,20 @@ HERE chmod +x metering.sh ./metering.sh & metering_pid=$! +echo "kill ${metering_pid} || true # Metering" >> cancel.sh + +# Stream logs +log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt" +touch ${log_path} +tail -f ${log_path} & +tail_pid=$! +echo "kill ${tail_pid} || true # Tail logs" >> cancel.sh # Run 3dcs SECONDS=0 eval "${dcs_run}" macroScript.txt kill ${metering_pid} -mv ${resource_jobdir}/usage/$(hostname)-${job_number}-merge ${resource_jobdir}/usage_completed/$(hostname)-${job_number}-merge +mv ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER}-merge ${PW_PARENT_JOB_DIR}/usage_completed/$(hostname)-${PW_JOB_NUMBER}-merge # Results is own by root diff --git a/resources/002_merge_executor/sensitivity.sh b/scripts/merge/sensitivity.sh similarity index 100% rename from resources/002_merge_executor/sensitivity.sh rename to scripts/merge/sensitivity.sh diff --git a/scripts/merge/transfer_inputs.sh b/scripts/merge/transfer_inputs.sh new file mode 100644 index 0000000..9f75e0e --- /dev/null +++ b/scripts/merge/transfer_inputs.sh @@ -0,0 +1,3 @@ + + +pw buckets cp -r ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER} . > bucket_download.log 2>&1 diff --git a/scripts/merge/transfer_outputs.sh b/scripts/merge/transfer_outputs.sh new file mode 100644 index 0000000..f6cad30 --- /dev/null +++ b/scripts/merge/transfer_outputs.sh @@ -0,0 +1,15 @@ + +pw buckets cp merge.sh ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/merge.sh > bucket_upload.log 2>&1 +echo >> bucket_upload.log + +# The pw CLI mis-signs S3 keys that contain spaces (403 SignatureDoesNotMatch), +# and 3DCS names some result files with spaces (e.g. "CMRailTraining .hsu"), +# which aborts the whole Results upload. Strip spaces before uploading. +find Results -depth -name '* *' | while IFS= read -r f; do + mv "$f" "$(dirname "$f")/$(basename "$f" | tr -d ' ')" +done + +pw buckets cp -r Results ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/Results >> bucket_upload.log 2>&1 +echo >> bucket_upload.log +pw buckets cp -r TempData ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/TempData >> bucket_upload.log 2>&1 + diff --git a/resources/001_simulation_executor/montecarlo.sh b/scripts/montecarlo.sh similarity index 100% rename from resources/001_simulation_executor/montecarlo.sh rename to scripts/montecarlo.sh diff --git a/scripts/plot_monitoring.sh b/scripts/plot_monitoring.sh new file mode 100644 index 0000000..06cda51 --- /dev/null +++ b/scripts/plot_monitoring.sh @@ -0,0 +1 @@ +python ${PW_PARENT_JOB_DIR}/scripts/cpu_and_memory_usage.py --plot-usage --txt ${monitoring_txt} diff --git a/scripts/run_dcs.sh b/scripts/run_dcs.sh new file mode 100644 index 0000000..2704946 --- /dev/null +++ b/scripts/run_dcs.sh @@ -0,0 +1,34 @@ +# run in bat file to get correct exit code from the software +#echo set DCS2FLMD_LICENSE_FILE="$DCS2FLMD_LICENSE_FILE" > run.bat + +# Create metering script +cat >> metering.sh <> ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER} + sleep \$((RANDOM % 30 + 30)) +done +HERE + +chmod +x metering.sh +./metering.sh & +metering_pid=$! +echo "kill ${metering_pid} || true # Metering" >> cancel.sh + +# Stream logs +log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt" +touch ${log_path} +tail -f ${log_path} & +tail_pid=$! +echo "kill ${tail_pid} || true # Tail logs" >> cancel.sh + +# Run 3dcs +SECONDS=0 +eval "${dcs_run}" macroScript.txt +kill ${metering_pid} +mv ${PW_PARENT_JOB_DIR}/usage/$(hostname)-${PW_JOB_NUMBER} ${PW_PARENT_JOB_DIR}/usage_completed/$(hostname)-${PW_JOB_NUMBER} + +# Results is own by root +sudo chmod 777 Results/ -R +echo ${SECONDS} > Results/dcs-runtime-${case_index}.txt + diff --git a/resources/001_simulation_executor/sensitivity.sh b/scripts/sensitivity.sh similarity index 100% rename from resources/001_simulation_executor/sensitivity.sh rename to scripts/sensitivity.sh diff --git a/resources/001_simulation_executor/transfer_inputs.sh b/scripts/transfer_and_process_inputs.sh similarity index 68% rename from resources/001_simulation_executor/transfer_inputs.sh rename to scripts/transfer_and_process_inputs.sh index 9022ae0..627ac6f 100644 --- a/resources/001_simulation_executor/transfer_inputs.sh +++ b/scripts/transfer_and_process_inputs.sh @@ -4,8 +4,7 @@ mkdir tmp-data-transfer cd tmp-data-transfer -# dcs_model_directory can end with or without / -aws s3 sync ${BUCKET_URI}/${dcs_model_directory} . --cli-read-timeout 60 +pw buckets cp -r ${BUCKET_URI}/${dcs_model_directory} . # User aws s3 cp --recursive ../test ${BUCKET_URI}/path/to/dir/test to transfer to the bucket @@ -30,25 +29,10 @@ else fi -# Get the count of directories in the current directory -dir_count=$(ls -d */ | wc -l) - -# If there's only one directory -if [ "$dir_count" -eq 1 ]; then - # Get the name of the directory - fea_dir=$(ls -d */ | head -n 1) - export fea_dir=${fea_dir%/} # Remove trailing slash - echo "Directory found: $fea_dir" -else - # If no directory or multiple directories exist - echo "Error: Either no FEA directory found or multiple directories exist." >&2 - exit 1 -fi - # Process WTX file to adapt the paths to the files retries=5 while true; do - python3 ../adapt_wtx_paths.py ${dcs_model_file} ${fea_dir} + python3 ../adapt_wtx_paths.py ${dcs_model_file} exit_code=$? if [ ${exit_code} -ne 0 ]; then retries=$((retries-1)) diff --git a/scripts/transfer_outputs.sh b/scripts/transfer_outputs.sh new file mode 100644 index 0000000..81d9e2d --- /dev/null +++ b/scripts/transfer_outputs.sh @@ -0,0 +1,6 @@ + +# Copy path/to/worker_ to bucket +set -x +pw buckets cp -r ${PWD} ${dcs_bucket_uri}/${dcs_output_directory}/${USER}/${PW_WORKFLOW_NAME}/${PW_JOB_NUMBER}/$(basename ${PWD}) > bucket_upload.log 2>&1 + + diff --git a/stream.sh b/stream.sh deleted file mode 100755 index 8e87a10..0000000 --- a/stream.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -while [ ! -f "SUBMITTED" ]; do - echo "$(date '+%Y-%m-%d %H:%M:%S') - Waiting for simulations to be submitted" - sleep 10 -done - -# Source the inputs file -source resources/001_simulation_executor/inputs.sh - -export sshcmd="ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=3 ${resource_publicIp}" - -log_path="TempData/dcsSimuMacro_SA_log_x64_$(echo ${dcs_version} | tr '.' '_').txt" - -wait_for_all_simulations_to_start() { - while true; do - n_running_workers=$(${sshcmd} ls -d ${resource_jobdir}/worker_*/${log_path} | wc -l) - if [ $? -ne 0 ]; then - n_running_workers=0 - fi - if [ "${n_running_workers}" -lt "${dcs_concurrency}" ]; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - ${n_running_workers}/${dcs_concurrency} simulations started" - echo "$(date '+%Y-%m-%d %H:%M:%S') - Waiting for all simulations to start..." - sleep 15 - else - break - fi - if [ -f "COMPLETED" ]; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - Simulations are completed" - exit 0 - fi - done -} - -wait_for_all_simulations_to_start - -echo "$(date '+%Y-%m-%d %H:%M:%S') - All simulations are started!" -echo; echo -echo "$(date '+%Y-%m-%d %H:%M:%S') - Initiating streaming" - -stream_logs() { - local max_retries=5 - local retry_delay=10 - local attempt=1 - - while [ $attempt -le $max_retries ]; do - if [ -f "COMPLETED" ]; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - COMPLETED file detected, stopping log streaming" - return 0 - fi - echo "$(date '+%Y-%m-%d %H:%M:%S') - Attempt $attempt of $max_retries to stream logs" - ${sshcmd} "tail -f ${resource_jobdir}/worker_*/${log_path}" - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - Streaming completed successfully" - return 0 - else - echo "$(date '+%Y-%m-%d %H:%M:%S') - SSH tail failed with exit code $exit_code. Retrying in $retry_delay seconds..." - sleep $retry_delay - attempt=$((attempt + 1)) - fi - done - - if [ $attempt -gt $max_retries ]; then - if [ -f "COMPLETED" ]; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - COMPLETED file detected, ignoring retry limit" - return 0 - else - echo "$(date '+%Y-%m-%d %H:%M:%S') - Failed to stream logs after $max_retries attempts" - return 1 - fi - fi -} - -# Start log streaming in the background -echo "$(date '+%Y-%m-%d %H:%M:%S') - Starting background log streaming" -stream_logs & -stream_pid=$! -echo "kill ${stream_pid}" > cancel_stream.sh - -# Wait for either COMPLETED file or stream_logs to exit -while [ ! -f "COMPLETED" ]; do - # Check if the stream_logs process is still running - if ! ps -p $stream_pid > /dev/null; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - Stream logs process (PID: $stream_pid) has exited" - break - fi - sleep 10 -done \ No newline at end of file diff --git a/update-3dcs-usage.py b/update-3dcs-usage.py index e318430..f77c2b4 100755 --- a/update-3dcs-usage.py +++ b/update-3dcs-usage.py @@ -35,20 +35,16 @@ def encode_string_to_base64(text): os.makedirs(DCS_PROCESSED_USAGE_DIR, exist_ok=True) LOCK_FILE_PATH = os.path.join(DCS_DIR, 'update-3dcs-usage.lock') -# See https://cloud.parallel.works/api/v2/organization +# See https://cloud.parallel.works/api/organizations CUSTOMER_ORG_ID = '63572a4c1129281e00477a0c' PW_PLATFORM_HOST = os.environ.get('PW_PLATFORM_HOST') PW_API_KEY = os.environ.get('PW_API_KEY') -CUSTOMER_ORG_NAME = os.environ.get('CUSTOMER_ORG_NAME') +CUSTOMER_ORG_NAME = 'honda' +HEADERS = {"Authorization": "Basic {}".format(encode_string_to_base64(PW_API_KEY))} -group_name_to_id_mapping = {} -HEADERS = {"Authorization": "Basic {}".format(encode_string_to_base64(os.environ['PW_API_KEY']))} - -GROUP_NAME: str = '3dcs-run-hours' -# ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams?organization={CUSTOMER_ORG_ID}' -# ORGANIZATION_URL: str = f'https://{PW_PLATFORM_HOST}/api/v2/organization/teams' +GROUP_NAME: str = 'japan-3dcs-run-hours' ORGANIZATION_URL = f'https://{PW_PLATFORM_HOST}/api/organizations/{CUSTOMER_ORG_NAME}/groups' - +#ORGANIZATION_URL = f'https://{PW_PLATFORM_HOST}/api/organization/{CUSTOMER_ORG_NAME}/groups/{GROUP_NAME}' CONNECTED_WORKERS = {} @@ -79,23 +75,24 @@ def get_group_info(): def get_allocation_used(group): if 'used' in group['allocations']: - return group['allocations']['used']['value'] - return 0 + return group['allocations']['used'] + return 0 def http_put_sync(url, payload): response = requests.put(url, json=payload, headers = HEADERS) return response.json() -def update_group_allocation_used(group_id, allocation_used): - #logger.info(f'Updating {group_id} used allocation to {allocation_used}') - url = f"https://{PW_PLATFORM_HOST}/api/v2/organization/teams/{group_id}" +def update_group_allocation_used(group_name, allocation_used): + #logger.info(f'Updating {group_name} used allocation to {allocation_used}') + url = f"https://{PW_PLATFORM_HOST}/api/organizations/{CUSTOMER_ORG_NAME}/groups/{group_name}/allocations" payload = { - "allocation_used": allocation_used + "allocation": float(group_info['allocations']['total']), + "allocationUsed": float(allocation_used) } - return http_put_sync(url, payload) + response = requests.patch(url, json=payload, headers=HEADERS) + return response.json() def get_group_id(group_name): - res = requests.get(ORGANIZATION_URL, headers = get_headers()) for group in res.json(): @@ -221,7 +218,7 @@ def process_worker_files(worker_files, allocation_used): if cached_usage > 0: allocation_used += cached_usage logger.info(f'Updating allocation used to {allocation_used}.') - update_group_allocation_used(group_id, round(allocation_used,2)) + update_group_allocation_used(GROUP_NAME, round(allocation_used,2)) return allocation_used diff --git a/windows-aws-credentials.yaml b/windows-aws-credentials.yaml deleted file mode 100644 index c5525b9..0000000 --- a/windows-aws-credentials.yaml +++ /dev/null @@ -1,18 +0,0 @@ -permissions: - - '*' -jobs: - main: - steps: - - name: Generate Windows S3 Credentials - run: | - source /etc/profile.d/parallelworks.sh - source /etc/profile.d/parallelworks-env.sh - source /pw/.miniconda3/etc/profile.d/conda.sh - python3 bucket_token_generator.py --bucket_id ${{ inputs.bucket_id }} --token_format text --platform windows -'on': - execute: - inputs: - bucket_id: - label: Bucket ID or namespace - type: string - tooltip: Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name] diff --git a/workflow.xml b/workflow.xml deleted file mode 100644 index a44814a..0000000 --- a/workflow.xml +++ /dev/null @@ -1,201 +0,0 @@ - - main.sh - cancel.sh - - - -
- - - - - - - - - - - - - - - -
-
- - - - - - - - - -
-
- - - - - - - - - -
-
-
diff --git a/workflow.yaml b/workflow.yaml deleted file mode 100644 index 55434b5..0000000 --- a/workflow.yaml +++ /dev/null @@ -1,172 +0,0 @@ -permissions: - - '*' -jobs: - main: - steps: - - name: Main - run: bash main.sh - cleanup: | - bash cancel.sh || true - exit 0 -'on': - execute: - inputs: - metering_user: - label: Metering User - type: string - default: __metering_user__ - hidden: true - metering_ip: - label: Metering IP - type: string - default: 34.132.102.65 - hidden: true - dcs: - type: group - label: 3DCS Options - items: - dry_run: - label: Dry run? - type: boolean - default: false - tooltip: Dry run generates the macroScript.txt file and executes every step of the workflow except running 3DCS - version: - label: 3DCS Version - type: dropdown - default: 8.0.0.2 - tooltip: Select the 3DCS version - options: - - value: 8.0.0.2 - label: 8.0.0.2 - - value: 7.10.0.2 - label: 7.10.0.2 - analysis_type: - label: Analysis Type - type: dropdown - tooltip: Select Montecarlo simulation or contributor analysis (sensitivity) - default: montecarlo - options: - - value: montecarlo - label: Monte Carlo Simulation - - value: sensitivity - label: Contributor Analysis - bucket_id: - label: Bucket ID or namespace - type: string - tooltip: Type in the bucket ID string or namespace in the format [bucket-owner]/[bucket-name] - model_directory: - label: Model Path - type: string - tooltip: Directory within the bucket with the model and all the required files - output_directory: - label: Output Path - type: string - tooltip: Output directory within the bucket to store outputs and logs. It cannot be inside the Model Directory! - num_seeds: - label: Number of Monte Carlo Simulations - type: number - min: 1 - max: 100000 - default: 2000 - hidden: ${{ 'montecarlo' !=inputs.dcs.analysis_type }} - ignore: ${{ .hidden }} - optional: ${{ .hidden }} - concurrency: - label: Number of Workers - type: number - min: 1 - max: 50 - default: 1 - tooltip: Number of workers used to run the simulations. A SLURM job is submitted for each worker. - thread: - label: Number of Threads per Worker - type: number - min: 1 - max: 64 - default: 1 - tooltip: Number of threads used to run the simulation. Be aware that N threads require more than N times the memory needed for a single thread. - - pwrl_001_simulation_executor: - type: group - label: Simulation Executor - items: - monitoring_conda_dir: - label: PW Conda Directory - type: string - default: /dcs/pw/miniconda - hidden: true - monitoring_conda_env: - label: PW Conda Environment - type: string - default: psutil - hidden: true - resource: - label: Resource - type: compute-clusters - tooltip: Resource to run the simulation task - include-workspace: false - jobschedulertype: - label: Select Controller, SLURM Partition or PBS Queue - type: string - default: SLURM - hidden: true - _sch__dd_partition_e_: - type: slurm-partitions - label: SLURM partition - hidden: ${{ 'SLURM' !=inputs.pwrl_001_simulation_executor.jobschedulertype }} - ignore: ${{ .hidden }} - optional: true - tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option. - resource: ${{ inputs.pwrl_001_simulation_executor.resource }} - _sch__dd_time_e_: - label: Walltime - type: string - default: '999:00:00' - tooltip: Maximum walltime per job - scheduler_directives: - label: Scheduler directives - type: string - default: '--exclusive' - tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword. - pwrl_002_merge_executor: - type: group - label: Merge Executor - items: - monitoring_conda_dir: - label: PW Conda Directory - type: string - default: /dcs/pw/miniconda - hidden: true - monitoring_conda_env: - label: PW Conda Environment - type: string - default: psutil - hidden: true - resource: - label: Resource - type: compute-clusters - tooltip: Resource to run the simulation task - include-workspace: false - jobschedulertype: - label: Select Controller, SLURM Partition or PBS Queue - type: string - default: SLURM - hidden: true - _sch__dd_partition_e_: - type: slurm-partitions - label: SLURM partition - hidden: ${{ 'SLURM' !=inputs.pwrl_002_merge_executor.jobschedulertype }} - ignore: ${{ .hidden }} - optional: true - tooltip: Partition to submit the job. Leave empty to let SLURM pick the optimal option. - resource: ${{ inputs.pwrl_002_merge_executor.resource }} - _sch__dd_time_e_: - label: Walltime - type: string - default: '999:00:00' - tooltip: Maximum walltime per job - scheduler_directives: - label: Scheduler directives - type: string - default: '--exclusive' - tooltip: e.g. --mem=1000;--gpus-per-node=1 - Use the semicolon character ; to separate parameters. Do not include the SBATCH keyword.