diff --git a/rerun_all_required_experiments_linux.sh b/rerun_all_required_experiments_linux.sh new file mode 100755 index 0000000..8636ceb --- /dev/null +++ b/rerun_all_required_experiments_linux.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +# +# Re-run every experiment invalidated by removing the assessment event token, +# requiring train-split RobustScale, and deleting the mixed distribution. +# +# The workflow runs: +# 1. seed-42 full factorial experiments; +# 2. seed-43/44 key-model experiments; +# 3. seed-42/43/44 disease-history T/O/S ablation; +# 4. seed-42/43/44 assessment + smoking + alcohol experiment; +# 5. AUC and calibration evaluation for all four campaigns. + +set -euo pipefail + +SCRIPT_DIR="${BASH_SOURCE[0]%/*}" +if [[ "$SCRIPT_DIR" == "${BASH_SOURCE[0]}" ]]; then + SCRIPT_DIR="." +fi +SCRIPT_DIR="$(cd -- "$SCRIPT_DIR" && pwd)" +cd "$SCRIPT_DIR" + +NUM_GPUS=1 +NUM_GPUS_SET=0 +GPU_CSV="" +NUM_CPUS=16 +PYTHON_BIN="${PYTHON_BIN:-python}" +BASH_BIN="${BASH:-bash}" +DRY_RUN=0 + +FULL_FACTORIAL_CAMPAIGN="full_factorial_smoking_alcohol_bmi" +KEY_MODELS_CAMPAIGN="key_models_multiseed_smoking_alcohol_bmi" +HISTORY_CAMPAIGN="disease_history_ablation_no_extra" +EXTRA_INFO_CAMPAIGN="extra_info_assessment_smoking_alcohol_robust_multiseed" + +usage() { + cat <<'EOF' +Usage: + bash rerun_all_required_experiments_linux.sh [options] + +Options: + --num-gpus N Number of GPUs; uses GPU ids 0 through N-1 (default: 1). + --gpus LIST Explicit comma-separated GPU ids, for example 0,2. + Cannot be combined with --num-gpus. + --num-cpus N Total CPU workers shared across active GPUs (default: 16). + --python PATH Python executable used by every child script. + --dry-run Print commands without training or evaluation. + -h, --help Show this help message. + +Examples: + bash rerun_all_required_experiments_linux.sh --num-gpus 1 --num-cpus 16 + bash rerun_all_required_experiments_linux.sh --num-gpus 4 --num-cpus 32 + bash rerun_all_required_experiments_linux.sh --gpus 0,2 --num-cpus 16 + +CPU allocation: + The script divides --num-cpus evenly across simultaneously active GPUs. + For example, 4 GPUs and 32 CPUs gives each GPU job 8 workers. +EOF +} + +while (($# > 0)); do + case "$1" in + --num-gpus) + [[ $# -ge 2 ]] || { + echo "ERROR: --num-gpus requires a value." >&2 + exit 2 + } + [[ -z "$GPU_CSV" ]] || { + echo "ERROR: --num-gpus cannot be combined with --gpus." >&2 + exit 2 + } + NUM_GPUS="$2" + NUM_GPUS_SET=1 + shift 2 + ;; + --gpus) + [[ $# -ge 2 ]] || { + echo "ERROR: --gpus requires a value." >&2 + exit 2 + } + ((NUM_GPUS_SET == 0)) || { + echo "ERROR: --gpus cannot be combined with --num-gpus." >&2 + exit 2 + } + GPU_CSV="$2" + shift 2 + ;; + --num-cpus) + [[ $# -ge 2 ]] || { + echo "ERROR: --num-cpus requires a value." >&2 + exit 2 + } + NUM_CPUS="$2" + shift 2 + ;; + --python) + [[ $# -ge 2 ]] || { + echo "ERROR: --python requires a value." >&2 + exit 2 + } + PYTHON_BIN="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +[[ "$NUM_GPUS" =~ ^[1-9][0-9]*$ ]] || { + echo "ERROR: --num-gpus must be a positive integer." >&2 + exit 2 +} +[[ "$NUM_CPUS" =~ ^[1-9][0-9]*$ ]] || { + echo "ERROR: --num-cpus must be a positive integer." >&2 + exit 2 +} + +if [[ -z "$GPU_CSV" ]]; then + for ((gpu_id = 0; gpu_id < NUM_GPUS; gpu_id++)); do + if [[ -n "$GPU_CSV" ]]; then + GPU_CSV+="," + fi + GPU_CSV+="$gpu_id" + done +else + IFS=',' read -r -a GPU_IDS <<< "$GPU_CSV" + NUM_GPUS="${#GPU_IDS[@]}" + ((NUM_GPUS > 0)) || { + echo "ERROR: --gpus must not be empty." >&2 + exit 2 + } + for gpu_id in "${GPU_IDS[@]}"; do + [[ "$gpu_id" =~ ^[0-9]+$ ]] || { + echo "ERROR: invalid GPU id: $gpu_id" >&2 + exit 2 + } + done +fi + +((NUM_CPUS >= NUM_GPUS)) || { + echo "ERROR: --num-cpus must be at least the number of GPUs." >&2 + exit 2 +} +WORKERS_PER_GPU=$((NUM_CPUS / NUM_GPUS)) + +required_scripts=( + train_batch_linux.sh + train_key_models_multiseed_linux.sh + train_disease_history_ablation_linux.sh + train_extra_info_assessment_all_multiseed_linux.sh + evaluate_all_runs_linux.sh + evaluate_calibration_all_runs_linux.sh +) +for script in "${required_scripts[@]}"; do + [[ -f "$SCRIPT_DIR/$script" ]] || { + echo "ERROR: missing required script: $SCRIPT_DIR/$script" >&2 + exit 2 + } +done + +run_command() { + printf '>>' + printf ' %q' "$@" + printf '\n' + if ((DRY_RUN == 0)); then + "$@" + fi +} + +echo "GPUs: $GPU_CSV ($NUM_GPUS total)" +echo "CPUs: $NUM_CPUS total; $WORKERS_PER_GPU workers per GPU job" +echo "Python: $PYTHON_BIN" + +run_command "$BASH_BIN" "$SCRIPT_DIR/train_batch_linux.sh" \ + --gpus "$GPU_CSV" \ + --seed 42 \ + --num-workers "$WORKERS_PER_GPU" \ + --python "$PYTHON_BIN" + +run_command "$BASH_BIN" "$SCRIPT_DIR/train_key_models_multiseed_linux.sh" \ + --gpus "$GPU_CSV" \ + --seeds 43,44 \ + --num-workers "$WORKERS_PER_GPU" \ + --python "$PYTHON_BIN" + +run_command "$BASH_BIN" "$SCRIPT_DIR/train_disease_history_ablation_linux.sh" \ + --gpus "$GPU_CSV" \ + --seeds 42,43,44 \ + --num-workers "$WORKERS_PER_GPU" \ + --python "$PYTHON_BIN" + +run_command "$BASH_BIN" "$SCRIPT_DIR/train_extra_info_assessment_all_multiseed_linux.sh" \ + --gpus "$GPU_CSV" \ + --seeds 42,43,44 \ + --num-workers "$WORKERS_PER_GPU" \ + --python "$PYTHON_BIN" + +campaigns=( + "$FULL_FACTORIAL_CAMPAIGN" + "$KEY_MODELS_CAMPAIGN" + "$HISTORY_CAMPAIGN" + "$EXTRA_INFO_CAMPAIGN" +) + +for campaign in "${campaigns[@]}"; do + runs_root="$SCRIPT_DIR/runs/$campaign" + run_command "$BASH_BIN" "$SCRIPT_DIR/evaluate_all_runs_linux.sh" \ + --gpus "$GPU_CSV" \ + --runs-root "$runs_root" \ + --log-root "$SCRIPT_DIR/batch_logs/evaluate_all_required/$campaign" \ + --python "$PYTHON_BIN" \ + --num-workers "$WORKERS_PER_GPU" \ + --num-workers-auc "$WORKERS_PER_GPU" + + run_command "$BASH_BIN" "$SCRIPT_DIR/evaluate_calibration_all_runs_linux.sh" \ + --gpus "$GPU_CSV" \ + --runs-root "$runs_root" \ + --log-root "$SCRIPT_DIR/batch_logs/evaluate_calibration_all_required/$campaign" \ + --python "$PYTHON_BIN" \ + --num-workers "$WORKERS_PER_GPU" \ + --num-workers-calibration "$WORKERS_PER_GPU" +done + +echo "All required re-training and evaluation workflows completed."