diff --git a/evaluate_all_runs_linux.sh b/evaluate_all_runs_linux.sh new file mode 100644 index 0000000..4c22bcf --- /dev/null +++ b/evaluate_all_runs_linux.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +# +# Recursively evaluate every completed run under runs/. +# +# A directory is considered a runnable run when it contains both: +# - train_config.json +# - best_model.pt +# +# Each missing report is scheduled independently: +# - evaluate_auc.py +# -> df_auc_delphi2m_report.csv +# - evaluate_auc_v2.py +# -> df_auc_landmark_delphi2m_report.csv +# +# Existing non-empty reports are skipped. Each evaluation task uses one GPU. +# Tasks on the same GPU run sequentially; different GPUs run in parallel. +# +# Examples: +# bash evaluate_all_runs_linux.sh --gpus 0 +# bash evaluate_all_runs_linux.sh --gpus 0,1,2,3 +# bash evaluate_all_runs_linux.sh --gpus 0,1 --dry-run +# + +set -uo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +RUNS_ROOT="$SCRIPT_DIR/runs" +LOG_ROOT="$SCRIPT_DIR/batch_logs/evaluate_all_runs" +GPU_CSV="0" +PYTHON_BIN="${PYTHON_BIN:-python}" +NUM_WORKERS=4 +NUM_WORKERS_AUC=4 +DRY_RUN=0 + +TOKEN_REPORT="df_auc_delphi2m_report.csv" +LANDMARK_REPORT="df_auc_landmark_delphi2m_report.csv" + +usage() { + cat <<'EOF' +Usage: + bash evaluate_all_runs_linux.sh [options] + +Options: + --gpus LIST Comma-separated GPU ids (default: 0). + --runs-root PATH Root directory scanned recursively (default: ./runs). + --log-root PATH Evaluation log root. + --python PATH Python executable (default: $PYTHON_BIN or python). + --num-workers N DataLoader workers per evaluation (default: 4). + --num-workers-auc N CPU AUC workers per evaluation (default: 4). + --dry-run Discover runs and print missing evaluations only. + -h, --help Show this help message. + +Completion files: + evaluate_auc.py df_auc_delphi2m_report.csv + evaluate_auc_v2.py df_auc_landmark_delphi2m_report.csv + +Existing non-empty completion files are skipped independently. +EOF +} + +while (($# > 0)); do + case "$1" in + --gpus) + [[ $# -ge 2 ]] || { + echo "ERROR: --gpus requires a value." >&2 + exit 2 + } + GPU_CSV="$2" + shift 2 + ;; + --runs-root) + [[ $# -ge 2 ]] || { + echo "ERROR: --runs-root requires a value." >&2 + exit 2 + } + RUNS_ROOT="$2" + shift 2 + ;; + --log-root) + [[ $# -ge 2 ]] || { + echo "ERROR: --log-root requires a value." >&2 + exit 2 + } + LOG_ROOT="$2" + shift 2 + ;; + --python) + [[ $# -ge 2 ]] || { + echo "ERROR: --python requires a value." >&2 + exit 2 + } + PYTHON_BIN="$2" + shift 2 + ;; + --num-workers) + [[ $# -ge 2 ]] || { + echo "ERROR: --num-workers requires a value." >&2 + exit 2 + } + NUM_WORKERS="$2" + shift 2 + ;; + --num-workers-auc) + [[ $# -ge 2 ]] || { + echo "ERROR: --num-workers-auc requires a value." >&2 + exit 2 + } + NUM_WORKERS_AUC="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +[[ -n "$GPU_CSV" ]] || { + echo "ERROR: --gpus must not be empty." >&2 + exit 2 +} +[[ "$NUM_WORKERS" =~ ^[0-9]+$ ]] || { + echo "ERROR: --num-workers must be a non-negative integer." >&2 + exit 2 +} +[[ "$NUM_WORKERS_AUC" =~ ^[1-9][0-9]*$ ]] || { + echo "ERROR: --num-workers-auc must be a positive integer." >&2 + exit 2 +} +[[ -d "$RUNS_ROOT" ]] || { + echo "ERROR: runs root does not exist: $RUNS_ROOT" >&2 + exit 2 +} +[[ -f "$SCRIPT_DIR/evaluate_auc.py" ]] || { + echo "ERROR: missing evaluator: $SCRIPT_DIR/evaluate_auc.py" >&2 + exit 2 +} +[[ -f "$SCRIPT_DIR/evaluate_auc_v2.py" ]] || { + echo "ERROR: missing evaluator: $SCRIPT_DIR/evaluate_auc_v2.py" >&2 + exit 2 +} +command -v "$PYTHON_BIN" >/dev/null 2>&1 || { + echo "ERROR: Python executable not found: $PYTHON_BIN" >&2 + exit 2 +} + +RUNS_ROOT="$(cd -- "$RUNS_ROOT" && pwd)" +if [[ "$LOG_ROOT" != /* ]]; then + LOG_ROOT="$SCRIPT_DIR/$LOG_ROOT" +fi + +IFS=',' read -r -a GPU_IDS <<< "$GPU_CSV" +declare -A SEEN_GPUS=() +for gpu in "${GPU_IDS[@]}"; do + [[ -n "$gpu" && "$gpu" =~ ^[A-Za-z0-9._:-]+$ ]] || { + echo "ERROR: invalid GPU id: $gpu" >&2 + exit 2 + } + [[ -z "${SEEN_GPUS[$gpu]+x}" ]] || { + echo "ERROR: duplicate GPU id: $gpu" >&2 + exit 2 + } + SEEN_GPUS["$gpu"]=1 +done + +declare -a JOB_RUN_DIRS=() +declare -a JOB_EVALUATORS=() +declare -a JOB_REPORTS=() +declare -a JOB_LOG_FILES=() + +add_job() { + local run_dir="$1" + local evaluator="$2" + local report_name="$3" + local relative_run="${run_dir#"$RUNS_ROOT"/}" + local evaluator_name="${evaluator%.py}" + + JOB_RUN_DIRS+=("$run_dir") + JOB_EVALUATORS+=("$evaluator") + JOB_REPORTS+=("$report_name") + JOB_LOG_FILES+=("$LOG_ROOT/$relative_run/$evaluator_name.log") +} + +run_count=0 +incomplete_run_count=0 +skipped_token_count=0 +skipped_landmark_count=0 + +while IFS= read -r -d '' config_path; do + run_dir="${config_path%/train_config.json}" + ((run_count += 1)) + + if [[ ! -f "$run_dir/best_model.pt" ]]; then + echo "[SKIP] Incomplete run without best_model.pt: $run_dir" + ((incomplete_run_count += 1)) + continue + fi + + if [[ -s "$run_dir/$TOKEN_REPORT" ]]; then + ((skipped_token_count += 1)) + else + add_job "$run_dir" "evaluate_auc.py" "$TOKEN_REPORT" + fi + + if [[ -s "$run_dir/$LANDMARK_REPORT" ]]; then + ((skipped_landmark_count += 1)) + else + add_job "$run_dir" "evaluate_auc_v2.py" "$LANDMARK_REPORT" + fi +done < <(find "$RUNS_ROOT" -type f -name "train_config.json" -print0) + +if ((!DRY_RUN)); then + mkdir -p "$LOG_ROOT" +fi + +print_command() { + printf '%q ' "$@" + printf '\n' +} + +run_job() { + local job_index="$1" + local gpu="$2" + local run_dir="${JOB_RUN_DIRS[$job_index]}" + local evaluator="${JOB_EVALUATORS[$job_index]}" + local report_name="${JOB_REPORTS[$job_index]}" + local log_file="${JOB_LOG_FILES[$job_index]}" + local -a command=( + "$PYTHON_BIN" + -u + "$SCRIPT_DIR/$evaluator" + --run_path "$run_dir" + --output_path "$run_dir" + --device cuda + --num_workers "$NUM_WORKERS" + --num_workers_auc "$NUM_WORKERS_AUC" + ) + + echo "[$(date '+%F %T')] START evaluator=$evaluator gpu=$gpu" + echo " run=$run_dir" + echo " report=$report_name" + + if ((DRY_RUN)); then + printf ' CUDA_VISIBLE_DEVICES=%q ' "$gpu" + print_command "${command[@]}" + return 0 + fi + + mkdir -p "$(dirname -- "$log_file")" + if CUDA_VISIBLE_DEVICES="$gpu" PYTHONUNBUFFERED=1 \ + "${command[@]}" >"$log_file" 2>&1; then + if [[ -s "$run_dir/$report_name" ]]; then + echo "[$(date '+%F %T')] DONE evaluator=$evaluator gpu=$gpu" + return 0 + fi + + echo "[$(date '+%F %T')] FAIL evaluator=$evaluator gpu=$gpu" >&2 + echo " Evaluator exited successfully but did not create: $run_dir/$report_name" >&2 + echo " See: $log_file" >&2 + return 1 + else + local exit_code=$? + echo "[$(date '+%F %T')] FAIL evaluator=$evaluator gpu=$gpu exit=$exit_code" >&2 + echo " See: $log_file" >&2 + return "$exit_code" + fi +} + +worker() { + local slot="$1" + local gpu="${GPU_IDS[$slot]}" + local job_index + local failed=0 + + for ((job_index = slot; job_index < ${#JOB_RUN_DIRS[@]}; job_index += ${#GPU_IDS[@]})); do + run_job "$job_index" "$gpu" || failed=1 + done + return "$failed" +} + +echo "Runs root: $RUNS_ROOT" +echo "GPUs: ${GPU_IDS[*]}" +echo "Runs discovered: $run_count" +echo "Incomplete runs skipped: $incomplete_run_count" +echo "Existing token reports skipped: $skipped_token_count" +echo "Existing landmark reports skipped: $skipped_landmark_count" +echo "Missing evaluation tasks: ${#JOB_RUN_DIRS[@]}" +echo "Log root: $LOG_ROOT" +echo + +if ((${#JOB_RUN_DIRS[@]} == 0)); then + echo "All discovered runs already have both AUC reports." + exit 0 +fi + +declare -a WORKER_PIDS=() +for ((slot = 0; slot < ${#GPU_IDS[@]}; slot++)); do + worker "$slot" & + WORKER_PIDS+=("$!") +done + +overall_status=0 +for pid in "${WORKER_PIDS[@]}"; do + wait "$pid" || overall_status=1 +done + +if ((overall_status != 0)); then + echo "One or more evaluation tasks failed. Inspect logs under: $LOG_ROOT" >&2 + exit 1 +fi + +if ((DRY_RUN)); then + echo "Dry run completed successfully." +else + echo "All missing AUC evaluations completed successfully." +fi