Add recursive AUC evaluation script
This commit is contained in:
327
evaluate_all_runs_linux.sh
Normal file
327
evaluate_all_runs_linux.sh
Normal file
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Recursively evaluate every completed run under runs/.
|
||||
#
|
||||
# A directory is considered a runnable run when it contains both:
|
||||
# - train_config.json
|
||||
# - best_model.pt
|
||||
#
|
||||
# Each missing report is scheduled independently:
|
||||
# - evaluate_auc.py
|
||||
# -> df_auc_delphi2m_report.csv
|
||||
# - evaluate_auc_v2.py
|
||||
# -> df_auc_landmark_delphi2m_report.csv
|
||||
#
|
||||
# Existing non-empty reports are skipped. Each evaluation task uses one GPU.
|
||||
# Tasks on the same GPU run sequentially; different GPUs run in parallel.
|
||||
#
|
||||
# Examples:
|
||||
# bash evaluate_all_runs_linux.sh --gpus 0
|
||||
# bash evaluate_all_runs_linux.sh --gpus 0,1,2,3
|
||||
# bash evaluate_all_runs_linux.sh --gpus 0,1 --dry-run
|
||||
#
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
RUNS_ROOT="$SCRIPT_DIR/runs"
|
||||
LOG_ROOT="$SCRIPT_DIR/batch_logs/evaluate_all_runs"
|
||||
GPU_CSV="0"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python}"
|
||||
NUM_WORKERS=4
|
||||
NUM_WORKERS_AUC=4
|
||||
DRY_RUN=0
|
||||
|
||||
TOKEN_REPORT="df_auc_delphi2m_report.csv"
|
||||
LANDMARK_REPORT="df_auc_landmark_delphi2m_report.csv"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
bash evaluate_all_runs_linux.sh [options]
|
||||
|
||||
Options:
|
||||
--gpus LIST Comma-separated GPU ids (default: 0).
|
||||
--runs-root PATH Root directory scanned recursively (default: ./runs).
|
||||
--log-root PATH Evaluation log root.
|
||||
--python PATH Python executable (default: $PYTHON_BIN or python).
|
||||
--num-workers N DataLoader workers per evaluation (default: 4).
|
||||
--num-workers-auc N CPU AUC workers per evaluation (default: 4).
|
||||
--dry-run Discover runs and print missing evaluations only.
|
||||
-h, --help Show this help message.
|
||||
|
||||
Completion files:
|
||||
evaluate_auc.py df_auc_delphi2m_report.csv
|
||||
evaluate_auc_v2.py df_auc_landmark_delphi2m_report.csv
|
||||
|
||||
Existing non-empty completion files are skipped independently.
|
||||
EOF
|
||||
}
|
||||
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--gpus)
|
||||
[[ $# -ge 2 ]] || {
|
||||
echo "ERROR: --gpus requires a value." >&2
|
||||
exit 2
|
||||
}
|
||||
GPU_CSV="$2"
|
||||
shift 2
|
||||
;;
|
||||
--runs-root)
|
||||
[[ $# -ge 2 ]] || {
|
||||
echo "ERROR: --runs-root requires a value." >&2
|
||||
exit 2
|
||||
}
|
||||
RUNS_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--log-root)
|
||||
[[ $# -ge 2 ]] || {
|
||||
echo "ERROR: --log-root requires a value." >&2
|
||||
exit 2
|
||||
}
|
||||
LOG_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--python)
|
||||
[[ $# -ge 2 ]] || {
|
||||
echo "ERROR: --python requires a value." >&2
|
||||
exit 2
|
||||
}
|
||||
PYTHON_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
--num-workers)
|
||||
[[ $# -ge 2 ]] || {
|
||||
echo "ERROR: --num-workers requires a value." >&2
|
||||
exit 2
|
||||
}
|
||||
NUM_WORKERS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--num-workers-auc)
|
||||
[[ $# -ge 2 ]] || {
|
||||
echo "ERROR: --num-workers-auc requires a value." >&2
|
||||
exit 2
|
||||
}
|
||||
NUM_WORKERS_AUC="$2"
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$GPU_CSV" ]] || {
|
||||
echo "ERROR: --gpus must not be empty." >&2
|
||||
exit 2
|
||||
}
|
||||
[[ "$NUM_WORKERS" =~ ^[0-9]+$ ]] || {
|
||||
echo "ERROR: --num-workers must be a non-negative integer." >&2
|
||||
exit 2
|
||||
}
|
||||
[[ "$NUM_WORKERS_AUC" =~ ^[1-9][0-9]*$ ]] || {
|
||||
echo "ERROR: --num-workers-auc must be a positive integer." >&2
|
||||
exit 2
|
||||
}
|
||||
[[ -d "$RUNS_ROOT" ]] || {
|
||||
echo "ERROR: runs root does not exist: $RUNS_ROOT" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ -f "$SCRIPT_DIR/evaluate_auc.py" ]] || {
|
||||
echo "ERROR: missing evaluator: $SCRIPT_DIR/evaluate_auc.py" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ -f "$SCRIPT_DIR/evaluate_auc_v2.py" ]] || {
|
||||
echo "ERROR: missing evaluator: $SCRIPT_DIR/evaluate_auc_v2.py" >&2
|
||||
exit 2
|
||||
}
|
||||
command -v "$PYTHON_BIN" >/dev/null 2>&1 || {
|
||||
echo "ERROR: Python executable not found: $PYTHON_BIN" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
RUNS_ROOT="$(cd -- "$RUNS_ROOT" && pwd)"
|
||||
if [[ "$LOG_ROOT" != /* ]]; then
|
||||
LOG_ROOT="$SCRIPT_DIR/$LOG_ROOT"
|
||||
fi
|
||||
|
||||
IFS=',' read -r -a GPU_IDS <<< "$GPU_CSV"
|
||||
declare -A SEEN_GPUS=()
|
||||
for gpu in "${GPU_IDS[@]}"; do
|
||||
[[ -n "$gpu" && "$gpu" =~ ^[A-Za-z0-9._:-]+$ ]] || {
|
||||
echo "ERROR: invalid GPU id: $gpu" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ -z "${SEEN_GPUS[$gpu]+x}" ]] || {
|
||||
echo "ERROR: duplicate GPU id: $gpu" >&2
|
||||
exit 2
|
||||
}
|
||||
SEEN_GPUS["$gpu"]=1
|
||||
done
|
||||
|
||||
declare -a JOB_RUN_DIRS=()
|
||||
declare -a JOB_EVALUATORS=()
|
||||
declare -a JOB_REPORTS=()
|
||||
declare -a JOB_LOG_FILES=()
|
||||
|
||||
add_job() {
|
||||
local run_dir="$1"
|
||||
local evaluator="$2"
|
||||
local report_name="$3"
|
||||
local relative_run="${run_dir#"$RUNS_ROOT"/}"
|
||||
local evaluator_name="${evaluator%.py}"
|
||||
|
||||
JOB_RUN_DIRS+=("$run_dir")
|
||||
JOB_EVALUATORS+=("$evaluator")
|
||||
JOB_REPORTS+=("$report_name")
|
||||
JOB_LOG_FILES+=("$LOG_ROOT/$relative_run/$evaluator_name.log")
|
||||
}
|
||||
|
||||
run_count=0
|
||||
incomplete_run_count=0
|
||||
skipped_token_count=0
|
||||
skipped_landmark_count=0
|
||||
|
||||
while IFS= read -r -d '' config_path; do
|
||||
run_dir="${config_path%/train_config.json}"
|
||||
((run_count += 1))
|
||||
|
||||
if [[ ! -f "$run_dir/best_model.pt" ]]; then
|
||||
echo "[SKIP] Incomplete run without best_model.pt: $run_dir"
|
||||
((incomplete_run_count += 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -s "$run_dir/$TOKEN_REPORT" ]]; then
|
||||
((skipped_token_count += 1))
|
||||
else
|
||||
add_job "$run_dir" "evaluate_auc.py" "$TOKEN_REPORT"
|
||||
fi
|
||||
|
||||
if [[ -s "$run_dir/$LANDMARK_REPORT" ]]; then
|
||||
((skipped_landmark_count += 1))
|
||||
else
|
||||
add_job "$run_dir" "evaluate_auc_v2.py" "$LANDMARK_REPORT"
|
||||
fi
|
||||
done < <(find "$RUNS_ROOT" -type f -name "train_config.json" -print0)
|
||||
|
||||
if ((!DRY_RUN)); then
|
||||
mkdir -p "$LOG_ROOT"
|
||||
fi
|
||||
|
||||
print_command() {
|
||||
printf '%q ' "$@"
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
run_job() {
|
||||
local job_index="$1"
|
||||
local gpu="$2"
|
||||
local run_dir="${JOB_RUN_DIRS[$job_index]}"
|
||||
local evaluator="${JOB_EVALUATORS[$job_index]}"
|
||||
local report_name="${JOB_REPORTS[$job_index]}"
|
||||
local log_file="${JOB_LOG_FILES[$job_index]}"
|
||||
local -a command=(
|
||||
"$PYTHON_BIN"
|
||||
-u
|
||||
"$SCRIPT_DIR/$evaluator"
|
||||
--run_path "$run_dir"
|
||||
--output_path "$run_dir"
|
||||
--device cuda
|
||||
--num_workers "$NUM_WORKERS"
|
||||
--num_workers_auc "$NUM_WORKERS_AUC"
|
||||
)
|
||||
|
||||
echo "[$(date '+%F %T')] START evaluator=$evaluator gpu=$gpu"
|
||||
echo " run=$run_dir"
|
||||
echo " report=$report_name"
|
||||
|
||||
if ((DRY_RUN)); then
|
||||
printf ' CUDA_VISIBLE_DEVICES=%q ' "$gpu"
|
||||
print_command "${command[@]}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname -- "$log_file")"
|
||||
if CUDA_VISIBLE_DEVICES="$gpu" PYTHONUNBUFFERED=1 \
|
||||
"${command[@]}" >"$log_file" 2>&1; then
|
||||
if [[ -s "$run_dir/$report_name" ]]; then
|
||||
echo "[$(date '+%F %T')] DONE evaluator=$evaluator gpu=$gpu"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "[$(date '+%F %T')] FAIL evaluator=$evaluator gpu=$gpu" >&2
|
||||
echo " Evaluator exited successfully but did not create: $run_dir/$report_name" >&2
|
||||
echo " See: $log_file" >&2
|
||||
return 1
|
||||
else
|
||||
local exit_code=$?
|
||||
echo "[$(date '+%F %T')] FAIL evaluator=$evaluator gpu=$gpu exit=$exit_code" >&2
|
||||
echo " See: $log_file" >&2
|
||||
return "$exit_code"
|
||||
fi
|
||||
}
|
||||
|
||||
worker() {
|
||||
local slot="$1"
|
||||
local gpu="${GPU_IDS[$slot]}"
|
||||
local job_index
|
||||
local failed=0
|
||||
|
||||
for ((job_index = slot; job_index < ${#JOB_RUN_DIRS[@]}; job_index += ${#GPU_IDS[@]})); do
|
||||
run_job "$job_index" "$gpu" || failed=1
|
||||
done
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
echo "Runs root: $RUNS_ROOT"
|
||||
echo "GPUs: ${GPU_IDS[*]}"
|
||||
echo "Runs discovered: $run_count"
|
||||
echo "Incomplete runs skipped: $incomplete_run_count"
|
||||
echo "Existing token reports skipped: $skipped_token_count"
|
||||
echo "Existing landmark reports skipped: $skipped_landmark_count"
|
||||
echo "Missing evaluation tasks: ${#JOB_RUN_DIRS[@]}"
|
||||
echo "Log root: $LOG_ROOT"
|
||||
echo
|
||||
|
||||
if ((${#JOB_RUN_DIRS[@]} == 0)); then
|
||||
echo "All discovered runs already have both AUC reports."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -a WORKER_PIDS=()
|
||||
for ((slot = 0; slot < ${#GPU_IDS[@]}; slot++)); do
|
||||
worker "$slot" &
|
||||
WORKER_PIDS+=("$!")
|
||||
done
|
||||
|
||||
overall_status=0
|
||||
for pid in "${WORKER_PIDS[@]}"; do
|
||||
wait "$pid" || overall_status=1
|
||||
done
|
||||
|
||||
if ((overall_status != 0)); then
|
||||
echo "One or more evaluation tasks failed. Inspect logs under: $LOG_ROOT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ((DRY_RUN)); then
|
||||
echo "Dry run completed successfully."
|
||||
else
|
||||
echo "All missing AUC evaluations completed successfully."
|
||||
fi
|
||||
Reference in New Issue
Block a user