From f1ca401783e762d6f9ffe8c3933cba0667836006 Mon Sep 17 00:00:00 2001
From: Jiarui Li <jiarui_li@outlook.com>
Date: Mon, 3 Aug 2026 14:02:09 +0800
Subject: [PATCH] feat: force recompute AUC batch evaluations

---
 evaluate_all_runs_linux.sh | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/evaluate_all_runs_linux.sh b/evaluate_all_runs_linux.sh
index 4c22bcf..467b7f8 100644
--- a/evaluate_all_runs_linux.sh
+++ b/evaluate_all_runs_linux.sh
@@ -12,12 +12,14 @@
 #   - evaluate_auc_v2.py
 #       -> df_auc_landmark_delphi2m_report.csv
 #
-# Existing non-empty reports are skipped. Each evaluation task uses one GPU.
+# Existing non-empty reports are skipped unless --force is supplied. Each
+# evaluation task uses one GPU.
 # Tasks on the same GPU run sequentially; different GPUs run in parallel.
 #
 # Examples:
 #   bash evaluate_all_runs_linux.sh --gpus 0
 #   bash evaluate_all_runs_linux.sh --gpus 0,1,2,3
+#   bash evaluate_all_runs_linux.sh --gpus 0,1 --force
 #   bash evaluate_all_runs_linux.sh --gpus 0,1 --dry-run
 #
 
@@ -32,6 +34,7 @@ GPU_CSV="0"
 PYTHON_BIN="${PYTHON_BIN:-python}"
 NUM_WORKERS=4
 NUM_WORKERS_AUC=4
+FORCE=0
 DRY_RUN=0
 
 TOKEN_REPORT="df_auc_delphi2m_report.csv"
@@ -49,14 +52,16 @@ Options:
   --python PATH          Python executable (default: $PYTHON_BIN or python).
   --num-workers N        DataLoader workers per evaluation (default: 4).
   --num-workers-auc N    CPU AUC workers per evaluation (default: 4).
-  --dry-run              Discover runs and print missing evaluations only.
+  --force                Recompute both AUC reports even when they exist.
+  --dry-run              Discover runs and print scheduled evaluations only.
   -h, --help             Show this help message.
 
 Completion files:
   evaluate_auc.py        df_auc_delphi2m_report.csv
   evaluate_auc_v2.py     df_auc_landmark_delphi2m_report.csv
 
-Existing non-empty completion files are skipped independently.
+Existing non-empty completion files are skipped independently unless --force
+is supplied.
 EOF
 }
 
@@ -110,6 +115,10 @@ while (($# > 0)); do
             NUM_WORKERS_AUC="$2"
             shift 2
             ;;
+        --force)
+            FORCE=1
+            shift
+            ;;
         --dry-run)
             DRY_RUN=1
             shift
@@ -207,13 +216,13 @@ while IFS= read -r -d '' config_path; do
         continue
     fi
 
-    if [[ -s "$run_dir/$TOKEN_REPORT" ]]; then
+    if ((!FORCE)) && [[ -s "$run_dir/$TOKEN_REPORT" ]]; then
         ((skipped_token_count += 1))
     else
         add_job "$run_dir" "evaluate_auc.py" "$TOKEN_REPORT"
     fi
 
-    if [[ -s "$run_dir/$LANDMARK_REPORT" ]]; then
+    if ((!FORCE)) && [[ -s "$run_dir/$LANDMARK_REPORT" ]]; then
         ((skipped_landmark_count += 1))
     else
         add_job "$run_dir" "evaluate_auc_v2.py" "$LANDMARK_REPORT"
@@ -289,18 +298,24 @@ worker() {
     return "$failed"
 }
 
+force_label="no"
+if ((FORCE)); then
+    force_label="yes"
+fi
+
 echo "Runs root: $RUNS_ROOT"
 echo "GPUs: ${GPU_IDS[*]}"
+echo "Force recompute: $force_label"
 echo "Runs discovered: $run_count"
 echo "Incomplete runs skipped: $incomplete_run_count"
 echo "Existing token reports skipped: $skipped_token_count"
 echo "Existing landmark reports skipped: $skipped_landmark_count"
-echo "Missing evaluation tasks: ${#JOB_RUN_DIRS[@]}"
+echo "Scheduled evaluation tasks: ${#JOB_RUN_DIRS[@]}"
 echo "Log root: $LOG_ROOT"
 echo
 
 if ((${#JOB_RUN_DIRS[@]} == 0)); then
-    echo "All discovered runs already have both AUC reports."
+    echo "No AUC evaluation tasks are scheduled."
     exit 0
 fi
 
@@ -323,5 +338,5 @@ fi
 if ((DRY_RUN)); then
     echo "Dry run completed successfully."
 else
-    echo "All missing AUC evaluations completed successfully."
+    echo "All scheduled AUC evaluations completed successfully."
 fi