Update missing evaluation runner

run_missing_evaluations.sh

@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run all non-wrapper evaluation scripts for every completed experiment under
+# runs/.  The script is written for Linux servers with bash 4.2.
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+PYTHON_BIN="${PYTHON_BIN:-python}"
+DEVICE="${DEVICE:-cuda}"
+EVAL_SPLIT="${EVAL_SPLIT:-test}"
+TAU="${TAU:-5}"
+NUM_WORKERS="${NUM_WORKERS:-4}"
+NUM_WORKERS_AUC="${NUM_WORKERS_AUC:-}"
+BATCH_SIZE="${BATCH_SIZE:-}"
+DATASET_SUBSET_SIZE="${DATASET_SUBSET_SIZE:-}"
+DRY_RUN="${DRY_RUN:-0}"
+
+# These attribution jobs can be expensive, but they are part of the evaluation
+# surface in this repository. Set either variable to 0 to leave that family out.
+RUN_EXTRA_INFO_ATTRIBUTION="${RUN_EXTRA_INFO_ATTRIBUTION:-1}"
+RUN_SINGLE_DISEASE_MORTALITY_ATTRIBUTION="${RUN_SINGLE_DISEASE_MORTALITY_ATTRIBUTION:-1}"
+TAU_LABEL="$("${PYTHON_BIN}" -c 'import sys; print(f"{float(sys.argv[1]):g}")' "${TAU}")"
+
+common_args_base() {
+  printf '%s\n' --run_path "$1" --eval_split "${EVAL_SPLIT}" --num_workers "${NUM_WORKERS}"
+  if [[ -n "${BATCH_SIZE}" ]]; then
+    printf '%s\n' --batch_size "${BATCH_SIZE}"
+  fi
+  if [[ -n "${DATASET_SUBSET_SIZE}" ]]; then
+    printf '%s\n' --dataset_subset_size "${DATASET_SUBSET_SIZE}"
+  fi
+}
+
+common_args_with_device() {
+  common_args_base "$1"
+  printf '%s\n' --device "${DEVICE}"
+}
+
+auc_args() {
+  if [[ -n "${NUM_WORKERS_AUC}" ]]; then
+    printf '%s\n' --num_workers_auc "${NUM_WORKERS_AUC}"
+  fi
+}
+
+has_completed_dir() {
+  local dir="$1"
+  shift
+  [[ -d "${dir}" ]] || return 1
+  local required
+  for required in "$@"; do
+    [[ -s "${dir}/${required}" ]] || return 1
+  done
+}
+
+run_command() {
+  echo "    run: $*"
+  if [[ "${DRY_RUN}" == "1" ]]; then
+    return 0
+  fi
+  "$@"
+}
+
+run_dir_result_if_missing() {
+  local label="$1"
+  local result_dir="$2"
+  local required_1="$3"
+  local required_2="$4"
+  shift 4
+
+  if has_completed_dir "${result_dir}" "${required_1}" "${required_2}"; then
+    echo "    skip ${label}: found ${result_dir}"
+    return 0
+  fi
+
+  run_command "$@"
+}
+
+run_has_extra_info() {
+  "${PYTHON_BIN}" - "$1" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+cfg_path = Path(sys.argv[1]) / "train_config.json"
+try:
+    cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
+except Exception:
+    raise SystemExit(1)
+
+extra = cfg.get("extra_info_types", [])
+raise SystemExit(0 if isinstance(extra, list) and len(extra) > 0 else 1)
+PY
+}
+
+for run_path in runs/*; do
+  [[ -d "${run_path}" ]] || continue
+
+  echo "==> ${run_path}"
+  if [[ ! -f "${run_path}/train_config.json" ]]; then
+    echo "    skip run: missing train_config.json"
+    continue
+  fi
+  if [[ ! -s "${run_path}/best_model.pt" ]]; then
+    echo "    skip run: missing best_model.pt"
+    continue
+  fi
+
+  common=()
+  while IFS= read -r arg; do common+=("${arg}"); done < <(common_args_with_device "${run_path}")
+
+  auc_extra=()
+  while IFS= read -r arg; do auc_extra+=("${arg}"); done < <(auc_args)
+
+  run_dir_result_if_missing \
+    "evaluate_auc.py" \
+    "${run_path}" \
+    "df_both.csv" \
+    "df_auc_unpooled.csv" \
+    "${PYTHON_BIN}" evaluate_auc.py "${common[@]}" "${auc_extra[@]}"
+
+  run_dir_result_if_missing \
+    "evaluate_auc_v2.py" \
+    "${run_path}" \
+    "df_auc_landmark.csv" \
+    "df_auc_landmark_unpooled.csv" \
+    "${PYTHON_BIN}" evaluate_auc_v2.py "${common[@]}" "${auc_extra[@]}"
+
+  if [[ "${RUN_EXTRA_INFO_ATTRIBUTION}" == "1" ]]; then
+    if run_has_extra_info "${run_path}"; then
+      run_dir_result_if_missing \
+        "evaluate_extra_info_attribution.py" \
+        "${run_path}/extra_info_attribution_${EVAL_SPLIT}_tau${TAU_LABEL}y" \
+        "manifest.json" \
+        "summary_extra_info_future_disease_risk.csv" \
+        "${PYTHON_BIN}" evaluate_extra_info_attribution.py "${common[@]}" --tau "${TAU}"
+    else
+      echo "    skip evaluate_extra_info_attribution.py: run has no extra-info types"
+    fi
+  fi
+
+  if [[ "${RUN_SINGLE_DISEASE_MORTALITY_ATTRIBUTION}" == "1" ]]; then
+    run_dir_result_if_missing \
+      "evaluate_single_disease_mortality_attribution.py" \
+      "${run_path}/single_disease_mortality_parameters_${EVAL_SPLIT}_all_diseases" \
+      "manifest.json" \
+      "summary_by_disease_age_sex.csv" \
+      "${PYTHON_BIN}" evaluate_single_disease_mortality_attribution.py "${common[@]}"
+  fi
+done
+
+echo "All missing evaluations are complete."