diff --git a/run_missing_training_runs.sh b/run_missing_training_runs.sh new file mode 100755 index 0000000..5aabb91 --- /dev/null +++ b/run_missing_training_runs.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Linux bash 5.2+ training-only script. +# +# Based on the existing runs, the objective/time/death-distribution checks are +# already covered. The remaining gap for the current proof chain is the +# extra-info ablation under the final candidate model: +# +# all_future + relative time + mixed death/risk head +# +# This script only launches those missing training jobs. It intentionally does +# not call evaluate_*.py and does not add extra random seeds. + +cd "$(dirname "${BASH_SOURCE[0]}")" + +PYTHON_BIN="${PYTHON_BIN:-python}" +DEVICE="${DEVICE:-cuda}" +NUM_WORKERS="${NUM_WORKERS:-4}" +PROGRESS_INTERVAL="${PROGRESS_INTERVAL:-20}" + +TIME_MODE="relative" +DIST_MODE="mixed" +SEED="42" +VALIDATION_QUERY_SEED="42" + +COMMON_ARGS=( + --data_prefix ukb + --labels_file labels.csv + --seed "${SEED}" + --validation_query_seed "${VALIDATION_QUERY_SEED}" + --train_eid_file ukb_train_eid.csv + --val_eid_file ukb_val_eid.csv + --test_eid_file ukb_test_eid.csv + --min_history_events 1 + --min_future_events 1 + --n_embd 120 + --n_head 10 + --n_hist_layer 12 + --n_tab_layer 4 + --n_bins 16 + --extra_pool_reduce mean + --dropout 0.0 + --batch_size 256 + --base_lr 0.0003 + --weight_decay 0.1 + --betas 0.9 0.99 + --grad_clip 1.0 + --max_epochs 200 + --warmup_epochs 10 + --patience 15 + --min_lr_ratio 0.1 + --num_workers "${NUM_WORKERS}" + --device "${DEVICE}" + --progress_interval "${PROGRESS_INTERVAL}" +) + +already_trained() { + local extra_file="$1" + "${PYTHON_BIN}" - "$TIME_MODE" "$DIST_MODE" "$extra_file" "$SEED" "$VALIDATION_QUERY_SEED" <<'PY' +import json +import sys +from pathlib import Path + +time_mode, dist_mode, extra_file, seed, validation_query_seed = sys.argv[1:6] +extra_name = Path(extra_file).name + +for config_path in Path("runs").glob("*/train_config.json"): + try: + cfg = json.loads(config_path.read_text(encoding="utf-8")) + except Exception: + continue + + observed_query_seed = cfg.get( + "all_future_validation_query_seed", + cfg.get("validation_query_seed", -1), + ) + + if ( + cfg.get("model_target_mode") == "all_future" + and cfg.get("time_mode") == time_mode + and cfg.get("dist_mode") == dist_mode + and Path(str(cfg.get("extra_info_types_file", ""))).name == extra_name + and int(cfg.get("seed", -1)) == int(seed) + and int(observed_query_seed) == int(validation_query_seed) + ): + print(config_path.parent) + raise SystemExit(0) + +raise SystemExit(1) +PY +} + +train_if_missing() { + local label="$1" + local extra_file="$2" + + if [[ ! -f "${extra_file}" ]]; then + echo "ERROR: missing extra-info type file: ${extra_file}" >&2 + return 2 + fi + + echo "==> Checking ${label}: ${TIME_MODE} ${DIST_MODE} all_future with ${extra_file}" + if existing_run="$(already_trained "$extra_file")"; then + echo " skip: already trained at ${existing_run}" + return 0 + fi + + echo " train: ${label}" + "${PYTHON_BIN}" train_all_future.py \ + "${COMMON_ARGS[@]}" \ + --time_mode "${TIME_MODE}" \ + --dist_mode "${DIST_MODE}" \ + --extra_info_types_file "${extra_file}" +} + +# Already present in runs/: +# - next-token objective checks under SAB, plus older absolute extra ablations +# - all-future absolute/relative x exponential/weibull/mixed under SAB +# +# Still needed: +# - final all-future relative+mixed extra-info ablations beyond the existing +# SAB baseline. These close the disease-only question without expanding seed +# count or running downstream evaluation. +train_if_missing "true_disease_only" "extra_info_types_none.txt" +train_if_missing "assessment_only_extra" "extra_info_types_assessment_only.txt" +train_if_missing "exposure_only_extra" "extra_info_types_exposure_only.txt" +train_if_missing "all_extra_info" "extra_info_types_all.txt" + +echo "All requested training-only missing configurations are done."