Train assessment smoking and alcohol model

extra_info_types_assessment_smoking_alcohol.txt

@@ -0,0 +1,71 @@
+# Assessment/body-measurement variables plus smoking and alcohol (field_type=1 plus types 66-67)
+# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering.
+# BMI is already included as assessment type 11 and is not duplicated.
+# Format: <extra_info_type_id> # <var_name> | <full_name>
+1 # waist_circumference | Waist circumference
+2 # hip_circumference | Hip circumference
+3 # standing_height | Standing height
+4 # fasting_time | Fasting time
+5 # pulse_rate | Pulse rate automated reading
+6 # dbp | Diastolic blood pressure automated reading
+7 # sbp | Systolic blood pressure automated reading
+8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure
+9 # fvc_best | Forced vital capacity (FVC) Best measure
+10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score
+11 # bmi | Body mass index (BMI)
+12 # WBC | White blood cell (leukocyte) count
+13 # RBC | Red blood cell (erythrocyte) count
+14 # hemoglobin | Haemoglobin concentration
+15 # hematocrit | Haematocrit percentage
+16 # MCV | Mean corpuscular volume
+17 # MCH | Mean corpuscular haemoglobin
+18 # MCHC | Mean corpuscular haemoglobin concentration
+19 # Pc | Platelet count
+20 # MPV | Mean platelet (thrombocyte) volume
+21 # LymC | Lymphocyte count
+22 # MonC | Monocyte count
+23 # NeuC | Neutrophill count
+24 # EosC | Eosinophill count
+25 # BasC | Basophill count
+26 # nRBC | Nucleated red blood cell count
+27 # RC | Reticulocyte count
+28 # MRV | Mean reticulocyte volume
+29 # MSCV | Mean sphered cell volume
+30 # IRF | Immature reticulocyte fraction
+31 # HLSRC | High light scatter reticulocyte count
+32 # MicU | Microalbumin in urine
+33 # CreaU | Creatinine (enzymatic) in urine
+34 # PotU | Potassium in urine
+35 # SodU | Sodium in urine
+36 # Alb | Albumin
+37 # ALP | Alkaline phosphatase
+38 # Alanine | Alanine aminotransferase
+39 # ApoA | Apolipoprotein A
+40 # ApoB | Apolipoprotein B
+41 # AA | Aspartate aminotransferase
+42 # DBil | Direct bilirubin
+43 # Urea | Urea
+44 # Calcium | Calcium
+45 # Cholesterol | Cholesterol
+46 # Creatinine | Creatinine
+47 # CRP | C-reactive protein
+48 # CystatinC | Cystatin C
+49 # GGT | Gamma glutamyltransferase
+50 # Glu | Glucose
+51 # HbA1c | Glycated haemoglobin (HbA1c)
+52 # HDL | HDL cholesterol
+53 # IGF1 | IGF-1
+54 # LDL | LDL direct
+55 # LpA | Lipoprotein A
+56 # Oestradiol | Oestradiol
+57 # Phosphate | Phosphate
+58 # Rheu | Rheumatoid factor
+59 # SHBG | SHBG
+60 # TotalBil | Total bilirubin
+61 # Testosterone | Testosterone
+62 # TotalProtein | Total protein
+63 # Tri | Triglycerides
+64 # Urate | Urate
+65 # VitaminD | Vitamin D
+66 # smoking | Current tobacco smoking
+67 # alcohol | Alcohol intake frequency.

train_extra_info_assessment_all_multiseed_linux.sh

@@ -1,15 +1,18 @@
 #!/usr/bin/env bash
 #
-# Train the two remaining extra-information experiments:
+# Train the assessment + smoking + alcohol extra-information experiment.
-#   1. assessment_only: 65 routine assessment/body/laboratory variables.
-#   2. all:             all 265 assessment and exposure variables.
 #
 # Fixed model:
 #   TrajMixer + all_future + relative + Weibull + timed disease history + sex
 #
-# A6000 48 GB defaults:
+# Extra information:
-#   assessment_only batch_size=256
+#   - 65 routine assessment/body/laboratory variables
-#   all             batch_size=128
+#   - smoking
+#   - alcohol
+#   - BMI is already included in the assessment variables
+#
+# A6000 48 GB default:
+#   batch_size=256
 #
 # Each task uses one GPU. Tasks assigned to the same GPU run sequentially;
 # different GPUs run in parallel.
@@ -18,7 +21,7 @@
 #   bash train_extra_info_assessment_all_multiseed_linux.sh --gpus 0
 #   bash train_extra_info_assessment_all_multiseed_linux.sh --gpus 0,1,2
 #   bash train_extra_info_assessment_all_multiseed_linux.sh \
-#       --gpus 0 --seeds 42 --types all --dry-run
+#       --gpus 0 --seeds 42 --dry-run
 #
 
 set -uo pipefail
@@ -28,17 +31,14 @@ cd "$SCRIPT_DIR"
 
 GPU_CSV=""
 SEED_CSV="42,43,44"
-TYPE_CSV="assessment_only,all"
 NUM_WORKERS=4
-ASSESSMENT_BATCH_SIZE=256
+BATCH_SIZE=256
-ALL_BATCH_SIZE=128
 PYTHON_BIN="${PYTHON_BIN:-python}"
-CAMPAIGN_NAME="extra_info_assessment_all_multiseed"
+CAMPAIGN_NAME="extra_info_assessment_smoking_alcohol_multiseed"
 DRY_RUN=0
 
 ENTRYPOINT="$SCRIPT_DIR/train_all_future.py"
-ASSESSMENT_FILE="$SCRIPT_DIR/extra_info_types_assessment_only.txt"
+EXTRA_INFO_TYPES_FILE="$SCRIPT_DIR/extra_info_types_assessment_smoking_alcohol.txt"
-ALL_FILE="$SCRIPT_DIR/extra_info_types_all.txt"
 
 usage() {
     cat <<'EOF'
@@ -51,9 +51,7 @@ Required:
 
 Options:
   --seeds LIST           Comma-separated seeds (default: 42,43,44).
-  --types LIST                 Subset of assessment_only,all (default: both).
+  --batch-size N         Batch size per task (default: 256).
-  --assessment-batch-size N    assessment_only batch size (default: 256).
-  --all-batch-size N           all batch size (default: 128 for A6000 48 GB).
   --num-workers N        DataLoader workers per task (default: 4).
   --python PATH          Python executable (default: $PYTHON_BIN or python).
   --campaign NAME        Output campaign name.
@@ -67,17 +65,14 @@ Fixed experiment settings:
   distribution           weibull
   disease history        timed
   sex                    enabled by the model
+  extra information      extra_info_types_assessment_smoking_alcohol.txt
 
-A6000 48 GB memory policy:
+A6000 48 GB default:
-  assessment_only              batch_size=256
+  batch_size             256
-  all                          batch_size=128
-
-If all still runs out of memory because of an unusually long padded batch,
-restart that experiment with --all-batch-size 64.
 
 Outputs:
   runs/<campaign>/seed_<seed>/traj_mixer_v5/...
-  batch_logs/<campaign>/seed_<seed>/<type>.log
+  batch_logs/<campaign>/seed_<seed>/assessment_smoking_alcohol.log
 EOF
 }
 
@@ -99,28 +94,12 @@ while (($# > 0)); do
             SEED_CSV="$2"
             shift 2
             ;;
-        --types)
+        --batch-size)
             [[ $# -ge 2 ]] || {
-                echo "ERROR: --types requires a value." >&2
+                echo "ERROR: --batch-size requires a value." >&2
                 exit 2
             }
-            TYPE_CSV="$2"
+            BATCH_SIZE="$2"
-            shift 2
-            ;;
-        --assessment-batch-size)
-            [[ $# -ge 2 ]] || {
-                echo "ERROR: --assessment-batch-size requires a value." >&2
-                exit 2
-            }
-            ASSESSMENT_BATCH_SIZE="$2"
-            shift 2
-            ;;
-        --all-batch-size)
-            [[ $# -ge 2 ]] || {
-                echo "ERROR: --all-batch-size requires a value." >&2
-                exit 2
-            }
-            ALL_BATCH_SIZE="$2"
             shift 2
             ;;
         --num-workers)
@@ -172,16 +151,8 @@ done
     echo "ERROR: --seeds must not be empty." >&2
     exit 2
 }
-[[ -n "$TYPE_CSV" ]] || {
+[[ "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]] || {
-    echo "ERROR: --types must not be empty." >&2
+    echo "ERROR: --batch-size must be a positive integer." >&2
-    exit 2
-}
-[[ "$ASSESSMENT_BATCH_SIZE" =~ ^[1-9][0-9]*$ ]] || {
-    echo "ERROR: --assessment-batch-size must be a positive integer." >&2
-    exit 2
-}
-[[ "$ALL_BATCH_SIZE" =~ ^[1-9][0-9]*$ ]] || {
-    echo "ERROR: --all-batch-size must be a positive integer." >&2
     exit 2
 }
 [[ "$NUM_WORKERS" =~ ^[0-9]+$ ]] || {
@@ -192,12 +163,14 @@ done
     echo "ERROR: --campaign may contain only letters, numbers, ., _, and -." >&2
     exit 2
 }
-for required_file in "$ENTRYPOINT" "$ASSESSMENT_FILE" "$ALL_FILE"; do
+[[ -f "$ENTRYPOINT" ]] || {
-    [[ -f "$required_file" ]] || {
+    echo "ERROR: missing training entrypoint: $ENTRYPOINT" >&2
-        echo "ERROR: missing required file: $required_file" >&2
     exit 2
-    }
+}
-done
+[[ -f "$EXTRA_INFO_TYPES_FILE" ]] || {
+    echo "ERROR: missing extra-info file: $EXTRA_INFO_TYPES_FILE" >&2
+    exit 2
+}
 command -v "$PYTHON_BIN" >/dev/null 2>&1 || {
     echo "ERROR: Python executable not found: $PYTHON_BIN" >&2
     exit 2
@@ -231,86 +204,44 @@ for seed in "${SEEDS[@]}"; do
     SEEN_SEEDS["$seed"]=1
 done
 
-IFS=',' read -r -a TYPES <<< "$TYPE_CSV"
-declare -A SEEN_TYPES=()
-for info_type in "${TYPES[@]}"; do
-    case "$info_type" in
-        assessment_only|all)
-            ;;
-        *)
-            echo "ERROR: invalid type: $info_type" >&2
-            echo "Expected assessment_only or all." >&2
-            exit 2
-            ;;
-    esac
-    [[ -z "${SEEN_TYPES[$info_type]+x}" ]] || {
-        echo "ERROR: duplicate type: $info_type" >&2
-        exit 2
-    }
-    SEEN_TYPES["$info_type"]=1
-done
-
 RUNS_ROOT="$SCRIPT_DIR/runs/$CAMPAIGN_NAME"
 LOG_ROOT="$SCRIPT_DIR/batch_logs/$CAMPAIGN_NAME"
 if ((!DRY_RUN)); then
     mkdir -p "$RUNS_ROOT" "$LOG_ROOT"
 fi
 
-declare -a JOB_SEEDS=()
-declare -a JOB_TYPES=()
-declare -a JOB_BATCH_SIZES=()
-declare -a JOB_EXTRA_FILES=()
-
-for seed in "${SEEDS[@]}"; do
-    for info_type in "${TYPES[@]}"; do
-        JOB_SEEDS+=("$seed")
-        JOB_TYPES+=("$info_type")
-        if [[ "$info_type" == "assessment_only" ]]; then
-            JOB_BATCH_SIZES+=("$ASSESSMENT_BATCH_SIZE")
-            JOB_EXTRA_FILES+=("$ASSESSMENT_FILE")
-        else
-            JOB_BATCH_SIZES+=("$ALL_BATCH_SIZE")
-            JOB_EXTRA_FILES+=("$ALL_FILE")
-        fi
-    done
-done
-
 print_command() {
     printf '%q ' "$@"
     printf '\n'
 }
 
 run_job() {
-    local job_index="$1"
+    local seed="$1"
     local gpu="$2"
-    local seed="${JOB_SEEDS[$job_index]}"
-    local info_type="${JOB_TYPES[$job_index]}"
-    local batch_size="${JOB_BATCH_SIZES[$job_index]}"
-    local extra_file="${JOB_EXTRA_FILES[$job_index]}"
     local seed_runs_root="$RUNS_ROOT/seed_$seed"
     local seed_log_root="$LOG_ROOT/seed_$seed"
-    local log_file="$seed_log_root/$info_type.log"
+    local log_file="$seed_log_root/assessment_smoking_alcohol.log"
     local -a command=(
         "$PYTHON_BIN"
         -u
         "$ENTRYPOINT"
         --runs_root "$seed_runs_root"
         --seed "$seed"
-        --batch_size "$batch_size"
+        --batch_size "$BATCH_SIZE"
         --num_workers "$NUM_WORKERS"
         --device cuda
         --model_architecture traj_mixer_v5
         --time_mode relative
         --dist_mode weibull
         --disease_history_mode timed
-        --extra_info_types_file "$extra_file"
+        --extra_info_types_file "$EXTRA_INFO_TYPES_FILE"
     )
 
     if ((!DRY_RUN)); then
         mkdir -p "$seed_runs_root" "$seed_log_root"
     fi
 
-    echo "[$(date '+%F %T')] START seed=$seed type=$info_type gpu=$gpu batch=$batch_size"
+    echo "[$(date '+%F %T')] START seed=$seed gpu=$gpu batch=$BATCH_SIZE"
     echo "  log=$log_file"
     if ((DRY_RUN)); then
         printf '  CUDA_VISIBLE_DEVICES=%q ' "$gpu"
@@ -320,15 +251,12 @@ run_job() {
 
     if CUDA_VISIBLE_DEVICES="$gpu" PYTHONUNBUFFERED=1 \
         "${command[@]}" >"$log_file" 2>&1; then
-        echo "[$(date '+%F %T')] DONE  seed=$seed type=$info_type gpu=$gpu"
+        echo "[$(date '+%F %T')] DONE  seed=$seed gpu=$gpu"
         return 0
     else
         local exit_code=$?
-        echo "[$(date '+%F %T')] FAIL seed=$seed type=$info_type gpu=$gpu exit=$exit_code" >&2
+        echo "[$(date '+%F %T')] FAIL seed=$seed gpu=$gpu exit=$exit_code" >&2
         echo "  See: $log_file" >&2
-        if [[ "$info_type" == "all" ]]; then
-            echo "  If this is CUDA OOM, retry with --all-batch-size 64." >&2
-        fi
         return "$exit_code"
     fi
 }
@@ -336,22 +264,21 @@ run_job() {
 worker() {
     local slot="$1"
     local gpu="${GPU_IDS[$slot]}"
-    local job_index
+    local seed_index
     local failed=0
 
-    for ((job_index = slot; job_index < ${#JOB_SEEDS[@]}; job_index += ${#GPU_IDS[@]})); do
+    for ((seed_index = slot; seed_index < ${#SEEDS[@]}; seed_index += ${#GPU_IDS[@]})); do
-        run_job "$job_index" "$gpu" || failed=1
+        run_job "${SEEDS[$seed_index]}" "$gpu" || failed=1
     done
     return "$failed"
 }
 
 echo "Campaign: $CAMPAIGN_NAME"
 echo "Seeds: ${SEEDS[*]}"
-echo "Extra-info types: ${TYPES[*]}"
 echo "GPUs: ${GPU_IDS[*]}"
-echo "assessment_only batch size: $ASSESSMENT_BATCH_SIZE"
+echo "Batch size: $BATCH_SIZE"
-echo "all batch size: $ALL_BATCH_SIZE"
+echo "Extra-info file: $EXTRA_INFO_TYPES_FILE"
-echo "Total tasks: ${#JOB_SEEDS[@]}"
+echo "Total tasks: ${#SEEDS[@]}"
 echo "Runs root: $RUNS_ROOT"
 echo "Log root: $LOG_ROOT"
 if command -v nvidia-smi >/dev/null 2>&1; then
@@ -385,5 +312,5 @@ fi
 if ((DRY_RUN)); then
     echo "Dry run completed successfully."
 else
-    echo "All assessment_only/all training tasks completed successfully."
+    echo "All assessment + smoking + alcohol tasks completed successfully."
 fi