diff --git a/extra_info_types_assessment_smoking_alcohol.txt b/extra_info_types_assessment_smoking_alcohol.txt new file mode 100644 index 0000000..6841d57 --- /dev/null +++ b/extra_info_types_assessment_smoking_alcohol.txt @@ -0,0 +1,71 @@ +# Assessment/body-measurement variables plus smoking and alcohol (field_type=1 plus types 66-67) +# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. +# BMI is already included as assessment type 11 and is not duplicated. +# Format: # | +1 # waist_circumference | Waist circumference +2 # hip_circumference | Hip circumference +3 # standing_height | Standing height +4 # fasting_time | Fasting time +5 # pulse_rate | Pulse rate automated reading +6 # dbp | Diastolic blood pressure automated reading +7 # sbp | Systolic blood pressure automated reading +8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure +9 # fvc_best | Forced vital capacity (FVC) Best measure +10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score +11 # bmi | Body mass index (BMI) +12 # WBC | White blood cell (leukocyte) count +13 # RBC | Red blood cell (erythrocyte) count +14 # hemoglobin | Haemoglobin concentration +15 # hematocrit | Haematocrit percentage +16 # MCV | Mean corpuscular volume +17 # MCH | Mean corpuscular haemoglobin +18 # MCHC | Mean corpuscular haemoglobin concentration +19 # Pc | Platelet count +20 # MPV | Mean platelet (thrombocyte) volume +21 # LymC | Lymphocyte count +22 # MonC | Monocyte count +23 # NeuC | Neutrophill count +24 # EosC | Eosinophill count +25 # BasC | Basophill count +26 # nRBC | Nucleated red blood cell count +27 # RC | Reticulocyte count +28 # MRV | Mean reticulocyte volume +29 # MSCV | Mean sphered cell volume +30 # IRF | Immature reticulocyte fraction +31 # HLSRC | High light scatter reticulocyte count +32 # MicU | Microalbumin in urine +33 # CreaU | Creatinine (enzymatic) in urine +34 # PotU | Potassium in urine +35 # SodU | Sodium in urine +36 # Alb | Albumin +37 # ALP | Alkaline phosphatase +38 # Alanine | Alanine aminotransferase +39 # ApoA | Apolipoprotein A +40 # ApoB | Apolipoprotein B +41 # AA | Aspartate aminotransferase +42 # DBil | Direct bilirubin +43 # Urea | Urea +44 # Calcium | Calcium +45 # Cholesterol | Cholesterol +46 # Creatinine | Creatinine +47 # CRP | C-reactive protein +48 # CystatinC | Cystatin C +49 # GGT | Gamma glutamyltransferase +50 # Glu | Glucose +51 # HbA1c | Glycated haemoglobin (HbA1c) +52 # HDL | HDL cholesterol +53 # IGF1 | IGF-1 +54 # LDL | LDL direct +55 # LpA | Lipoprotein A +56 # Oestradiol | Oestradiol +57 # Phosphate | Phosphate +58 # Rheu | Rheumatoid factor +59 # SHBG | SHBG +60 # TotalBil | Total bilirubin +61 # Testosterone | Testosterone +62 # TotalProtein | Total protein +63 # Tri | Triglycerides +64 # Urate | Urate +65 # VitaminD | Vitamin D +66 # smoking | Current tobacco smoking +67 # alcohol | Alcohol intake frequency. diff --git a/train_extra_info_assessment_all_multiseed_linux.sh b/train_extra_info_assessment_all_multiseed_linux.sh index 9f166ec..86f3144 100755 --- a/train_extra_info_assessment_all_multiseed_linux.sh +++ b/train_extra_info_assessment_all_multiseed_linux.sh @@ -1,15 +1,18 @@ #!/usr/bin/env bash # -# Train the two remaining extra-information experiments: -# 1. assessment_only: 65 routine assessment/body/laboratory variables. -# 2. all: all 265 assessment and exposure variables. +# Train the assessment + smoking + alcohol extra-information experiment. # # Fixed model: # TrajMixer + all_future + relative + Weibull + timed disease history + sex # -# A6000 48 GB defaults: -# assessment_only batch_size=256 -# all batch_size=128 +# Extra information: +# - 65 routine assessment/body/laboratory variables +# - smoking +# - alcohol +# - BMI is already included in the assessment variables +# +# A6000 48 GB default: +# batch_size=256 # # Each task uses one GPU. Tasks assigned to the same GPU run sequentially; # different GPUs run in parallel. @@ -18,7 +21,7 @@ # bash train_extra_info_assessment_all_multiseed_linux.sh --gpus 0 # bash train_extra_info_assessment_all_multiseed_linux.sh --gpus 0,1,2 # bash train_extra_info_assessment_all_multiseed_linux.sh \ -# --gpus 0 --seeds 42 --types all --dry-run +# --gpus 0 --seeds 42 --dry-run # set -uo pipefail @@ -28,17 +31,14 @@ cd "$SCRIPT_DIR" GPU_CSV="" SEED_CSV="42,43,44" -TYPE_CSV="assessment_only,all" NUM_WORKERS=4 -ASSESSMENT_BATCH_SIZE=256 -ALL_BATCH_SIZE=128 +BATCH_SIZE=256 PYTHON_BIN="${PYTHON_BIN:-python}" -CAMPAIGN_NAME="extra_info_assessment_all_multiseed" +CAMPAIGN_NAME="extra_info_assessment_smoking_alcohol_multiseed" DRY_RUN=0 ENTRYPOINT="$SCRIPT_DIR/train_all_future.py" -ASSESSMENT_FILE="$SCRIPT_DIR/extra_info_types_assessment_only.txt" -ALL_FILE="$SCRIPT_DIR/extra_info_types_all.txt" +EXTRA_INFO_TYPES_FILE="$SCRIPT_DIR/extra_info_types_assessment_smoking_alcohol.txt" usage() { cat <<'EOF' @@ -47,37 +47,32 @@ Usage: --gpus GPU_LIST [options] Required: - --gpus LIST Comma-separated GPU ids, for example 0,1,2. + --gpus LIST Comma-separated GPU ids, for example 0,1,2. Options: - --seeds LIST Comma-separated seeds (default: 42,43,44). - --types LIST Subset of assessment_only,all (default: both). - --assessment-batch-size N assessment_only batch size (default: 256). - --all-batch-size N all batch size (default: 128 for A6000 48 GB). - --num-workers N DataLoader workers per task (default: 4). - --python PATH Python executable (default: $PYTHON_BIN or python). - --campaign NAME Output campaign name. - --dry-run Print commands without creating files or training. - -h, --help Show this help message. + --seeds LIST Comma-separated seeds (default: 42,43,44). + --batch-size N Batch size per task (default: 256). + --num-workers N DataLoader workers per task (default: 4). + --python PATH Python executable (default: $PYTHON_BIN or python). + --campaign NAME Output campaign name. + --dry-run Print commands without creating files or training. + -h, --help Show this help message. Fixed experiment settings: - architecture traj_mixer_v5 - target all_future - time mode relative - distribution weibull - disease history timed - sex enabled by the model + architecture traj_mixer_v5 + target all_future + time mode relative + distribution weibull + disease history timed + sex enabled by the model + extra information extra_info_types_assessment_smoking_alcohol.txt -A6000 48 GB memory policy: - assessment_only batch_size=256 - all batch_size=128 - -If all still runs out of memory because of an unusually long padded batch, -restart that experiment with --all-batch-size 64. +A6000 48 GB default: + batch_size 256 Outputs: runs//seed_/traj_mixer_v5/... - batch_logs//seed_/.log + batch_logs//seed_/assessment_smoking_alcohol.log EOF } @@ -99,28 +94,12 @@ while (($# > 0)); do SEED_CSV="$2" shift 2 ;; - --types) + --batch-size) [[ $# -ge 2 ]] || { - echo "ERROR: --types requires a value." >&2 + echo "ERROR: --batch-size requires a value." >&2 exit 2 } - TYPE_CSV="$2" - shift 2 - ;; - --assessment-batch-size) - [[ $# -ge 2 ]] || { - echo "ERROR: --assessment-batch-size requires a value." >&2 - exit 2 - } - ASSESSMENT_BATCH_SIZE="$2" - shift 2 - ;; - --all-batch-size) - [[ $# -ge 2 ]] || { - echo "ERROR: --all-batch-size requires a value." >&2 - exit 2 - } - ALL_BATCH_SIZE="$2" + BATCH_SIZE="$2" shift 2 ;; --num-workers) @@ -172,16 +151,8 @@ done echo "ERROR: --seeds must not be empty." >&2 exit 2 } -[[ -n "$TYPE_CSV" ]] || { - echo "ERROR: --types must not be empty." >&2 - exit 2 -} -[[ "$ASSESSMENT_BATCH_SIZE" =~ ^[1-9][0-9]*$ ]] || { - echo "ERROR: --assessment-batch-size must be a positive integer." >&2 - exit 2 -} -[[ "$ALL_BATCH_SIZE" =~ ^[1-9][0-9]*$ ]] || { - echo "ERROR: --all-batch-size must be a positive integer." >&2 +[[ "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]] || { + echo "ERROR: --batch-size must be a positive integer." >&2 exit 2 } [[ "$NUM_WORKERS" =~ ^[0-9]+$ ]] || { @@ -192,12 +163,14 @@ done echo "ERROR: --campaign may contain only letters, numbers, ., _, and -." >&2 exit 2 } -for required_file in "$ENTRYPOINT" "$ASSESSMENT_FILE" "$ALL_FILE"; do - [[ -f "$required_file" ]] || { - echo "ERROR: missing required file: $required_file" >&2 - exit 2 - } -done +[[ -f "$ENTRYPOINT" ]] || { + echo "ERROR: missing training entrypoint: $ENTRYPOINT" >&2 + exit 2 +} +[[ -f "$EXTRA_INFO_TYPES_FILE" ]] || { + echo "ERROR: missing extra-info file: $EXTRA_INFO_TYPES_FILE" >&2 + exit 2 +} command -v "$PYTHON_BIN" >/dev/null 2>&1 || { echo "ERROR: Python executable not found: $PYTHON_BIN" >&2 exit 2 @@ -231,86 +204,44 @@ for seed in "${SEEDS[@]}"; do SEEN_SEEDS["$seed"]=1 done -IFS=',' read -r -a TYPES <<< "$TYPE_CSV" -declare -A SEEN_TYPES=() -for info_type in "${TYPES[@]}"; do - case "$info_type" in - assessment_only|all) - ;; - *) - echo "ERROR: invalid type: $info_type" >&2 - echo "Expected assessment_only or all." >&2 - exit 2 - ;; - esac - [[ -z "${SEEN_TYPES[$info_type]+x}" ]] || { - echo "ERROR: duplicate type: $info_type" >&2 - exit 2 - } - SEEN_TYPES["$info_type"]=1 -done - RUNS_ROOT="$SCRIPT_DIR/runs/$CAMPAIGN_NAME" LOG_ROOT="$SCRIPT_DIR/batch_logs/$CAMPAIGN_NAME" if ((!DRY_RUN)); then mkdir -p "$RUNS_ROOT" "$LOG_ROOT" fi -declare -a JOB_SEEDS=() -declare -a JOB_TYPES=() -declare -a JOB_BATCH_SIZES=() -declare -a JOB_EXTRA_FILES=() - -for seed in "${SEEDS[@]}"; do - for info_type in "${TYPES[@]}"; do - JOB_SEEDS+=("$seed") - JOB_TYPES+=("$info_type") - if [[ "$info_type" == "assessment_only" ]]; then - JOB_BATCH_SIZES+=("$ASSESSMENT_BATCH_SIZE") - JOB_EXTRA_FILES+=("$ASSESSMENT_FILE") - else - JOB_BATCH_SIZES+=("$ALL_BATCH_SIZE") - JOB_EXTRA_FILES+=("$ALL_FILE") - fi - done -done - print_command() { printf '%q ' "$@" printf '\n' } run_job() { - local job_index="$1" + local seed="$1" local gpu="$2" - local seed="${JOB_SEEDS[$job_index]}" - local info_type="${JOB_TYPES[$job_index]}" - local batch_size="${JOB_BATCH_SIZES[$job_index]}" - local extra_file="${JOB_EXTRA_FILES[$job_index]}" local seed_runs_root="$RUNS_ROOT/seed_$seed" local seed_log_root="$LOG_ROOT/seed_$seed" - local log_file="$seed_log_root/$info_type.log" + local log_file="$seed_log_root/assessment_smoking_alcohol.log" local -a command=( "$PYTHON_BIN" -u "$ENTRYPOINT" --runs_root "$seed_runs_root" --seed "$seed" - --batch_size "$batch_size" + --batch_size "$BATCH_SIZE" --num_workers "$NUM_WORKERS" --device cuda --model_architecture traj_mixer_v5 --time_mode relative --dist_mode weibull --disease_history_mode timed - --extra_info_types_file "$extra_file" + --extra_info_types_file "$EXTRA_INFO_TYPES_FILE" ) if ((!DRY_RUN)); then mkdir -p "$seed_runs_root" "$seed_log_root" fi - echo "[$(date '+%F %T')] START seed=$seed type=$info_type gpu=$gpu batch=$batch_size" + echo "[$(date '+%F %T')] START seed=$seed gpu=$gpu batch=$BATCH_SIZE" echo " log=$log_file" if ((DRY_RUN)); then printf ' CUDA_VISIBLE_DEVICES=%q ' "$gpu" @@ -320,15 +251,12 @@ run_job() { if CUDA_VISIBLE_DEVICES="$gpu" PYTHONUNBUFFERED=1 \ "${command[@]}" >"$log_file" 2>&1; then - echo "[$(date '+%F %T')] DONE seed=$seed type=$info_type gpu=$gpu" + echo "[$(date '+%F %T')] DONE seed=$seed gpu=$gpu" return 0 else local exit_code=$? - echo "[$(date '+%F %T')] FAIL seed=$seed type=$info_type gpu=$gpu exit=$exit_code" >&2 + echo "[$(date '+%F %T')] FAIL seed=$seed gpu=$gpu exit=$exit_code" >&2 echo " See: $log_file" >&2 - if [[ "$info_type" == "all" ]]; then - echo " If this is CUDA OOM, retry with --all-batch-size 64." >&2 - fi return "$exit_code" fi } @@ -336,22 +264,21 @@ run_job() { worker() { local slot="$1" local gpu="${GPU_IDS[$slot]}" - local job_index + local seed_index local failed=0 - for ((job_index = slot; job_index < ${#JOB_SEEDS[@]}; job_index += ${#GPU_IDS[@]})); do - run_job "$job_index" "$gpu" || failed=1 + for ((seed_index = slot; seed_index < ${#SEEDS[@]}; seed_index += ${#GPU_IDS[@]})); do + run_job "${SEEDS[$seed_index]}" "$gpu" || failed=1 done return "$failed" } echo "Campaign: $CAMPAIGN_NAME" echo "Seeds: ${SEEDS[*]}" -echo "Extra-info types: ${TYPES[*]}" echo "GPUs: ${GPU_IDS[*]}" -echo "assessment_only batch size: $ASSESSMENT_BATCH_SIZE" -echo "all batch size: $ALL_BATCH_SIZE" -echo "Total tasks: ${#JOB_SEEDS[@]}" +echo "Batch size: $BATCH_SIZE" +echo "Extra-info file: $EXTRA_INFO_TYPES_FILE" +echo "Total tasks: ${#SEEDS[@]}" echo "Runs root: $RUNS_ROOT" echo "Log root: $LOG_ROOT" if command -v nvidia-smi >/dev/null 2>&1; then @@ -385,5 +312,5 @@ fi if ((DRY_RUN)); then echo "Dry run completed successfully." else - echo "All assessment_only/all training tasks completed successfully." + echo "All assessment + smoking + alcohol tasks completed successfully." fi