diff --git a/README.md b/README.md index 2488f97..eda11e8 100644 --- a/README.md +++ b/README.md @@ -243,10 +243,108 @@ python train.py \ 选择额外信息变量: ```bash -python train.py --extra_info_types 1 3 7 +python train.py --extra_info_types_file extra_info_types_smoking_alcohol_bmi.txt ``` -如果不传 `--extra_info_types`,默认使用全部 other-info type。 +`train.py` 只接受 `--extra_info_types_file` 指定变量列表,不接受在 CLI 里直接输入 type id。文件可以每行一个 type id,也可以带 `#` 注释;如果不传 `--extra_info_types_file`,默认使用全部 other-info type。 + +训练输出的 `train_config.json` 会记录: + +- `extra_info_types_file`:训练时使用的列表文件名 +- `extra_info_types`:解析后的实际 type id 列表,用于评估脚本复现变量选择 + +## 评估 AUC + +当前提供两个 AUC 评估入口,二者都已适配新的 `DeepHealth` 模型和统一的 other-info token 输入;AUC 的 DeLong 计算、病例/对照筛选和分层聚合逻辑保持原评估脚本口径。 + +### `evaluate_auc.py` + +`evaluate_auc.py` 评估的是 **next-step / token-level 预测位置上的疾病 AUC**。 + +核心流程: + +- 按训练配置重新构建 `HealthDataset` 和 `DeepHealth`。 +- 对评估 split 中的患者做一次模型推理,缓存每个 disease-token readout hidden。 +- 对疾病 token 分块投影到 `risk_head`,避免一次性保存全词表 logits。 +- 对每个疾病、性别、年龄段、prediction offset 分别计算 AUC。 +- 输出未池化分层结果和按疾病汇总后的结果。 + +典型用法: + +```bash +python evaluate_auc.py \ + --run_path runs/your_run_dir \ + --eval_split test \ + --offsets 0.1,1,5,10 +``` + +主要输出: + +- `df_auc_unpooled.csv` + - 疾病 token 在 sex、age bracket、offset 分层下的 AUC。 +- `df_both.csv` + - 按疾病 token 和 offset 聚合后的 AUC。 + +适合回答的问题: + +- “模型在历史序列中的某个预测 token 上,提前 offset 年预测未来疾病的区分能力如何?” +- “不同年龄段、性别、提前量下,next-step 训练模型的疾病预测 AUC 如何?” + +### `evaluate_auc_v2.py` + +`evaluate_auc_v2.py` 评估的是 **landmark fixed-horizon incident disease AUC**。 + +它不是使用已有序列中的普通 readout 位置,而是在指定 landmark age 人工插入一个 `` query token,然后评估该 landmark 后固定 horizon 内是否发生 incident disease。 + +核心流程: + +- 为每个患者和 landmark age 构造 landmark query 样本。 +- 在 landmark age 插入 `` token,取该位置 hidden。 +- 对疾病 token 分块投影到 `risk_head`。 +- 按疾病、性别、landmark age、horizon 计算 incident disease AUC。 +- 可选择排除 horizon 内先于目标疾病发生的死亡竞争风险。 + +典型用法: + +```bash +python evaluate_auc_v2.py \ + --run_path runs/your_run_dir \ + --eval_split test \ + --landmark_start 40 \ + --landmark_stop 80 \ + --landmark_step 5 \ + --horizons 1,5,10 +``` + +主要输出: + +- `df_auc_landmark_unpooled.csv` + - 疾病 token 在 sex、landmark age、horizon 分层下的 AUC。 +- `df_auc_landmark.csv` + - 按疾病 token 和 horizon 聚合后的 landmark AUC。 + +适合回答的问题: + +- “一个人在 40/45/50/... 岁这个固定年龄点,如果此前未患某病,未来 1/5/10 年内发生该病的风险区分能力如何?” +- “模型能否作为 landmark risk prediction 模型使用?” + +### 两者区别 + +| 项目 | `evaluate_auc.py` | `evaluate_auc_v2.py` | +| --- | --- | --- | +| 评估口径 | next-step/token-level 预测点 | landmark fixed-horizon incident risk | +| 查询位置 | 原始序列中满足 offset 条件的最新 readout token | 人工插入的 `` landmark token | +| 时间参数 | `offsets`:预测点至少早于目标事件多少年 | `landmark_*` 和 `horizons`:固定年龄点与未来窗口 | +| 病例定义 | target table 中出现目标疾病的患者/事件 | landmark 后 horizon 内首次发生目标疾病 | +| 对照定义 | 从未出现该疾病的患者的 eligible target occurrence | landmark 时未患病,且 horizon 内未发病并有足够随访 | +| 分层 | sex + age bracket + offset | sex + landmark age + horizon | +| 输出文件 | `df_auc_unpooled.csv`, `df_both.csv` | `df_auc_landmark_unpooled.csv`, `df_auc_landmark.csv` | +| 适用问题 | 提前若干年预测未来目标事件的 token-level AUC | 固定年龄点未来固定年限 incident disease risk AUC | + +简单选择: + +- 想复现/延续旧的 next-token Delphi 风格 AUC:用 `evaluate_auc.py`。 +- 想做临床上更像 “某年龄点未来 N 年发病风险” 的 landmark AUC:用 `evaluate_auc_v2.py`。 ## 主要文件 @@ -278,3 +376,11 @@ python train.py --extra_info_types 1 3 7 - token readout - same-time group readout - last-valid readout + +- `evaluate_auc.py` + - next-step/token-level 疾病 AUC 评估 + - 使用 prediction offset、sex、age bracket 分层 + +- `evaluate_auc_v2.py` + - landmark fixed-horizon incident disease AUC 评估 + - 通过插入 `` landmark token 查询固定年龄点风险 diff --git a/evaluate_auc.py b/evaluate_auc.py index d96e70c..1449059 100644 --- a/evaluate_auc.py +++ b/evaluate_auc.py @@ -403,6 +403,32 @@ def make_eval_subset(dataset: HealthDataset, args: argparse.Namespace | Dict[str return Subset(dataset, indices.tolist()), np.asarray(indices, dtype=np.int64) +def validate_dataset_metadata(dataset: HealthDataset, cfg: Dict[str, Any]) -> None: + meta = cfg.get("dataset_metadata") + if not isinstance(meta, dict): + return + + actual: Dict[str, Any] = { + "vocab_size": int(dataset.vocab_size), + "n_types": int(dataset.n_types), + "n_cont_types": int(dataset.n_cont_types), + "n_categories": int(dataset.n_categories), + "cont_type_ids": [int(x) for x in dataset.cont_type_ids], + "extra_info_types": [int(x) for x in dataset.extra_info_types], + } + mismatches = [ + f"{key}: train_config={meta.get(key)!r}, current_dataset={value!r}" + for key, value in actual.items() + if key in meta and meta.get(key) != value + ] + if mismatches: + raise RuntimeError( + "Current dataset metadata does not match train_config.json. " + "Use the same prepared data and extra_info_types as training. " + + "; ".join(mismatches) + ) + + # --------------------------------------------------------------------------- # Batched inference + cached hidden states # --------------------------------------------------------------------------- @@ -426,7 +452,6 @@ def infer_readout_hidden( model: DeepHealth, loader: DataLoader, device: torch.device, - attn_mask_mode: str, readout_name: str, readout_reduce: str, use_amp: bool, @@ -736,7 +761,6 @@ def _init_auc_worker_flat( p_sex: np.ndarray, age_groups: np.ndarray, n_patients: int, - use_delong: bool, ): # Prevent BLAS/OpenMP oversubscription when many worker processes are active. os.environ.setdefault("OMP_NUM_THREADS", "1") @@ -759,7 +783,6 @@ def _init_auc_worker_flat( "p_sex": p_sex, "age_groups": age_groups, "n_patients": int(n_patients), - "use_delong": bool(use_delong), }) @@ -793,7 +816,6 @@ def _calibration_auc_one_disease_flat(task: Tuple[int, int]) -> List[Dict[str, A p_sex = _WORKER["p_sex"] age_groups = _WORKER["age_groups"] n_patients = _WORKER["n_patients"] - use_delong = _WORKER["use_delong"] case_idx = _case_indices_for_token(int(token)) if case_idx.size < 2: @@ -838,12 +860,7 @@ def _calibration_auc_one_disease_flat(task: Tuple[int, int]) -> List[Dict[str, A if case_scores.size == 0 or control_scores.size == 0: continue - if use_delong: - auc_value, auc_var = get_auc_delong_var( - control_scores, case_scores) - else: - auc_value, auc_var = get_auc_delong_var( - control_scores, case_scores) + auc_value, auc_var = get_auc_delong_var(control_scores, case_scores) out.append({ "token": int(token), @@ -890,7 +907,6 @@ def compute_auc_chunk_parallel( offset: float, valid_target_min_id: int, num_workers: int, - use_delong: bool, auc_task_chunk_size: int = 0, ) -> List[Dict[str, Any]]: sex_mask = arrays["sex"] == sex_value @@ -928,8 +944,7 @@ def compute_auc_chunk_parallel( flat["patient"], flat["target_event"], flat["pred_idx"], flat["age_bin"], flat["target_time"], flat["sort_order"], flat["sorted_target_event"], flat["raw_patient"], flat["raw_sort_order"], flat["raw_sorted_target_event"], - flat["p_sex"], flat["age_groups"], int( - flat["n_patients"]), use_delong, + flat["p_sex"], flat["age_groups"], int(flat["n_patients"]), ) nested = [_calibration_auc_one_disease_flat(t) for t in tqdm( tasks, desc=f"AUC {sex_name}", leave=False, dynamic_ncols=True)] @@ -946,8 +961,7 @@ def compute_auc_chunk_parallel( flat["patient"], flat["target_event"], flat["pred_idx"], flat["age_bin"], flat["target_time"], flat["sort_order"], flat["sorted_target_event"], flat["raw_patient"], flat["raw_sort_order"], flat["raw_sorted_target_event"], - flat["p_sex"], flat["age_groups"], int( - flat["n_patients"]), use_delong, + flat["p_sex"], flat["age_groups"], int(flat["n_patients"]), ), ) as ex: nested = list(tqdm( @@ -985,7 +999,6 @@ def evaluate_auc_pipeline( age_groups: np.ndarray, offsets: Sequence[float], device: torch.device, - attn_mask_mode: str, readout_name: str, readout_reduce: str, num_workers_auc: int, @@ -1045,7 +1058,6 @@ def evaluate_auc_pipeline( model=model, loader=loader, device=device, - attn_mask_mode=attn_mask_mode, readout_name=readout_name, readout_reduce=readout_reduce, use_amp=use_amp, @@ -1075,7 +1087,6 @@ def evaluate_auc_pipeline( offset=float(offset), valid_target_min_id=valid_target_min_id, num_workers=num_workers_auc, - use_delong=True, auc_task_chunk_size=auc_task_chunk_size, ) for r in rows: @@ -1131,6 +1142,14 @@ def parse_int_list(s: Any) -> Optional[List[int]]: text = str(s).strip() if text == "": return None + if text.startswith("["): + try: + values = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid integer list: {text!r}") from exc + if not isinstance(values, list): + raise ValueError(f"Expected a JSON list, got {type(values).__name__}") + return [int(x) for x in values] return [int(x.strip()) for x in text.split(",") if x.strip()] @@ -1142,6 +1161,14 @@ def parse_float_list(s: Any) -> Optional[List[float]]: text = str(s).strip() if text == "": return None + if text.startswith("["): + try: + values = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid float list: {text!r}") from exc + if not isinstance(values, list): + raise ValueError(f"Expected a JSON list, got {type(values).__name__}") + return [float(x) for x in values] return [float(x.strip()) for x in text.split(",") if x.strip()] @@ -1231,8 +1258,6 @@ def main() -> None: target_mode = cfg.get("target_mode", "uts") dist_mode_cfg = cfg.get("dist_mode", "exponential") - attn_mask_mode = cfg.get( - "attn_mask_mode", "non_strict_time" if target_mode == "uts" else "target_aware") readout_name = cfg.get( "readout_name", "same_time_group_end" if target_mode == "uts" else "token") readout_reduce = cfg.get("readout_reduce", "mean") @@ -1250,6 +1275,7 @@ def main() -> None: include_no_event_in_uts_target=include_no_event, extra_info_types=parse_int_list(cfg.get("extra_info_types", None)), ) + validate_dataset_metadata(dataset, cfg) subset, subset_indices = make_eval_subset(dataset, args, cfg) print(f"Dataset: {len(dataset)} samples, vocab_size={dataset.vocab_size}") @@ -1309,7 +1335,6 @@ def main() -> None: age_groups=age_groups, offsets=auc_offsets, device=device, - attn_mask_mode=attn_mask_mode, readout_name=readout_name, readout_reduce=readout_reduce, num_workers_auc=int(cfg_get(args, cfg, "num_workers_auc", max( diff --git a/evaluate_auc_v2.py b/evaluate_auc_v2.py index a98cfce..e09b1d2 100644 --- a/evaluate_auc_v2.py +++ b/evaluate_auc_v2.py @@ -56,6 +56,14 @@ def parse_int_list(value: Any) -> Optional[List[int]]: text = str(value).strip() if text == "": return None + if text.startswith("["): + try: + values = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid integer list: {text!r}") from exc + if not isinstance(values, list): + raise ValueError(f"Expected a JSON list, got {type(values).__name__}") + return [int(x) for x in values] return [int(x.strip()) for x in text.split(",") if x.strip()] @@ -67,6 +75,14 @@ def parse_float_list(value: Any) -> Optional[List[float]]: text = str(value).strip() if text == "": return None + if text.startswith("["): + try: + values = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid float list: {text!r}") from exc + if not isinstance(values, list): + raise ValueError(f"Expected a JSON list, got {type(values).__name__}") + return [float(x) for x in values] return [float(x.strip()) for x in text.split(",") if x.strip()] @@ -155,6 +171,32 @@ def load_model_state(model: torch.nn.Module, state_dict: Dict[str, Any]) -> None f"[WARN] load_state_dict strict=False: missing={missing[:10]}, unexpected={unexpected[:10]}") +def validate_dataset_metadata(dataset: HealthDataset, cfg: Dict[str, Any]) -> None: + meta = cfg.get("dataset_metadata") + if not isinstance(meta, dict): + return + + actual: Dict[str, Any] = { + "vocab_size": int(dataset.vocab_size), + "n_types": int(dataset.n_types), + "n_cont_types": int(dataset.n_cont_types), + "n_categories": int(dataset.n_categories), + "cont_type_ids": [int(x) for x in dataset.cont_type_ids], + "extra_info_types": [int(x) for x in dataset.extra_info_types], + } + mismatches = [ + f"{key}: train_config={meta.get(key)!r}, current_dataset={value!r}" + for key, value in actual.items() + if key in meta and meta.get(key) != value + ] + if mismatches: + raise RuntimeError( + "Current dataset metadata does not match train_config.json. " + "Use the same prepared data and extra_info_types as training. " + + "; ".join(mismatches) + ) + + # --------------------------------------------------------------------------- # DeLong AUC utilities # --------------------------------------------------------------------------- @@ -525,10 +567,8 @@ class LandmarkDataset(Dataset): np.array([np.float32(landmark_age)], dtype=np.float32), ] ) - - target_time_seq = time_seq_landmark.copy() if self.attn_mask_mode in _TARGET_AWARE_MODES: - target_time_seq[-1] = np.nextafter( + time_seq_landmark[-1] = np.nextafter( np.float32(landmark_age), np.float32(np.inf), dtype=np.float32 ) @@ -546,7 +586,6 @@ class LandmarkDataset(Dataset): "landmark_pos": int(len(event_seq_landmark) - 1), "event_seq": event_seq_landmark, "time_seq": time_seq_landmark, - "target_time_seq": target_time_seq, "readout_mask": readout_mask, "other_type": np.asarray(s["other_type"], dtype=np.int64), "other_value": np.asarray(s["other_value"], dtype=np.float32), @@ -570,7 +609,6 @@ class LandmarkDataset(Dataset): return { "event_seq": torch.from_numpy(s["event_seq"]).long(), "time_seq": torch.from_numpy(s["time_seq"]).float(), - "target_time_seq": torch.from_numpy(s["target_time_seq"]).float(), "readout_mask": torch.from_numpy(s["readout_mask"]), "sex": torch.tensor(s["sex"], dtype=torch.long), "other_type": torch.from_numpy(s["other_type"]).long(), @@ -590,8 +628,6 @@ def collate_landmark_fn(batch: List[Dict[str, torch.Tensor]]) -> Dict[str, torch [x["event_seq"] for x in batch], batch_first=True, padding_value=PAD_IDX) time_seq = pad_sequence([x["time_seq"] for x in batch], batch_first=True, padding_value=0.0) - target_time_seq = pad_sequence( - [x["target_time_seq"] for x in batch], batch_first=True, padding_value=0.0) readout_mask = pad_sequence( [x["readout_mask"] for x in batch], batch_first=True, padding_value=False) other_type = pad_sequence( @@ -606,7 +642,6 @@ def collate_landmark_fn(batch: List[Dict[str, torch.Tensor]]) -> Dict[str, torch return { "event_seq": event_seq, "time_seq": time_seq, - "target_time_seq": target_time_seq, "padding_mask": event_seq > PAD_IDX, "readout_mask": readout_mask, "sex": torch.stack([x["sex"] for x in batch]), @@ -637,7 +672,6 @@ def infer_landmark_hidden( model: DeepHealth, loader: DataLoader, device: torch.device, - attn_mask_mode: str, readout_name: str, readout_reduce: str, use_amp: bool, @@ -939,7 +973,6 @@ def evaluate_landmark_auc( score_mode: str, horizons: np.ndarray, device: torch.device, - attn_mask_mode: str, readout_name: str, readout_reduce: str, num_workers_auc: int, @@ -957,7 +990,6 @@ def evaluate_landmark_auc( model=model, loader=loader, device=device, - attn_mask_mode=attn_mask_mode, readout_name=readout_name, readout_reduce=readout_reduce, use_amp=use_amp, @@ -1193,6 +1225,7 @@ def main() -> None: include_no_event_in_uts_target=bool(include_no_event_in_uts_target), extra_info_types=parse_int_list(cfg.get("extra_info_types", None)), ) + validate_dataset_metadata(dataset, cfg) has_no_event = ( NO_EVENT_IDX in dataset.label_id_to_code @@ -1381,7 +1414,6 @@ def main() -> None: score_mode=score_mode, horizons=horizons, device=device, - attn_mask_mode=attn_mask_mode, readout_name=readout_name, readout_reduce=readout_reduce, num_workers_auc=num_workers_auc, diff --git a/extra_info_types_all.txt b/extra_info_types_all.txt new file mode 100644 index 0000000..076e0b8 --- /dev/null +++ b/extra_info_types_all.txt @@ -0,0 +1,268 @@ +# All other-info variables (field_type=1 and field_type=2) +# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. +# Format: # | +1 # waist_circumference | Waist circumference +2 # hip_circumference | Hip circumference +3 # standing_height | Standing height +4 # fasting_time | Fasting time +5 # pulse_rate | Pulse rate automated reading +6 # dbp | Diastolic blood pressure automated reading +7 # sbp | Systolic blood pressure automated reading +8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure +9 # fvc_best | Forced vital capacity (FVC) Best measure +10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score +11 # bmi | Body mass index (BMI) +12 # WBC | White blood cell (leukocyte) count +13 # RBC | Red blood cell (erythrocyte) count +14 # hemoglobin | Haemoglobin concentration +15 # hematocrit | Haematocrit percentage +16 # MCV | Mean corpuscular volume +17 # MCH | Mean corpuscular haemoglobin +18 # MCHC | Mean corpuscular haemoglobin concentration +19 # Pc | Platelet count +20 # MPV | Mean platelet (thrombocyte) volume +21 # LymC | Lymphocyte count +22 # MonC | Monocyte count +23 # NeuC | Neutrophill count +24 # EosC | Eosinophill count +25 # BasC | Basophill count +26 # nRBC | Nucleated red blood cell count +27 # RC | Reticulocyte count +28 # MRV | Mean reticulocyte volume +29 # MSCV | Mean sphered cell volume +30 # IRF | Immature reticulocyte fraction +31 # HLSRC | High light scatter reticulocyte count +32 # MicU | Microalbumin in urine +33 # CreaU | Creatinine (enzymatic) in urine +34 # PotU | Potassium in urine +35 # SodU | Sodium in urine +36 # Alb | Albumin +37 # ALP | Alkaline phosphatase +38 # Alanine | Alanine aminotransferase +39 # ApoA | Apolipoprotein A +40 # ApoB | Apolipoprotein B +41 # AA | Aspartate aminotransferase +42 # DBil | Direct bilirubin +43 # Urea | Urea +44 # Calcium | Calcium +45 # Cholesterol | Cholesterol +46 # Creatinine | Creatinine +47 # CRP | C-reactive protein +48 # CystatinC | Cystatin C +49 # GGT | Gamma glutamyltransferase +50 # Glu | Glucose +51 # HbA1c | Glycated haemoglobin (HbA1c) +52 # HDL | HDL cholesterol +53 # IGF1 | IGF-1 +54 # LDL | LDL direct +55 # LpA | Lipoprotein A +56 # Oestradiol | Oestradiol +57 # Phosphate | Phosphate +58 # Rheu | Rheumatoid factor +59 # SHBG | SHBG +60 # TotalBil | Total bilirubin +61 # Testosterone | Testosterone +62 # TotalProtein | Total protein +63 # Tri | Triglycerides +64 # Urate | Urate +65 # VitaminD | Vitamin D +66 # smoking | Current tobacco smoking +67 # alcohol | Alcohol intake frequency. +68 # ipaq_activity_group | IPAQ activity group +69 # moderate_activity_met_minutes_week | MET minutes per week for moderate activity +70 # vigorous_activity_met_minutes_week | MET minutes per week for vigorous activity +71 # walking_met_minutes_week | MET minutes per week for walking +72 # total_activity_met_minutes_week | Summed MET minutes per week for all activity +73 # total_activity_days | Summed days activity +74 # total_activity_minutes | Summed minutes activity +75 # heavy_diy_duration | Duration of heavy DIY +76 # light_diy_duration | Duration of light DIY +77 # moderate_activity_duration | Duration of moderate activity +78 # other_exercise_duration | Duration of other exercises +79 # strenuous_sport_duration | Duration of strenuous sports +80 # vigorous_activity_duration | Duration of vigorous activity +81 # walking_duration | Duration of walks +82 # pleasure_walking_duration | Duration walking for pleasure +83 # heavy_diy_frequency_4_weeks | Frequency of heavy DIY in last 4 weeks +84 # light_diy_frequency_4_weeks | Frequency of light DIY in last 4 weeks +85 # other_exercise_frequency_4_weeks | Frequency of other exercises in last 4 weeks +86 # stair_climbing_frequency_4_weeks | Frequency of stair climbing in last 4 weeks +87 # strenuous_sport_frequency_4_weeks | Frequency of strenuous sports in last 4 weeks +88 # pleasure_walking_frequency_4_weeks | Frequency of walking for pleasure in last 4 weeks +89 # moderate_activity_days_week_10min | Number of days/week of moderate physical activity 10+ minutes +90 # vigorous_activity_days_week_10min | Number of days/week of vigorous physical activity 10+ minutes +91 # walking_days_week_10min | Number of days/week walked 10+ minutes +92 # driving_time | Time spent driving +93 # computer_use_time | Time spent using computer +94 # tv_watching_time | Time spent watching television (TV) +95 # physical_activity_types_4_weeks | Types of physical activity in last 4 weeks +96 # nonwork_transport_types | Types of transport used (excluding work) +97 # usual_walking_pace | Usual walking pace +98 # mobile_phone_use_duration | Length of mobile phone use +99 # mobile_phone_use_weekly_3_months | Weekly usage of mobile phone in last 3 months +100 # computer_game_playing | Plays computer games +101 # sleep_duration | Sleep duration +102 # chronotype | Morning/evening person (chronotype) +103 # daytime_napping | Nap during day +104 # insomnia | Sleeplessness / insomnia +105 # daytime_dozing | Daytime dozing / sleeping +106 # ever_smoked | Ever smoked +107 # smoking_pack_years | Pack years of smoking +108 # smoking_status | Smoking status +109 # past_tobacco_smoking | Past tobacco smoking +110 # lifetime_smoking_100_plus | Light smokers, at least 100 smokes in lifetime +111 # current_tobacco_type | Type of tobacco currently smoked +112 # current_cigarettes_per_day | Number of cigarettes currently smoked daily (current cigarette smokers) +113 # previous_cigarettes_per_day_current_cigar_pipe_smokers | Number of cigarettes previously smoked daily (current cigar/pipe smokers) +114 # time_to_first_cigarette | Time from waking to first cigarette +115 # ever_tried_smoking_cessation | Ever tried to stop smoking +116 # smoking_change_vs_10_years_ago | Smoking compared to 10 years previous +117 # previous_tobacco_type | Type of tobacco previously smoked +118 # previous_cigarettes_per_day | Number of cigarettes previously smoked daily +119 # ever_stopped_smoking_6_months | Ever stopped smoking for 6+ months +120 # household_smokers | Smoking/smokers in household +121 # home_secondhand_smoke_exposure | Exposure to tobacco smoke at home +122 # nonhome_secondhand_smoke_exposure | Exposure to tobacco smoke outside home +123 # cooked_vegetable_intake | Cooked vegetable intake +124 # raw_vegetable_intake | Salad / raw vegetable intake +125 # fresh_fruit_intake | Fresh fruit intake +126 # dried_fruit_intake | Dried fruit intake +127 # oily_fish_intake | Oily fish intake +128 # non_oily_fish_intake | Non-oily fish intake +129 # processed_meat_intake | Processed meat intake +130 # poultry_intake | Poultry intake +131 # beef_intake | Beef intake +132 # lamb_mutton_intake | Lamb/mutton intake +133 # pork_intake | Pork intake +134 # age_last_ate_meat | Age when last ate meat +135 # food_avoidance_eggs_dairy_wheat_sugar | Never eat eggs, dairy, wheat, sugar +136 # cheese_intake | Cheese intake +137 # milk_type | Milk type used +138 # spread_type | Spread type +139 # bread_intake | Bread intake +140 # bread_type | Bread type +141 # cereal_intake | Cereal intake +142 # cereal_type | Cereal type +143 # added_salt | Salt added to food +144 # tea_intake | Tea intake +145 # coffee_intake | Coffee intake +146 # coffee_type | Coffee type +147 # hot_drink_temperature | Hot drink temperature +148 # water_intake | Water intake +149 # diet_variation | Variation in diet +150 # alcohol_drinker_status | Alcohol drinker status +151 # former_alcohol_drinker | Former alcohol drinker +152 # red_wine_intake_monthly | Average monthly red wine intake +153 # champagne_white_wine_intake_monthly | Average monthly champagne plus white wine intake +154 # beer_cider_intake_monthly | Average monthly beer plus cider intake +155 # spirits_intake_monthly | Average monthly spirits intake +156 # fortified_wine_intake_monthly | Average monthly fortified wine intake +157 # other_alcohol_intake_monthly | Average monthly intake of other alcoholic drinks +158 # red_wine_intake_weekly | Average weekly red wine intake +159 # champagne_white_wine_intake_weekly | Average weekly champagne plus white wine intake +160 # beer_cider_intake_weekly | Average weekly beer plus cider intake +161 # spirits_intake_weekly | Average weekly spirits intake +162 # fortified_wine_intake_weekly | Average weekly fortified wine intake +163 # other_alcohol_intake_weekly | Average weekly intake of other alcoholic drinks +164 # alcohol_with_meals | Alcohol usually taken with meals +165 # country_of_birth_uk_elsewhere | Country of birth (UK/elsewhere) +166 # breastfed_in_infancy | Breastfed as a baby +167 # comparative_body_size_age_10 | Comparative body size at age 10 +168 # comparative_height_age_10 | Comparative height size at age 10 +169 # handedness | Handedness (chirality/laterality) +170 # adopted_as_child | Adopted as a child +171 # multiple_birth | Part of a multiple birth +172 # maternal_smoking_around_birth | Maternal smoking around birth +173 # accommodation_type | Type of accommodation lived in +174 # housing_tenure | Own or rent accommodation lived in +175 # gas_solid_fuel_use | Gas or solid-fuel cooking/heating +176 # home_heating_types | Heating type(s) in home +177 # household_vehicle_count | Number of vehicles in household +178 # household_income_before_tax | Average total household income before tax +179 # current_employment_status | Current employment status +180 # current_employment_status_corrected | Current employment status - corrected +181 # home_work_distance | Distance between home and job workplace +182 # main_job_hours_week | Length of working week for main job +183 # commuting_frequency | Frequency of travelling from home to job workplace +184 # commuting_transport_type | Transport type for commuting to job workplace +185 # job_walking_standing | Job involves mainly walking or standing +186 # job_heavy_manual_work | Job involves heavy manual or physical work +187 # job_shift_work | Job involves shift work +188 # job_night_shift_work | Job involves night shift work +189 # educational_qualifications | Qualifications +190 # age_completed_full_time_education | Age completed full time education +191 # friend_family_visit_frequency | Frequency of friend/family visits +192 # leisure_social_activities | Leisure/social activities +193 # ability_to_confide | Able to confide +194 # bipolar_major_depression_status | Bipolar and major depression status +195 # neuroticism_score | Neuroticism score +196 # mood_swings | Mood swings +197 # miserableness | Miserableness +198 # irritability | Irritability +199 # sensitivity_hurt_feelings | Sensitivity / hurt feelings +200 # fed_up_feelings | Fed-up feelings +201 # nervous_feelings | Nervous feelings +202 # worry_anxiety_feelings | Worrier / anxious feelings +203 # tenseness_highly_strung | Tense / 'highly strung' +204 # suffering_from_nerves | Suffer from 'nerves' +205 # loneliness_isolation | Loneliness, isolation +206 # guilty_feelings | Guilty feelings +207 # risk_taking | Risk taking +208 # happiness | Happiness +209 # job_satisfaction | Work/job satisfaction +210 # health_satisfaction | Health satisfaction +211 # family_relationship_satisfaction | Family relationship satisfaction +212 # friendship_satisfaction | Friendships satisfaction +213 # financial_situation_satisfaction | Financial situation satisfaction +214 # depressed_mood_frequency_2_weeks | Frequency of depressed mood in last 2 weeks +215 # disinterest_frequency_2_weeks | Frequency of unenthusiasm / disinterest in last 2 weeks +216 # tenseness_restlessness_frequency_2_weeks | Frequency of tenseness / restlessness in last 2 weeks +217 # tiredness_lethargy_frequency_2_weeks | Frequency of tiredness / lethargy in last 2 weeks +218 # ever_depressed_full_week | Ever depressed for a whole week +219 # longest_depression_duration | Longest period of depression +220 # depression_episode_count | Number of depression episodes +221 # longest_disinterest_duration | Longest period of unenthusiasm / disinterest +222 # disinterest_episode_count | Number of unenthusiastic/disinterested episodes +223 # ever_manic_hyper_2_days | Ever manic/hyper for 2 days +224 # ever_irritable_argumentative_2_days | Ever highly irritable/argumentative for 2 days +225 # manic_hyper_symptoms | Manic/hyper symptoms +226 # longest_manic_irritable_episode_duration | Length of longest manic/irritable episode +227 # manic_irritable_episode_severity | Severity of manic/irritable episodes +228 # adverse_life_events_2_years | Illness, injury, bereavement, stress in last 2 years +229 # outdoor_time_summer | Time spend outdoors in summer +230 # outdoor_time_winter | Time spent outdoors in winter +231 # skin_tanning_ease | Ease of skin tanning +232 # childhood_sunburn_frequency | Childhood sunburn occasions +233 # sun_uv_protection_use | Use of sun/uv protection +234 # solarium_sunlamp_frequency | Frequency of solarium/sunlamp use +235 # proximity_to_major_road | Close to major road +236 # inverse_distance_nearest_major_road | Inverse distance to the nearest major road +237 # inverse_distance_nearest_road | Inverse distance to the nearest road +238 # no2_2005 | Nitrogen dioxide air pollution; 2005 +239 # no2_2006 | Nitrogen dioxide air pollution; 2006 +240 # no2_2007 | Nitrogen dioxide air pollution; 2007 +241 # no2_2010 | Nitrogen dioxide air pollution; 2010 +242 # nox_2010 | Nitrogen oxides air pollution; 2010 +243 # pm10_2007 | Particulate matter air pollution (pm10); 2007 +244 # pm10_2010 | Particulate matter air pollution (pm10); 2010 +245 # pm25_absorbance_2010 | Particulate matter air pollution (pm2.5) absorbance; 2010 +246 # pm25_2010 | Particulate matter air pollution (pm2.5); 2010 +247 # pm25_10_2010 | Particulate matter air pollution 2.5-10um; 2010 +248 # major_road_length_100m | Sum of road length of major roads within 100m +249 # major_road_traffic_load | Total traffic load on major roads +250 # nearest_major_road_traffic_intensity | Traffic intensity on the nearest major road +251 # nearest_road_traffic_intensity | Traffic intensity on the nearest road +252 # noise_level_16h | Average 16-hour sound level of noise pollution +253 # noise_level_24h | Average 24-hour sound level of noise pollution +254 # noise_level_daytime | Average daytime sound level of noise pollution +255 # noise_level_evening | Average evening sound level of noise pollution +256 # noise_level_nighttime | Average night-time sound level of noise pollution +257 # natural_environment_percent_1000m | Natural environment percentage, buffer 1000m +258 # natural_environment_percent_300m | Natural environment percentage, buffer 300m +259 # greenspace_percent_1000m | Greenspace percentage, buffer 1000m +260 # greenspace_percent_300m | Greenspace percentage, buffer 300m +261 # domestic_garden_percent_1000m | Domestic garden percentage, buffer 1000m +262 # domestic_garden_percent_300m | Domestic garden percentage, buffer 300m +263 # water_percent_1000m | Water percentage, buffer 1000m +264 # water_percent_300m | Water percentage, buffer 300m +265 # distance_to_coast | Distance (Euclidean) to coast diff --git a/extra_info_types_assessment_only.txt b/extra_info_types_assessment_only.txt new file mode 100644 index 0000000..7535941 --- /dev/null +++ b/extra_info_types_assessment_only.txt @@ -0,0 +1,68 @@ +# Only assessment/body-measurement variables (field_type=1) +# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. +# Format: # | +1 # waist_circumference | Waist circumference +2 # hip_circumference | Hip circumference +3 # standing_height | Standing height +4 # fasting_time | Fasting time +5 # pulse_rate | Pulse rate automated reading +6 # dbp | Diastolic blood pressure automated reading +7 # sbp | Systolic blood pressure automated reading +8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure +9 # fvc_best | Forced vital capacity (FVC) Best measure +10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score +11 # bmi | Body mass index (BMI) +12 # WBC | White blood cell (leukocyte) count +13 # RBC | Red blood cell (erythrocyte) count +14 # hemoglobin | Haemoglobin concentration +15 # hematocrit | Haematocrit percentage +16 # MCV | Mean corpuscular volume +17 # MCH | Mean corpuscular haemoglobin +18 # MCHC | Mean corpuscular haemoglobin concentration +19 # Pc | Platelet count +20 # MPV | Mean platelet (thrombocyte) volume +21 # LymC | Lymphocyte count +22 # MonC | Monocyte count +23 # NeuC | Neutrophill count +24 # EosC | Eosinophill count +25 # BasC | Basophill count +26 # nRBC | Nucleated red blood cell count +27 # RC | Reticulocyte count +28 # MRV | Mean reticulocyte volume +29 # MSCV | Mean sphered cell volume +30 # IRF | Immature reticulocyte fraction +31 # HLSRC | High light scatter reticulocyte count +32 # MicU | Microalbumin in urine +33 # CreaU | Creatinine (enzymatic) in urine +34 # PotU | Potassium in urine +35 # SodU | Sodium in urine +36 # Alb | Albumin +37 # ALP | Alkaline phosphatase +38 # Alanine | Alanine aminotransferase +39 # ApoA | Apolipoprotein A +40 # ApoB | Apolipoprotein B +41 # AA | Aspartate aminotransferase +42 # DBil | Direct bilirubin +43 # Urea | Urea +44 # Calcium | Calcium +45 # Cholesterol | Cholesterol +46 # Creatinine | Creatinine +47 # CRP | C-reactive protein +48 # CystatinC | Cystatin C +49 # GGT | Gamma glutamyltransferase +50 # Glu | Glucose +51 # HbA1c | Glycated haemoglobin (HbA1c) +52 # HDL | HDL cholesterol +53 # IGF1 | IGF-1 +54 # LDL | LDL direct +55 # LpA | Lipoprotein A +56 # Oestradiol | Oestradiol +57 # Phosphate | Phosphate +58 # Rheu | Rheumatoid factor +59 # SHBG | SHBG +60 # TotalBil | Total bilirubin +61 # Testosterone | Testosterone +62 # TotalProtein | Total protein +63 # Tri | Triglycerides +64 # Urate | Urate +65 # VitaminD | Vitamin D diff --git a/extra_info_types_exposure_only.txt b/extra_info_types_exposure_only.txt new file mode 100644 index 0000000..0fd3f94 --- /dev/null +++ b/extra_info_types_exposure_only.txt @@ -0,0 +1,203 @@ +# Only environment/lifestyle exposure variables (field_type=2) +# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. +# Format: # | +66 # smoking | Current tobacco smoking +67 # alcohol | Alcohol intake frequency. +68 # ipaq_activity_group | IPAQ activity group +69 # moderate_activity_met_minutes_week | MET minutes per week for moderate activity +70 # vigorous_activity_met_minutes_week | MET minutes per week for vigorous activity +71 # walking_met_minutes_week | MET minutes per week for walking +72 # total_activity_met_minutes_week | Summed MET minutes per week for all activity +73 # total_activity_days | Summed days activity +74 # total_activity_minutes | Summed minutes activity +75 # heavy_diy_duration | Duration of heavy DIY +76 # light_diy_duration | Duration of light DIY +77 # moderate_activity_duration | Duration of moderate activity +78 # other_exercise_duration | Duration of other exercises +79 # strenuous_sport_duration | Duration of strenuous sports +80 # vigorous_activity_duration | Duration of vigorous activity +81 # walking_duration | Duration of walks +82 # pleasure_walking_duration | Duration walking for pleasure +83 # heavy_diy_frequency_4_weeks | Frequency of heavy DIY in last 4 weeks +84 # light_diy_frequency_4_weeks | Frequency of light DIY in last 4 weeks +85 # other_exercise_frequency_4_weeks | Frequency of other exercises in last 4 weeks +86 # stair_climbing_frequency_4_weeks | Frequency of stair climbing in last 4 weeks +87 # strenuous_sport_frequency_4_weeks | Frequency of strenuous sports in last 4 weeks +88 # pleasure_walking_frequency_4_weeks | Frequency of walking for pleasure in last 4 weeks +89 # moderate_activity_days_week_10min | Number of days/week of moderate physical activity 10+ minutes +90 # vigorous_activity_days_week_10min | Number of days/week of vigorous physical activity 10+ minutes +91 # walking_days_week_10min | Number of days/week walked 10+ minutes +92 # driving_time | Time spent driving +93 # computer_use_time | Time spent using computer +94 # tv_watching_time | Time spent watching television (TV) +95 # physical_activity_types_4_weeks | Types of physical activity in last 4 weeks +96 # nonwork_transport_types | Types of transport used (excluding work) +97 # usual_walking_pace | Usual walking pace +98 # mobile_phone_use_duration | Length of mobile phone use +99 # mobile_phone_use_weekly_3_months | Weekly usage of mobile phone in last 3 months +100 # computer_game_playing | Plays computer games +101 # sleep_duration | Sleep duration +102 # chronotype | Morning/evening person (chronotype) +103 # daytime_napping | Nap during day +104 # insomnia | Sleeplessness / insomnia +105 # daytime_dozing | Daytime dozing / sleeping +106 # ever_smoked | Ever smoked +107 # smoking_pack_years | Pack years of smoking +108 # smoking_status | Smoking status +109 # past_tobacco_smoking | Past tobacco smoking +110 # lifetime_smoking_100_plus | Light smokers, at least 100 smokes in lifetime +111 # current_tobacco_type | Type of tobacco currently smoked +112 # current_cigarettes_per_day | Number of cigarettes currently smoked daily (current cigarette smokers) +113 # previous_cigarettes_per_day_current_cigar_pipe_smokers | Number of cigarettes previously smoked daily (current cigar/pipe smokers) +114 # time_to_first_cigarette | Time from waking to first cigarette +115 # ever_tried_smoking_cessation | Ever tried to stop smoking +116 # smoking_change_vs_10_years_ago | Smoking compared to 10 years previous +117 # previous_tobacco_type | Type of tobacco previously smoked +118 # previous_cigarettes_per_day | Number of cigarettes previously smoked daily +119 # ever_stopped_smoking_6_months | Ever stopped smoking for 6+ months +120 # household_smokers | Smoking/smokers in household +121 # home_secondhand_smoke_exposure | Exposure to tobacco smoke at home +122 # nonhome_secondhand_smoke_exposure | Exposure to tobacco smoke outside home +123 # cooked_vegetable_intake | Cooked vegetable intake +124 # raw_vegetable_intake | Salad / raw vegetable intake +125 # fresh_fruit_intake | Fresh fruit intake +126 # dried_fruit_intake | Dried fruit intake +127 # oily_fish_intake | Oily fish intake +128 # non_oily_fish_intake | Non-oily fish intake +129 # processed_meat_intake | Processed meat intake +130 # poultry_intake | Poultry intake +131 # beef_intake | Beef intake +132 # lamb_mutton_intake | Lamb/mutton intake +133 # pork_intake | Pork intake +134 # age_last_ate_meat | Age when last ate meat +135 # food_avoidance_eggs_dairy_wheat_sugar | Never eat eggs, dairy, wheat, sugar +136 # cheese_intake | Cheese intake +137 # milk_type | Milk type used +138 # spread_type | Spread type +139 # bread_intake | Bread intake +140 # bread_type | Bread type +141 # cereal_intake | Cereal intake +142 # cereal_type | Cereal type +143 # added_salt | Salt added to food +144 # tea_intake | Tea intake +145 # coffee_intake | Coffee intake +146 # coffee_type | Coffee type +147 # hot_drink_temperature | Hot drink temperature +148 # water_intake | Water intake +149 # diet_variation | Variation in diet +150 # alcohol_drinker_status | Alcohol drinker status +151 # former_alcohol_drinker | Former alcohol drinker +152 # red_wine_intake_monthly | Average monthly red wine intake +153 # champagne_white_wine_intake_monthly | Average monthly champagne plus white wine intake +154 # beer_cider_intake_monthly | Average monthly beer plus cider intake +155 # spirits_intake_monthly | Average monthly spirits intake +156 # fortified_wine_intake_monthly | Average monthly fortified wine intake +157 # other_alcohol_intake_monthly | Average monthly intake of other alcoholic drinks +158 # red_wine_intake_weekly | Average weekly red wine intake +159 # champagne_white_wine_intake_weekly | Average weekly champagne plus white wine intake +160 # beer_cider_intake_weekly | Average weekly beer plus cider intake +161 # spirits_intake_weekly | Average weekly spirits intake +162 # fortified_wine_intake_weekly | Average weekly fortified wine intake +163 # other_alcohol_intake_weekly | Average weekly intake of other alcoholic drinks +164 # alcohol_with_meals | Alcohol usually taken with meals +165 # country_of_birth_uk_elsewhere | Country of birth (UK/elsewhere) +166 # breastfed_in_infancy | Breastfed as a baby +167 # comparative_body_size_age_10 | Comparative body size at age 10 +168 # comparative_height_age_10 | Comparative height size at age 10 +169 # handedness | Handedness (chirality/laterality) +170 # adopted_as_child | Adopted as a child +171 # multiple_birth | Part of a multiple birth +172 # maternal_smoking_around_birth | Maternal smoking around birth +173 # accommodation_type | Type of accommodation lived in +174 # housing_tenure | Own or rent accommodation lived in +175 # gas_solid_fuel_use | Gas or solid-fuel cooking/heating +176 # home_heating_types | Heating type(s) in home +177 # household_vehicle_count | Number of vehicles in household +178 # household_income_before_tax | Average total household income before tax +179 # current_employment_status | Current employment status +180 # current_employment_status_corrected | Current employment status - corrected +181 # home_work_distance | Distance between home and job workplace +182 # main_job_hours_week | Length of working week for main job +183 # commuting_frequency | Frequency of travelling from home to job workplace +184 # commuting_transport_type | Transport type for commuting to job workplace +185 # job_walking_standing | Job involves mainly walking or standing +186 # job_heavy_manual_work | Job involves heavy manual or physical work +187 # job_shift_work | Job involves shift work +188 # job_night_shift_work | Job involves night shift work +189 # educational_qualifications | Qualifications +190 # age_completed_full_time_education | Age completed full time education +191 # friend_family_visit_frequency | Frequency of friend/family visits +192 # leisure_social_activities | Leisure/social activities +193 # ability_to_confide | Able to confide +194 # bipolar_major_depression_status | Bipolar and major depression status +195 # neuroticism_score | Neuroticism score +196 # mood_swings | Mood swings +197 # miserableness | Miserableness +198 # irritability | Irritability +199 # sensitivity_hurt_feelings | Sensitivity / hurt feelings +200 # fed_up_feelings | Fed-up feelings +201 # nervous_feelings | Nervous feelings +202 # worry_anxiety_feelings | Worrier / anxious feelings +203 # tenseness_highly_strung | Tense / 'highly strung' +204 # suffering_from_nerves | Suffer from 'nerves' +205 # loneliness_isolation | Loneliness, isolation +206 # guilty_feelings | Guilty feelings +207 # risk_taking | Risk taking +208 # happiness | Happiness +209 # job_satisfaction | Work/job satisfaction +210 # health_satisfaction | Health satisfaction +211 # family_relationship_satisfaction | Family relationship satisfaction +212 # friendship_satisfaction | Friendships satisfaction +213 # financial_situation_satisfaction | Financial situation satisfaction +214 # depressed_mood_frequency_2_weeks | Frequency of depressed mood in last 2 weeks +215 # disinterest_frequency_2_weeks | Frequency of unenthusiasm / disinterest in last 2 weeks +216 # tenseness_restlessness_frequency_2_weeks | Frequency of tenseness / restlessness in last 2 weeks +217 # tiredness_lethargy_frequency_2_weeks | Frequency of tiredness / lethargy in last 2 weeks +218 # ever_depressed_full_week | Ever depressed for a whole week +219 # longest_depression_duration | Longest period of depression +220 # depression_episode_count | Number of depression episodes +221 # longest_disinterest_duration | Longest period of unenthusiasm / disinterest +222 # disinterest_episode_count | Number of unenthusiastic/disinterested episodes +223 # ever_manic_hyper_2_days | Ever manic/hyper for 2 days +224 # ever_irritable_argumentative_2_days | Ever highly irritable/argumentative for 2 days +225 # manic_hyper_symptoms | Manic/hyper symptoms +226 # longest_manic_irritable_episode_duration | Length of longest manic/irritable episode +227 # manic_irritable_episode_severity | Severity of manic/irritable episodes +228 # adverse_life_events_2_years | Illness, injury, bereavement, stress in last 2 years +229 # outdoor_time_summer | Time spend outdoors in summer +230 # outdoor_time_winter | Time spent outdoors in winter +231 # skin_tanning_ease | Ease of skin tanning +232 # childhood_sunburn_frequency | Childhood sunburn occasions +233 # sun_uv_protection_use | Use of sun/uv protection +234 # solarium_sunlamp_frequency | Frequency of solarium/sunlamp use +235 # proximity_to_major_road | Close to major road +236 # inverse_distance_nearest_major_road | Inverse distance to the nearest major road +237 # inverse_distance_nearest_road | Inverse distance to the nearest road +238 # no2_2005 | Nitrogen dioxide air pollution; 2005 +239 # no2_2006 | Nitrogen dioxide air pollution; 2006 +240 # no2_2007 | Nitrogen dioxide air pollution; 2007 +241 # no2_2010 | Nitrogen dioxide air pollution; 2010 +242 # nox_2010 | Nitrogen oxides air pollution; 2010 +243 # pm10_2007 | Particulate matter air pollution (pm10); 2007 +244 # pm10_2010 | Particulate matter air pollution (pm10); 2010 +245 # pm25_absorbance_2010 | Particulate matter air pollution (pm2.5) absorbance; 2010 +246 # pm25_2010 | Particulate matter air pollution (pm2.5); 2010 +247 # pm25_10_2010 | Particulate matter air pollution 2.5-10um; 2010 +248 # major_road_length_100m | Sum of road length of major roads within 100m +249 # major_road_traffic_load | Total traffic load on major roads +250 # nearest_major_road_traffic_intensity | Traffic intensity on the nearest major road +251 # nearest_road_traffic_intensity | Traffic intensity on the nearest road +252 # noise_level_16h | Average 16-hour sound level of noise pollution +253 # noise_level_24h | Average 24-hour sound level of noise pollution +254 # noise_level_daytime | Average daytime sound level of noise pollution +255 # noise_level_evening | Average evening sound level of noise pollution +256 # noise_level_nighttime | Average night-time sound level of noise pollution +257 # natural_environment_percent_1000m | Natural environment percentage, buffer 1000m +258 # natural_environment_percent_300m | Natural environment percentage, buffer 300m +259 # greenspace_percent_1000m | Greenspace percentage, buffer 1000m +260 # greenspace_percent_300m | Greenspace percentage, buffer 300m +261 # domestic_garden_percent_1000m | Domestic garden percentage, buffer 1000m +262 # domestic_garden_percent_300m | Domestic garden percentage, buffer 300m +263 # water_percent_1000m | Water percentage, buffer 1000m +264 # water_percent_300m | Water percentage, buffer 300m +265 # distance_to_coast | Distance (Euclidean) to coast diff --git a/extra_info_types_smoking_alcohol_bmi.txt b/extra_info_types_smoking_alcohol_bmi.txt new file mode 100644 index 0000000..67f5532 --- /dev/null +++ b/extra_info_types_smoking_alcohol_bmi.txt @@ -0,0 +1,6 @@ +# Only smoking, alcohol, and BMI variables +# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. +# Format: # | +11 # bmi | Body mass index (BMI) +66 # smoking | Current tobacco smoking +67 # alcohol | Alcohol intake frequency. \ No newline at end of file diff --git a/models.py b/models.py index 4026226..e74db4b 100644 --- a/models.py +++ b/models.py @@ -94,7 +94,7 @@ class DeepHealth(nn.Module): ]) self.rope = None self.rbf = None - if time_mode == "relative": + elif time_mode == "relative": self.age_encoding = None self.blocks = nn.ModuleList([ GPTBlock( @@ -154,6 +154,9 @@ class DeepHealth(nn.Module): other_time: torch.FloatTensor | None = None, **unused_kwargs, ) -> torch.Tensor: + if unused_kwargs: + unknown = ", ".join(sorted(unused_kwargs)) + raise TypeError(f"Unexpected DeepHealth forward arguments: {unknown}") if mode not in {"next_token", "all_future"}: raise ValueError("mode must be either 'next_token' or 'all_future'") if mode == "all_future" and t_query is None: diff --git a/train.py b/train.py index 50b7c99..de2a947 100644 --- a/train.py +++ b/train.py @@ -20,7 +20,7 @@ import sys import time from datetime import datetime from pathlib import Path -from typing import Any, Dict, Iterable, Tuple +from typing import Any, Dict, Tuple import numpy as np import torch @@ -135,22 +135,6 @@ def load_extra_info_types_file(path: str) -> list[int]: return parsed -def merge_extra_info_types(*sources: Iterable[int] | None) -> list[int] | None: - """Merge optional type-id lists while preserving first-seen order.""" - merged: list[int] = [] - seen: set[int] = set() - for source in sources: - if source is None: - continue - for raw_type in source: - type_id = int(raw_type) - if type_id in seen: - continue - seen.add(type_id) - merged.append(type_id) - return merged or None - - def configure_torch_for_training(device: torch.device) -> None: """Enable backend settings that can improve training throughput on CUDA.""" if device.type == "cuda": @@ -573,19 +557,68 @@ def save_checkpoint( torch.save(model.state_dict(), checkpoint_path) -def save_config(args: argparse.Namespace, config_path: Path) -> None: +def save_config( + args: argparse.Namespace, + config_path: Path, + extra: Dict[str, Any] | None = None, +) -> None: """Save training config as JSON.""" - config_dict = vars(args) - # Convert non-serializable types - config_dict = { - k: str(v) if not isinstance( - v, (int, float, str, bool, type(None))) else v - for k, v in config_dict.items() - } + config_dict: Dict[str, Any] = {} + for key, value in vars(args).items(): + if isinstance(value, tuple): + config_dict[key] = list(value) + elif isinstance(value, list): + config_dict[key] = value + elif isinstance(value, (int, float, str, bool, type(None))): + config_dict[key] = value + else: + config_dict[key] = str(value) + if extra: + config_dict.update(extra) with open(config_path, "w") as f: json.dump(config_dict, f, indent=2) +def build_run_metadata( + args: argparse.Namespace, + dataset: HealthDataset, + train_subset: Subset, + val_subset: Subset, + test_subset: Subset, + run_name: str, +) -> Dict[str, Any]: + """Collect resolved training facts needed to rebuild the model for evaluation.""" + return { + "run_name": run_name, + "dataset_class": "NextStepHealthDataset", + "collate_fn": "next_step_collate_fn", + "model_class": "DeepHealth", + "model_target_mode": "next_token", + "dist_mode": "exponential", + "extra_info_types_file": ( + Path(args.extra_info_types_file).name + if args.extra_info_types_file is not None + else None + ), + "extra_info_types": dataset.extra_info_types, + "dataset_metadata": { + "vocab_size": int(dataset.vocab_size), + "n_types": int(dataset.n_types), + "n_cont_types": int(dataset.n_cont_types), + "n_categories": int(dataset.n_categories), + "cont_type_ids": [int(x) for x in dataset.cont_type_ids], + "extra_info_types": [int(x) for x in dataset.extra_info_types], + }, + "split_sizes": { + "train": int(len(train_subset)), + "val": int(len(val_subset)), + "test": int(len(test_subset)), + }, + "resolved_readout_name": args.readout_name, + "resolved_loss_name": args.loss_name, + } + + def normalize_training_config(args: argparse.Namespace) -> None: """Fill in and validate training options that depend on other flags.""" if args.target_mode not in {"delphi2m", "uts"}: @@ -661,8 +694,6 @@ def main(): help="Time encoding mode for disease history") parser.add_argument("--dropout", type=float, default=0.0, help="Dropout rate") - parser.add_argument("--extra_info_types", type=int, nargs="*", default=None, - help="Optional list of other-information type ids to include") parser.add_argument("--extra_info_types_file", type=str, default=None, help="Optional file containing other-information type ids to include") @@ -715,15 +746,11 @@ def main(): help="Device to use for training: cpu, cuda, or cuda:") args = parser.parse_args() - file_extra_info_types = ( + args.extra_info_types = ( load_extra_info_types_file(args.extra_info_types_file) if args.extra_info_types_file is not None else None ) - args.extra_info_types = merge_extra_info_types( - args.extra_info_types, - file_extra_info_types, - ) # ---- Setup ---- set_seed(args.seed) @@ -893,7 +920,18 @@ def main(): raise ValueError(f"Unknown loss: {args.loss_name}") # ---- Save Config ---- - save_config(args, run_dir / "train_config.json") + save_config( + args, + run_dir / "train_config.json", + extra=build_run_metadata( + args=args, + dataset=dataset, + train_subset=train_subset, + val_subset=val_subset, + test_subset=test_subset, + run_name=run_name, + ), + ) logger.info(f"Config saved to {run_dir / 'train_config.json'}") # ---- Training Loop ----