From a0379daf2967648957c0f151233603075d5c4fa8 Mon Sep 17 00:00:00 2001 From: Jiarui Li Date: Tue, 7 Jul 2026 16:57:49 +0800 Subject: [PATCH] Remove extra info token pathway --- backbones.py | 51 ----- dataset.py | 147 +------------ eval_data.py | 37 +--- evaluate_auc.py | 23 +- extra_info_types_all.txt | 268 ----------------------- extra_info_types_assessment_only.txt | 68 ------ extra_info_types_exposure_only.txt | 203 ----------------- extra_info_types_none.txt | 3 - extra_info_types_smoking_alcohol_bmi.txt | 6 - models.py | 246 +-------------------- prepare_data.py | 126 +---------- train_next_step.py | 192 +--------------- train_util.py | 38 +--- 13 files changed, 18 insertions(+), 1390 deletions(-) delete mode 100644 extra_info_types_all.txt delete mode 100644 extra_info_types_assessment_only.txt delete mode 100644 extra_info_types_exposure_only.txt delete mode 100644 extra_info_types_none.txt delete mode 100644 extra_info_types_smoking_alcohol_bmi.txt diff --git a/backbones.py b/backbones.py index 4a95694..66f3bd7 100644 --- a/backbones.py +++ b/backbones.py @@ -247,57 +247,6 @@ class GPTBlock(nn.Module): return x -class TokenAutoDiscretization(nn.Module): - def __init__( - self, - n_cont_types: int, - n_bins: int, - n_embd: int, - ): - super().__init__() - if n_cont_types <= 0: - raise ValueError(f"n_cont_types must be > 0, got {n_cont_types}") - if n_bins <= 1: - raise ValueError(f"n_bins must be > 1, got {n_bins}") - if n_embd <= 0: - raise ValueError(f"n_embd must be > 0, got {n_embd}") - - self.n_cont_types = n_cont_types - self.n_bins = n_bins - self.n_embd = n_embd - self.weight = nn.Parameter(torch.empty(n_cont_types, n_bins)) - self.bias = nn.Parameter(torch.empty(n_cont_types, n_bins)) - self.bin_emb = nn.Parameter(torch.empty(n_cont_types, n_bins, n_embd)) - self.reset_parameters() - - def reset_parameters(self) -> None: - nn.init.normal_(self.weight, mean=0.0, std=0.02) - nn.init.zeros_(self.bias) - nn.init.normal_(self.bin_emb, mean=0.0, std=0.02) - - def forward( - self, - cont_type_idx: torch.LongTensor, # (N,) - value: torch.Tensor, # (N,) - ) -> torch.Tensor: - if cont_type_idx.dim() != 1: - raise ValueError( - f"cont_type_idx must be 1D, got {tuple(cont_type_idx.shape)}" - ) - if value.dim() != 1: - raise ValueError(f"value must be 1D, got {tuple(value.shape)}") - if cont_type_idx.numel() != value.numel(): - raise ValueError("cont_type_idx and value must have the same length") - - w = self.weight[cont_type_idx] # (N, n_bins) - b = self.bias[cont_type_idx] # (N, n_bins) - e = self.bin_emb[cont_type_idx] # (N, n_bins, D) - logits = value.unsqueeze(-1) * w + b - probs = torch.softmax(logits, dim=-1) - return torch.einsum("nb,nbd->nd", probs, e) - - - class AgeSinusoidalEncoding(nn.Module): def __init__(self, embedding_dim: int): diff --git a/dataset.py b/dataset.py index 2743c73..0691fdf 100644 --- a/dataset.py +++ b/dataset.py @@ -1,7 +1,7 @@ # dataset.py from __future__ import annotations -from typing import Dict, Iterable, List, Literal, Optional, Tuple +from typing import Dict, List, Literal, Optional, Tuple import numpy as np import pandas as pd @@ -87,17 +87,11 @@ class _ExpoBaseDataset(Dataset): labels_file: str = "labels.csv", no_event_interval_years: float = 5.0, include_no_event_in_uts_target: bool = False, - extra_info_types: Iterable[int] | None = None, ) -> None: self.data_prefix = data_prefix self.labels_file = labels_file self.no_event_interval_years = float(no_event_interval_years) self.include_no_event_in_uts_target = bool(include_no_event_in_uts_target) - self.requested_extra_info_types = ( - None - if extra_info_types is None - else list(dict.fromkeys(int(t) for t in extra_info_types)) - ) self.label_code_to_id, self.label_id_to_code = load_label_vocab( labels_file, @@ -112,26 +106,12 @@ class _ExpoBaseDataset(Dataset): self.event_data = event_data[order] basic_table = pd.read_csv(f"{data_prefix}_basic_info.csv", index_col=0) - other_info = np.load(f"{data_prefix}_other_info.npy") - if other_info.ndim != 2 or other_info.shape[1] != 5: - raise ValueError( - f"other_info must have shape (N, 5), got {other_info.shape}" - ) - cate_types = pd.read_csv("cate_types.csv") - required_cate_cols = {"type", "name", "n_categories"} - missing_cate_cols = required_cate_cols - set(cate_types.columns) - if missing_cate_cols: - raise ValueError( - f"cate_types.csv is missing columns: {sorted(missing_cate_cols)}" - ) - basic_table.index = basic_table.index.astype(np.int64) unique_eids = np.unique(self.event_data[:, 0].astype(np.int64)) basic_table = basic_table.loc[unique_eids] self._prepare_sex(basic_table, unique_eids) - self._prepare_other_info(other_info, cate_types, unique_eids) max_id_in_vocab = max(self.label_id_to_code.keys()) max_id_in_data = int(self.event_data[:, 2].max()) if len(self.event_data) > 0 else 0 @@ -160,95 +140,6 @@ class _ExpoBaseDataset(Dataset): ) self.sex_mapping = {int(eid): int(s) for eid, s in zip(unique_eids, sex01)} - def _prepare_other_info( - self, - other_info: np.ndarray, - cate_types: pd.DataFrame, - unique_eids: np.ndarray, - ) -> None: - other_info = other_info.copy() - other_info[:, 0] = other_info[:, 0].astype(np.int64) - other_info[:, 1] = other_info[:, 1].astype(np.int64) - other_info[:, 3] = other_info[:, 3].astype(np.int64) - - available_types = sorted( - int(t) for t in np.unique(other_info[:, 1]) if int(t) > 0 - ) - if self.requested_extra_info_types is None: - selected_types = available_types - else: - selected_types = self.requested_extra_info_types - missing = sorted(set(selected_types) - set(available_types)) - if missing: - raise ValueError(f"Requested extra_info_types not found: {missing}") - - keep = np.isin(other_info[:, 0].astype(np.int64), unique_eids) - keep &= np.isin(other_info[:, 1].astype(np.int64), selected_types) - other_info = other_info[keep] - - cate_counts = { - int(row["type"]): int(row["n_categories"]) - for _, row in cate_types.iterrows() - } - cate_offsets: Dict[int, int] = {} - next_offset = 0 - for type_id in selected_types: - if type_id in cate_counts: - cate_offsets[type_id] = next_offset - next_offset += cate_counts[type_id] - - kinds = other_info[:, 3].astype(np.int64) - types = other_info[:, 1].astype(np.int64) - cate_rows = kinds == 2 - for type_id in np.unique(types[cate_rows]): - type_id = int(type_id) - if type_id not in cate_offsets: - raise ValueError( - f"type {type_id} appears categorical but is missing from cate_types.csv" - ) - row_mask = cate_rows & (types == type_id) - local_value = other_info[row_mask, 2].astype(np.int64) - other_info[row_mask, 2] = local_value + cate_offsets[type_id] - - cont_type_ids = [ - int(t) - for t in selected_types - if np.any((types == int(t)) & (kinds == 1)) - ] - - self.extra_info_types = selected_types - self.cate_type_offsets = cate_offsets - self.n_types = (max(selected_types) + 1) if selected_types else 1 - self.cont_type_ids = cont_type_ids - self.n_cont_types = len(cont_type_ids) - self.n_categories = next_offset + 1 - - order = np.lexsort((other_info[:, 4], other_info[:, 1], other_info[:, 0])) - other_info = other_info[order] - self.other_info_by_eid: Dict[int, Dict[str, np.ndarray]] = {} - - for eid in unique_eids.astype(np.int64): - self.other_info_by_eid[int(eid)] = { - "other_type": np.zeros(0, dtype=np.int64), - "other_value": np.zeros(0, dtype=np.float32), - "other_value_kind": np.zeros(0, dtype=np.int64), - "other_time": np.zeros(0, dtype=np.float32), - } - - if len(other_info) == 0: - return - - eids, starts = np.unique(other_info[:, 0].astype(np.int64), return_index=True) - ends = np.concatenate([starts[1:], [len(other_info)]]) - for eid_raw, start, end in zip(eids, starts, ends): - rows = other_info[start:end] - self.other_info_by_eid[int(eid_raw)] = { - "other_type": rows[:, 1].astype(np.int64), - "other_value": rows[:, 2].astype(np.float32), - "other_value_kind": rows[:, 3].astype(np.int64), - "other_time": (rows[:, 4].astype(np.float32) / DAYS_PER_YEAR), - } - def _iter_patient_events( self, *, @@ -279,12 +170,10 @@ class _ExpoBaseDataset(Dataset): yield eid, times_days, labels def _split_features(self, eid: int) -> Optional[Dict]: - other_info = self.other_info_by_eid.get(eid) - if other_info is None: + if eid not in self.sex_mapping: return None return { "sex": self.sex_mapping[eid], - **other_info, } class NextStepHealthDataset(_ExpoBaseDataset): @@ -305,14 +194,12 @@ class NextStepHealthDataset(_ExpoBaseDataset): labels_file: str = "labels.csv", no_event_interval_years: float = 5.0, include_no_event_in_uts_target: bool = False, - extra_info_types: Iterable[int] | None = None, ) -> None: super().__init__( data_prefix=data_prefix, labels_file=labels_file, no_event_interval_years=no_event_interval_years, include_no_event_in_uts_target=include_no_event_in_uts_target, - extra_info_types=extra_info_types, ) self.samples: List[Dict] = [] @@ -355,10 +242,6 @@ class NextStepHealthDataset(_ExpoBaseDataset): "event_seq": torch.from_numpy(s["event_seq"]).long(), "time_seq": torch.from_numpy(s["time_seq"]).float(), "sex": torch.tensor(s["sex"], dtype=torch.long), - "other_type": torch.from_numpy(s["other_type"]).long(), - "other_value": torch.from_numpy(s["other_value"]).float(), - "other_value_kind": torch.from_numpy(s["other_value_kind"]).long(), - "other_time": torch.from_numpy(s["other_time"]).float(), "target_event_seq": torch.from_numpy(s["target_event_seq"]).long(), "target_time_seq": torch.from_numpy(s["target_time_seq"]).float(), "readout_mask": torch.from_numpy(s["readout_mask"]).bool(), @@ -390,7 +273,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset): min_history_events: int = 1, min_future_events: int = 1, validation_query_seed: int = 42, - extra_info_types: Iterable[int] | None = None, ) -> None: if split not in {"train", "valid", "test"}: raise ValueError(f"split must be train/valid/test, got {split!r}") @@ -400,7 +282,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset): labels_file=labels_file, no_event_interval_years=no_event_interval_years, include_no_event_in_uts_target=include_no_event_in_uts_target, - extra_info_types=extra_info_types, ) self.split = split @@ -537,10 +418,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset): "future_dt": torch.from_numpy(times[fut] - np.float32(t_query)).float(), "exposure": torch.tensor(np.float32(patient["t_obs"] - t_query), dtype=torch.float32), "sex": torch.tensor(patient["sex"], dtype=torch.long), - "other_type": torch.from_numpy(patient["other_type"]).long(), - "other_value": torch.from_numpy(patient["other_value"]).float(), - "other_value_kind": torch.from_numpy(patient["other_value_kind"]).long(), - "other_time": torch.from_numpy(patient["other_time"]).float(), } def __len__(self) -> int: @@ -561,26 +438,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset): def _collate_common_static(batch: List[Dict]) -> Dict: return { "sex": torch.stack([s["sex"] for s in batch]), - "other_type": pad_sequence( - [s["other_type"] for s in batch], - batch_first=True, - padding_value=0, - ), - "other_value": pad_sequence( - [s["other_value"] for s in batch], - batch_first=True, - padding_value=0.0, - ), - "other_value_kind": pad_sequence( - [s["other_value_kind"] for s in batch], - batch_first=True, - padding_value=0, - ), - "other_time": pad_sequence( - [s["other_time"] for s in batch], - batch_first=True, - padding_value=0.0, - ), } diff --git a/eval_data.py b/eval_data.py index f139b3c..7b45a48 100644 --- a/eval_data.py +++ b/eval_data.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, Iterable, List +from typing import Any, Dict, List import numpy as np import torch @@ -25,7 +25,6 @@ class AllFutureSequenceEvalDataset: labels_file: str, min_history_events: int = 1, min_future_events: int = 1, - extra_info_types: Iterable[int] | None = None, ) -> None: base = AllFutureHealthDataset( data_prefix=data_prefix, @@ -33,18 +32,12 @@ class AllFutureSequenceEvalDataset: split="train", min_history_events=min_history_events, min_future_events=min_future_events, - extra_info_types=extra_info_types, ) self.base = base self.label_code_to_id = base.label_code_to_id self.label_id_to_code = base.label_id_to_code self.vocab_size = base.vocab_size - self.n_types = base.n_types - self.n_cont_types = base.n_cont_types - self.n_categories = base.n_categories - self.cont_type_ids = base.cont_type_ids - self.extra_info_types = base.extra_info_types self.samples: List[Dict[str, Any]] = [] for patient in base.patients: @@ -62,10 +55,6 @@ class AllFutureSequenceEvalDataset: "target_time_seq": times[1:], "readout_mask": np.ones(input_len, dtype=bool), "sex": int(patient["sex"]), - "other_type": np.asarray(patient["other_type"], dtype=np.int64), - "other_value": np.asarray(patient["other_value"], dtype=np.float32), - "other_value_kind": np.asarray(patient["other_value_kind"], dtype=np.int64), - "other_time": np.asarray(patient["other_time"], dtype=np.float32), } ) @@ -81,10 +70,6 @@ class AllFutureSequenceEvalDataset: "target_time_seq": torch.from_numpy(s["target_time_seq"]).float(), "readout_mask": torch.from_numpy(s["readout_mask"]).bool(), "sex": torch.tensor(s["sex"], dtype=torch.long), - "other_type": torch.from_numpy(s["other_type"]).long(), - "other_value": torch.from_numpy(s["other_value"]).float(), - "other_value_kind": torch.from_numpy(s["other_value_kind"]).long(), - "other_time": torch.from_numpy(s["other_time"]).float(), } @@ -97,7 +82,6 @@ def load_sequence_eval_dataset( include_no_event_in_uts_target: bool, min_history_events: int, min_future_events: int, - extra_info_types: Iterable[int] | None, ): mode = str(model_target_mode).lower() if mode == "next_token": @@ -106,7 +90,6 @@ def load_sequence_eval_dataset( labels_file=labels_file, no_event_interval_years=no_event_interval_years, include_no_event_in_uts_target=include_no_event_in_uts_target, - extra_info_types=extra_info_types, ) if mode == "all_future": return AllFutureSequenceEvalDataset( @@ -114,7 +97,6 @@ def load_sequence_eval_dataset( labels_file=labels_file, min_history_events=min_history_events, min_future_events=min_future_events, - extra_info_types=extra_info_types, ) raise ValueError(f"Unknown model_target_mode: {model_target_mode!r}") @@ -135,19 +117,6 @@ def sequence_eval_collate_fn(batch: List[Dict[str, torch.Tensor]]) -> Dict[str, readout_mask = pad_sequence( [s["readout_mask"] for s in batch], batch_first=True, padding_value=False ) - other_type = pad_sequence( - [s["other_type"] for s in batch], batch_first=True, padding_value=0 - ) - other_value = pad_sequence( - [s["other_value"] for s in batch], batch_first=True, padding_value=0.0 - ) - other_value_kind = pad_sequence( - [s["other_value_kind"] for s in batch], batch_first=True, padding_value=0 - ) - other_time = pad_sequence( - [s["other_time"] for s in batch], batch_first=True, padding_value=0.0 - ) - return { "event_seq": event_seq, "time_seq": time_seq, @@ -156,8 +125,4 @@ def sequence_eval_collate_fn(batch: List[Dict[str, torch.Tensor]]) -> Dict[str, "target_time_seq": target_time_seq, "readout_mask": readout_mask, "sex": torch.stack([s["sex"] for s in batch]), - "other_type": other_type, - "other_value": other_value, - "other_value_kind": other_value_kind, - "other_time": other_time, } diff --git a/evaluate_auc.py b/evaluate_auc.py index 96d05b7..d8a0a69 100644 --- a/evaluate_auc.py +++ b/evaluate_auc.py @@ -320,13 +320,6 @@ def build_model_from_dataset(args: argparse.Namespace, cfg: Dict[str, Any], data n_embd=int(cfg_get(args, cfg, "n_embd", 120)), n_head=int(cfg_get(args, cfg, "n_head", 10)), n_hist_layer=int(cfg_get(args, cfg, "n_hist_layer", 12)), - n_tab_layer=int(cfg_get(args, cfg, "n_tab_layer", 4)), - n_types=dataset.n_types, - n_cont_types=dataset.n_cont_types, - n_categories=dataset.n_categories, - cont_type_ids=dataset.cont_type_ids, - n_bins=int(cfg_get(args, cfg, "n_bins", 16)), - extra_pool_reduce=str(cfg_get(args, cfg, "extra_pool_reduce", "mean")), target_mode=model_target_mode, time_mode=str(cfg_get(args, cfg, "time_mode", "relative")), dist_mode=str(cfg_get(args, cfg, "dist_mode", "exponential")), @@ -424,11 +417,6 @@ def validate_dataset_metadata(dataset: HealthDataset, cfg: Dict[str, Any]) -> No actual: Dict[str, Any] = { "vocab_size": int(dataset.vocab_size), - "n_types": int(dataset.n_types), - "n_cont_types": int(dataset.n_cont_types), - "n_categories": int(dataset.n_categories), - "cont_type_ids": [int(x) for x in dataset.cont_type_ids], - "extra_info_types": [int(x) for x in dataset.extra_info_types], } mismatches = [ f"{key}: train_config={meta.get(key)!r}, current_dataset={value!r}" @@ -438,7 +426,7 @@ def validate_dataset_metadata(dataset: HealthDataset, cfg: Dict[str, Any]) -> No if mismatches: raise RuntimeError( "Current dataset metadata does not match train_config.json. " - "Use the same prepared data and extra_info_types as training. " + "Use the same prepared data as training. " + "; ".join(mismatches) ) @@ -536,10 +524,6 @@ def infer_readout_hidden( sex=batch_dev["sex"][active], padding_mask=padding_mask[active], t_query=time_seq[active, pos], - other_type=batch_dev["other_type"][active], - other_value=batch_dev["other_value"][active], - other_value_kind=batch_dev["other_value_kind"][active], - other_time=batch_dev["other_time"][active], target_mode="all_future", ) hidden[active, pos, :] = hidden_pos.float() @@ -550,10 +534,6 @@ def infer_readout_hidden( time_seq=time_seq, sex=batch_dev["sex"], padding_mask=padding_mask, - other_type=batch_dev["other_type"], - other_value=batch_dev["other_value"], - other_value_kind=batch_dev["other_value_kind"], - other_time=batch_dev["other_time"], target_mode="next_token", ) ro = readout( @@ -1337,7 +1317,6 @@ def main() -> None: include_no_event_in_uts_target=include_no_event, min_history_events=int(cfg.get("all_future_min_history_events", 1)), min_future_events=int(cfg.get("all_future_min_future_events", 1)), - extra_info_types=parse_int_list(cfg.get("extra_info_types", None)), ) validate_dataset_metadata(dataset, cfg) diff --git a/extra_info_types_all.txt b/extra_info_types_all.txt deleted file mode 100644 index 076e0b8..0000000 --- a/extra_info_types_all.txt +++ /dev/null @@ -1,268 +0,0 @@ -# All other-info variables (field_type=1 and field_type=2) -# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. -# Format: # | -1 # waist_circumference | Waist circumference -2 # hip_circumference | Hip circumference -3 # standing_height | Standing height -4 # fasting_time | Fasting time -5 # pulse_rate | Pulse rate automated reading -6 # dbp | Diastolic blood pressure automated reading -7 # sbp | Systolic blood pressure automated reading -8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure -9 # fvc_best | Forced vital capacity (FVC) Best measure -10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score -11 # bmi | Body mass index (BMI) -12 # WBC | White blood cell (leukocyte) count -13 # RBC | Red blood cell (erythrocyte) count -14 # hemoglobin | Haemoglobin concentration -15 # hematocrit | Haematocrit percentage -16 # MCV | Mean corpuscular volume -17 # MCH | Mean corpuscular haemoglobin -18 # MCHC | Mean corpuscular haemoglobin concentration -19 # Pc | Platelet count -20 # MPV | Mean platelet (thrombocyte) volume -21 # LymC | Lymphocyte count -22 # MonC | Monocyte count -23 # NeuC | Neutrophill count -24 # EosC | Eosinophill count -25 # BasC | Basophill count -26 # nRBC | Nucleated red blood cell count -27 # RC | Reticulocyte count -28 # MRV | Mean reticulocyte volume -29 # MSCV | Mean sphered cell volume -30 # IRF | Immature reticulocyte fraction -31 # HLSRC | High light scatter reticulocyte count -32 # MicU | Microalbumin in urine -33 # CreaU | Creatinine (enzymatic) in urine -34 # PotU | Potassium in urine -35 # SodU | Sodium in urine -36 # Alb | Albumin -37 # ALP | Alkaline phosphatase -38 # Alanine | Alanine aminotransferase -39 # ApoA | Apolipoprotein A -40 # ApoB | Apolipoprotein B -41 # AA | Aspartate aminotransferase -42 # DBil | Direct bilirubin -43 # Urea | Urea -44 # Calcium | Calcium -45 # Cholesterol | Cholesterol -46 # Creatinine | Creatinine -47 # CRP | C-reactive protein -48 # CystatinC | Cystatin C -49 # GGT | Gamma glutamyltransferase -50 # Glu | Glucose -51 # HbA1c | Glycated haemoglobin (HbA1c) -52 # HDL | HDL cholesterol -53 # IGF1 | IGF-1 -54 # LDL | LDL direct -55 # LpA | Lipoprotein A -56 # Oestradiol | Oestradiol -57 # Phosphate | Phosphate -58 # Rheu | Rheumatoid factor -59 # SHBG | SHBG -60 # TotalBil | Total bilirubin -61 # Testosterone | Testosterone -62 # TotalProtein | Total protein -63 # Tri | Triglycerides -64 # Urate | Urate -65 # VitaminD | Vitamin D -66 # smoking | Current tobacco smoking -67 # alcohol | Alcohol intake frequency. -68 # ipaq_activity_group | IPAQ activity group -69 # moderate_activity_met_minutes_week | MET minutes per week for moderate activity -70 # vigorous_activity_met_minutes_week | MET minutes per week for vigorous activity -71 # walking_met_minutes_week | MET minutes per week for walking -72 # total_activity_met_minutes_week | Summed MET minutes per week for all activity -73 # total_activity_days | Summed days activity -74 # total_activity_minutes | Summed minutes activity -75 # heavy_diy_duration | Duration of heavy DIY -76 # light_diy_duration | Duration of light DIY -77 # moderate_activity_duration | Duration of moderate activity -78 # other_exercise_duration | Duration of other exercises -79 # strenuous_sport_duration | Duration of strenuous sports -80 # vigorous_activity_duration | Duration of vigorous activity -81 # walking_duration | Duration of walks -82 # pleasure_walking_duration | Duration walking for pleasure -83 # heavy_diy_frequency_4_weeks | Frequency of heavy DIY in last 4 weeks -84 # light_diy_frequency_4_weeks | Frequency of light DIY in last 4 weeks -85 # other_exercise_frequency_4_weeks | Frequency of other exercises in last 4 weeks -86 # stair_climbing_frequency_4_weeks | Frequency of stair climbing in last 4 weeks -87 # strenuous_sport_frequency_4_weeks | Frequency of strenuous sports in last 4 weeks -88 # pleasure_walking_frequency_4_weeks | Frequency of walking for pleasure in last 4 weeks -89 # moderate_activity_days_week_10min | Number of days/week of moderate physical activity 10+ minutes -90 # vigorous_activity_days_week_10min | Number of days/week of vigorous physical activity 10+ minutes -91 # walking_days_week_10min | Number of days/week walked 10+ minutes -92 # driving_time | Time spent driving -93 # computer_use_time | Time spent using computer -94 # tv_watching_time | Time spent watching television (TV) -95 # physical_activity_types_4_weeks | Types of physical activity in last 4 weeks -96 # nonwork_transport_types | Types of transport used (excluding work) -97 # usual_walking_pace | Usual walking pace -98 # mobile_phone_use_duration | Length of mobile phone use -99 # mobile_phone_use_weekly_3_months | Weekly usage of mobile phone in last 3 months -100 # computer_game_playing | Plays computer games -101 # sleep_duration | Sleep duration -102 # chronotype | Morning/evening person (chronotype) -103 # daytime_napping | Nap during day -104 # insomnia | Sleeplessness / insomnia -105 # daytime_dozing | Daytime dozing / sleeping -106 # ever_smoked | Ever smoked -107 # smoking_pack_years | Pack years of smoking -108 # smoking_status | Smoking status -109 # past_tobacco_smoking | Past tobacco smoking -110 # lifetime_smoking_100_plus | Light smokers, at least 100 smokes in lifetime -111 # current_tobacco_type | Type of tobacco currently smoked -112 # current_cigarettes_per_day | Number of cigarettes currently smoked daily (current cigarette smokers) -113 # previous_cigarettes_per_day_current_cigar_pipe_smokers | Number of cigarettes previously smoked daily (current cigar/pipe smokers) -114 # time_to_first_cigarette | Time from waking to first cigarette -115 # ever_tried_smoking_cessation | Ever tried to stop smoking -116 # smoking_change_vs_10_years_ago | Smoking compared to 10 years previous -117 # previous_tobacco_type | Type of tobacco previously smoked -118 # previous_cigarettes_per_day | Number of cigarettes previously smoked daily -119 # ever_stopped_smoking_6_months | Ever stopped smoking for 6+ months -120 # household_smokers | Smoking/smokers in household -121 # home_secondhand_smoke_exposure | Exposure to tobacco smoke at home -122 # nonhome_secondhand_smoke_exposure | Exposure to tobacco smoke outside home -123 # cooked_vegetable_intake | Cooked vegetable intake -124 # raw_vegetable_intake | Salad / raw vegetable intake -125 # fresh_fruit_intake | Fresh fruit intake -126 # dried_fruit_intake | Dried fruit intake -127 # oily_fish_intake | Oily fish intake -128 # non_oily_fish_intake | Non-oily fish intake -129 # processed_meat_intake | Processed meat intake -130 # poultry_intake | Poultry intake -131 # beef_intake | Beef intake -132 # lamb_mutton_intake | Lamb/mutton intake -133 # pork_intake | Pork intake -134 # age_last_ate_meat | Age when last ate meat -135 # food_avoidance_eggs_dairy_wheat_sugar | Never eat eggs, dairy, wheat, sugar -136 # cheese_intake | Cheese intake -137 # milk_type | Milk type used -138 # spread_type | Spread type -139 # bread_intake | Bread intake -140 # bread_type | Bread type -141 # cereal_intake | Cereal intake -142 # cereal_type | Cereal type -143 # added_salt | Salt added to food -144 # tea_intake | Tea intake -145 # coffee_intake | Coffee intake -146 # coffee_type | Coffee type -147 # hot_drink_temperature | Hot drink temperature -148 # water_intake | Water intake -149 # diet_variation | Variation in diet -150 # alcohol_drinker_status | Alcohol drinker status -151 # former_alcohol_drinker | Former alcohol drinker -152 # red_wine_intake_monthly | Average monthly red wine intake -153 # champagne_white_wine_intake_monthly | Average monthly champagne plus white wine intake -154 # beer_cider_intake_monthly | Average monthly beer plus cider intake -155 # spirits_intake_monthly | Average monthly spirits intake -156 # fortified_wine_intake_monthly | Average monthly fortified wine intake -157 # other_alcohol_intake_monthly | Average monthly intake of other alcoholic drinks -158 # red_wine_intake_weekly | Average weekly red wine intake -159 # champagne_white_wine_intake_weekly | Average weekly champagne plus white wine intake -160 # beer_cider_intake_weekly | Average weekly beer plus cider intake -161 # spirits_intake_weekly | Average weekly spirits intake -162 # fortified_wine_intake_weekly | Average weekly fortified wine intake -163 # other_alcohol_intake_weekly | Average weekly intake of other alcoholic drinks -164 # alcohol_with_meals | Alcohol usually taken with meals -165 # country_of_birth_uk_elsewhere | Country of birth (UK/elsewhere) -166 # breastfed_in_infancy | Breastfed as a baby -167 # comparative_body_size_age_10 | Comparative body size at age 10 -168 # comparative_height_age_10 | Comparative height size at age 10 -169 # handedness | Handedness (chirality/laterality) -170 # adopted_as_child | Adopted as a child -171 # multiple_birth | Part of a multiple birth -172 # maternal_smoking_around_birth | Maternal smoking around birth -173 # accommodation_type | Type of accommodation lived in -174 # housing_tenure | Own or rent accommodation lived in -175 # gas_solid_fuel_use | Gas or solid-fuel cooking/heating -176 # home_heating_types | Heating type(s) in home -177 # household_vehicle_count | Number of vehicles in household -178 # household_income_before_tax | Average total household income before tax -179 # current_employment_status | Current employment status -180 # current_employment_status_corrected | Current employment status - corrected -181 # home_work_distance | Distance between home and job workplace -182 # main_job_hours_week | Length of working week for main job -183 # commuting_frequency | Frequency of travelling from home to job workplace -184 # commuting_transport_type | Transport type for commuting to job workplace -185 # job_walking_standing | Job involves mainly walking or standing -186 # job_heavy_manual_work | Job involves heavy manual or physical work -187 # job_shift_work | Job involves shift work -188 # job_night_shift_work | Job involves night shift work -189 # educational_qualifications | Qualifications -190 # age_completed_full_time_education | Age completed full time education -191 # friend_family_visit_frequency | Frequency of friend/family visits -192 # leisure_social_activities | Leisure/social activities -193 # ability_to_confide | Able to confide -194 # bipolar_major_depression_status | Bipolar and major depression status -195 # neuroticism_score | Neuroticism score -196 # mood_swings | Mood swings -197 # miserableness | Miserableness -198 # irritability | Irritability -199 # sensitivity_hurt_feelings | Sensitivity / hurt feelings -200 # fed_up_feelings | Fed-up feelings -201 # nervous_feelings | Nervous feelings -202 # worry_anxiety_feelings | Worrier / anxious feelings -203 # tenseness_highly_strung | Tense / 'highly strung' -204 # suffering_from_nerves | Suffer from 'nerves' -205 # loneliness_isolation | Loneliness, isolation -206 # guilty_feelings | Guilty feelings -207 # risk_taking | Risk taking -208 # happiness | Happiness -209 # job_satisfaction | Work/job satisfaction -210 # health_satisfaction | Health satisfaction -211 # family_relationship_satisfaction | Family relationship satisfaction -212 # friendship_satisfaction | Friendships satisfaction -213 # financial_situation_satisfaction | Financial situation satisfaction -214 # depressed_mood_frequency_2_weeks | Frequency of depressed mood in last 2 weeks -215 # disinterest_frequency_2_weeks | Frequency of unenthusiasm / disinterest in last 2 weeks -216 # tenseness_restlessness_frequency_2_weeks | Frequency of tenseness / restlessness in last 2 weeks -217 # tiredness_lethargy_frequency_2_weeks | Frequency of tiredness / lethargy in last 2 weeks -218 # ever_depressed_full_week | Ever depressed for a whole week -219 # longest_depression_duration | Longest period of depression -220 # depression_episode_count | Number of depression episodes -221 # longest_disinterest_duration | Longest period of unenthusiasm / disinterest -222 # disinterest_episode_count | Number of unenthusiastic/disinterested episodes -223 # ever_manic_hyper_2_days | Ever manic/hyper for 2 days -224 # ever_irritable_argumentative_2_days | Ever highly irritable/argumentative for 2 days -225 # manic_hyper_symptoms | Manic/hyper symptoms -226 # longest_manic_irritable_episode_duration | Length of longest manic/irritable episode -227 # manic_irritable_episode_severity | Severity of manic/irritable episodes -228 # adverse_life_events_2_years | Illness, injury, bereavement, stress in last 2 years -229 # outdoor_time_summer | Time spend outdoors in summer -230 # outdoor_time_winter | Time spent outdoors in winter -231 # skin_tanning_ease | Ease of skin tanning -232 # childhood_sunburn_frequency | Childhood sunburn occasions -233 # sun_uv_protection_use | Use of sun/uv protection -234 # solarium_sunlamp_frequency | Frequency of solarium/sunlamp use -235 # proximity_to_major_road | Close to major road -236 # inverse_distance_nearest_major_road | Inverse distance to the nearest major road -237 # inverse_distance_nearest_road | Inverse distance to the nearest road -238 # no2_2005 | Nitrogen dioxide air pollution; 2005 -239 # no2_2006 | Nitrogen dioxide air pollution; 2006 -240 # no2_2007 | Nitrogen dioxide air pollution; 2007 -241 # no2_2010 | Nitrogen dioxide air pollution; 2010 -242 # nox_2010 | Nitrogen oxides air pollution; 2010 -243 # pm10_2007 | Particulate matter air pollution (pm10); 2007 -244 # pm10_2010 | Particulate matter air pollution (pm10); 2010 -245 # pm25_absorbance_2010 | Particulate matter air pollution (pm2.5) absorbance; 2010 -246 # pm25_2010 | Particulate matter air pollution (pm2.5); 2010 -247 # pm25_10_2010 | Particulate matter air pollution 2.5-10um; 2010 -248 # major_road_length_100m | Sum of road length of major roads within 100m -249 # major_road_traffic_load | Total traffic load on major roads -250 # nearest_major_road_traffic_intensity | Traffic intensity on the nearest major road -251 # nearest_road_traffic_intensity | Traffic intensity on the nearest road -252 # noise_level_16h | Average 16-hour sound level of noise pollution -253 # noise_level_24h | Average 24-hour sound level of noise pollution -254 # noise_level_daytime | Average daytime sound level of noise pollution -255 # noise_level_evening | Average evening sound level of noise pollution -256 # noise_level_nighttime | Average night-time sound level of noise pollution -257 # natural_environment_percent_1000m | Natural environment percentage, buffer 1000m -258 # natural_environment_percent_300m | Natural environment percentage, buffer 300m -259 # greenspace_percent_1000m | Greenspace percentage, buffer 1000m -260 # greenspace_percent_300m | Greenspace percentage, buffer 300m -261 # domestic_garden_percent_1000m | Domestic garden percentage, buffer 1000m -262 # domestic_garden_percent_300m | Domestic garden percentage, buffer 300m -263 # water_percent_1000m | Water percentage, buffer 1000m -264 # water_percent_300m | Water percentage, buffer 300m -265 # distance_to_coast | Distance (Euclidean) to coast diff --git a/extra_info_types_assessment_only.txt b/extra_info_types_assessment_only.txt deleted file mode 100644 index 7535941..0000000 --- a/extra_info_types_assessment_only.txt +++ /dev/null @@ -1,68 +0,0 @@ -# Only assessment/body-measurement variables (field_type=1) -# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. -# Format: # | -1 # waist_circumference | Waist circumference -2 # hip_circumference | Hip circumference -3 # standing_height | Standing height -4 # fasting_time | Fasting time -5 # pulse_rate | Pulse rate automated reading -6 # dbp | Diastolic blood pressure automated reading -7 # sbp | Systolic blood pressure automated reading -8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure -9 # fvc_best | Forced vital capacity (FVC) Best measure -10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score -11 # bmi | Body mass index (BMI) -12 # WBC | White blood cell (leukocyte) count -13 # RBC | Red blood cell (erythrocyte) count -14 # hemoglobin | Haemoglobin concentration -15 # hematocrit | Haematocrit percentage -16 # MCV | Mean corpuscular volume -17 # MCH | Mean corpuscular haemoglobin -18 # MCHC | Mean corpuscular haemoglobin concentration -19 # Pc | Platelet count -20 # MPV | Mean platelet (thrombocyte) volume -21 # LymC | Lymphocyte count -22 # MonC | Monocyte count -23 # NeuC | Neutrophill count -24 # EosC | Eosinophill count -25 # BasC | Basophill count -26 # nRBC | Nucleated red blood cell count -27 # RC | Reticulocyte count -28 # MRV | Mean reticulocyte volume -29 # MSCV | Mean sphered cell volume -30 # IRF | Immature reticulocyte fraction -31 # HLSRC | High light scatter reticulocyte count -32 # MicU | Microalbumin in urine -33 # CreaU | Creatinine (enzymatic) in urine -34 # PotU | Potassium in urine -35 # SodU | Sodium in urine -36 # Alb | Albumin -37 # ALP | Alkaline phosphatase -38 # Alanine | Alanine aminotransferase -39 # ApoA | Apolipoprotein A -40 # ApoB | Apolipoprotein B -41 # AA | Aspartate aminotransferase -42 # DBil | Direct bilirubin -43 # Urea | Urea -44 # Calcium | Calcium -45 # Cholesterol | Cholesterol -46 # Creatinine | Creatinine -47 # CRP | C-reactive protein -48 # CystatinC | Cystatin C -49 # GGT | Gamma glutamyltransferase -50 # Glu | Glucose -51 # HbA1c | Glycated haemoglobin (HbA1c) -52 # HDL | HDL cholesterol -53 # IGF1 | IGF-1 -54 # LDL | LDL direct -55 # LpA | Lipoprotein A -56 # Oestradiol | Oestradiol -57 # Phosphate | Phosphate -58 # Rheu | Rheumatoid factor -59 # SHBG | SHBG -60 # TotalBil | Total bilirubin -61 # Testosterone | Testosterone -62 # TotalProtein | Total protein -63 # Tri | Triglycerides -64 # Urate | Urate -65 # VitaminD | Vitamin D diff --git a/extra_info_types_exposure_only.txt b/extra_info_types_exposure_only.txt deleted file mode 100644 index 0fd3f94..0000000 --- a/extra_info_types_exposure_only.txt +++ /dev/null @@ -1,203 +0,0 @@ -# Only environment/lifestyle exposure variables (field_type=2) -# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. -# Format: # | -66 # smoking | Current tobacco smoking -67 # alcohol | Alcohol intake frequency. -68 # ipaq_activity_group | IPAQ activity group -69 # moderate_activity_met_minutes_week | MET minutes per week for moderate activity -70 # vigorous_activity_met_minutes_week | MET minutes per week for vigorous activity -71 # walking_met_minutes_week | MET minutes per week for walking -72 # total_activity_met_minutes_week | Summed MET minutes per week for all activity -73 # total_activity_days | Summed days activity -74 # total_activity_minutes | Summed minutes activity -75 # heavy_diy_duration | Duration of heavy DIY -76 # light_diy_duration | Duration of light DIY -77 # moderate_activity_duration | Duration of moderate activity -78 # other_exercise_duration | Duration of other exercises -79 # strenuous_sport_duration | Duration of strenuous sports -80 # vigorous_activity_duration | Duration of vigorous activity -81 # walking_duration | Duration of walks -82 # pleasure_walking_duration | Duration walking for pleasure -83 # heavy_diy_frequency_4_weeks | Frequency of heavy DIY in last 4 weeks -84 # light_diy_frequency_4_weeks | Frequency of light DIY in last 4 weeks -85 # other_exercise_frequency_4_weeks | Frequency of other exercises in last 4 weeks -86 # stair_climbing_frequency_4_weeks | Frequency of stair climbing in last 4 weeks -87 # strenuous_sport_frequency_4_weeks | Frequency of strenuous sports in last 4 weeks -88 # pleasure_walking_frequency_4_weeks | Frequency of walking for pleasure in last 4 weeks -89 # moderate_activity_days_week_10min | Number of days/week of moderate physical activity 10+ minutes -90 # vigorous_activity_days_week_10min | Number of days/week of vigorous physical activity 10+ minutes -91 # walking_days_week_10min | Number of days/week walked 10+ minutes -92 # driving_time | Time spent driving -93 # computer_use_time | Time spent using computer -94 # tv_watching_time | Time spent watching television (TV) -95 # physical_activity_types_4_weeks | Types of physical activity in last 4 weeks -96 # nonwork_transport_types | Types of transport used (excluding work) -97 # usual_walking_pace | Usual walking pace -98 # mobile_phone_use_duration | Length of mobile phone use -99 # mobile_phone_use_weekly_3_months | Weekly usage of mobile phone in last 3 months -100 # computer_game_playing | Plays computer games -101 # sleep_duration | Sleep duration -102 # chronotype | Morning/evening person (chronotype) -103 # daytime_napping | Nap during day -104 # insomnia | Sleeplessness / insomnia -105 # daytime_dozing | Daytime dozing / sleeping -106 # ever_smoked | Ever smoked -107 # smoking_pack_years | Pack years of smoking -108 # smoking_status | Smoking status -109 # past_tobacco_smoking | Past tobacco smoking -110 # lifetime_smoking_100_plus | Light smokers, at least 100 smokes in lifetime -111 # current_tobacco_type | Type of tobacco currently smoked -112 # current_cigarettes_per_day | Number of cigarettes currently smoked daily (current cigarette smokers) -113 # previous_cigarettes_per_day_current_cigar_pipe_smokers | Number of cigarettes previously smoked daily (current cigar/pipe smokers) -114 # time_to_first_cigarette | Time from waking to first cigarette -115 # ever_tried_smoking_cessation | Ever tried to stop smoking -116 # smoking_change_vs_10_years_ago | Smoking compared to 10 years previous -117 # previous_tobacco_type | Type of tobacco previously smoked -118 # previous_cigarettes_per_day | Number of cigarettes previously smoked daily -119 # ever_stopped_smoking_6_months | Ever stopped smoking for 6+ months -120 # household_smokers | Smoking/smokers in household -121 # home_secondhand_smoke_exposure | Exposure to tobacco smoke at home -122 # nonhome_secondhand_smoke_exposure | Exposure to tobacco smoke outside home -123 # cooked_vegetable_intake | Cooked vegetable intake -124 # raw_vegetable_intake | Salad / raw vegetable intake -125 # fresh_fruit_intake | Fresh fruit intake -126 # dried_fruit_intake | Dried fruit intake -127 # oily_fish_intake | Oily fish intake -128 # non_oily_fish_intake | Non-oily fish intake -129 # processed_meat_intake | Processed meat intake -130 # poultry_intake | Poultry intake -131 # beef_intake | Beef intake -132 # lamb_mutton_intake | Lamb/mutton intake -133 # pork_intake | Pork intake -134 # age_last_ate_meat | Age when last ate meat -135 # food_avoidance_eggs_dairy_wheat_sugar | Never eat eggs, dairy, wheat, sugar -136 # cheese_intake | Cheese intake -137 # milk_type | Milk type used -138 # spread_type | Spread type -139 # bread_intake | Bread intake -140 # bread_type | Bread type -141 # cereal_intake | Cereal intake -142 # cereal_type | Cereal type -143 # added_salt | Salt added to food -144 # tea_intake | Tea intake -145 # coffee_intake | Coffee intake -146 # coffee_type | Coffee type -147 # hot_drink_temperature | Hot drink temperature -148 # water_intake | Water intake -149 # diet_variation | Variation in diet -150 # alcohol_drinker_status | Alcohol drinker status -151 # former_alcohol_drinker | Former alcohol drinker -152 # red_wine_intake_monthly | Average monthly red wine intake -153 # champagne_white_wine_intake_monthly | Average monthly champagne plus white wine intake -154 # beer_cider_intake_monthly | Average monthly beer plus cider intake -155 # spirits_intake_monthly | Average monthly spirits intake -156 # fortified_wine_intake_monthly | Average monthly fortified wine intake -157 # other_alcohol_intake_monthly | Average monthly intake of other alcoholic drinks -158 # red_wine_intake_weekly | Average weekly red wine intake -159 # champagne_white_wine_intake_weekly | Average weekly champagne plus white wine intake -160 # beer_cider_intake_weekly | Average weekly beer plus cider intake -161 # spirits_intake_weekly | Average weekly spirits intake -162 # fortified_wine_intake_weekly | Average weekly fortified wine intake -163 # other_alcohol_intake_weekly | Average weekly intake of other alcoholic drinks -164 # alcohol_with_meals | Alcohol usually taken with meals -165 # country_of_birth_uk_elsewhere | Country of birth (UK/elsewhere) -166 # breastfed_in_infancy | Breastfed as a baby -167 # comparative_body_size_age_10 | Comparative body size at age 10 -168 # comparative_height_age_10 | Comparative height size at age 10 -169 # handedness | Handedness (chirality/laterality) -170 # adopted_as_child | Adopted as a child -171 # multiple_birth | Part of a multiple birth -172 # maternal_smoking_around_birth | Maternal smoking around birth -173 # accommodation_type | Type of accommodation lived in -174 # housing_tenure | Own or rent accommodation lived in -175 # gas_solid_fuel_use | Gas or solid-fuel cooking/heating -176 # home_heating_types | Heating type(s) in home -177 # household_vehicle_count | Number of vehicles in household -178 # household_income_before_tax | Average total household income before tax -179 # current_employment_status | Current employment status -180 # current_employment_status_corrected | Current employment status - corrected -181 # home_work_distance | Distance between home and job workplace -182 # main_job_hours_week | Length of working week for main job -183 # commuting_frequency | Frequency of travelling from home to job workplace -184 # commuting_transport_type | Transport type for commuting to job workplace -185 # job_walking_standing | Job involves mainly walking or standing -186 # job_heavy_manual_work | Job involves heavy manual or physical work -187 # job_shift_work | Job involves shift work -188 # job_night_shift_work | Job involves night shift work -189 # educational_qualifications | Qualifications -190 # age_completed_full_time_education | Age completed full time education -191 # friend_family_visit_frequency | Frequency of friend/family visits -192 # leisure_social_activities | Leisure/social activities -193 # ability_to_confide | Able to confide -194 # bipolar_major_depression_status | Bipolar and major depression status -195 # neuroticism_score | Neuroticism score -196 # mood_swings | Mood swings -197 # miserableness | Miserableness -198 # irritability | Irritability -199 # sensitivity_hurt_feelings | Sensitivity / hurt feelings -200 # fed_up_feelings | Fed-up feelings -201 # nervous_feelings | Nervous feelings -202 # worry_anxiety_feelings | Worrier / anxious feelings -203 # tenseness_highly_strung | Tense / 'highly strung' -204 # suffering_from_nerves | Suffer from 'nerves' -205 # loneliness_isolation | Loneliness, isolation -206 # guilty_feelings | Guilty feelings -207 # risk_taking | Risk taking -208 # happiness | Happiness -209 # job_satisfaction | Work/job satisfaction -210 # health_satisfaction | Health satisfaction -211 # family_relationship_satisfaction | Family relationship satisfaction -212 # friendship_satisfaction | Friendships satisfaction -213 # financial_situation_satisfaction | Financial situation satisfaction -214 # depressed_mood_frequency_2_weeks | Frequency of depressed mood in last 2 weeks -215 # disinterest_frequency_2_weeks | Frequency of unenthusiasm / disinterest in last 2 weeks -216 # tenseness_restlessness_frequency_2_weeks | Frequency of tenseness / restlessness in last 2 weeks -217 # tiredness_lethargy_frequency_2_weeks | Frequency of tiredness / lethargy in last 2 weeks -218 # ever_depressed_full_week | Ever depressed for a whole week -219 # longest_depression_duration | Longest period of depression -220 # depression_episode_count | Number of depression episodes -221 # longest_disinterest_duration | Longest period of unenthusiasm / disinterest -222 # disinterest_episode_count | Number of unenthusiastic/disinterested episodes -223 # ever_manic_hyper_2_days | Ever manic/hyper for 2 days -224 # ever_irritable_argumentative_2_days | Ever highly irritable/argumentative for 2 days -225 # manic_hyper_symptoms | Manic/hyper symptoms -226 # longest_manic_irritable_episode_duration | Length of longest manic/irritable episode -227 # manic_irritable_episode_severity | Severity of manic/irritable episodes -228 # adverse_life_events_2_years | Illness, injury, bereavement, stress in last 2 years -229 # outdoor_time_summer | Time spend outdoors in summer -230 # outdoor_time_winter | Time spent outdoors in winter -231 # skin_tanning_ease | Ease of skin tanning -232 # childhood_sunburn_frequency | Childhood sunburn occasions -233 # sun_uv_protection_use | Use of sun/uv protection -234 # solarium_sunlamp_frequency | Frequency of solarium/sunlamp use -235 # proximity_to_major_road | Close to major road -236 # inverse_distance_nearest_major_road | Inverse distance to the nearest major road -237 # inverse_distance_nearest_road | Inverse distance to the nearest road -238 # no2_2005 | Nitrogen dioxide air pollution; 2005 -239 # no2_2006 | Nitrogen dioxide air pollution; 2006 -240 # no2_2007 | Nitrogen dioxide air pollution; 2007 -241 # no2_2010 | Nitrogen dioxide air pollution; 2010 -242 # nox_2010 | Nitrogen oxides air pollution; 2010 -243 # pm10_2007 | Particulate matter air pollution (pm10); 2007 -244 # pm10_2010 | Particulate matter air pollution (pm10); 2010 -245 # pm25_absorbance_2010 | Particulate matter air pollution (pm2.5) absorbance; 2010 -246 # pm25_2010 | Particulate matter air pollution (pm2.5); 2010 -247 # pm25_10_2010 | Particulate matter air pollution 2.5-10um; 2010 -248 # major_road_length_100m | Sum of road length of major roads within 100m -249 # major_road_traffic_load | Total traffic load on major roads -250 # nearest_major_road_traffic_intensity | Traffic intensity on the nearest major road -251 # nearest_road_traffic_intensity | Traffic intensity on the nearest road -252 # noise_level_16h | Average 16-hour sound level of noise pollution -253 # noise_level_24h | Average 24-hour sound level of noise pollution -254 # noise_level_daytime | Average daytime sound level of noise pollution -255 # noise_level_evening | Average evening sound level of noise pollution -256 # noise_level_nighttime | Average night-time sound level of noise pollution -257 # natural_environment_percent_1000m | Natural environment percentage, buffer 1000m -258 # natural_environment_percent_300m | Natural environment percentage, buffer 300m -259 # greenspace_percent_1000m | Greenspace percentage, buffer 1000m -260 # greenspace_percent_300m | Greenspace percentage, buffer 300m -261 # domestic_garden_percent_1000m | Domestic garden percentage, buffer 1000m -262 # domestic_garden_percent_300m | Domestic garden percentage, buffer 300m -263 # water_percent_1000m | Water percentage, buffer 1000m -264 # water_percent_300m | Water percentage, buffer 300m -265 # distance_to_coast | Distance (Euclidean) to coast diff --git a/extra_info_types_none.txt b/extra_info_types_none.txt deleted file mode 100644 index 95cb9f8..0000000 --- a/extra_info_types_none.txt +++ /dev/null @@ -1,3 +0,0 @@ -# No extra-info variables. -# Use this file with --extra_info_types_file to train/evaluate with disease history only. -# Keep this file free of numeric type ids; the loader parses it as an empty list. diff --git a/extra_info_types_smoking_alcohol_bmi.txt b/extra_info_types_smoking_alcohol_bmi.txt deleted file mode 100644 index 67f5532..0000000 --- a/extra_info_types_smoking_alcohol_bmi.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Only smoking, alcohol, and BMI variables -# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering. -# Format: # | -11 # bmi | Body mass index (BMI) -66 # smoking | Current tobacco smoking -67 # alcohol | Alcohol intake frequency. \ No newline at end of file diff --git a/models.py b/models.py index 04e59af..9495980 100644 --- a/models.py +++ b/models.py @@ -10,7 +10,6 @@ from backbones import ( GaussianRBFTimeBasis, TimesNetExposureEncoder, TimeRoPE, - TokenAutoDiscretization, ) from targets import PAD_IDX @@ -23,125 +22,6 @@ class DeepHealthOutput: event_len: int -class OtherInfoTokenizer(nn.Module): - PAD_KIND = 0 - CONT_KIND = 1 - CATE_KIND = 2 - - def __init__( - self, - n_embd: int, - n_types: int, - n_cont_types: int, - n_categories: int, - cont_type_ids: list[int], - n_value_kinds: int = 3, - n_bins: int = 16, - ): - super().__init__() - if len(cont_type_ids) != n_cont_types: - raise ValueError( - "cont_type_ids length must match n_cont_types, got " - f"{len(cont_type_ids)} vs {n_cont_types}" - ) - if n_types <= 0: - raise ValueError(f"n_types must include PAD and be > 0, got {n_types}") - if n_categories <= 0: - raise ValueError( - f"n_categories must include PAD and be > 0, got {n_categories}" - ) - if n_value_kinds <= self.CATE_KIND: - raise ValueError( - f"n_value_kinds must be > {self.CATE_KIND}, got {n_value_kinds}" - ) - - self.type_emb = nn.Embedding(n_types, n_embd, padding_idx=0) - self.kind_emb = nn.Embedding(n_value_kinds, n_embd, padding_idx=0) - self.cont_value_encoder = ( - TokenAutoDiscretization( - n_cont_types=n_cont_types, - n_bins=n_bins, - n_embd=n_embd, - ) - if n_cont_types > 0 - else None - ) - self.cate_value_emb = nn.Embedding( - n_categories, - n_embd, - padding_idx=0, - ) - - cont_type_index = torch.full((n_types,), -1, dtype=torch.long) - for idx, type_id in enumerate(cont_type_ids): - if type_id <= 0 or type_id >= n_types: - raise ValueError( - f"continuous type id {type_id} must be in [1, {n_types})" - ) - cont_type_index[type_id] = idx - self.register_buffer( - "cont_type_index", - cont_type_index, - persistent=False, - ) - self.reset_parameters() - - def reset_parameters(self) -> None: - nn.init.normal_(self.type_emb.weight, mean=0.0, std=0.02) - nn.init.zeros_(self.type_emb.weight[0]) - nn.init.normal_(self.kind_emb.weight, mean=0.0, std=0.02) - nn.init.zeros_(self.kind_emb.weight[0]) - nn.init.normal_(self.cate_value_emb.weight, mean=0.0, std=0.02) - nn.init.zeros_(self.cate_value_emb.weight[0]) - - def forward( - self, - other_type: torch.LongTensor, - other_value: torch.Tensor, - other_value_kind: torch.LongTensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - if other_type.shape != other_value.shape: - raise ValueError( - "other_type and other_value must have the same shape, got " - f"{tuple(other_type.shape)} vs {tuple(other_value.shape)}" - ) - if other_type.shape != other_value_kind.shape: - raise ValueError( - "other_type and other_value_kind must have the same shape, got " - f"{tuple(other_type.shape)} vs {tuple(other_value_kind.shape)}" - ) - - other_valid = other_type > 0 - type_emb = self.type_emb(other_type) - kind_emb = self.kind_emb(other_value_kind) - value_emb = torch.zeros_like(type_emb) - - cont_pos = other_valid & (other_value_kind == self.CONT_KIND) - if cont_pos.any(): - if self.cont_value_encoder is None: - raise ValueError("continuous tokens found but n_cont_types is 0") - cont_idx = self.cont_type_index[other_type[cont_pos]] - if (cont_idx < 0).any(): - bad_type = other_type[cont_pos][cont_idx < 0][0].item() - raise ValueError( - f"type_id={bad_type} is marked continuous but is not in " - "cont_type_ids" - ) - value_emb[cont_pos] = self.cont_value_encoder( - cont_type_idx=cont_idx, - value=other_value[cont_pos].to(type_emb.dtype), - ) - - cate_pos = other_valid & (other_value_kind == self.CATE_KIND) - if cate_pos.any(): - cate_id = other_value[cate_pos].long() - value_emb[cate_pos] = self.cate_value_emb(cate_id) - - out = type_emb + kind_emb + value_emb - out = out * other_valid.unsqueeze(-1).to(out.dtype) - return out, other_valid - - class DeepHealth(nn.Module): def __init__( self, @@ -149,17 +29,9 @@ class DeepHealth(nn.Module): n_embd: int, n_head: int, n_hist_layer: int, - n_tab_layer: int, - n_types: int, - n_cont_types: int, - n_categories: int, - cont_type_ids: list[int], - n_value_kinds: int = 3, - n_bins: int = 16, target_mode: str = "next_token", # "next_token" or "all_future" time_mode: str = "relative", # "relative" or "absolute" dist_mode: str = "exponential", # "exponential", "weibull" or "mixed" - extra_pool_reduce: str = "mean", dropout: float = 0.0, use_exposure_encoder: bool = False, exposure_daily_input_dim: int = 4, @@ -182,24 +54,12 @@ class DeepHealth(nn.Module): if dist_mode not in ["exponential", "weibull", "mixed"]: raise ValueError( "dist_mode must be either 'exponential', 'weibull' or 'mixed'") - if extra_pool_reduce not in {"mean", "sum"}: - raise ValueError("extra_pool_reduce must be either 'mean' or 'sum'") self.token_embedding = nn.Embedding(vocab_size, n_embd, padding_idx=0) self.gender_embedding = nn.Embedding( 2, n_embd) # Assuming binary gender - self.tokenizer = OtherInfoTokenizer( - n_embd=n_embd, - n_types=n_types, - n_cont_types=n_cont_types, - n_categories=n_categories, - cont_type_ids=cont_type_ids, - n_value_kinds=n_value_kinds, - n_bins=n_bins, - ) self.target_mode = target_mode self.time_mode = time_mode self.dist_mode = dist_mode - self.extra_pool_reduce = extra_pool_reduce self.use_exposure_encoder = use_exposure_encoder self.n_embd = n_embd self.vocab_size = vocab_size @@ -283,69 +143,6 @@ class DeepHealth(nn.Module): dtype=dtype, ).masked_fill(~valid, -1e4)[:, None, :, :] - def _pool_other_by_time( - self, - h_other: torch.Tensor, - other_time: torch.Tensor, - other_mask: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - batch_size, n_other, n_embd = h_other.shape - if n_other == 0: - empty_h = h_other.new_zeros(batch_size, 0, n_embd) - empty_t = other_time.new_zeros(batch_size, 0) - empty_m = torch.zeros(batch_size, 0, dtype=torch.bool, device=h_other.device) - return empty_h, empty_t, empty_m - - masked_time = other_time.masked_fill(~other_mask, float("inf")) - _sorted_time_with_pad, order = masked_time.sort(dim=1) - sorted_time = other_time.gather(1, order) - sorted_mask = other_mask.gather(1, order) - sorted_h = h_other.gather(1, order.unsqueeze(-1).expand(-1, -1, n_embd)) - - group_start = torch.zeros_like(sorted_mask) - group_start[:, 0] = sorted_mask[:, 0] - group_start[:, 1:] = sorted_mask[:, 1:] & ( - sorted_time[:, 1:] != sorted_time[:, :-1] - ) - group_id = group_start.long().cumsum(dim=1) - 1 - max_groups = int(group_start.sum(dim=1).max().item()) - - pooled_h = h_other.new_zeros(batch_size, max_groups, n_embd) - pooled_time = other_time.new_zeros(batch_size, max_groups) - pooled_mask = torch.zeros( - batch_size, - max_groups, - dtype=torch.bool, - device=h_other.device, - ) - if max_groups == 0: - return pooled_h, pooled_time, pooled_mask - - safe_group_id = group_id.clamp_min(0) - pooled_h.scatter_add_( - 1, - safe_group_id.unsqueeze(-1).expand_as(sorted_h), - sorted_h * sorted_mask.unsqueeze(-1).to(sorted_h.dtype), - ) - if self.extra_pool_reduce == "mean": - counts = h_other.new_zeros(batch_size, max_groups, 1) - counts.scatter_add_( - 1, - safe_group_id.unsqueeze(-1), - sorted_mask.unsqueeze(-1).to(h_other.dtype), - ) - pooled_h = pooled_h / counts.clamp_min(1.0) - - pooled_time.scatter_add_( - 1, - safe_group_id, - sorted_time * group_start.to(sorted_time.dtype), - ) - group_count = group_start.sum(dim=1) - arange_groups = torch.arange(max_groups, device=h_other.device) - pooled_mask = arange_groups.unsqueeze(0) < group_count.unsqueeze(1) - return pooled_h, pooled_time, pooled_mask - def _encode_event_exposure( self, exposure_daily: torch.Tensor | None, @@ -442,10 +239,6 @@ class DeepHealth(nn.Module): mode: str, padding_mask: torch.Tensor | None = None, t_query: torch.FloatTensor | None = None, - other_type: torch.LongTensor | None = None, - other_value: torch.Tensor | None = None, - other_value_kind: torch.LongTensor | None = None, - other_time: torch.FloatTensor | None = None, exposure_daily: torch.Tensor | None = None, exposure_monthly: torch.Tensor | None = None, exposure_daily_mask: torch.Tensor | None = None, @@ -460,16 +253,6 @@ class DeepHealth(nn.Module): raise ValueError("mode must be either 'next_token' or 'all_future'") if mode == "all_future" and t_query is None: raise ValueError("t_query is required when mode='all_future'") - if ( - other_type is None - or other_value is None - or other_value_kind is None - or other_time is None - ): - raise ValueError( - "DeepHealth expects other_type, other_value, " - "other_value_kind, and other_time." - ) if padding_mask is None: padding_mask = event_seq > PAD_IDX @@ -489,24 +272,6 @@ class DeepHealth(nn.Module): h_exposure = h_exposure.to(device=event_seq.device, dtype=h_disease.dtype) h_disease = h_disease + h_exposure t_disease = time_seq - - if other_time.shape != other_type.shape: - raise ValueError( - "other_time must have the same shape as other_type, got " - f"{tuple(other_time.shape)} vs {tuple(other_type.shape)}" - ) - other_time = other_time.to(device=event_seq.device, dtype=time_seq.dtype) - h_other, other_mask = self.tokenizer( - other_type=other_type, - other_value=other_value, - other_value_kind=other_value_kind, - ) - h_other = h_other.to(device=event_seq.device) - other_mask = other_mask.to(device=event_seq.device, dtype=torch.bool) - - h_disease = torch.cat([h_disease, h_other], dim=1) - t_disease = torch.cat([t_disease, other_time], dim=1) - padding_mask = torch.cat([padding_mask, other_mask], dim=1) h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype) if mode == "all_future": @@ -571,15 +336,10 @@ class DeepHealth(nn.Module): h_event = h_disease[:, :event_len, :] t_event = t_disease[:, :event_len] event_mask = padding_mask[:, :event_len] - h_extra, t_extra, extra_mask = self._pool_other_by_time( - h_other=h_disease[:, event_len:, :], - other_time=t_disease[:, event_len:], - other_mask=padding_mask[:, event_len:], - ) return DeepHealthOutput( - hidden=torch.cat([h_event, h_extra], dim=1), - time_seq=torch.cat([t_event, t_extra], dim=1), - padding_mask=torch.cat([event_mask, extra_mask], dim=1), + hidden=h_event, + time_seq=t_event, + padding_mask=event_mask, event_len=event_len, ) return h_disease[:, :event_len, :] diff --git a/prepare_data.py b/prepare_data.py index 2838a70..b100767 100644 --- a/prepare_data.py +++ b/prepare_data.py @@ -6,13 +6,6 @@ DeepHealth: * ``ukb_event_data.npy``: ``(N, 3)`` uint32 array of ``(eid, days, label)`` disease/death/checkup events sorted by patient then time. * ``ukb_basic_info.csv``: basic patient table indexed by ``eid`` with ``sex``. -* ``ukb_other_info.npy``: ``(M, 5)`` float64 array of - ``(eid, type, value, value_kind, time)`` rows. ``type=0`` is reserved for - padding, ``value_kind=1`` means continuous, and ``value_kind=2`` means - categorical. -* ``cate_types.csv``: categorical-variable metadata with - ``type,name,n_categories``. Dataset code computes global category ids after - experiment-specific variable selection. Processing steps ---------------- @@ -21,10 +14,7 @@ Processing steps 3. Extract ICD-10 date fields and cancer date/type fields into long-form events and map codes to integer labels via ``labels.csv``. 4. De-duplicate events per ``(eid, label)`` keeping the earliest occurrence. -5. Convert available non-sex tabular fields into unified other-information - tokens timestamped by ``date_of_assessment``. -6. Write event data, sex, unified other-information tokens, and categorical - type metadata. +5. Write event data and sex. Usage ----- @@ -45,108 +35,6 @@ import pandas as pd # Pandas for data manipulation import tqdm # Progress bar for chunk processing -CONT_VALUE_KIND = 1 -CATE_VALUE_KIND = 2 - - -def _sort_values(values): - """Sort mixed pandas/numpy scalar values deterministically.""" - try: - return sorted(values) - except TypeError: - return sorted(values, key=lambda x: str(x)) - - -def _build_other_info_tokens( - table: pd.DataFrame, - feature_fields: list[str], - *, - time_col: str = "date_of_assessment", -) -> tuple[np.ndarray, pd.DataFrame]: - """Convert wide tabular features into (eid, type, value, kind, time) rows.""" - rows = [] - cate_meta = [] - present_features = [ - col for col in feature_fields - if col in table.columns and col not in {time_col, "sex"} - ] - - if time_col not in table.columns: - raise ValueError( - f"{time_col!r} is required to timestamp other-information tokens" - ) - - token_times = pd.to_numeric(table[time_col], errors="coerce") - - for type_idx, col in enumerate(present_features, start=1): - series = table[col] - n_unique = series.dropna().nunique() - valid_time = token_times.notna() - - if n_unique > 10: - numeric = pd.to_numeric(series, errors="coerce") - valid = numeric.notna() & valid_time - if not valid.any(): - continue - rows.append( - np.column_stack( - ( - table.index[valid].to_numpy(dtype=np.float64), - np.full(valid.sum(), type_idx, dtype=np.float64), - numeric[valid].to_numpy(dtype=np.float64), - np.full(valid.sum(), CONT_VALUE_KIND, dtype=np.float64), - token_times[valid].to_numpy(dtype=np.float64), - ) - ) - ) - else: - unique_vals = _sort_values(series.dropna().unique()) - value_map = {val: idx + 1 for idx, val in enumerate(unique_vals)} - mapped = series.map(value_map) - valid = mapped.notna() & valid_time - n_categories = len(unique_vals) - cate_meta.append( - { - "type": type_idx, - "name": col, - "n_categories": n_categories, - } - ) - if not valid.any(): - continue - rows.append( - np.column_stack( - ( - table.index[valid].to_numpy(dtype=np.float64), - np.full(valid.sum(), type_idx, dtype=np.float64), - mapped[valid].to_numpy(dtype=np.float64), - np.full(valid.sum(), CATE_VALUE_KIND, dtype=np.float64), - token_times[valid].to_numpy(dtype=np.float64), - ) - ) - ) - - cate_types = pd.DataFrame( - cate_meta, - columns=["type", "name", "n_categories"], - ) - if not rows: - return np.empty((0, 5), dtype=np.float64), cate_types - out = np.vstack(rows) - return out[np.lexsort((out[:, 3], out[:, 1], out[:, 0]))], cate_types - - -def _unique_preserve_order(values): - """Return unique values while preserving first-seen order.""" - seen = set() - out = [] - for value in values: - if value not in seen: - seen.add(value) - out.append(value) - return out - - # CSV mapping field IDs to human-readable names field_map_file = "field_ids_enriched.csv" @@ -369,17 +257,5 @@ final_tabular = final_tabular.convert_dtypes() basic_info = final_tabular[["sex"]].copy() basic_info.to_csv("ukb_basic_info.csv") -# Save unified other-information tokens. Missing values simply produce no token. -other_info_fields = _unique_preserve_order( - basic_info_fields + assessment_fields + exposure_fields -) -other_info, cate_types = _build_other_info_tokens( - final_tabular, - other_info_fields, - time_col="date_of_assessment", -) -np.save("ukb_other_info.npy", other_info) -cate_types.to_csv("cate_types.csv", index=False) - # Save event data np.save("ukb_event_data.npy", data) diff --git a/train_next_step.py b/train_next_step.py index 35f86ff..87521ea 100644 --- a/train_next_step.py +++ b/train_next_step.py @@ -12,7 +12,6 @@ import json import logging import math import time -from pathlib import Path from typing import Any, Dict import numpy as np @@ -30,8 +29,6 @@ from targets import CHECKUP_IDX, NO_EVENT_IDX, PAD_IDX from train_util import ( configure_torch_for_training, create_unique_run_dir, - format_extra_info_types, - load_extra_info_types_file, resolve_device, save_checkpoint, save_config, @@ -48,10 +45,6 @@ MODEL_INPUT_KEYS = ( "time_seq", "sex", "padding_mask", - "other_type", - "other_value", - "other_value_kind", - "other_time", ) @@ -62,7 +55,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--data_prefix", type=str, default="ukb") parser.add_argument("--labels_file", type=str, default="labels.csv") parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--extra_info_types_file", type=str, default=None) parser.add_argument("--no_event_interval_years", type=float, default=5.0) parser.add_argument("--include_no_event_in_uts_target", action="store_true") @@ -76,10 +68,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--n_embd", type=int, default=120) parser.add_argument("--n_head", type=int, default=10) parser.add_argument("--n_hist_layer", type=int, default=12) - parser.add_argument("--n_tab_layer", type=int, default=4) - parser.add_argument("--n_bins", type=int, default=16) - parser.add_argument("--extra_pool_reduce", type=str, default="mean", - choices=["mean", "sum"]) parser.add_argument("--time_mode", type=str, default="relative", choices=["relative", "absolute"]) parser.add_argument("--dropout", type=float, default=0.0) @@ -121,11 +109,6 @@ def parse_args() -> argparse.Namespace: args.include_no_event_in_uts_target = True else: args.readout_name = args.readout_name or "token" - args.extra_info_types = ( - load_extra_info_types_file(args.extra_info_types_file) - if args.extra_info_types_file is not None - else None - ) return args @@ -153,13 +136,6 @@ def build_model(args: argparse.Namespace, dataset: HealthDataset) -> DeepHealth: n_embd=args.n_embd, n_head=args.n_head, n_hist_layer=args.n_hist_layer, - n_tab_layer=args.n_tab_layer, - n_types=dataset.n_types, - n_cont_types=dataset.n_cont_types, - n_categories=dataset.n_categories, - cont_type_ids=dataset.cont_type_ids, - n_bins=args.n_bins, - extra_pool_reduce=args.extra_pool_reduce, target_mode="next_token", time_mode=args.time_mode, dist_mode="exponential", @@ -199,153 +175,20 @@ def build_augmented_next_step_targets( model_out: DeepHealthOutput, include_uts_targets: bool, ) -> Dict[str, torch.Tensor]: - hidden_len = model_out.hidden.size(1) - event_len = int(model_out.event_len) - extra_len = hidden_len - event_len device = model_out.hidden.device non_blocking = device.type == "cuda" - if extra_len <= 0: - targets = { - "target_event_seq": batch_cpu["target_event_seq"].to(device, non_blocking=non_blocking), - "target_time_seq": batch_cpu["target_time_seq"].to(device, non_blocking=non_blocking), - "readout_mask": batch_cpu["readout_mask"].to(device, non_blocking=non_blocking), - } - if include_uts_targets: - targets["target_dt_unique"] = batch_cpu["target_dt_unique"].to( - device, non_blocking=non_blocking - ) - targets["target_multi_hot"] = batch_cpu["target_multi_hot"].to( - device, non_blocking=non_blocking - ) - return targets - - bsz = batch_cpu["target_event_seq"].size(0) - vocab_size = ( - batch_cpu["target_multi_hot"].size(2) - if include_uts_targets - else None - ) - other_valid = batch_cpu["other_type"] > 0 - extra_time = batch_cpu["other_time"].new_zeros(bsz, extra_len) - extra_mask = torch.zeros(bsz, extra_len, dtype=torch.bool) - for b in range(bsz): - unique_time = torch.unique(batch_cpu["other_time"][b, other_valid[b]], sorted=True) - n_time = min(int(unique_time.numel()), extra_len) - if n_time > 0: - extra_time[b, :n_time] = unique_time[:n_time] - extra_mask[b, :n_time] = True - - target_event_seq = torch.cat( - [ - batch_cpu["target_event_seq"], - torch.full( - (bsz, extra_len), - PAD_IDX, - dtype=batch_cpu["target_event_seq"].dtype, - ), - ], - dim=1, - ) - target_time_seq = torch.cat( - [ - batch_cpu["target_time_seq"], - torch.zeros( - bsz, - extra_len, - dtype=batch_cpu["target_time_seq"].dtype, - ), - ], - dim=1, - ) - readout_mask = torch.cat([batch_cpu["readout_mask"], extra_mask], dim=1) - target_dt_unique = None - target_multi_hot = None - if include_uts_targets: - target_dt_unique = torch.cat( - [ - batch_cpu["target_dt_unique"], - torch.zeros( - bsz, - extra_len, - dtype=batch_cpu["target_dt_unique"].dtype, - ), - ], - dim=1, - ) - target_multi_hot = torch.cat( - [ - batch_cpu["target_multi_hot"], - torch.zeros( - bsz, - extra_len, - vocab_size, - dtype=batch_cpu["target_multi_hot"].dtype, - ), - ], - dim=1, - ) - - for b in range(bsz): - valid_event = batch_cpu["padding_mask"][b].bool() - if not valid_event.any(): - continue - n_event = int(valid_event.sum().item()) - events = torch.cat( - [ - batch_cpu["event_seq"][b, :n_event], - batch_cpu["target_event_seq"][b, n_event - 1:n_event], - ] - ) - times = torch.cat( - [ - batch_cpu["time_seq"][b, :n_event], - batch_cpu["target_time_seq"][b, n_event - 1:n_event], - ] - ) - valid_full = events > PAD_IDX - events = events[valid_full] - times = times[valid_full] - if events.numel() == 0: - continue - - for j in range(extra_len): - if not bool(extra_mask[b, j]): - continue - pos = event_len + j - t = extra_time[b, j] - future = times > t - if not future.any(): - readout_mask[b, pos] = False - continue - - first_idx = int(torch.nonzero(future, as_tuple=False)[0].item()) - next_time = times[first_idx] - next_event = events[first_idx] - target_event_seq[b, pos] = next_event - target_time_seq[b, pos] = next_time - - if not include_uts_targets: - continue - - same_next_time = times == next_time - next_events = events[same_next_time] - valid_next_events = next_events[ - (next_events > PAD_IDX) & (next_events < vocab_size) - ].long() - if valid_next_events.numel() == 0: - readout_mask[b, pos] = False - continue - target_multi_hot[b, pos, valid_next_events] = True - target_dt_unique[b, pos] = next_time - t - targets = { - "target_event_seq": target_event_seq.to(device, non_blocking=non_blocking), - "target_time_seq": target_time_seq.to(device, non_blocking=non_blocking), - "readout_mask": readout_mask.to(device, non_blocking=non_blocking), + "target_event_seq": batch_cpu["target_event_seq"].to(device, non_blocking=non_blocking), + "target_time_seq": batch_cpu["target_time_seq"].to(device, non_blocking=non_blocking), + "readout_mask": batch_cpu["readout_mask"].to(device, non_blocking=non_blocking), } if include_uts_targets: - targets["target_dt_unique"] = target_dt_unique.to(device, non_blocking=non_blocking) - targets["target_multi_hot"] = target_multi_hot.to(device, non_blocking=non_blocking) + targets["target_dt_unique"] = batch_cpu["target_dt_unique"].to( + device, non_blocking=non_blocking + ) + targets["target_multi_hot"] = batch_cpu["target_multi_hot"].to( + device, non_blocking=non_blocking + ) return targets @@ -367,10 +210,6 @@ def compute_next_step_loss( time_seq=batch["time_seq"], sex=batch["sex"], padding_mask=batch["padding_mask"], - other_type=batch["other_type"], - other_value=batch["other_value"], - other_value_kind=batch["other_value_kind"], - other_time=batch["other_time"], target_mode="next_token", return_output=True, ) @@ -487,19 +326,8 @@ def build_metadata( "model_target_mode": "next_token", "target_mode": args.target_mode, "dist_mode": "exponential", - "extra_info_types_file": ( - Path(args.extra_info_types_file).name - if args.extra_info_types_file is not None - else None - ), - "extra_info_types": [int(x) for x in dataset.extra_info_types], "dataset_metadata": { "vocab_size": int(dataset.vocab_size), - "n_types": int(dataset.n_types), - "n_cont_types": int(dataset.n_cont_types), - "n_categories": int(dataset.n_categories), - "cont_type_ids": [int(x) for x in dataset.cont_type_ids], - "extra_info_types": [int(x) for x in dataset.extra_info_types], }, "split_sizes": { "train": int(len(train_subset)), @@ -527,7 +355,6 @@ def main() -> None: logger.info(f"Starting next-step training run: {run_name}") logger.info(f"Device: {device}") - logger.info(f"extra_info_types: {format_extra_info_types(args.extra_info_types)}") logger.info(f"readout={args.readout_name}, target_mode={args.target_mode}") dataset = HealthDataset( @@ -535,7 +362,6 @@ def main() -> None: labels_file=args.labels_file, no_event_interval_years=args.no_event_interval_years, include_no_event_in_uts_target=args.include_no_event_in_uts_target, - extra_info_types=args.extra_info_types, ) if args.train_eid_file and args.val_eid_file and args.test_eid_file: train_subset, val_subset, test_subset = split_dataset_by_eid_files( diff --git a/train_util.py b/train_util.py index 1fe578c..e91a6bc 100644 --- a/train_util.py +++ b/train_util.py @@ -1,13 +1,12 @@ from __future__ import annotations -import json import logging import sys import time import csv from datetime import datetime from pathlib import Path -from typing import Any, Dict, Iterable, Tuple +from typing import Any, Dict, Tuple import numpy as np import torch @@ -61,41 +60,6 @@ def set_seed(seed: int) -> None: torch.cuda.manual_seed(seed) -def load_extra_info_types_file(path: str) -> list[int]: - file_path = Path(path) - if not file_path.is_file(): - raise FileNotFoundError(f"extra_info_types_file not found: {path}") - - text = file_path.read_text(encoding="utf-8").strip() - if not text: - return [] - - if text.startswith("["): - raw_items = json.loads(text) - if not isinstance(raw_items, list): - raise ValueError("extra_info_types_file JSON must be a list") - else: - raw_items = [] - for line in text.splitlines(): - line = line.split("#", 1)[0].strip() - if line: - raw_items.extend(line.replace(",", " ").replace(";", " ").split()) - - try: - return [int(x) for x in raw_items] - except (TypeError, ValueError) as exc: - raise ValueError(f"Invalid extra info type id in {path}") from exc - - -def format_extra_info_types(extra_info_types: Iterable[int] | None) -> str: - if extra_info_types is None: - return "all" - values = [int(x) for x in extra_info_types] - if not values: - return "none" - return str(values) - - def load_eid_file(path: str | Path) -> set[int]: file_path = Path(path) if not file_path.is_file():