JiaruiLi/DeepHealthExpo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove extra info token pathway

Browse Source
This commit is contained in:
Jiarui Li
2026-07-07 16:57:49 +08:00
parent 6dfeb5a696
commit a0379daf29
13 changed files with 18 additions and 1390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
backbones.py
View File

@@ -247,57 +247,6 @@ class GPTBlock(nn.Module):
return x return x
class TokenAutoDiscretization(nn.Module):
def __init__(
self,
n_cont_types: int,
n_bins: int,
n_embd: int,
):
super().__init__()
if n_cont_types <= 0:
raise ValueError(f"n_cont_types must be > 0, got {n_cont_types}")
if n_bins <= 1:
raise ValueError(f"n_bins must be > 1, got {n_bins}")
if n_embd <= 0:
raise ValueError(f"n_embd must be > 0, got {n_embd}")
self.n_cont_types = n_cont_types
self.n_bins = n_bins
self.n_embd = n_embd
self.weight = nn.Parameter(torch.empty(n_cont_types, n_bins))
self.bias = nn.Parameter(torch.empty(n_cont_types, n_bins))
self.bin_emb = nn.Parameter(torch.empty(n_cont_types, n_bins, n_embd))
self.reset_parameters()
def reset_parameters(self) -> None:
nn.init.normal_(self.weight, mean=0.0, std=0.02)
nn.init.zeros_(self.bias)
nn.init.normal_(self.bin_emb, mean=0.0, std=0.02)
def forward(
self,
cont_type_idx: torch.LongTensor, # (N,)
value: torch.Tensor, # (N,)
) -> torch.Tensor:
if cont_type_idx.dim() != 1:
raise ValueError(
f"cont_type_idx must be 1D, got {tuple(cont_type_idx.shape)}"
)
if value.dim() != 1:
raise ValueError(f"value must be 1D, got {tuple(value.shape)}")
if cont_type_idx.numel() != value.numel():
raise ValueError("cont_type_idx and value must have the same length")
w = self.weight[cont_type_idx] # (N, n_bins)
b = self.bias[cont_type_idx] # (N, n_bins)
e = self.bin_emb[cont_type_idx] # (N, n_bins, D)
logits = value.unsqueeze(-1) * w + b
probs = torch.softmax(logits, dim=-1)
return torch.einsum("nb,nbd->nd", probs, e)
class AgeSinusoidalEncoding(nn.Module): class AgeSinusoidalEncoding(nn.Module):
def __init__(self, embedding_dim: int): def __init__(self, embedding_dim: int):

147
dataset.py
View File

@@ -1,7 +1,7 @@
# dataset.py # dataset.py
from __future__ import annotations from __future__ import annotations
from typing import Dict, Iterable, List, Literal, Optional, Tuple from typing import Dict, List, Literal, Optional, Tuple
import numpy as np import numpy as np
import pandas as pd import pandas as pd
@@ -87,17 +87,11 @@ class _ExpoBaseDataset(Dataset):
labels_file: str = "labels.csv", labels_file: str = "labels.csv",
no_event_interval_years: float = 5.0, no_event_interval_years: float = 5.0,
include_no_event_in_uts_target: bool = False, include_no_event_in_uts_target: bool = False,
extra_info_types: Iterable[int] | None = None,
) -> None: ) -> None:
self.data_prefix = data_prefix self.data_prefix = data_prefix
self.labels_file = labels_file self.labels_file = labels_file
self.no_event_interval_years = float(no_event_interval_years) self.no_event_interval_years = float(no_event_interval_years)
self.include_no_event_in_uts_target = bool(include_no_event_in_uts_target) self.include_no_event_in_uts_target = bool(include_no_event_in_uts_target)
self.requested_extra_info_types = (
None
if extra_info_types is None
else list(dict.fromkeys(int(t) for t in extra_info_types))
)
self.label_code_to_id, self.label_id_to_code = load_label_vocab( self.label_code_to_id, self.label_id_to_code = load_label_vocab(
labels_file, labels_file,
@@ -112,26 +106,12 @@ class _ExpoBaseDataset(Dataset):
self.event_data = event_data[order] self.event_data = event_data[order]
basic_table = pd.read_csv(f"{data_prefix}_basic_info.csv", index_col=0) basic_table = pd.read_csv(f"{data_prefix}_basic_info.csv", index_col=0)
other_info = np.load(f"{data_prefix}_other_info.npy")
if other_info.ndim != 2 or other_info.shape[1] != 5:
raise ValueError(
f"other_info must have shape (N, 5), got {other_info.shape}"
)
cate_types = pd.read_csv("cate_types.csv")
required_cate_cols = {"type", "name", "n_categories"}
missing_cate_cols = required_cate_cols - set(cate_types.columns)
if missing_cate_cols:
raise ValueError(
f"cate_types.csv is missing columns: {sorted(missing_cate_cols)}"
)
basic_table.index = basic_table.index.astype(np.int64) basic_table.index = basic_table.index.astype(np.int64)
unique_eids = np.unique(self.event_data[:, 0].astype(np.int64)) unique_eids = np.unique(self.event_data[:, 0].astype(np.int64))
basic_table = basic_table.loc[unique_eids] basic_table = basic_table.loc[unique_eids]
self._prepare_sex(basic_table, unique_eids) self._prepare_sex(basic_table, unique_eids)
self._prepare_other_info(other_info, cate_types, unique_eids)
max_id_in_vocab = max(self.label_id_to_code.keys()) max_id_in_vocab = max(self.label_id_to_code.keys())
max_id_in_data = int(self.event_data[:, 2].max()) if len(self.event_data) > 0 else 0 max_id_in_data = int(self.event_data[:, 2].max()) if len(self.event_data) > 0 else 0
@@ -160,95 +140,6 @@ class _ExpoBaseDataset(Dataset):
) )
self.sex_mapping = {int(eid): int(s) for eid, s in zip(unique_eids, sex01)} self.sex_mapping = {int(eid): int(s) for eid, s in zip(unique_eids, sex01)}
def _prepare_other_info(
self,
other_info: np.ndarray,
cate_types: pd.DataFrame,
unique_eids: np.ndarray,
) -> None:
other_info = other_info.copy()
other_info[:, 0] = other_info[:, 0].astype(np.int64)
other_info[:, 1] = other_info[:, 1].astype(np.int64)
other_info[:, 3] = other_info[:, 3].astype(np.int64)
available_types = sorted(
int(t) for t in np.unique(other_info[:, 1]) if int(t) > 0
)
if self.requested_extra_info_types is None:
selected_types = available_types
else:
selected_types = self.requested_extra_info_types
missing = sorted(set(selected_types) - set(available_types))
if missing:
raise ValueError(f"Requested extra_info_types not found: {missing}")
keep = np.isin(other_info[:, 0].astype(np.int64), unique_eids)
keep &= np.isin(other_info[:, 1].astype(np.int64), selected_types)
other_info = other_info[keep]
cate_counts = {
int(row["type"]): int(row["n_categories"])
for _, row in cate_types.iterrows()
}
cate_offsets: Dict[int, int] = {}
next_offset = 0
for type_id in selected_types:
if type_id in cate_counts:
cate_offsets[type_id] = next_offset
next_offset += cate_counts[type_id]
kinds = other_info[:, 3].astype(np.int64)
types = other_info[:, 1].astype(np.int64)
cate_rows = kinds == 2
for type_id in np.unique(types[cate_rows]):
type_id = int(type_id)
if type_id not in cate_offsets:
raise ValueError(
f"type {type_id} appears categorical but is missing from cate_types.csv"
)
row_mask = cate_rows & (types == type_id)
local_value = other_info[row_mask, 2].astype(np.int64)
other_info[row_mask, 2] = local_value + cate_offsets[type_id]
cont_type_ids = [
int(t)
for t in selected_types
if np.any((types == int(t)) & (kinds == 1))
]
self.extra_info_types = selected_types
self.cate_type_offsets = cate_offsets
self.n_types = (max(selected_types) + 1) if selected_types else 1
self.cont_type_ids = cont_type_ids
self.n_cont_types = len(cont_type_ids)
self.n_categories = next_offset + 1
order = np.lexsort((other_info[:, 4], other_info[:, 1], other_info[:, 0]))
other_info = other_info[order]
self.other_info_by_eid: Dict[int, Dict[str, np.ndarray]] = {}
for eid in unique_eids.astype(np.int64):
self.other_info_by_eid[int(eid)] = {
"other_type": np.zeros(0, dtype=np.int64),
"other_value": np.zeros(0, dtype=np.float32),
"other_value_kind": np.zeros(0, dtype=np.int64),
"other_time": np.zeros(0, dtype=np.float32),
}
if len(other_info) == 0:
return
eids, starts = np.unique(other_info[:, 0].astype(np.int64), return_index=True)
ends = np.concatenate([starts[1:], [len(other_info)]])
for eid_raw, start, end in zip(eids, starts, ends):
rows = other_info[start:end]
self.other_info_by_eid[int(eid_raw)] = {
"other_type": rows[:, 1].astype(np.int64),
"other_value": rows[:, 2].astype(np.float32),
"other_value_kind": rows[:, 3].astype(np.int64),
"other_time": (rows[:, 4].astype(np.float32) / DAYS_PER_YEAR),
}
def _iter_patient_events( def _iter_patient_events(
self, self,
*, *,
@@ -279,12 +170,10 @@ class _ExpoBaseDataset(Dataset):
yield eid, times_days, labels yield eid, times_days, labels
def _split_features(self, eid: int) -> Optional[Dict]: def _split_features(self, eid: int) -> Optional[Dict]:
other_info = self.other_info_by_eid.get(eid) if eid not in self.sex_mapping:
if other_info is None:
return None return None
return { return {
"sex": self.sex_mapping[eid], "sex": self.sex_mapping[eid],
**other_info,
} }
class NextStepHealthDataset(_ExpoBaseDataset): class NextStepHealthDataset(_ExpoBaseDataset):
@@ -305,14 +194,12 @@ class NextStepHealthDataset(_ExpoBaseDataset):
labels_file: str = "labels.csv", labels_file: str = "labels.csv",
no_event_interval_years: float = 5.0, no_event_interval_years: float = 5.0,
include_no_event_in_uts_target: bool = False, include_no_event_in_uts_target: bool = False,
extra_info_types: Iterable[int] | None = None,
) -> None: ) -> None:
super().__init__( super().__init__(
data_prefix=data_prefix, data_prefix=data_prefix,
labels_file=labels_file, labels_file=labels_file,
no_event_interval_years=no_event_interval_years, no_event_interval_years=no_event_interval_years,
include_no_event_in_uts_target=include_no_event_in_uts_target, include_no_event_in_uts_target=include_no_event_in_uts_target,
extra_info_types=extra_info_types,
) )
self.samples: List[Dict] = [] self.samples: List[Dict] = []
@@ -355,10 +242,6 @@ class NextStepHealthDataset(_ExpoBaseDataset):
"event_seq": torch.from_numpy(s["event_seq"]).long(), "event_seq": torch.from_numpy(s["event_seq"]).long(),
"time_seq": torch.from_numpy(s["time_seq"]).float(), "time_seq": torch.from_numpy(s["time_seq"]).float(),
"sex": torch.tensor(s["sex"], dtype=torch.long), "sex": torch.tensor(s["sex"], dtype=torch.long),
"other_type": torch.from_numpy(s["other_type"]).long(),
"other_value": torch.from_numpy(s["other_value"]).float(),
"other_value_kind": torch.from_numpy(s["other_value_kind"]).long(),
"other_time": torch.from_numpy(s["other_time"]).float(),
"target_event_seq": torch.from_numpy(s["target_event_seq"]).long(), "target_event_seq": torch.from_numpy(s["target_event_seq"]).long(),
"target_time_seq": torch.from_numpy(s["target_time_seq"]).float(), "target_time_seq": torch.from_numpy(s["target_time_seq"]).float(),
"readout_mask": torch.from_numpy(s["readout_mask"]).bool(), "readout_mask": torch.from_numpy(s["readout_mask"]).bool(),
@@ -390,7 +273,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset):
min_history_events: int = 1, min_history_events: int = 1,
min_future_events: int = 1, min_future_events: int = 1,
validation_query_seed: int = 42, validation_query_seed: int = 42,
extra_info_types: Iterable[int] | None = None,
) -> None: ) -> None:
if split not in {"train", "valid", "test"}: if split not in {"train", "valid", "test"}:
raise ValueError(f"split must be train/valid/test, got {split!r}") raise ValueError(f"split must be train/valid/test, got {split!r}")
@@ -400,7 +282,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset):
labels_file=labels_file, labels_file=labels_file,
no_event_interval_years=no_event_interval_years, no_event_interval_years=no_event_interval_years,
include_no_event_in_uts_target=include_no_event_in_uts_target, include_no_event_in_uts_target=include_no_event_in_uts_target,
extra_info_types=extra_info_types,
) )
self.split = split self.split = split
@@ -537,10 +418,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset):
"future_dt": torch.from_numpy(times[fut] - np.float32(t_query)).float(), "future_dt": torch.from_numpy(times[fut] - np.float32(t_query)).float(),
"exposure": torch.tensor(np.float32(patient["t_obs"] - t_query), dtype=torch.float32), "exposure": torch.tensor(np.float32(patient["t_obs"] - t_query), dtype=torch.float32),
"sex": torch.tensor(patient["sex"], dtype=torch.long), "sex": torch.tensor(patient["sex"], dtype=torch.long),
"other_type": torch.from_numpy(patient["other_type"]).long(),
"other_value": torch.from_numpy(patient["other_value"]).float(),
"other_value_kind": torch.from_numpy(patient["other_value_kind"]).long(),
"other_time": torch.from_numpy(patient["other_time"]).float(),
} }
def __len__(self) -> int: def __len__(self) -> int:
@@ -561,26 +438,6 @@ class AllFutureHealthDataset(_ExpoBaseDataset):
def _collate_common_static(batch: List[Dict]) -> Dict: def _collate_common_static(batch: List[Dict]) -> Dict:
return { return {
"sex": torch.stack([s["sex"] for s in batch]), "sex": torch.stack([s["sex"] for s in batch]),
"other_type": pad_sequence(
[s["other_type"] for s in batch],
batch_first=True,
padding_value=0,
),
"other_value": pad_sequence(
[s["other_value"] for s in batch],
batch_first=True,
padding_value=0.0,
),
"other_value_kind": pad_sequence(
[s["other_value_kind"] for s in batch],
batch_first=True,
padding_value=0,
),
"other_time": pad_sequence(
[s["other_time"] for s in batch],
batch_first=True,
padding_value=0.0,
),
} }

37
eval_data.py
View File

@@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, Iterable, List from typing import Any, Dict, List
import numpy as np import numpy as np
import torch import torch
@@ -25,7 +25,6 @@ class AllFutureSequenceEvalDataset:
labels_file: str, labels_file: str,
min_history_events: int = 1, min_history_events: int = 1,
min_future_events: int = 1, min_future_events: int = 1,
extra_info_types: Iterable[int] | None = None,
) -> None: ) -> None:
base = AllFutureHealthDataset( base = AllFutureHealthDataset(
data_prefix=data_prefix, data_prefix=data_prefix,
@@ -33,18 +32,12 @@ class AllFutureSequenceEvalDataset:
split="train", split="train",
min_history_events=min_history_events, min_history_events=min_history_events,
min_future_events=min_future_events, min_future_events=min_future_events,
extra_info_types=extra_info_types,
) )
self.base = base self.base = base
self.label_code_to_id = base.label_code_to_id self.label_code_to_id = base.label_code_to_id
self.label_id_to_code = base.label_id_to_code self.label_id_to_code = base.label_id_to_code
self.vocab_size = base.vocab_size self.vocab_size = base.vocab_size
self.n_types = base.n_types
self.n_cont_types = base.n_cont_types
self.n_categories = base.n_categories
self.cont_type_ids = base.cont_type_ids
self.extra_info_types = base.extra_info_types
self.samples: List[Dict[str, Any]] = [] self.samples: List[Dict[str, Any]] = []
for patient in base.patients: for patient in base.patients:
@@ -62,10 +55,6 @@ class AllFutureSequenceEvalDataset:
"target_time_seq": times[1:], "target_time_seq": times[1:],
"readout_mask": np.ones(input_len, dtype=bool), "readout_mask": np.ones(input_len, dtype=bool),
"sex": int(patient["sex"]), "sex": int(patient["sex"]),
"other_type": np.asarray(patient["other_type"], dtype=np.int64),
"other_value": np.asarray(patient["other_value"], dtype=np.float32),
"other_value_kind": np.asarray(patient["other_value_kind"], dtype=np.int64),
"other_time": np.asarray(patient["other_time"], dtype=np.float32),
} }
) )
@@ -81,10 +70,6 @@ class AllFutureSequenceEvalDataset:
"target_time_seq": torch.from_numpy(s["target_time_seq"]).float(), "target_time_seq": torch.from_numpy(s["target_time_seq"]).float(),
"readout_mask": torch.from_numpy(s["readout_mask"]).bool(), "readout_mask": torch.from_numpy(s["readout_mask"]).bool(),
"sex": torch.tensor(s["sex"], dtype=torch.long), "sex": torch.tensor(s["sex"], dtype=torch.long),
"other_type": torch.from_numpy(s["other_type"]).long(),
"other_value": torch.from_numpy(s["other_value"]).float(),
"other_value_kind": torch.from_numpy(s["other_value_kind"]).long(),
"other_time": torch.from_numpy(s["other_time"]).float(),
} }
@@ -97,7 +82,6 @@ def load_sequence_eval_dataset(
include_no_event_in_uts_target: bool, include_no_event_in_uts_target: bool,
min_history_events: int, min_history_events: int,
min_future_events: int, min_future_events: int,
extra_info_types: Iterable[int] | None,
): ):
mode = str(model_target_mode).lower() mode = str(model_target_mode).lower()
if mode == "next_token": if mode == "next_token":
@@ -106,7 +90,6 @@ def load_sequence_eval_dataset(
labels_file=labels_file, labels_file=labels_file,
no_event_interval_years=no_event_interval_years, no_event_interval_years=no_event_interval_years,
include_no_event_in_uts_target=include_no_event_in_uts_target, include_no_event_in_uts_target=include_no_event_in_uts_target,
extra_info_types=extra_info_types,
) )
if mode == "all_future": if mode == "all_future":
return AllFutureSequenceEvalDataset( return AllFutureSequenceEvalDataset(
@@ -114,7 +97,6 @@ def load_sequence_eval_dataset(
labels_file=labels_file, labels_file=labels_file,
min_history_events=min_history_events, min_history_events=min_history_events,
min_future_events=min_future_events, min_future_events=min_future_events,
extra_info_types=extra_info_types,
) )
raise ValueError(f"Unknown model_target_mode: {model_target_mode!r}") raise ValueError(f"Unknown model_target_mode: {model_target_mode!r}")
@@ -135,19 +117,6 @@ def sequence_eval_collate_fn(batch: List[Dict[str, torch.Tensor]]) -> Dict[str,
readout_mask = pad_sequence( readout_mask = pad_sequence(
[s["readout_mask"] for s in batch], batch_first=True, padding_value=False [s["readout_mask"] for s in batch], batch_first=True, padding_value=False
) )
other_type = pad_sequence(
[s["other_type"] for s in batch], batch_first=True, padding_value=0
)
other_value = pad_sequence(
[s["other_value"] for s in batch], batch_first=True, padding_value=0.0
)
other_value_kind = pad_sequence(
[s["other_value_kind"] for s in batch], batch_first=True, padding_value=0
)
other_time = pad_sequence(
[s["other_time"] for s in batch], batch_first=True, padding_value=0.0
)
return { return {
"event_seq": event_seq, "event_seq": event_seq,
"time_seq": time_seq, "time_seq": time_seq,
@@ -156,8 +125,4 @@ def sequence_eval_collate_fn(batch: List[Dict[str, torch.Tensor]]) -> Dict[str,
"target_time_seq": target_time_seq, "target_time_seq": target_time_seq,
"readout_mask": readout_mask, "readout_mask": readout_mask,
"sex": torch.stack([s["sex"] for s in batch]), "sex": torch.stack([s["sex"] for s in batch]),
"other_type": other_type,
"other_value": other_value,
"other_value_kind": other_value_kind,
"other_time": other_time,
} }

23
evaluate_auc.py
View File

@@ -320,13 +320,6 @@ def build_model_from_dataset(args: argparse.Namespace, cfg: Dict[str, Any], data
n_embd=int(cfg_get(args, cfg, "n_embd", 120)), n_embd=int(cfg_get(args, cfg, "n_embd", 120)),
n_head=int(cfg_get(args, cfg, "n_head", 10)), n_head=int(cfg_get(args, cfg, "n_head", 10)),
n_hist_layer=int(cfg_get(args, cfg, "n_hist_layer", 12)), n_hist_layer=int(cfg_get(args, cfg, "n_hist_layer", 12)),
n_tab_layer=int(cfg_get(args, cfg, "n_tab_layer", 4)),
n_types=dataset.n_types,
n_cont_types=dataset.n_cont_types,
n_categories=dataset.n_categories,
cont_type_ids=dataset.cont_type_ids,
n_bins=int(cfg_get(args, cfg, "n_bins", 16)),
extra_pool_reduce=str(cfg_get(args, cfg, "extra_pool_reduce", "mean")),
target_mode=model_target_mode, target_mode=model_target_mode,
time_mode=str(cfg_get(args, cfg, "time_mode", "relative")), time_mode=str(cfg_get(args, cfg, "time_mode", "relative")),
dist_mode=str(cfg_get(args, cfg, "dist_mode", "exponential")), dist_mode=str(cfg_get(args, cfg, "dist_mode", "exponential")),
@@ -424,11 +417,6 @@ def validate_dataset_metadata(dataset: HealthDataset, cfg: Dict[str, Any]) -> No
actual: Dict[str, Any] = { actual: Dict[str, Any] = {
"vocab_size": int(dataset.vocab_size), "vocab_size": int(dataset.vocab_size),
"n_types": int(dataset.n_types),
"n_cont_types": int(dataset.n_cont_types),
"n_categories": int(dataset.n_categories),
"cont_type_ids": [int(x) for x in dataset.cont_type_ids],
"extra_info_types": [int(x) for x in dataset.extra_info_types],
} }
mismatches = [ mismatches = [
f"{key}: train_config={meta.get(key)!r}, current_dataset={value!r}" f"{key}: train_config={meta.get(key)!r}, current_dataset={value!r}"
@@ -438,7 +426,7 @@ def validate_dataset_metadata(dataset: HealthDataset, cfg: Dict[str, Any]) -> No
if mismatches: if mismatches:
raise RuntimeError( raise RuntimeError(
"Current dataset metadata does not match train_config.json. " "Current dataset metadata does not match train_config.json. "
"Use the same prepared data and extra_info_types as training. " "Use the same prepared data as training. "
+ "; ".join(mismatches) + "; ".join(mismatches)
) )
@@ -536,10 +524,6 @@ def infer_readout_hidden(
sex=batch_dev["sex"][active], sex=batch_dev["sex"][active],
padding_mask=padding_mask[active], padding_mask=padding_mask[active],
t_query=time_seq[active, pos], t_query=time_seq[active, pos],
other_type=batch_dev["other_type"][active],
other_value=batch_dev["other_value"][active],
other_value_kind=batch_dev["other_value_kind"][active],
other_time=batch_dev["other_time"][active],
target_mode="all_future", target_mode="all_future",
) )
hidden[active, pos, :] = hidden_pos.float() hidden[active, pos, :] = hidden_pos.float()
@@ -550,10 +534,6 @@ def infer_readout_hidden(
time_seq=time_seq, time_seq=time_seq,
sex=batch_dev["sex"], sex=batch_dev["sex"],
padding_mask=padding_mask, padding_mask=padding_mask,
other_type=batch_dev["other_type"],
other_value=batch_dev["other_value"],
other_value_kind=batch_dev["other_value_kind"],
other_time=batch_dev["other_time"],
target_mode="next_token", target_mode="next_token",
) )
ro = readout( ro = readout(
@@ -1337,7 +1317,6 @@ def main() -> None:
include_no_event_in_uts_target=include_no_event, include_no_event_in_uts_target=include_no_event,
min_history_events=int(cfg.get("all_future_min_history_events", 1)), min_history_events=int(cfg.get("all_future_min_history_events", 1)),
min_future_events=int(cfg.get("all_future_min_future_events", 1)), min_future_events=int(cfg.get("all_future_min_future_events", 1)),
extra_info_types=parse_int_list(cfg.get("extra_info_types", None)),
) )
validate_dataset_metadata(dataset, cfg) validate_dataset_metadata(dataset, cfg)

268
extra_info_types_all.txt
View File

@@ -1,268 +0,0 @@
# All other-info variables (field_type=1 and field_type=2)
# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering.
# Format: <extra_info_type_id> # <var_name> | <full_name>
1 # waist_circumference | Waist circumference
2 # hip_circumference | Hip circumference
3 # standing_height | Standing height
4 # fasting_time | Fasting time
5 # pulse_rate | Pulse rate automated reading
6 # dbp | Diastolic blood pressure automated reading
7 # sbp | Systolic blood pressure automated reading
8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure
9 # fvc_best | Forced vital capacity (FVC) Best measure
10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score
11 # bmi | Body mass index (BMI)
12 # WBC | White blood cell (leukocyte) count
13 # RBC | Red blood cell (erythrocyte) count
14 # hemoglobin | Haemoglobin concentration
15 # hematocrit | Haematocrit percentage
16 # MCV | Mean corpuscular volume
17 # MCH | Mean corpuscular haemoglobin
18 # MCHC | Mean corpuscular haemoglobin concentration
19 # Pc | Platelet count
20 # MPV | Mean platelet (thrombocyte) volume
21 # LymC | Lymphocyte count
22 # MonC | Monocyte count
23 # NeuC | Neutrophill count
24 # EosC | Eosinophill count
25 # BasC | Basophill count
26 # nRBC | Nucleated red blood cell count
27 # RC | Reticulocyte count
28 # MRV | Mean reticulocyte volume
29 # MSCV | Mean sphered cell volume
30 # IRF | Immature reticulocyte fraction
31 # HLSRC | High light scatter reticulocyte count
32 # MicU | Microalbumin in urine
33 # CreaU | Creatinine (enzymatic) in urine
34 # PotU | Potassium in urine
35 # SodU | Sodium in urine
36 # Alb | Albumin
37 # ALP | Alkaline phosphatase
38 # Alanine | Alanine aminotransferase
39 # ApoA | Apolipoprotein A
40 # ApoB | Apolipoprotein B
41 # AA | Aspartate aminotransferase
42 # DBil | Direct bilirubin
43 # Urea | Urea
44 # Calcium | Calcium
45 # Cholesterol | Cholesterol
46 # Creatinine | Creatinine
47 # CRP | C-reactive protein
48 # CystatinC | Cystatin C
49 # GGT | Gamma glutamyltransferase
50 # Glu | Glucose
51 # HbA1c | Glycated haemoglobin (HbA1c)
52 # HDL | HDL cholesterol
53 # IGF1 | IGF-1
54 # LDL | LDL direct
55 # LpA | Lipoprotein A
56 # Oestradiol | Oestradiol
57 # Phosphate | Phosphate
58 # Rheu | Rheumatoid factor
59 # SHBG | SHBG
60 # TotalBil | Total bilirubin
61 # Testosterone | Testosterone
62 # TotalProtein | Total protein
63 # Tri | Triglycerides
64 # Urate | Urate
65 # VitaminD | Vitamin D
66 # smoking | Current tobacco smoking
67 # alcohol | Alcohol intake frequency.
68 # ipaq_activity_group | IPAQ activity group
69 # moderate_activity_met_minutes_week | MET minutes per week for moderate activity
70 # vigorous_activity_met_minutes_week | MET minutes per week for vigorous activity
71 # walking_met_minutes_week | MET minutes per week for walking
72 # total_activity_met_minutes_week | Summed MET minutes per week for all activity
73 # total_activity_days | Summed days activity
74 # total_activity_minutes | Summed minutes activity
75 # heavy_diy_duration | Duration of heavy DIY
76 # light_diy_duration | Duration of light DIY
77 # moderate_activity_duration | Duration of moderate activity
78 # other_exercise_duration | Duration of other exercises
79 # strenuous_sport_duration | Duration of strenuous sports
80 # vigorous_activity_duration | Duration of vigorous activity
81 # walking_duration | Duration of walks
82 # pleasure_walking_duration | Duration walking for pleasure
83 # heavy_diy_frequency_4_weeks | Frequency of heavy DIY in last 4 weeks
84 # light_diy_frequency_4_weeks | Frequency of light DIY in last 4 weeks
85 # other_exercise_frequency_4_weeks | Frequency of other exercises in last 4 weeks
86 # stair_climbing_frequency_4_weeks | Frequency of stair climbing in last 4 weeks
87 # strenuous_sport_frequency_4_weeks | Frequency of strenuous sports in last 4 weeks
88 # pleasure_walking_frequency_4_weeks | Frequency of walking for pleasure in last 4 weeks
89 # moderate_activity_days_week_10min | Number of days/week of moderate physical activity 10+ minutes
90 # vigorous_activity_days_week_10min | Number of days/week of vigorous physical activity 10+ minutes
91 # walking_days_week_10min | Number of days/week walked 10+ minutes
92 # driving_time | Time spent driving
93 # computer_use_time | Time spent using computer
94 # tv_watching_time | Time spent watching television (TV)
95 # physical_activity_types_4_weeks | Types of physical activity in last 4 weeks
96 # nonwork_transport_types | Types of transport used (excluding work)
97 # usual_walking_pace | Usual walking pace
98 # mobile_phone_use_duration | Length of mobile phone use
99 # mobile_phone_use_weekly_3_months | Weekly usage of mobile phone in last 3 months
100 # computer_game_playing | Plays computer games
101 # sleep_duration | Sleep duration
102 # chronotype | Morning/evening person (chronotype)
103 # daytime_napping | Nap during day
104 # insomnia | Sleeplessness / insomnia
105 # daytime_dozing | Daytime dozing / sleeping
106 # ever_smoked | Ever smoked
107 # smoking_pack_years | Pack years of smoking
108 # smoking_status | Smoking status
109 # past_tobacco_smoking | Past tobacco smoking
110 # lifetime_smoking_100_plus | Light smokers, at least 100 smokes in lifetime
111 # current_tobacco_type | Type of tobacco currently smoked
112 # current_cigarettes_per_day | Number of cigarettes currently smoked daily (current cigarette smokers)
113 # previous_cigarettes_per_day_current_cigar_pipe_smokers | Number of cigarettes previously smoked daily (current cigar/pipe smokers)
114 # time_to_first_cigarette | Time from waking to first cigarette
115 # ever_tried_smoking_cessation | Ever tried to stop smoking
116 # smoking_change_vs_10_years_ago | Smoking compared to 10 years previous
117 # previous_tobacco_type | Type of tobacco previously smoked
118 # previous_cigarettes_per_day | Number of cigarettes previously smoked daily
119 # ever_stopped_smoking_6_months | Ever stopped smoking for 6+ months
120 # household_smokers | Smoking/smokers in household
121 # home_secondhand_smoke_exposure | Exposure to tobacco smoke at home
122 # nonhome_secondhand_smoke_exposure | Exposure to tobacco smoke outside home
123 # cooked_vegetable_intake | Cooked vegetable intake
124 # raw_vegetable_intake | Salad / raw vegetable intake
125 # fresh_fruit_intake | Fresh fruit intake
126 # dried_fruit_intake | Dried fruit intake
127 # oily_fish_intake | Oily fish intake
128 # non_oily_fish_intake | Non-oily fish intake
129 # processed_meat_intake | Processed meat intake
130 # poultry_intake | Poultry intake
131 # beef_intake | Beef intake
132 # lamb_mutton_intake | Lamb/mutton intake
133 # pork_intake | Pork intake
134 # age_last_ate_meat | Age when last ate meat
135 # food_avoidance_eggs_dairy_wheat_sugar | Never eat eggs, dairy, wheat, sugar
136 # cheese_intake | Cheese intake
137 # milk_type | Milk type used
138 # spread_type | Spread type
139 # bread_intake | Bread intake
140 # bread_type | Bread type
141 # cereal_intake | Cereal intake
142 # cereal_type | Cereal type
143 # added_salt | Salt added to food
144 # tea_intake | Tea intake
145 # coffee_intake | Coffee intake
146 # coffee_type | Coffee type
147 # hot_drink_temperature | Hot drink temperature
148 # water_intake | Water intake
149 # diet_variation | Variation in diet
150 # alcohol_drinker_status | Alcohol drinker status
151 # former_alcohol_drinker | Former alcohol drinker
152 # red_wine_intake_monthly | Average monthly red wine intake
153 # champagne_white_wine_intake_monthly | Average monthly champagne plus white wine intake
154 # beer_cider_intake_monthly | Average monthly beer plus cider intake
155 # spirits_intake_monthly | Average monthly spirits intake
156 # fortified_wine_intake_monthly | Average monthly fortified wine intake
157 # other_alcohol_intake_monthly | Average monthly intake of other alcoholic drinks
158 # red_wine_intake_weekly | Average weekly red wine intake
159 # champagne_white_wine_intake_weekly | Average weekly champagne plus white wine intake
160 # beer_cider_intake_weekly | Average weekly beer plus cider intake
161 # spirits_intake_weekly | Average weekly spirits intake
162 # fortified_wine_intake_weekly | Average weekly fortified wine intake
163 # other_alcohol_intake_weekly | Average weekly intake of other alcoholic drinks
164 # alcohol_with_meals | Alcohol usually taken with meals
165 # country_of_birth_uk_elsewhere | Country of birth (UK/elsewhere)
166 # breastfed_in_infancy | Breastfed as a baby
167 # comparative_body_size_age_10 | Comparative body size at age 10
168 # comparative_height_age_10 | Comparative height size at age 10
169 # handedness | Handedness (chirality/laterality)
170 # adopted_as_child | Adopted as a child
171 # multiple_birth | Part of a multiple birth
172 # maternal_smoking_around_birth | Maternal smoking around birth
173 # accommodation_type | Type of accommodation lived in
174 # housing_tenure | Own or rent accommodation lived in
175 # gas_solid_fuel_use | Gas or solid-fuel cooking/heating
176 # home_heating_types | Heating type(s) in home
177 # household_vehicle_count | Number of vehicles in household
178 # household_income_before_tax | Average total household income before tax
179 # current_employment_status | Current employment status
180 # current_employment_status_corrected | Current employment status - corrected
181 # home_work_distance | Distance between home and job workplace
182 # main_job_hours_week | Length of working week for main job
183 # commuting_frequency | Frequency of travelling from home to job workplace
184 # commuting_transport_type | Transport type for commuting to job workplace
185 # job_walking_standing | Job involves mainly walking or standing
186 # job_heavy_manual_work | Job involves heavy manual or physical work
187 # job_shift_work | Job involves shift work
188 # job_night_shift_work | Job involves night shift work
189 # educational_qualifications | Qualifications
190 # age_completed_full_time_education | Age completed full time education
191 # friend_family_visit_frequency | Frequency of friend/family visits
192 # leisure_social_activities | Leisure/social activities
193 # ability_to_confide | Able to confide
194 # bipolar_major_depression_status | Bipolar and major depression status
195 # neuroticism_score | Neuroticism score
196 # mood_swings | Mood swings
197 # miserableness | Miserableness
198 # irritability | Irritability
199 # sensitivity_hurt_feelings | Sensitivity / hurt feelings
200 # fed_up_feelings | Fed-up feelings
201 # nervous_feelings | Nervous feelings
202 # worry_anxiety_feelings | Worrier / anxious feelings
203 # tenseness_highly_strung | Tense / 'highly strung'
204 # suffering_from_nerves | Suffer from 'nerves'
205 # loneliness_isolation | Loneliness, isolation
206 # guilty_feelings | Guilty feelings
207 # risk_taking | Risk taking
208 # happiness | Happiness
209 # job_satisfaction | Work/job satisfaction
210 # health_satisfaction | Health satisfaction
211 # family_relationship_satisfaction | Family relationship satisfaction
212 # friendship_satisfaction | Friendships satisfaction
213 # financial_situation_satisfaction | Financial situation satisfaction
214 # depressed_mood_frequency_2_weeks | Frequency of depressed mood in last 2 weeks
215 # disinterest_frequency_2_weeks | Frequency of unenthusiasm / disinterest in last 2 weeks
216 # tenseness_restlessness_frequency_2_weeks | Frequency of tenseness / restlessness in last 2 weeks
217 # tiredness_lethargy_frequency_2_weeks | Frequency of tiredness / lethargy in last 2 weeks
218 # ever_depressed_full_week | Ever depressed for a whole week
219 # longest_depression_duration | Longest period of depression
220 # depression_episode_count | Number of depression episodes
221 # longest_disinterest_duration | Longest period of unenthusiasm / disinterest
222 # disinterest_episode_count | Number of unenthusiastic/disinterested episodes
223 # ever_manic_hyper_2_days | Ever manic/hyper for 2 days
224 # ever_irritable_argumentative_2_days | Ever highly irritable/argumentative for 2 days
225 # manic_hyper_symptoms | Manic/hyper symptoms
226 # longest_manic_irritable_episode_duration | Length of longest manic/irritable episode
227 # manic_irritable_episode_severity | Severity of manic/irritable episodes
228 # adverse_life_events_2_years | Illness, injury, bereavement, stress in last 2 years
229 # outdoor_time_summer | Time spend outdoors in summer
230 # outdoor_time_winter | Time spent outdoors in winter
231 # skin_tanning_ease | Ease of skin tanning
232 # childhood_sunburn_frequency | Childhood sunburn occasions
233 # sun_uv_protection_use | Use of sun/uv protection
234 # solarium_sunlamp_frequency | Frequency of solarium/sunlamp use
235 # proximity_to_major_road | Close to major road
236 # inverse_distance_nearest_major_road | Inverse distance to the nearest major road
237 # inverse_distance_nearest_road | Inverse distance to the nearest road
238 # no2_2005 | Nitrogen dioxide air pollution; 2005
239 # no2_2006 | Nitrogen dioxide air pollution; 2006
240 # no2_2007 | Nitrogen dioxide air pollution; 2007
241 # no2_2010 | Nitrogen dioxide air pollution; 2010
242 # nox_2010 | Nitrogen oxides air pollution; 2010
243 # pm10_2007 | Particulate matter air pollution (pm10); 2007
244 # pm10_2010 | Particulate matter air pollution (pm10); 2010
245 # pm25_absorbance_2010 | Particulate matter air pollution (pm2.5) absorbance; 2010
246 # pm25_2010 | Particulate matter air pollution (pm2.5); 2010
247 # pm25_10_2010 | Particulate matter air pollution 2.5-10um; 2010
248 # major_road_length_100m | Sum of road length of major roads within 100m
249 # major_road_traffic_load | Total traffic load on major roads
250 # nearest_major_road_traffic_intensity | Traffic intensity on the nearest major road
251 # nearest_road_traffic_intensity | Traffic intensity on the nearest road
252 # noise_level_16h | Average 16-hour sound level of noise pollution
253 # noise_level_24h | Average 24-hour sound level of noise pollution
254 # noise_level_daytime | Average daytime sound level of noise pollution
255 # noise_level_evening | Average evening sound level of noise pollution
256 # noise_level_nighttime | Average night-time sound level of noise pollution
257 # natural_environment_percent_1000m | Natural environment percentage, buffer 1000m
258 # natural_environment_percent_300m | Natural environment percentage, buffer 300m
259 # greenspace_percent_1000m | Greenspace percentage, buffer 1000m
260 # greenspace_percent_300m | Greenspace percentage, buffer 300m
261 # domestic_garden_percent_1000m | Domestic garden percentage, buffer 1000m
262 # domestic_garden_percent_300m | Domestic garden percentage, buffer 300m
263 # water_percent_1000m | Water percentage, buffer 1000m
264 # water_percent_300m | Water percentage, buffer 300m
265 # distance_to_coast | Distance (Euclidean) to coast

68
extra_info_types_assessment_only.txt
View File

@@ -1,68 +0,0 @@
# Only assessment/body-measurement variables (field_type=1)
# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering.
# Format: <extra_info_type_id> # <var_name> | <full_name>
1 # waist_circumference | Waist circumference
2 # hip_circumference | Hip circumference
3 # standing_height | Standing height
4 # fasting_time | Fasting time
5 # pulse_rate | Pulse rate automated reading
6 # dbp | Diastolic blood pressure automated reading
7 # sbp | Systolic blood pressure automated reading
8 # fev1_best | Forced expiratory volume in 1-second (FEV1) Best measure
9 # fvc_best | Forced vital capacity (FVC) Best measure
10 # fev1_fvc_ratio | FEV1/ FVC ratio Z-score
11 # bmi | Body mass index (BMI)
12 # WBC | White blood cell (leukocyte) count
13 # RBC | Red blood cell (erythrocyte) count
14 # hemoglobin | Haemoglobin concentration
15 # hematocrit | Haematocrit percentage
16 # MCV | Mean corpuscular volume
17 # MCH | Mean corpuscular haemoglobin
18 # MCHC | Mean corpuscular haemoglobin concentration
19 # Pc | Platelet count
20 # MPV | Mean platelet (thrombocyte) volume
21 # LymC | Lymphocyte count
22 # MonC | Monocyte count
23 # NeuC | Neutrophill count
24 # EosC | Eosinophill count
25 # BasC | Basophill count
26 # nRBC | Nucleated red blood cell count
27 # RC | Reticulocyte count
28 # MRV | Mean reticulocyte volume
29 # MSCV | Mean sphered cell volume
30 # IRF | Immature reticulocyte fraction
31 # HLSRC | High light scatter reticulocyte count
32 # MicU | Microalbumin in urine
33 # CreaU | Creatinine (enzymatic) in urine
34 # PotU | Potassium in urine
35 # SodU | Sodium in urine
36 # Alb | Albumin
37 # ALP | Alkaline phosphatase
38 # Alanine | Alanine aminotransferase
39 # ApoA | Apolipoprotein A
40 # ApoB | Apolipoprotein B
41 # AA | Aspartate aminotransferase
42 # DBil | Direct bilirubin
43 # Urea | Urea
44 # Calcium | Calcium
45 # Cholesterol | Cholesterol
46 # Creatinine | Creatinine
47 # CRP | C-reactive protein
48 # CystatinC | Cystatin C
49 # GGT | Gamma glutamyltransferase
50 # Glu | Glucose
51 # HbA1c | Glycated haemoglobin (HbA1c)
52 # HDL | HDL cholesterol
53 # IGF1 | IGF-1
54 # LDL | LDL direct
55 # LpA | Lipoprotein A
56 # Oestradiol | Oestradiol
57 # Phosphate | Phosphate
58 # Rheu | Rheumatoid factor
59 # SHBG | SHBG
60 # TotalBil | Total bilirubin
61 # Testosterone | Testosterone
62 # TotalProtein | Total protein
63 # Tri | Triglycerides
64 # Urate | Urate
65 # VitaminD | Vitamin D

203
extra_info_types_exposure_only.txt
View File

@@ -1,203 +0,0 @@
# Only environment/lifestyle exposure variables (field_type=2)
# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering.
# Format: <extra_info_type_id> # <var_name> | <full_name>
66 # smoking | Current tobacco smoking
67 # alcohol | Alcohol intake frequency.
68 # ipaq_activity_group | IPAQ activity group
69 # moderate_activity_met_minutes_week | MET minutes per week for moderate activity
70 # vigorous_activity_met_minutes_week | MET minutes per week for vigorous activity
71 # walking_met_minutes_week | MET minutes per week for walking
72 # total_activity_met_minutes_week | Summed MET minutes per week for all activity
73 # total_activity_days | Summed days activity
74 # total_activity_minutes | Summed minutes activity
75 # heavy_diy_duration | Duration of heavy DIY
76 # light_diy_duration | Duration of light DIY
77 # moderate_activity_duration | Duration of moderate activity
78 # other_exercise_duration | Duration of other exercises
79 # strenuous_sport_duration | Duration of strenuous sports
80 # vigorous_activity_duration | Duration of vigorous activity
81 # walking_duration | Duration of walks
82 # pleasure_walking_duration | Duration walking for pleasure
83 # heavy_diy_frequency_4_weeks | Frequency of heavy DIY in last 4 weeks
84 # light_diy_frequency_4_weeks | Frequency of light DIY in last 4 weeks
85 # other_exercise_frequency_4_weeks | Frequency of other exercises in last 4 weeks
86 # stair_climbing_frequency_4_weeks | Frequency of stair climbing in last 4 weeks
87 # strenuous_sport_frequency_4_weeks | Frequency of strenuous sports in last 4 weeks
88 # pleasure_walking_frequency_4_weeks | Frequency of walking for pleasure in last 4 weeks
89 # moderate_activity_days_week_10min | Number of days/week of moderate physical activity 10+ minutes
90 # vigorous_activity_days_week_10min | Number of days/week of vigorous physical activity 10+ minutes
91 # walking_days_week_10min | Number of days/week walked 10+ minutes
92 # driving_time | Time spent driving
93 # computer_use_time | Time spent using computer
94 # tv_watching_time | Time spent watching television (TV)
95 # physical_activity_types_4_weeks | Types of physical activity in last 4 weeks
96 # nonwork_transport_types | Types of transport used (excluding work)
97 # usual_walking_pace | Usual walking pace
98 # mobile_phone_use_duration | Length of mobile phone use
99 # mobile_phone_use_weekly_3_months | Weekly usage of mobile phone in last 3 months
100 # computer_game_playing | Plays computer games
101 # sleep_duration | Sleep duration
102 # chronotype | Morning/evening person (chronotype)
103 # daytime_napping | Nap during day
104 # insomnia | Sleeplessness / insomnia
105 # daytime_dozing | Daytime dozing / sleeping
106 # ever_smoked | Ever smoked
107 # smoking_pack_years | Pack years of smoking
108 # smoking_status | Smoking status
109 # past_tobacco_smoking | Past tobacco smoking
110 # lifetime_smoking_100_plus | Light smokers, at least 100 smokes in lifetime
111 # current_tobacco_type | Type of tobacco currently smoked
112 # current_cigarettes_per_day | Number of cigarettes currently smoked daily (current cigarette smokers)
113 # previous_cigarettes_per_day_current_cigar_pipe_smokers | Number of cigarettes previously smoked daily (current cigar/pipe smokers)
114 # time_to_first_cigarette | Time from waking to first cigarette
115 # ever_tried_smoking_cessation | Ever tried to stop smoking
116 # smoking_change_vs_10_years_ago | Smoking compared to 10 years previous
117 # previous_tobacco_type | Type of tobacco previously smoked
118 # previous_cigarettes_per_day | Number of cigarettes previously smoked daily
119 # ever_stopped_smoking_6_months | Ever stopped smoking for 6+ months
120 # household_smokers | Smoking/smokers in household
121 # home_secondhand_smoke_exposure | Exposure to tobacco smoke at home
122 # nonhome_secondhand_smoke_exposure | Exposure to tobacco smoke outside home
123 # cooked_vegetable_intake | Cooked vegetable intake
124 # raw_vegetable_intake | Salad / raw vegetable intake
125 # fresh_fruit_intake | Fresh fruit intake
126 # dried_fruit_intake | Dried fruit intake
127 # oily_fish_intake | Oily fish intake
128 # non_oily_fish_intake | Non-oily fish intake
129 # processed_meat_intake | Processed meat intake
130 # poultry_intake | Poultry intake
131 # beef_intake | Beef intake
132 # lamb_mutton_intake | Lamb/mutton intake
133 # pork_intake | Pork intake
134 # age_last_ate_meat | Age when last ate meat
135 # food_avoidance_eggs_dairy_wheat_sugar | Never eat eggs, dairy, wheat, sugar
136 # cheese_intake | Cheese intake
137 # milk_type | Milk type used
138 # spread_type | Spread type
139 # bread_intake | Bread intake
140 # bread_type | Bread type
141 # cereal_intake | Cereal intake
142 # cereal_type | Cereal type
143 # added_salt | Salt added to food
144 # tea_intake | Tea intake
145 # coffee_intake | Coffee intake
146 # coffee_type | Coffee type
147 # hot_drink_temperature | Hot drink temperature
148 # water_intake | Water intake
149 # diet_variation | Variation in diet
150 # alcohol_drinker_status | Alcohol drinker status
151 # former_alcohol_drinker | Former alcohol drinker
152 # red_wine_intake_monthly | Average monthly red wine intake
153 # champagne_white_wine_intake_monthly | Average monthly champagne plus white wine intake
154 # beer_cider_intake_monthly | Average monthly beer plus cider intake
155 # spirits_intake_monthly | Average monthly spirits intake
156 # fortified_wine_intake_monthly | Average monthly fortified wine intake
157 # other_alcohol_intake_monthly | Average monthly intake of other alcoholic drinks
158 # red_wine_intake_weekly | Average weekly red wine intake
159 # champagne_white_wine_intake_weekly | Average weekly champagne plus white wine intake
160 # beer_cider_intake_weekly | Average weekly beer plus cider intake
161 # spirits_intake_weekly | Average weekly spirits intake
162 # fortified_wine_intake_weekly | Average weekly fortified wine intake
163 # other_alcohol_intake_weekly | Average weekly intake of other alcoholic drinks
164 # alcohol_with_meals | Alcohol usually taken with meals
165 # country_of_birth_uk_elsewhere | Country of birth (UK/elsewhere)
166 # breastfed_in_infancy | Breastfed as a baby
167 # comparative_body_size_age_10 | Comparative body size at age 10
168 # comparative_height_age_10 | Comparative height size at age 10
169 # handedness | Handedness (chirality/laterality)
170 # adopted_as_child | Adopted as a child
171 # multiple_birth | Part of a multiple birth
172 # maternal_smoking_around_birth | Maternal smoking around birth
173 # accommodation_type | Type of accommodation lived in
174 # housing_tenure | Own or rent accommodation lived in
175 # gas_solid_fuel_use | Gas or solid-fuel cooking/heating
176 # home_heating_types | Heating type(s) in home
177 # household_vehicle_count | Number of vehicles in household
178 # household_income_before_tax | Average total household income before tax
179 # current_employment_status | Current employment status
180 # current_employment_status_corrected | Current employment status - corrected
181 # home_work_distance | Distance between home and job workplace
182 # main_job_hours_week | Length of working week for main job
183 # commuting_frequency | Frequency of travelling from home to job workplace
184 # commuting_transport_type | Transport type for commuting to job workplace
185 # job_walking_standing | Job involves mainly walking or standing
186 # job_heavy_manual_work | Job involves heavy manual or physical work
187 # job_shift_work | Job involves shift work
188 # job_night_shift_work | Job involves night shift work
189 # educational_qualifications | Qualifications
190 # age_completed_full_time_education | Age completed full time education
191 # friend_family_visit_frequency | Frequency of friend/family visits
192 # leisure_social_activities | Leisure/social activities
193 # ability_to_confide | Able to confide
194 # bipolar_major_depression_status | Bipolar and major depression status
195 # neuroticism_score | Neuroticism score
196 # mood_swings | Mood swings
197 # miserableness | Miserableness
198 # irritability | Irritability
199 # sensitivity_hurt_feelings | Sensitivity / hurt feelings
200 # fed_up_feelings | Fed-up feelings
201 # nervous_feelings | Nervous feelings
202 # worry_anxiety_feelings | Worrier / anxious feelings
203 # tenseness_highly_strung | Tense / 'highly strung'
204 # suffering_from_nerves | Suffer from 'nerves'
205 # loneliness_isolation | Loneliness, isolation
206 # guilty_feelings | Guilty feelings
207 # risk_taking | Risk taking
208 # happiness | Happiness
209 # job_satisfaction | Work/job satisfaction
210 # health_satisfaction | Health satisfaction
211 # family_relationship_satisfaction | Family relationship satisfaction
212 # friendship_satisfaction | Friendships satisfaction
213 # financial_situation_satisfaction | Financial situation satisfaction
214 # depressed_mood_frequency_2_weeks | Frequency of depressed mood in last 2 weeks
215 # disinterest_frequency_2_weeks | Frequency of unenthusiasm / disinterest in last 2 weeks
216 # tenseness_restlessness_frequency_2_weeks | Frequency of tenseness / restlessness in last 2 weeks
217 # tiredness_lethargy_frequency_2_weeks | Frequency of tiredness / lethargy in last 2 weeks
218 # ever_depressed_full_week | Ever depressed for a whole week
219 # longest_depression_duration | Longest period of depression
220 # depression_episode_count | Number of depression episodes
221 # longest_disinterest_duration | Longest period of unenthusiasm / disinterest
222 # disinterest_episode_count | Number of unenthusiastic/disinterested episodes
223 # ever_manic_hyper_2_days | Ever manic/hyper for 2 days
224 # ever_irritable_argumentative_2_days | Ever highly irritable/argumentative for 2 days
225 # manic_hyper_symptoms | Manic/hyper symptoms
226 # longest_manic_irritable_episode_duration | Length of longest manic/irritable episode
227 # manic_irritable_episode_severity | Severity of manic/irritable episodes
228 # adverse_life_events_2_years | Illness, injury, bereavement, stress in last 2 years
229 # outdoor_time_summer | Time spend outdoors in summer
230 # outdoor_time_winter | Time spent outdoors in winter
231 # skin_tanning_ease | Ease of skin tanning
232 # childhood_sunburn_frequency | Childhood sunburn occasions
233 # sun_uv_protection_use | Use of sun/uv protection
234 # solarium_sunlamp_frequency | Frequency of solarium/sunlamp use
235 # proximity_to_major_road | Close to major road
236 # inverse_distance_nearest_major_road | Inverse distance to the nearest major road
237 # inverse_distance_nearest_road | Inverse distance to the nearest road
238 # no2_2005 | Nitrogen dioxide air pollution; 2005
239 # no2_2006 | Nitrogen dioxide air pollution; 2006
240 # no2_2007 | Nitrogen dioxide air pollution; 2007
241 # no2_2010 | Nitrogen dioxide air pollution; 2010
242 # nox_2010 | Nitrogen oxides air pollution; 2010
243 # pm10_2007 | Particulate matter air pollution (pm10); 2007
244 # pm10_2010 | Particulate matter air pollution (pm10); 2010
245 # pm25_absorbance_2010 | Particulate matter air pollution (pm2.5) absorbance; 2010
246 # pm25_2010 | Particulate matter air pollution (pm2.5); 2010
247 # pm25_10_2010 | Particulate matter air pollution 2.5-10um; 2010
248 # major_road_length_100m | Sum of road length of major roads within 100m
249 # major_road_traffic_load | Total traffic load on major roads
250 # nearest_major_road_traffic_intensity | Traffic intensity on the nearest major road
251 # nearest_road_traffic_intensity | Traffic intensity on the nearest road
252 # noise_level_16h | Average 16-hour sound level of noise pollution
253 # noise_level_24h | Average 24-hour sound level of noise pollution
254 # noise_level_daytime | Average daytime sound level of noise pollution
255 # noise_level_evening | Average evening sound level of noise pollution
256 # noise_level_nighttime | Average night-time sound level of noise pollution
257 # natural_environment_percent_1000m | Natural environment percentage, buffer 1000m
258 # natural_environment_percent_300m | Natural environment percentage, buffer 300m
259 # greenspace_percent_1000m | Greenspace percentage, buffer 1000m
260 # greenspace_percent_300m | Greenspace percentage, buffer 300m
261 # domestic_garden_percent_1000m | Domestic garden percentage, buffer 1000m
262 # domestic_garden_percent_300m | Domestic garden percentage, buffer 300m
263 # water_percent_1000m | Water percentage, buffer 1000m
264 # water_percent_300m | Water percentage, buffer 300m
265 # distance_to_coast | Distance (Euclidean) to coast

3
extra_info_types_none.txt
View File

@@ -1,3 +0,0 @@
# No extra-info variables.
# Use this file with --extra_info_types_file to train/evaluate with disease history only.
# Keep this file free of numeric type ids; the loader parses it as an empty list.

6
extra_info_types_smoking_alcohol_bmi.txt
View File

@@ -1,6 +0,0 @@
# Only smoking, alcohol, and BMI variables
# Generated from field_ids_enriched.csv using prepare_data.py other-info type ordering.
# Format: <extra_info_type_id> # <var_name> | <full_name>
11 # bmi | Body mass index (BMI)
66 # smoking | Current tobacco smoking
67 # alcohol | Alcohol intake frequency.

246
models.py
View File

@@ -10,7 +10,6 @@ from backbones import (
GaussianRBFTimeBasis, GaussianRBFTimeBasis,
TimesNetExposureEncoder, TimesNetExposureEncoder,
TimeRoPE, TimeRoPE,
TokenAutoDiscretization,
) )
from targets import PAD_IDX from targets import PAD_IDX
@@ -23,125 +22,6 @@ class DeepHealthOutput:
event_len: int event_len: int
class OtherInfoTokenizer(nn.Module):
PAD_KIND = 0
CONT_KIND = 1
CATE_KIND = 2
def __init__(
self,
n_embd: int,
n_types: int,
n_cont_types: int,
n_categories: int,
cont_type_ids: list[int],
n_value_kinds: int = 3,
n_bins: int = 16,
):
super().__init__()
if len(cont_type_ids) != n_cont_types:
raise ValueError(
"cont_type_ids length must match n_cont_types, got "
f"{len(cont_type_ids)} vs {n_cont_types}"
)
if n_types <= 0:
raise ValueError(f"n_types must include PAD and be > 0, got {n_types}")
if n_categories <= 0:
raise ValueError(
f"n_categories must include PAD and be > 0, got {n_categories}"
)
if n_value_kinds <= self.CATE_KIND:
raise ValueError(
f"n_value_kinds must be > {self.CATE_KIND}, got {n_value_kinds}"
)
self.type_emb = nn.Embedding(n_types, n_embd, padding_idx=0)
self.kind_emb = nn.Embedding(n_value_kinds, n_embd, padding_idx=0)
self.cont_value_encoder = (
TokenAutoDiscretization(
n_cont_types=n_cont_types,
n_bins=n_bins,
n_embd=n_embd,
)
if n_cont_types > 0
else None
)
self.cate_value_emb = nn.Embedding(
n_categories,
n_embd,
padding_idx=0,
)
cont_type_index = torch.full((n_types,), -1, dtype=torch.long)
for idx, type_id in enumerate(cont_type_ids):
if type_id <= 0 or type_id >= n_types:
raise ValueError(
f"continuous type id {type_id} must be in [1, {n_types})"
)
cont_type_index[type_id] = idx
self.register_buffer(
"cont_type_index",
cont_type_index,
persistent=False,
)
self.reset_parameters()
def reset_parameters(self) -> None:
nn.init.normal_(self.type_emb.weight, mean=0.0, std=0.02)
nn.init.zeros_(self.type_emb.weight[0])
nn.init.normal_(self.kind_emb.weight, mean=0.0, std=0.02)
nn.init.zeros_(self.kind_emb.weight[0])
nn.init.normal_(self.cate_value_emb.weight, mean=0.0, std=0.02)
nn.init.zeros_(self.cate_value_emb.weight[0])
def forward(
self,
other_type: torch.LongTensor,
other_value: torch.Tensor,
other_value_kind: torch.LongTensor,
) -> tuple[torch.Tensor, torch.Tensor]:
if other_type.shape != other_value.shape:
raise ValueError(
"other_type and other_value must have the same shape, got "
f"{tuple(other_type.shape)} vs {tuple(other_value.shape)}"
)
if other_type.shape != other_value_kind.shape:
raise ValueError(
"other_type and other_value_kind must have the same shape, got "
f"{tuple(other_type.shape)} vs {tuple(other_value_kind.shape)}"
)
other_valid = other_type > 0
type_emb = self.type_emb(other_type)
kind_emb = self.kind_emb(other_value_kind)
value_emb = torch.zeros_like(type_emb)
cont_pos = other_valid & (other_value_kind == self.CONT_KIND)
if cont_pos.any():
if self.cont_value_encoder is None:
raise ValueError("continuous tokens found but n_cont_types is 0")
cont_idx = self.cont_type_index[other_type[cont_pos]]
if (cont_idx < 0).any():
bad_type = other_type[cont_pos][cont_idx < 0][0].item()
raise ValueError(
f"type_id={bad_type} is marked continuous but is not in "
"cont_type_ids"
)
value_emb[cont_pos] = self.cont_value_encoder(
cont_type_idx=cont_idx,
value=other_value[cont_pos].to(type_emb.dtype),
)
cate_pos = other_valid & (other_value_kind == self.CATE_KIND)
if cate_pos.any():
cate_id = other_value[cate_pos].long()
value_emb[cate_pos] = self.cate_value_emb(cate_id)
out = type_emb + kind_emb + value_emb
out = out * other_valid.unsqueeze(-1).to(out.dtype)
return out, other_valid
class DeepHealth(nn.Module): class DeepHealth(nn.Module):
def __init__( def __init__(
self, self,
@@ -149,17 +29,9 @@ class DeepHealth(nn.Module):
n_embd: int, n_embd: int,
n_head: int, n_head: int,
n_hist_layer: int, n_hist_layer: int,
n_tab_layer: int,
n_types: int,
n_cont_types: int,
n_categories: int,
cont_type_ids: list[int],
n_value_kinds: int = 3,
n_bins: int = 16,
target_mode: str = "next_token", # "next_token" or "all_future" target_mode: str = "next_token", # "next_token" or "all_future"
time_mode: str = "relative", # "relative" or "absolute" time_mode: str = "relative", # "relative" or "absolute"
dist_mode: str = "exponential", # "exponential", "weibull" or "mixed" dist_mode: str = "exponential", # "exponential", "weibull" or "mixed"
extra_pool_reduce: str = "mean",
dropout: float = 0.0, dropout: float = 0.0,
use_exposure_encoder: bool = False, use_exposure_encoder: bool = False,
exposure_daily_input_dim: int = 4, exposure_daily_input_dim: int = 4,
@@ -182,24 +54,12 @@ class DeepHealth(nn.Module):
if dist_mode not in ["exponential", "weibull", "mixed"]: if dist_mode not in ["exponential", "weibull", "mixed"]:
raise ValueError( raise ValueError(
"dist_mode must be either 'exponential', 'weibull' or 'mixed'") "dist_mode must be either 'exponential', 'weibull' or 'mixed'")
if extra_pool_reduce not in {"mean", "sum"}:
raise ValueError("extra_pool_reduce must be either 'mean' or 'sum'")
self.token_embedding = nn.Embedding(vocab_size, n_embd, padding_idx=0) self.token_embedding = nn.Embedding(vocab_size, n_embd, padding_idx=0)
self.gender_embedding = nn.Embedding( self.gender_embedding = nn.Embedding(
2, n_embd) # Assuming binary gender 2, n_embd) # Assuming binary gender
self.tokenizer = OtherInfoTokenizer(
n_embd=n_embd,
n_types=n_types,
n_cont_types=n_cont_types,
n_categories=n_categories,
cont_type_ids=cont_type_ids,
n_value_kinds=n_value_kinds,
n_bins=n_bins,
)
self.target_mode = target_mode self.target_mode = target_mode
self.time_mode = time_mode self.time_mode = time_mode
self.dist_mode = dist_mode self.dist_mode = dist_mode
self.extra_pool_reduce = extra_pool_reduce
self.use_exposure_encoder = use_exposure_encoder self.use_exposure_encoder = use_exposure_encoder
self.n_embd = n_embd self.n_embd = n_embd
self.vocab_size = vocab_size self.vocab_size = vocab_size
@@ -283,69 +143,6 @@ class DeepHealth(nn.Module):
dtype=dtype, dtype=dtype,
).masked_fill(~valid, -1e4)[:, None, :, :] ).masked_fill(~valid, -1e4)[:, None, :, :]
def _pool_other_by_time(
self,
h_other: torch.Tensor,
other_time: torch.Tensor,
other_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
batch_size, n_other, n_embd = h_other.shape
if n_other == 0:
empty_h = h_other.new_zeros(batch_size, 0, n_embd)
empty_t = other_time.new_zeros(batch_size, 0)
empty_m = torch.zeros(batch_size, 0, dtype=torch.bool, device=h_other.device)
return empty_h, empty_t, empty_m
masked_time = other_time.masked_fill(~other_mask, float("inf"))
_sorted_time_with_pad, order = masked_time.sort(dim=1)
sorted_time = other_time.gather(1, order)
sorted_mask = other_mask.gather(1, order)
sorted_h = h_other.gather(1, order.unsqueeze(-1).expand(-1, -1, n_embd))
group_start = torch.zeros_like(sorted_mask)
group_start[:, 0] = sorted_mask[:, 0]
group_start[:, 1:] = sorted_mask[:, 1:] & (
sorted_time[:, 1:] != sorted_time[:, :-1]
)
group_id = group_start.long().cumsum(dim=1) - 1
max_groups = int(group_start.sum(dim=1).max().item())
pooled_h = h_other.new_zeros(batch_size, max_groups, n_embd)
pooled_time = other_time.new_zeros(batch_size, max_groups)
pooled_mask = torch.zeros(
batch_size,
max_groups,
dtype=torch.bool,
device=h_other.device,
)
if max_groups == 0:
return pooled_h, pooled_time, pooled_mask
safe_group_id = group_id.clamp_min(0)
pooled_h.scatter_add_(
1,
safe_group_id.unsqueeze(-1).expand_as(sorted_h),
sorted_h * sorted_mask.unsqueeze(-1).to(sorted_h.dtype),
)
if self.extra_pool_reduce == "mean":
counts = h_other.new_zeros(batch_size, max_groups, 1)
counts.scatter_add_(
1,
safe_group_id.unsqueeze(-1),
sorted_mask.unsqueeze(-1).to(h_other.dtype),
)
pooled_h = pooled_h / counts.clamp_min(1.0)
pooled_time.scatter_add_(
1,
safe_group_id,
sorted_time * group_start.to(sorted_time.dtype),
)
group_count = group_start.sum(dim=1)
arange_groups = torch.arange(max_groups, device=h_other.device)
pooled_mask = arange_groups.unsqueeze(0) < group_count.unsqueeze(1)
return pooled_h, pooled_time, pooled_mask
def _encode_event_exposure( def _encode_event_exposure(
self, self,
exposure_daily: torch.Tensor | None, exposure_daily: torch.Tensor | None,
@@ -442,10 +239,6 @@ class DeepHealth(nn.Module):
mode: str, mode: str,
padding_mask: torch.Tensor | None = None, padding_mask: torch.Tensor | None = None,
t_query: torch.FloatTensor | None = None, t_query: torch.FloatTensor | None = None,
other_type: torch.LongTensor | None = None,
other_value: torch.Tensor | None = None,
other_value_kind: torch.LongTensor | None = None,
other_time: torch.FloatTensor | None = None,
exposure_daily: torch.Tensor | None = None, exposure_daily: torch.Tensor | None = None,
exposure_monthly: torch.Tensor | None = None, exposure_monthly: torch.Tensor | None = None,
exposure_daily_mask: torch.Tensor | None = None, exposure_daily_mask: torch.Tensor | None = None,
@@ -460,16 +253,6 @@ class DeepHealth(nn.Module):
raise ValueError("mode must be either 'next_token' or 'all_future'") raise ValueError("mode must be either 'next_token' or 'all_future'")
if mode == "all_future" and t_query is None: if mode == "all_future" and t_query is None:
raise ValueError("t_query is required when mode='all_future'") raise ValueError("t_query is required when mode='all_future'")
if (
other_type is None
or other_value is None
or other_value_kind is None
or other_time is None
):
raise ValueError(
"DeepHealth expects other_type, other_value, "
"other_value_kind, and other_time."
)
if padding_mask is None: if padding_mask is None:
padding_mask = event_seq > PAD_IDX padding_mask = event_seq > PAD_IDX
@@ -489,24 +272,6 @@ class DeepHealth(nn.Module):
h_exposure = h_exposure.to(device=event_seq.device, dtype=h_disease.dtype) h_exposure = h_exposure.to(device=event_seq.device, dtype=h_disease.dtype)
h_disease = h_disease + h_exposure h_disease = h_disease + h_exposure
t_disease = time_seq t_disease = time_seq
if other_time.shape != other_type.shape:
raise ValueError(
"other_time must have the same shape as other_type, got "
f"{tuple(other_time.shape)} vs {tuple(other_type.shape)}"
)
other_time = other_time.to(device=event_seq.device, dtype=time_seq.dtype)
h_other, other_mask = self.tokenizer(
other_type=other_type,
other_value=other_value,
other_value_kind=other_value_kind,
)
h_other = h_other.to(device=event_seq.device)
other_mask = other_mask.to(device=event_seq.device, dtype=torch.bool)
h_disease = torch.cat([h_disease, h_other], dim=1)
t_disease = torch.cat([t_disease, other_time], dim=1)
padding_mask = torch.cat([padding_mask, other_mask], dim=1)
h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype) h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
if mode == "all_future": if mode == "all_future":
@@ -571,15 +336,10 @@ class DeepHealth(nn.Module):
h_event = h_disease[:, :event_len, :] h_event = h_disease[:, :event_len, :]
t_event = t_disease[:, :event_len] t_event = t_disease[:, :event_len]
event_mask = padding_mask[:, :event_len] event_mask = padding_mask[:, :event_len]
h_extra, t_extra, extra_mask = self._pool_other_by_time(
h_other=h_disease[:, event_len:, :],
other_time=t_disease[:, event_len:],
other_mask=padding_mask[:, event_len:],
)
return DeepHealthOutput( return DeepHealthOutput(
hidden=torch.cat([h_event, h_extra], dim=1), hidden=h_event,
time_seq=torch.cat([t_event, t_extra], dim=1), time_seq=t_event,
padding_mask=torch.cat([event_mask, extra_mask], dim=1), padding_mask=event_mask,
event_len=event_len, event_len=event_len,
) )
return h_disease[:, :event_len, :] return h_disease[:, :event_len, :]

126
prepare_data.py
View File

@@ -6,13 +6,6 @@ DeepHealth:
* ``ukb_event_data.npy``: ``(N, 3)`` uint32 array of ``(eid, days, label)`` * ``ukb_event_data.npy``: ``(N, 3)`` uint32 array of ``(eid, days, label)``
disease/death/checkup events sorted by patient then time. disease/death/checkup events sorted by patient then time.
* ``ukb_basic_info.csv``: basic patient table indexed by ``eid`` with ``sex``. * ``ukb_basic_info.csv``: basic patient table indexed by ``eid`` with ``sex``.
* ``ukb_other_info.npy``: ``(M, 5)`` float64 array of
``(eid, type, value, value_kind, time)`` rows. ``type=0`` is reserved for
padding, ``value_kind=1`` means continuous, and ``value_kind=2`` means
categorical.
* ``cate_types.csv``: categorical-variable metadata with
``type,name,n_categories``. Dataset code computes global category ids after
experiment-specific variable selection.
Processing steps Processing steps
---------------- ----------------
@@ -21,10 +14,7 @@ Processing steps
3. Extract ICD-10 date fields and cancer date/type fields into long-form 3. Extract ICD-10 date fields and cancer date/type fields into long-form
events and map codes to integer labels via ``labels.csv``. events and map codes to integer labels via ``labels.csv``.
4. De-duplicate events per ``(eid, label)`` keeping the earliest occurrence. 4. De-duplicate events per ``(eid, label)`` keeping the earliest occurrence.
5. Convert available non-sex tabular fields into unified other-information 5. Write event data and sex.
tokens timestamped by ``date_of_assessment``.
6. Write event data, sex, unified other-information tokens, and categorical
type metadata.
Usage Usage
----- -----
@@ -45,108 +35,6 @@ import pandas as pd # Pandas for data manipulation
import tqdm # Progress bar for chunk processing import tqdm # Progress bar for chunk processing
CONT_VALUE_KIND = 1
CATE_VALUE_KIND = 2
def _sort_values(values):
"""Sort mixed pandas/numpy scalar values deterministically."""
try:
return sorted(values)
except TypeError:
return sorted(values, key=lambda x: str(x))
def _build_other_info_tokens(
table: pd.DataFrame,
feature_fields: list[str],
*,
time_col: str = "date_of_assessment",
) -> tuple[np.ndarray, pd.DataFrame]:
"""Convert wide tabular features into (eid, type, value, kind, time) rows."""
rows = []
cate_meta = []
present_features = [
col for col in feature_fields
if col in table.columns and col not in {time_col, "sex"}
]
if time_col not in table.columns:
raise ValueError(
f"{time_col!r} is required to timestamp other-information tokens"
)
token_times = pd.to_numeric(table[time_col], errors="coerce")
for type_idx, col in enumerate(present_features, start=1):
series = table[col]
n_unique = series.dropna().nunique()
valid_time = token_times.notna()
if n_unique > 10:
numeric = pd.to_numeric(series, errors="coerce")
valid = numeric.notna() & valid_time
if not valid.any():
continue
rows.append(
np.column_stack(
(
table.index[valid].to_numpy(dtype=np.float64),
np.full(valid.sum(), type_idx, dtype=np.float64),
numeric[valid].to_numpy(dtype=np.float64),
np.full(valid.sum(), CONT_VALUE_KIND, dtype=np.float64),
token_times[valid].to_numpy(dtype=np.float64),
)
)
)
else:
unique_vals = _sort_values(series.dropna().unique())
value_map = {val: idx + 1 for idx, val in enumerate(unique_vals)}
mapped = series.map(value_map)
valid = mapped.notna() & valid_time
n_categories = len(unique_vals)
cate_meta.append(
{
"type": type_idx,
"name": col,
"n_categories": n_categories,
}
)
if not valid.any():
continue
rows.append(
np.column_stack(
(
table.index[valid].to_numpy(dtype=np.float64),
np.full(valid.sum(), type_idx, dtype=np.float64),
mapped[valid].to_numpy(dtype=np.float64),
np.full(valid.sum(), CATE_VALUE_KIND, dtype=np.float64),
token_times[valid].to_numpy(dtype=np.float64),
)
)
)
cate_types = pd.DataFrame(
cate_meta,
columns=["type", "name", "n_categories"],
)
if not rows:
return np.empty((0, 5), dtype=np.float64), cate_types
out = np.vstack(rows)
return out[np.lexsort((out[:, 3], out[:, 1], out[:, 0]))], cate_types
def _unique_preserve_order(values):
"""Return unique values while preserving first-seen order."""
seen = set()
out = []
for value in values:
if value not in seen:
seen.add(value)
out.append(value)
return out
# CSV mapping field IDs to human-readable names # CSV mapping field IDs to human-readable names
field_map_file = "field_ids_enriched.csv" field_map_file = "field_ids_enriched.csv"
@@ -369,17 +257,5 @@ final_tabular = final_tabular.convert_dtypes()
basic_info = final_tabular[["sex"]].copy() basic_info = final_tabular[["sex"]].copy()
basic_info.to_csv("ukb_basic_info.csv") basic_info.to_csv("ukb_basic_info.csv")
# Save unified other-information tokens. Missing values simply produce no token.
other_info_fields = _unique_preserve_order(
basic_info_fields + assessment_fields + exposure_fields
)
other_info, cate_types = _build_other_info_tokens(
final_tabular,
other_info_fields,
time_col="date_of_assessment",
)
np.save("ukb_other_info.npy", other_info)
cate_types.to_csv("cate_types.csv", index=False)
# Save event data # Save event data
np.save("ukb_event_data.npy", data) np.save("ukb_event_data.npy", data)

174
train_next_step.py
View File

@@ -12,7 +12,6 @@ import json
import logging import logging
import math import math
import time import time
from pathlib import Path
from typing import Any, Dict from typing import Any, Dict
import numpy as np import numpy as np
@@ -30,8 +29,6 @@ from targets import CHECKUP_IDX, NO_EVENT_IDX, PAD_IDX
from train_util import ( from train_util import (
configure_torch_for_training, configure_torch_for_training,
create_unique_run_dir, create_unique_run_dir,
format_extra_info_types,
load_extra_info_types_file,
resolve_device, resolve_device,
save_checkpoint, save_checkpoint,
save_config, save_config,
@@ -48,10 +45,6 @@ MODEL_INPUT_KEYS = (
"time_seq", "time_seq",
"sex", "sex",
"padding_mask", "padding_mask",
"other_type",
"other_value",
"other_value_kind",
"other_time",
) )
@@ -62,7 +55,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--data_prefix", type=str, default="ukb") parser.add_argument("--data_prefix", type=str, default="ukb")
parser.add_argument("--labels_file", type=str, default="labels.csv") parser.add_argument("--labels_file", type=str, default="labels.csv")
parser.add_argument("--seed", type=int, default=42) parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--extra_info_types_file", type=str, default=None)
parser.add_argument("--no_event_interval_years", type=float, default=5.0) parser.add_argument("--no_event_interval_years", type=float, default=5.0)
parser.add_argument("--include_no_event_in_uts_target", action="store_true") parser.add_argument("--include_no_event_in_uts_target", action="store_true")
@@ -76,10 +68,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--n_embd", type=int, default=120) parser.add_argument("--n_embd", type=int, default=120)
parser.add_argument("--n_head", type=int, default=10) parser.add_argument("--n_head", type=int, default=10)
parser.add_argument("--n_hist_layer", type=int, default=12) parser.add_argument("--n_hist_layer", type=int, default=12)
parser.add_argument("--n_tab_layer", type=int, default=4)
parser.add_argument("--n_bins", type=int, default=16)
parser.add_argument("--extra_pool_reduce", type=str, default="mean",
choices=["mean", "sum"])
parser.add_argument("--time_mode", type=str, default="relative", parser.add_argument("--time_mode", type=str, default="relative",
choices=["relative", "absolute"]) choices=["relative", "absolute"])
parser.add_argument("--dropout", type=float, default=0.0) parser.add_argument("--dropout", type=float, default=0.0)
@@ -121,11 +109,6 @@ def parse_args() -> argparse.Namespace:
args.include_no_event_in_uts_target = True args.include_no_event_in_uts_target = True
else: else:
args.readout_name = args.readout_name or "token" args.readout_name = args.readout_name or "token"
args.extra_info_types = (
load_extra_info_types_file(args.extra_info_types_file)
if args.extra_info_types_file is not None
else None
)
return args return args
@@ -153,13 +136,6 @@ def build_model(args: argparse.Namespace, dataset: HealthDataset) -> DeepHealth:
n_embd=args.n_embd, n_embd=args.n_embd,
n_head=args.n_head, n_head=args.n_head,
n_hist_layer=args.n_hist_layer, n_hist_layer=args.n_hist_layer,
n_tab_layer=args.n_tab_layer,
n_types=dataset.n_types,
n_cont_types=dataset.n_cont_types,
n_categories=dataset.n_categories,
cont_type_ids=dataset.cont_type_ids,
n_bins=args.n_bins,
extra_pool_reduce=args.extra_pool_reduce,
target_mode="next_token", target_mode="next_token",
time_mode=args.time_mode, time_mode=args.time_mode,
dist_mode="exponential", dist_mode="exponential",
@@ -199,12 +175,8 @@ def build_augmented_next_step_targets(
model_out: DeepHealthOutput, model_out: DeepHealthOutput,
include_uts_targets: bool, include_uts_targets: bool,
) -> Dict[str, torch.Tensor]: ) -> Dict[str, torch.Tensor]:
hidden_len = model_out.hidden.size(1)
event_len = int(model_out.event_len)
extra_len = hidden_len - event_len
device = model_out.hidden.device device = model_out.hidden.device
non_blocking = device.type == "cuda" non_blocking = device.type == "cuda"
if extra_len <= 0:
targets = { targets = {
"target_event_seq": batch_cpu["target_event_seq"].to(device, non_blocking=non_blocking), "target_event_seq": batch_cpu["target_event_seq"].to(device, non_blocking=non_blocking),
"target_time_seq": batch_cpu["target_time_seq"].to(device, non_blocking=non_blocking), "target_time_seq": batch_cpu["target_time_seq"].to(device, non_blocking=non_blocking),
@@ -219,135 +191,6 @@ def build_augmented_next_step_targets(
) )
return targets return targets
bsz = batch_cpu["target_event_seq"].size(0)
vocab_size = (
batch_cpu["target_multi_hot"].size(2)
if include_uts_targets
else None
)
other_valid = batch_cpu["other_type"] > 0
extra_time = batch_cpu["other_time"].new_zeros(bsz, extra_len)
extra_mask = torch.zeros(bsz, extra_len, dtype=torch.bool)
for b in range(bsz):
unique_time = torch.unique(batch_cpu["other_time"][b, other_valid[b]], sorted=True)
n_time = min(int(unique_time.numel()), extra_len)
if n_time > 0:
extra_time[b, :n_time] = unique_time[:n_time]
extra_mask[b, :n_time] = True
target_event_seq = torch.cat(
[
batch_cpu["target_event_seq"],
torch.full(
(bsz, extra_len),
PAD_IDX,
dtype=batch_cpu["target_event_seq"].dtype,
),
],
dim=1,
)
target_time_seq = torch.cat(
[
batch_cpu["target_time_seq"],
torch.zeros(
bsz,
extra_len,
dtype=batch_cpu["target_time_seq"].dtype,
),
],
dim=1,
)
readout_mask = torch.cat([batch_cpu["readout_mask"], extra_mask], dim=1)
target_dt_unique = None
target_multi_hot = None
if include_uts_targets:
target_dt_unique = torch.cat(
[
batch_cpu["target_dt_unique"],
torch.zeros(
bsz,
extra_len,
dtype=batch_cpu["target_dt_unique"].dtype,
),
],
dim=1,
)
target_multi_hot = torch.cat(
[
batch_cpu["target_multi_hot"],
torch.zeros(
bsz,
extra_len,
vocab_size,
dtype=batch_cpu["target_multi_hot"].dtype,
),
],
dim=1,
)
for b in range(bsz):
valid_event = batch_cpu["padding_mask"][b].bool()
if not valid_event.any():
continue
n_event = int(valid_event.sum().item())
events = torch.cat(
[
batch_cpu["event_seq"][b, :n_event],
batch_cpu["target_event_seq"][b, n_event - 1:n_event],
]
)
times = torch.cat(
[
batch_cpu["time_seq"][b, :n_event],
batch_cpu["target_time_seq"][b, n_event - 1:n_event],
]
)
valid_full = events > PAD_IDX
events = events[valid_full]
times = times[valid_full]
if events.numel() == 0:
continue
for j in range(extra_len):
if not bool(extra_mask[b, j]):
continue
pos = event_len + j
t = extra_time[b, j]
future = times > t
if not future.any():
readout_mask[b, pos] = False
continue
first_idx = int(torch.nonzero(future, as_tuple=False)[0].item())
next_time = times[first_idx]
next_event = events[first_idx]
target_event_seq[b, pos] = next_event
target_time_seq[b, pos] = next_time
if not include_uts_targets:
continue
same_next_time = times == next_time
next_events = events[same_next_time]
valid_next_events = next_events[
(next_events > PAD_IDX) & (next_events < vocab_size)
].long()
if valid_next_events.numel() == 0:
readout_mask[b, pos] = False
continue
target_multi_hot[b, pos, valid_next_events] = True
target_dt_unique[b, pos] = next_time - t
targets = {
"target_event_seq": target_event_seq.to(device, non_blocking=non_blocking),
"target_time_seq": target_time_seq.to(device, non_blocking=non_blocking),
"readout_mask": readout_mask.to(device, non_blocking=non_blocking),
}
if include_uts_targets:
targets["target_dt_unique"] = target_dt_unique.to(device, non_blocking=non_blocking)
targets["target_multi_hot"] = target_multi_hot.to(device, non_blocking=non_blocking)
return targets
def compute_next_step_loss( def compute_next_step_loss(
args: argparse.Namespace, args: argparse.Namespace,
@@ -367,10 +210,6 @@ def compute_next_step_loss(
time_seq=batch["time_seq"], time_seq=batch["time_seq"],
sex=batch["sex"], sex=batch["sex"],
padding_mask=batch["padding_mask"], padding_mask=batch["padding_mask"],
other_type=batch["other_type"],
other_value=batch["other_value"],
other_value_kind=batch["other_value_kind"],
other_time=batch["other_time"],
target_mode="next_token", target_mode="next_token",
return_output=True, return_output=True,
) )
@@ -487,19 +326,8 @@ def build_metadata(
"model_target_mode": "next_token", "model_target_mode": "next_token",
"target_mode": args.target_mode, "target_mode": args.target_mode,
"dist_mode": "exponential", "dist_mode": "exponential",
"extra_info_types_file": (
Path(args.extra_info_types_file).name
if args.extra_info_types_file is not None
else None
),
"extra_info_types": [int(x) for x in dataset.extra_info_types],
"dataset_metadata": { "dataset_metadata": {
"vocab_size": int(dataset.vocab_size), "vocab_size": int(dataset.vocab_size),
"n_types": int(dataset.n_types),
"n_cont_types": int(dataset.n_cont_types),
"n_categories": int(dataset.n_categories),
"cont_type_ids": [int(x) for x in dataset.cont_type_ids],
"extra_info_types": [int(x) for x in dataset.extra_info_types],
}, },
"split_sizes": { "split_sizes": {
"train": int(len(train_subset)), "train": int(len(train_subset)),
@@ -527,7 +355,6 @@ def main() -> None:
logger.info(f"Starting next-step training run: {run_name}") logger.info(f"Starting next-step training run: {run_name}")
logger.info(f"Device: {device}") logger.info(f"Device: {device}")
logger.info(f"extra_info_types: {format_extra_info_types(args.extra_info_types)}")
logger.info(f"readout={args.readout_name}, target_mode={args.target_mode}") logger.info(f"readout={args.readout_name}, target_mode={args.target_mode}")
dataset = HealthDataset( dataset = HealthDataset(
@@ -535,7 +362,6 @@ def main() -> None:
labels_file=args.labels_file, labels_file=args.labels_file,
no_event_interval_years=args.no_event_interval_years, no_event_interval_years=args.no_event_interval_years,
include_no_event_in_uts_target=args.include_no_event_in_uts_target, include_no_event_in_uts_target=args.include_no_event_in_uts_target,
extra_info_types=args.extra_info_types,
) )
if args.train_eid_file and args.val_eid_file and args.test_eid_file: if args.train_eid_file and args.val_eid_file and args.test_eid_file:
train_subset, val_subset, test_subset = split_dataset_by_eid_files( train_subset, val_subset, test_subset = split_dataset_by_eid_files(

38
train_util.py
View File

@@ -1,13 +1,12 @@
from __future__ import annotations from __future__ import annotations
import json
import logging import logging
import sys import sys
import time import time
import csv import csv
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Iterable, Tuple from typing import Any, Dict, Tuple
import numpy as np import numpy as np
import torch import torch
@@ -61,41 +60,6 @@ def set_seed(seed: int) -> None:
torch.cuda.manual_seed(seed) torch.cuda.manual_seed(seed)
def load_extra_info_types_file(path: str) -> list[int]:
file_path = Path(path)
if not file_path.is_file():
raise FileNotFoundError(f"extra_info_types_file not found: {path}")
text = file_path.read_text(encoding="utf-8").strip()
if not text:
return []
if text.startswith("["):
raw_items = json.loads(text)
if not isinstance(raw_items, list):
raise ValueError("extra_info_types_file JSON must be a list")
else:
raw_items = []
for line in text.splitlines():
line = line.split("#", 1)[0].strip()
if line:
raw_items.extend(line.replace(",", " ").replace(";", " ").split())
try:
return [int(x) for x in raw_items]
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid extra info type id in {path}") from exc
def format_extra_info_types(extra_info_types: Iterable[int] | None) -> str:
if extra_info_types is None:
return "all"
values = [int(x) for x in extra_info_types]
if not values:
return "none"
return str(values)
def load_eid_file(path: str | Path) -> set[int]: def load_eid_file(path: str | Path) -> set[int]:
file_path = Path(path) file_path = Path(path)
if not file_path.is_file(): if not file_path.is_file():