Jiarui Li
1757bcd25b
- Removed BaselineEncoder and CrossAttention classes from models.py. - Introduced OtherInfoTokenizer for handling additional token types. - Updated DeepHealth class to integrate OtherInfoTokenizer and manage extra pooling logic. - Added support for extra_pool_reduce parameter to control pooling behavior. - Modified forward methods to return structured output using DeepHealthOutput dataclass. - Updated training scripts to accommodate changes in model architecture and output handling. - Enhanced error handling and validation for input shapes and types.
470 lines
17 KiB
Python
470 lines
17 KiB
Python
from dataclasses import dataclass
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
|
|
from backbones import (
|
|
AgeSinusoidalEncoding,
|
|
GPTBlock,
|
|
GaussianRBFTimeBasis,
|
|
TimeRoPE,
|
|
TokenAutoDiscretization,
|
|
)
|
|
from targets import PAD_IDX
|
|
|
|
|
|
@dataclass
|
|
class DeepHealthOutput:
|
|
hidden: torch.Tensor
|
|
time_seq: torch.Tensor
|
|
padding_mask: torch.Tensor
|
|
event_len: int
|
|
|
|
|
|
class OtherInfoTokenizer(nn.Module):
|
|
PAD_KIND = 0
|
|
CONT_KIND = 1
|
|
CATE_KIND = 2
|
|
|
|
def __init__(
|
|
self,
|
|
n_embd: int,
|
|
n_types: int,
|
|
n_cont_types: int,
|
|
n_categories: int,
|
|
cont_type_ids: list[int],
|
|
n_value_kinds: int = 3,
|
|
n_bins: int = 16,
|
|
):
|
|
super().__init__()
|
|
if len(cont_type_ids) != n_cont_types:
|
|
raise ValueError(
|
|
"cont_type_ids length must match n_cont_types, got "
|
|
f"{len(cont_type_ids)} vs {n_cont_types}"
|
|
)
|
|
if n_types <= 0:
|
|
raise ValueError(f"n_types must include PAD and be > 0, got {n_types}")
|
|
if n_categories <= 0:
|
|
raise ValueError(
|
|
f"n_categories must include PAD and be > 0, got {n_categories}"
|
|
)
|
|
if n_value_kinds <= self.CATE_KIND:
|
|
raise ValueError(
|
|
f"n_value_kinds must be > {self.CATE_KIND}, got {n_value_kinds}"
|
|
)
|
|
|
|
self.type_emb = nn.Embedding(n_types, n_embd, padding_idx=0)
|
|
self.kind_emb = nn.Embedding(n_value_kinds, n_embd, padding_idx=0)
|
|
self.cont_value_encoder = (
|
|
TokenAutoDiscretization(
|
|
n_cont_types=n_cont_types,
|
|
n_bins=n_bins,
|
|
n_embd=n_embd,
|
|
)
|
|
if n_cont_types > 0
|
|
else None
|
|
)
|
|
self.cate_value_emb = nn.Embedding(
|
|
n_categories,
|
|
n_embd,
|
|
padding_idx=0,
|
|
)
|
|
|
|
cont_type_index = torch.full((n_types,), -1, dtype=torch.long)
|
|
for idx, type_id in enumerate(cont_type_ids):
|
|
if type_id <= 0 or type_id >= n_types:
|
|
raise ValueError(
|
|
f"continuous type id {type_id} must be in [1, {n_types})"
|
|
)
|
|
cont_type_index[type_id] = idx
|
|
self.register_buffer(
|
|
"cont_type_index",
|
|
cont_type_index,
|
|
persistent=False,
|
|
)
|
|
self.reset_parameters()
|
|
|
|
def reset_parameters(self) -> None:
|
|
nn.init.normal_(self.type_emb.weight, mean=0.0, std=0.02)
|
|
nn.init.zeros_(self.type_emb.weight[0])
|
|
nn.init.normal_(self.kind_emb.weight, mean=0.0, std=0.02)
|
|
nn.init.zeros_(self.kind_emb.weight[0])
|
|
nn.init.normal_(self.cate_value_emb.weight, mean=0.0, std=0.02)
|
|
nn.init.zeros_(self.cate_value_emb.weight[0])
|
|
|
|
def forward(
|
|
self,
|
|
other_type: torch.LongTensor,
|
|
other_value: torch.Tensor,
|
|
other_value_kind: torch.LongTensor,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
if other_type.shape != other_value.shape:
|
|
raise ValueError(
|
|
"other_type and other_value must have the same shape, got "
|
|
f"{tuple(other_type.shape)} vs {tuple(other_value.shape)}"
|
|
)
|
|
if other_type.shape != other_value_kind.shape:
|
|
raise ValueError(
|
|
"other_type and other_value_kind must have the same shape, got "
|
|
f"{tuple(other_type.shape)} vs {tuple(other_value_kind.shape)}"
|
|
)
|
|
|
|
other_valid = other_type > 0
|
|
type_emb = self.type_emb(other_type)
|
|
kind_emb = self.kind_emb(other_value_kind)
|
|
value_emb = torch.zeros_like(type_emb)
|
|
|
|
cont_pos = other_valid & (other_value_kind == self.CONT_KIND)
|
|
if cont_pos.any():
|
|
if self.cont_value_encoder is None:
|
|
raise ValueError("continuous tokens found but n_cont_types is 0")
|
|
cont_idx = self.cont_type_index[other_type[cont_pos]]
|
|
if (cont_idx < 0).any():
|
|
bad_type = other_type[cont_pos][cont_idx < 0][0].item()
|
|
raise ValueError(
|
|
f"type_id={bad_type} is marked continuous but is not in "
|
|
"cont_type_ids"
|
|
)
|
|
value_emb[cont_pos] = self.cont_value_encoder(
|
|
cont_type_idx=cont_idx,
|
|
value=other_value[cont_pos].to(type_emb.dtype),
|
|
)
|
|
|
|
cate_pos = other_valid & (other_value_kind == self.CATE_KIND)
|
|
if cate_pos.any():
|
|
cate_id = other_value[cate_pos].long()
|
|
value_emb[cate_pos] = self.cate_value_emb(cate_id)
|
|
|
|
out = type_emb + kind_emb + value_emb
|
|
out = out * other_valid.unsqueeze(-1).to(out.dtype)
|
|
return out, other_valid
|
|
|
|
|
|
class DeepHealth(nn.Module):
|
|
def __init__(
|
|
self,
|
|
vocab_size: int,
|
|
n_embd: int,
|
|
n_head: int,
|
|
n_hist_layer: int,
|
|
n_tab_layer: int,
|
|
n_types: int,
|
|
n_cont_types: int,
|
|
n_categories: int,
|
|
cont_type_ids: list[int],
|
|
n_value_kinds: int = 3,
|
|
n_bins: int = 16,
|
|
target_mode: str = "next_token", # "next_token" or "all_future"
|
|
time_mode: str = "relative", # "relative" or "absolute"
|
|
dist_mode: str = "exponential", # "exponential", "weibull" or "mixed"
|
|
extra_pool_reduce: str = "mean",
|
|
dropout: float = 0.0,
|
|
):
|
|
super().__init__()
|
|
if target_mode not in ["next_token", "all_future"]:
|
|
raise ValueError(
|
|
"target_mode must be either 'next_token' or 'all_future'")
|
|
if time_mode not in ["relative", "absolute"]:
|
|
raise ValueError(
|
|
"time_mode must be either 'relative' or 'absolute'")
|
|
if dist_mode not in ["exponential", "weibull", "mixed"]:
|
|
raise ValueError(
|
|
"dist_mode must be either 'exponential', 'weibull' or 'mixed'")
|
|
if extra_pool_reduce not in {"mean", "sum"}:
|
|
raise ValueError("extra_pool_reduce must be either 'mean' or 'sum'")
|
|
self.token_embedding = nn.Embedding(vocab_size, n_embd, padding_idx=0)
|
|
self.gender_embedding = nn.Embedding(
|
|
2, n_embd) # Assuming binary gender
|
|
self.tokenizer = OtherInfoTokenizer(
|
|
n_embd=n_embd,
|
|
n_types=n_types,
|
|
n_cont_types=n_cont_types,
|
|
n_categories=n_categories,
|
|
cont_type_ids=cont_type_ids,
|
|
n_value_kinds=n_value_kinds,
|
|
n_bins=n_bins,
|
|
)
|
|
self.target_mode = target_mode
|
|
self.time_mode = time_mode
|
|
self.dist_mode = dist_mode
|
|
self.extra_pool_reduce = extra_pool_reduce
|
|
self.n_embd = n_embd
|
|
self.vocab_size = vocab_size
|
|
nn.init.normal_(self.token_embedding.weight, mean=0.0, std=0.02)
|
|
nn.init.zeros_(self.token_embedding.weight[0])
|
|
nn.init.normal_(self.gender_embedding.weight, mean=0.0, std=0.02)
|
|
if dist_mode == "weibull":
|
|
self.rho_head = nn.Linear(n_embd, vocab_size)
|
|
nn.init.zeros_(self.rho_head.weight)
|
|
nn.init.constant_(self.rho_head.bias, 0.5413)
|
|
|
|
if dist_mode == "mixed":
|
|
self.death_idx = vocab_size - 1
|
|
self.rho_death_head = nn.Linear(n_embd, 1)
|
|
nn.init.zeros_(self.rho_death_head.weight)
|
|
nn.init.constant_(self.rho_death_head.bias, 0.5413)
|
|
|
|
if time_mode == "absolute":
|
|
self.age_encoding = AgeSinusoidalEncoding(n_embd)
|
|
self.blocks = nn.ModuleList([
|
|
GPTBlock(
|
|
n_embd=n_embd,
|
|
n_head=n_head,
|
|
use_time_rope=False,
|
|
use_rbf_bias=False,
|
|
mlp_dropout=dropout,
|
|
) for _ in range(n_hist_layer)
|
|
])
|
|
self.rope = None
|
|
self.rbf = None
|
|
elif time_mode == "relative":
|
|
self.age_encoding = None
|
|
self.blocks = nn.ModuleList([
|
|
GPTBlock(
|
|
n_embd=n_embd,
|
|
n_head=n_head,
|
|
use_time_rope=True,
|
|
use_rbf_bias=True,
|
|
mlp_dropout=dropout,
|
|
) for _ in range(n_hist_layer)
|
|
])
|
|
self.rope = TimeRoPE(n_embd // n_head)
|
|
self.rbf = GaussianRBFTimeBasis(n_bases=16, max_time_diff=40.0)
|
|
|
|
self.final_ln = nn.LayerNorm(n_embd)
|
|
self.risk_head = nn.Linear(n_embd, vocab_size, bias=False)
|
|
if target_mode == "next_token":
|
|
self.risk_head.weight = self.token_embedding.weight
|
|
self.query_token = nn.Parameter(torch.zeros(n_embd))
|
|
nn.init.normal_(self.query_token, mean=0.0, std=0.02)
|
|
|
|
def _make_history_attn_mask(
|
|
self,
|
|
padding_mask: torch.Tensor,
|
|
time_seq: torch.Tensor,
|
|
dtype: torch.dtype,
|
|
) -> torch.Tensor:
|
|
valid_key = padding_mask[:, None, :] # (B, 1, L)
|
|
visible_by_time = time_seq[:, None, :] <= time_seq[:, :, None]
|
|
valid = valid_key & visible_by_time
|
|
return torch.zeros(
|
|
valid.shape,
|
|
device=valid.device,
|
|
dtype=dtype,
|
|
).masked_fill(~valid, -1e4)[:, None, :, :]
|
|
|
|
def _pool_other_by_time(
|
|
self,
|
|
h_other: torch.Tensor,
|
|
other_time: torch.Tensor,
|
|
other_mask: torch.Tensor,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
batch_size, _n_other, n_embd = h_other.shape
|
|
group_counts = [
|
|
int(torch.unique(other_time[b, other_mask[b]], sorted=True).numel())
|
|
for b in range(batch_size)
|
|
]
|
|
max_groups = max(group_counts, default=0)
|
|
pooled_h = h_other.new_zeros(batch_size, max_groups, n_embd)
|
|
pooled_time = other_time.new_zeros(batch_size, max_groups)
|
|
pooled_mask = torch.zeros(
|
|
batch_size,
|
|
max_groups,
|
|
dtype=torch.bool,
|
|
device=h_other.device,
|
|
)
|
|
if max_groups == 0:
|
|
return pooled_h, pooled_time, pooled_mask
|
|
|
|
for b in range(batch_size):
|
|
valid_time = other_time[b, other_mask[b]]
|
|
if valid_time.numel() == 0:
|
|
continue
|
|
valid_h = h_other[b, other_mask[b]]
|
|
unique_time, inverse = torch.unique(
|
|
valid_time,
|
|
sorted=True,
|
|
return_inverse=True,
|
|
)
|
|
n_groups = unique_time.numel()
|
|
group_h = valid_h.new_zeros(n_groups, n_embd)
|
|
group_h.scatter_add_(
|
|
0,
|
|
inverse[:, None].expand(-1, n_embd),
|
|
valid_h,
|
|
)
|
|
if self.extra_pool_reduce == "mean":
|
|
counts = valid_h.new_zeros(n_groups, 1)
|
|
counts.scatter_add_(
|
|
0,
|
|
inverse[:, None],
|
|
torch.ones_like(valid_h[:, :1]),
|
|
)
|
|
group_h = group_h / counts.clamp_min(1.0)
|
|
|
|
pooled_h[b, :n_groups] = group_h
|
|
pooled_time[b, :n_groups] = unique_time
|
|
pooled_mask[b, :n_groups] = True
|
|
return pooled_h, pooled_time, pooled_mask
|
|
|
|
def _forward_shared(
|
|
self,
|
|
event_seq: torch.LongTensor,
|
|
time_seq: torch.FloatTensor,
|
|
sex: torch.LongTensor,
|
|
mode: str,
|
|
padding_mask: torch.Tensor | None = None,
|
|
t_query: torch.FloatTensor | None = None,
|
|
other_type: torch.LongTensor | None = None,
|
|
other_value: torch.Tensor | None = None,
|
|
other_value_kind: torch.LongTensor | None = None,
|
|
other_time: torch.FloatTensor | None = None,
|
|
return_output: bool = False,
|
|
**unused_kwargs,
|
|
) -> torch.Tensor:
|
|
if unused_kwargs:
|
|
unknown = ", ".join(sorted(unused_kwargs))
|
|
raise TypeError(f"Unexpected DeepHealth forward arguments: {unknown}")
|
|
if mode not in {"next_token", "all_future"}:
|
|
raise ValueError("mode must be either 'next_token' or 'all_future'")
|
|
if mode == "all_future" and t_query is None:
|
|
raise ValueError("t_query is required when mode='all_future'")
|
|
if (
|
|
other_type is None
|
|
or other_value is None
|
|
or other_value_kind is None
|
|
or other_time is None
|
|
):
|
|
raise ValueError(
|
|
"DeepHealth expects other_type, other_value, "
|
|
"other_value_kind, and other_time."
|
|
)
|
|
|
|
if padding_mask is None:
|
|
padding_mask = event_seq > PAD_IDX
|
|
else:
|
|
padding_mask = padding_mask.to(device=event_seq.device, dtype=torch.bool)
|
|
|
|
event_len = event_seq.size(1)
|
|
h_disease = self.token_embedding(event_seq)
|
|
t_disease = time_seq
|
|
|
|
if other_time.shape != other_type.shape:
|
|
raise ValueError(
|
|
"other_time must have the same shape as other_type, got "
|
|
f"{tuple(other_time.shape)} vs {tuple(other_type.shape)}"
|
|
)
|
|
other_time = other_time.to(device=event_seq.device, dtype=time_seq.dtype)
|
|
h_other, other_mask = self.tokenizer(
|
|
other_type=other_type,
|
|
other_value=other_value,
|
|
other_value_kind=other_value_kind,
|
|
)
|
|
h_other = h_other.to(device=event_seq.device)
|
|
other_mask = other_mask.to(device=event_seq.device, dtype=torch.bool)
|
|
h_other, other_time, other_mask = self._pool_other_by_time(
|
|
h_other=h_other,
|
|
other_time=other_time,
|
|
other_mask=other_mask,
|
|
)
|
|
|
|
h_disease = torch.cat([h_disease, h_other], dim=1)
|
|
t_disease = torch.cat([t_disease, other_time], dim=1)
|
|
padding_mask = torch.cat([padding_mask, other_mask], dim=1)
|
|
h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
|
|
|
|
if mode == "all_future":
|
|
batch_size = event_seq.size(0)
|
|
query = self.query_token.view(1, 1, -1).expand(batch_size, 1, -1)
|
|
h_disease = torch.cat([h_disease, query], dim=1)
|
|
t_disease = torch.cat([t_disease, t_query[:, None]], dim=1)
|
|
query_mask = torch.ones(
|
|
batch_size,
|
|
1,
|
|
dtype=torch.bool,
|
|
device=event_seq.device,
|
|
)
|
|
padding_mask = torch.cat([padding_mask, query_mask], dim=1)
|
|
|
|
sex_emb = self.gender_embedding(sex)[:, None, :]
|
|
h_disease = h_disease + sex_emb
|
|
h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
|
|
|
|
rope_cache = None
|
|
rbf_cache = None
|
|
if self.time_mode == "absolute":
|
|
h_disease = h_disease + self.age_encoding(t_disease)
|
|
h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
|
|
elif self.time_mode == "relative":
|
|
rope_cache = self.rope.precompute_cache(t_disease)
|
|
rbf_cache = self.rbf.precompute_cache(t_disease)
|
|
|
|
attn_mask = self._make_history_attn_mask(
|
|
padding_mask=padding_mask,
|
|
time_seq=t_disease,
|
|
dtype=h_disease.dtype,
|
|
)
|
|
for block in self.blocks:
|
|
h_disease = block(
|
|
h_disease,
|
|
rope_cache=rope_cache,
|
|
rbf_cache=rbf_cache,
|
|
attn_mask=attn_mask,
|
|
)
|
|
h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
|
|
|
|
h_disease = self.final_ln(h_disease)
|
|
h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
|
|
|
|
if mode == "all_future":
|
|
hidden = h_disease[:, -1, :]
|
|
if return_output:
|
|
return DeepHealthOutput(
|
|
hidden=hidden,
|
|
time_seq=t_query[:, None],
|
|
padding_mask=torch.ones(
|
|
hidden.size(0),
|
|
1,
|
|
dtype=torch.bool,
|
|
device=hidden.device,
|
|
),
|
|
event_len=event_len,
|
|
)
|
|
return hidden
|
|
if return_output:
|
|
return DeepHealthOutput(
|
|
hidden=h_disease,
|
|
time_seq=t_disease,
|
|
padding_mask=padding_mask,
|
|
event_len=event_len,
|
|
)
|
|
return h_disease[:, :event_len, :]
|
|
|
|
def forward_next_token(self, **kwargs) -> torch.Tensor:
|
|
return self._forward_shared(mode="next_token", **kwargs)
|
|
|
|
def forward_all_future(self, **kwargs) -> torch.Tensor:
|
|
return self._forward_shared(mode="all_future", **kwargs)
|
|
|
|
def forward(self, target_mode: str | None = None, **kwargs) -> torch.Tensor:
|
|
mode = self.target_mode if target_mode is None else target_mode
|
|
return self._forward_shared(mode=mode, **kwargs)
|
|
|
|
def calc_risk(self, x: torch.Tensor) -> torch.Tensor:
|
|
return self.risk_head(x)
|
|
|
|
def calc_weibull_rho(self, x: torch.Tensor) -> torch.Tensor:
|
|
if self.dist_mode != "weibull":
|
|
raise RuntimeError(
|
|
f"calc_weibull_rho called with dist_mode={self.dist_mode!r}"
|
|
)
|
|
return F.softplus(self.rho_head(x)) + 1e-6
|
|
|
|
def calc_death_rho(self, x: torch.Tensor) -> torch.Tensor:
|
|
if self.dist_mode != "mixed":
|
|
raise RuntimeError(
|
|
f"calc_death_rho called with dist_mode={self.dist_mode!r}"
|
|
)
|
|
return F.softplus(self.rho_death_head(x)).squeeze(-1) + 1e-6
|