DeepHealth/models.py

import torch
import torch.nn as nn
import torch.nn.functional as F

from backbones import (
    AgeSinusoidalEncoding,
    BaselineEncoder,
    CrossAttention,
    GPTBlock,
    GaussianRBFTimeBasis,
    TimeRoPE,
)


class DeepHealth(nn.Module):
    def __init__(
        self,
        vocab_size: int,
        n_embd: int,
        n_head: int,
        n_hist_layer: int,
        n_tab_layer: int,
        n_types: int,
        n_cont_types: int,
        n_categories: int,
        cont_type_ids: list[int],
        n_value_kinds: int = 3,
        n_bins: int = 16,
        target_mode: str = "next_token",  # "next_token" or "all_future"
        time_mode: str = "relative",  # "relative" or "absolute"
        dist_mode: str = "exponential",  # "exponential", "weibull" or "mixed"
        dropout: float = 0.0,
    ):
        super().__init__()
        if target_mode not in ["next_token", "all_future"]:
            raise ValueError(
                "target_mode must be either 'next_token' or 'all_future'")
        if time_mode not in ["relative", "absolute"]:
            raise ValueError(
                "time_mode must be either 'relative' or 'absolute'")
        if dist_mode not in ["exponential", "weibull", "mixed"]:
            raise ValueError(
                "dist_mode must be either 'exponential', 'weibull' or 'mixed'")
        self.token_embedding = nn.Embedding(vocab_size, n_embd, padding_idx=0)
        self.gender_embedding = nn.Embedding(
            2, n_embd)  # Assuming binary gender
        self.token_encoder = BaselineEncoder(
            n_embd=n_embd,
            n_head=n_head,
            n_types=n_types,
            n_cont_types=n_cont_types,
            n_categories=n_categories,
            cont_type_ids=cont_type_ids,
            n_value_kinds=n_value_kinds,
            n_bins=n_bins,
            n_tab_layer=n_tab_layer,
            dropout=dropout,
        )
        self.cross_attention = CrossAttention(
            n_embd=n_embd,
            n_head=n_head,
            dropout=dropout,
            n_rbf_bases=16,
        )
        self.target_mode = target_mode
        self.time_mode = time_mode
        self.dist_mode = dist_mode
        self.n_embd = n_embd
        self.vocab_size = vocab_size
        nn.init.normal_(self.token_embedding.weight, mean=0.0, std=0.02)
        nn.init.zeros_(self.token_embedding.weight[0])
        nn.init.normal_(self.gender_embedding.weight, mean=0.0, std=0.02)
        if dist_mode == "weibull":
            self.rho_head = nn.Linear(n_embd, vocab_size)
            nn.init.zeros_(self.rho_head.weight)
            nn.init.constant_(self.rho_head.bias, 0.5413)

        if dist_mode == "mixed":
            self.death_idx = vocab_size - 1
            self.rho_death_head = nn.Linear(n_embd, 1)
            nn.init.zeros_(self.rho_death_head.weight)
            nn.init.constant_(self.rho_death_head.bias, 0.5413)

        if time_mode == "absolute":
            self.age_encoding = AgeSinusoidalEncoding(n_embd)
            self.blocks = nn.ModuleList([
                GPTBlock(
                    n_embd=n_embd,
                    n_head=n_head,
                    use_time_rope=False,
                    use_rbf_bias=False,
                    mlp_dropout=dropout,
                ) for _ in range(n_hist_layer)
            ])
            self.rope = None
            self.rbf = None
        elif time_mode == "relative":
            self.age_encoding = None
            self.blocks = nn.ModuleList([
                GPTBlock(
                    n_embd=n_embd,
                    n_head=n_head,
                    use_time_rope=True,
                    use_rbf_bias=True,
                    mlp_dropout=dropout,
                ) for _ in range(n_hist_layer)
            ])
            self.rope = TimeRoPE(n_embd // n_head)
            self.rbf = GaussianRBFTimeBasis(n_bases=16, max_time_diff=40.0)

        self.final_ln = nn.LayerNorm(n_embd)
        self.risk_head = nn.Linear(n_embd, vocab_size, bias=False)
        self.query_token = nn.Parameter(torch.zeros(n_embd))
        nn.init.normal_(self.query_token, mean=0.0, std=0.02)

    def _make_history_attn_mask(
        self,
        padding_mask: torch.Tensor,
        time_seq: torch.Tensor,
        dtype: torch.dtype,
    ) -> torch.Tensor:
        valid_key = padding_mask[:, None, :]  # (B, 1, L)
        visible_by_time = time_seq[:, None, :] <= time_seq[:, :, None]
        valid = valid_key & visible_by_time
        return torch.zeros(
            valid.shape,
            device=valid.device,
            dtype=dtype,
        ).masked_fill(~valid, -1e4)[:, None, :, :]

    def _encode_other_tokens(
        self,
        other_type: torch.LongTensor,
        other_value: torch.Tensor,
        other_value_kind: torch.LongTensor,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        return self.token_encoder(
            other_type=other_type,
            other_value=other_value,
            other_value_kind=other_value_kind,
        )

    def _forward_shared(
        self,
        event_seq: torch.LongTensor,
        time_seq: torch.FloatTensor,
        sex: torch.LongTensor,
        mode: str,
        padding_mask: torch.Tensor | None = None,
        t_query: torch.FloatTensor | None = None,
        other_type: torch.LongTensor | None = None,
        other_value: torch.Tensor | None = None,
        other_value_kind: torch.LongTensor | None = None,
        other_time: torch.FloatTensor | None = None,
        **unused_kwargs,
    ) -> torch.Tensor:
        if unused_kwargs:
            unknown = ", ".join(sorted(unused_kwargs))
            raise TypeError(f"Unexpected DeepHealth forward arguments: {unknown}")
        if mode not in {"next_token", "all_future"}:
            raise ValueError("mode must be either 'next_token' or 'all_future'")
        if mode == "all_future" and t_query is None:
            raise ValueError("t_query is required when mode='all_future'")
        if (
            other_type is None
            or other_value is None
            or other_value_kind is None
            or other_time is None
        ):
            raise ValueError(
                "DeepHealth expects other_type, other_value, "
                "other_value_kind, and other_time."
            )

        if padding_mask is None:
            padding_mask = event_seq > 0
        else:
            padding_mask = padding_mask.to(device=event_seq.device, dtype=torch.bool)
        h_disease = self.token_embedding(event_seq)
        t_disease = time_seq

        if mode == "all_future":
            batch_size = event_seq.size(0)
            query = self.query_token.view(1, 1, -1).expand(batch_size, 1, -1)
            h_disease = torch.cat([h_disease, query], dim=1)
            t_disease = torch.cat([time_seq, t_query[:, None]], dim=1)
            query_mask = torch.ones(
                batch_size,
                1,
                dtype=torch.bool,
                device=event_seq.device,
            )
            padding_mask = torch.cat([padding_mask, query_mask], dim=1)

        sex_emb = self.gender_embedding(sex)[:, None, :]
        h_disease = h_disease + sex_emb
        h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)

        rope_cache = None
        rbf_cache = None
        if self.time_mode == "absolute":
            h_disease = h_disease + self.age_encoding(t_disease)
            h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)
        elif self.time_mode == "relative":
            rope_cache = self.rope.precompute_cache(t_disease)
            rbf_cache = self.rbf.precompute_cache(t_disease)

        attn_mask = self._make_history_attn_mask(
            padding_mask=padding_mask,
            time_seq=t_disease,
            dtype=h_disease.dtype,
        )
        for block in self.blocks:
            h_disease = block(
                h_disease,
                rope_cache=rope_cache,
                rbf_cache=rbf_cache,
                attn_mask=attn_mask,
            )
            h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)

        h_disease = self.final_ln(h_disease)
        h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)

        h_token, token_mask = self._encode_other_tokens(
            other_type=other_type,
            other_value=other_value,
            other_value_kind=other_value_kind,
        )
        if other_time.shape != other_type.shape:
            raise ValueError(
                "other_time must have the same shape as other_type, got "
                f"{tuple(other_time.shape)} vs {tuple(other_type.shape)}"
            )
        token_time = other_time.to(device=h_token.device, dtype=t_disease.dtype)

        h_disease = self.cross_attention(
            h_disease=h_disease,
            t_disease=t_disease,
            h_token=h_token,
            t_token=token_time,
            token_mask=token_mask,
        )
        h_disease = h_disease * padding_mask.unsqueeze(-1).to(h_disease.dtype)

        if mode == "all_future":
            return h_disease[:, -1, :]
        return h_disease

    def forward_next_token(self, **kwargs) -> torch.Tensor:
        return self._forward_shared(mode="next_token", **kwargs)

    def forward_all_future(self, **kwargs) -> torch.Tensor:
        return self._forward_shared(mode="all_future", **kwargs)

    def forward(self, target_mode: str | None = None, **kwargs) -> torch.Tensor:
        mode = self.target_mode if target_mode is None else target_mode
        return self._forward_shared(mode=mode, **kwargs)

    def calc_risk(self, x: torch.Tensor) -> torch.Tensor:
        return self.risk_head(x)

    def calc_weibull_rho(self, x: torch.Tensor) -> torch.Tensor:
        if self.dist_mode != "weibull":
            raise RuntimeError(
                f"calc_weibull_rho called with dist_mode={self.dist_mode!r}"
            )
        return F.softplus(self.rho_head(x)) + 1e-6

    def calc_death_rho(self, x: torch.Tensor) -> torch.Tensor:
        if self.dist_mode != "mixed":
            raise RuntimeError(
                f"calc_death_rho called with dist_mode={self.dist_mode!r}"
            )
        return F.softplus(self.rho_death_head(x)).squeeze(-1) + 1e-6