Source code for pyhealth.models.base_model

from abc import ABC
from collections.abc import Iterable, Sequence
from typing import Callable, Any, Optional
import functools
import inspect
import logging

import torch
import torch.nn as nn
import torch.nn.functional as F

from ..datasets import SampleDataset
from ..processors import PROCESSOR_REGISTRY

logger = logging.getLogger(__name__)

# Label kinds understood by the loss helpers, Trainer and the calibration methods.
# The timeseries modes are labels per time step; they need an output processor
# whose process() returns (value, mask), with mask 0 for steps without a label.
VALID_MODES = (
    "binary",
    "multiclass",
    "multilabel",
    "regression",
    "timeseries_binary",
    "timeseries_regression",
)


def _masked_step_loss(
    loss_fn: Callable,
    logits: torch.Tensor,
    target: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
    """Averages a per-step loss over the time steps that have a label.

    Args:
        loss_fn: Elementwise loss accepting ``reduction="none"``.
        logits: Per-step logits of shape (batch, steps, 1).
        target: Batched ``(value, mask)`` of shape (batch, label steps), where
            padding and unlabeled steps have mask 0. Values at those steps are
            ignored and may be NaN.
    """
    # Temporary workaround: per-step labels arrive as a (value, mask) tuple so
    # the existing single-tensor label API stays unchanged. A more unified
    # handling of padded/masked label tensors across models is a follow-up.
    value, mask = target
    if value.shape[1] > logits.shape[1]:
        raise ValueError(
            f"The label has {value.shape[1]} time steps but the input only "
            f"{logits.shape[1]}; label and input must share the time grid."
        )
    logits = logits[:, : value.shape[1], 0]
    labeled = mask > 0
    # Replace unlabeled values before the loss so a NaN there reaches neither
    # the loss nor its gradient, then drop those steps from the sum.
    value = torch.where(labeled, value, torch.zeros_like(value))
    loss = loss_fn(logits, value, reduction="none")
    loss = torch.where(labeled, loss, torch.zeros_like(loss))
    return loss.sum() / labeled.sum().clamp(min=1)


[docs]class BaseModel(ABC, nn.Module): """Abstract class for PyTorch models. Args: dataset (SampleDataset): The dataset to train the model. It is used to query certain information such as the set of all tokens. Interpretability -------- To use a model with interpretability methods, the model must implement a method `forward_from_embedding` that takes in embeddings as input instead of raw features; for the models that already take in dense features as input, this method can simply call the existing `forward` method. For certain gradient-based interpretability methods (e.g., DeepLIFT), the model must also ensure all non-linearity (e.g. ReLU, Sigmoid, Softmax) are using nn.Module versions instead of functional versions (e.g., F.relu, F.sigmoid, F.softmax) so that hooks can be registered properly. Mode -------- ``mode`` is always one of ``"binary"``, ``"multiclass"``, ``"multilabel"``, ``"regression"``, ``"timeseries_binary"``, ``"timeseries_regression"`` or ``None``. It is resolved from the single label's ``output_schema`` entry, which may be a string, a processor class, a processor instance or a ``(name, kwargs)`` tuple. Assigning ``self.mode`` in a subclass resolves the value the same way. Examples: >>> from pyhealth.datasets import create_sample_dataset >>> from pyhealth.models import RNN >>> from pyhealth.processors import MultiLabelProcessor >>> samples = [ ... {"patient_id": "p0", "codes": ["a", "b"], "labels": ["x"]}, ... {"patient_id": "p1", "codes": ["b"], "labels": ["x", "y"]}, ... ] >>> dataset = create_sample_dataset( ... samples=samples, ... input_schema={"codes": "sequence"}, ... output_schema={"labels": MultiLabelProcessor}, ... ) >>> RNN(dataset=dataset).mode 'multilabel' """
[docs] def __init__(self, dataset: SampleDataset): """ Initializes the BaseModel. Args: dataset (SampleDataset): The dataset to train the model. """ super(BaseModel, self).__init__() self.dataset = dataset self.feature_keys = [] self.label_keys = [] # Keep a mode a subclass may have set before calling this __init__. self._mode: str | None = self.__dict__.get("_mode") if dataset: self.feature_keys = list(dataset.input_schema.keys()) self.label_keys = list(dataset.output_schema.keys()) # if single label, resolve mode for Trainer and calibration usage if len(self.label_keys) == 1: try: m = self._resolve_mode(dataset.output_schema[self.label_keys[0]]) except ValueError: m = None if m in VALID_MODES: self._mode = m # used to query the device of the model self._dummy_param = nn.Parameter(torch.empty(0))
@property def mode(self) -> str | None: """The label kind: one of ``VALID_MODES``, or ``None`` if not applicable.""" return self.__dict__.get("_mode") @mode.setter def mode(self, value: Any) -> None: # Subclasses often assign the raw output_schema entry (e.g. a processor # class); resolve it so consumers can always compare against strings. if value is None: self._mode = None return try: resolved = self._resolve_mode(value) except ValueError: resolved = None if resolved not in VALID_MODES: logger.warning( "Cannot use %r as a model mode (expected one of %s); " "setting mode to None.", value, ", ".join(VALID_MODES), ) resolved = None self._mode = resolved
[docs] def forward(self, **kwargs: torch.Tensor | tuple[torch.Tensor, ...] ) -> dict[str, torch.Tensor]: """Forward pass of the model. Args: **kwargs: A variable number of keyword arguments representing input features. Each keyword argument is a tensor or a tuple of tensors of shape (batch_size, ...). Returns: A dictionary with the following keys: logit: a tensor of predicted logits. y_prob: a tensor of predicted probabilities. loss [optional]: a scalar tensor representing the final loss, if self.label_keys in kwargs. y_true [optional]: a tensor representing the true labels, if self.label_keys in kwargs. """ raise NotImplementedError
# ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ def _resolve_mode(self, schema_entry: Any) -> str: """Resolve a mode string from an output_schema entry. Supports: - direct string ("binary", ...) - processor class - processor instance - (string or processor class, kwargs) tuple Returns the registered processor name if found. """ if isinstance(schema_entry, tuple): schema_entry = schema_entry[0] if isinstance(schema_entry, str): return schema_entry.lower() # Get class reference cls = schema_entry if inspect.isclass(schema_entry) else schema_entry.__class__ for name, registered_cls in PROCESSOR_REGISTRY.items(): if cls is registered_cls or issubclass( cls, registered_cls ): # allow subclassing return name.lower() raise ValueError( f"Cannot resolve mode from output_schema entry {schema_entry}. Use a supported string" ) @property def device(self) -> torch.device: """ Gets the device of the model. Returns: torch.device: The device on which the model is located. """ return self._dummy_param.device
[docs] def get_output_size(self) -> int: """ Gets the default output size using the label tokenizer and `self.mode`. If the mode is "binary", the output size is 1. If the mode is "multiclass" or "multilabel", the output size is the number of classes or labels. Returns: int: The output size of the model. """ assert ( len(self.label_keys) == 1 ), "Only one label key is supported if get_output_size is called" output_size = self.dataset.output_processors[self.label_keys[0]].size() return output_size
@property def pos_weight(self) -> torch.Tensor | None: """Weight of positive examples in the binary/multilabel loss, or None.""" return self.__dict__.get("_pos_weight")
[docs] def set_pos_weight( self, pos_weight: float | Sequence[float] | torch.Tensor | str | None, dataset: Iterable[dict] | None = None, ) -> None: """Weights positive examples in the default loss, for rare outcomes. Applies to binary and multilabel labels, through :meth:`get_loss_function` (``pos_weight`` of ``F.binary_cross_entropy_with_logits``). Models that compute their own loss (e.g. the drug-recommendation models) are not affected. Weighting changes the scale of the predicted probabilities, so check calibration when you use it. Args: pos_weight: A number (binary), one number per label (multilabel), ``"balanced"`` for negatives / positives in ``dataset``, or None to remove the weight. dataset: The training samples, required for ``"balanced"``. Pass the training split, not the full dataset, so validation and test labels do not set the weight. Raises: ValueError: If the label is not binary or multilabel, or ``"balanced"`` is given without a dataset. Examples: >>> model.set_pos_weight(4.0) # doctest: +SKIP >>> model.set_pos_weight("balanced", train_dataset) # doctest: +SKIP """ if pos_weight is None: self._pos_weight = None return if self.mode not in ("binary", "multilabel"): raise ValueError( "pos_weight applies to binary or multilabel labels; this " f"model's mode is {self.mode!r}." ) if isinstance(pos_weight, str): if pos_weight != "balanced": raise ValueError( "pos_weight must be a number, a sequence, a tensor, " f"'balanced' or None, not {pos_weight!r}." ) if dataset is None: raise ValueError( "pos_weight='balanced' needs the training dataset: " "set_pos_weight('balanced', train_dataset)." ) label_key = self.label_keys[0] positives, count = None, 0 for sample in dataset: y = torch.as_tensor(sample[label_key], dtype=torch.float32).reshape(-1) positives = y.clone() if positives is None else positives + y count += 1 if count == 0: raise ValueError("pos_weight='balanced' got an empty dataset.") if bool((positives == 0).any()): logger.warning( "pos_weight='balanced': some labels have no positive " "examples in the dataset; their weight is set to 1." ) weight = torch.where( positives > 0, (count - positives) / positives.clamp(min=1), 1.0 ) else: weight = torch.as_tensor(pos_weight, dtype=torch.float32).reshape(-1) self._pos_weight = weight
[docs] def get_loss_function(self) -> Callable: """ Gets the default loss function using `self.mode`. The default loss functions are: - binary: `F.binary_cross_entropy_with_logits` - multiclass: `F.cross_entropy` - multilabel: `F.binary_cross_entropy_with_logits` - regression: `F.mse_loss` - timeseries_binary: `F.binary_cross_entropy_with_logits` per time step, averaged over the labeled steps - timeseries_regression: `F.mse_loss` per time step, averaged over the labeled steps The timeseries losses take per-step logits of shape (batch, steps, 1) and the batched ``(value, mask)`` label, each of shape (batch, steps), produced by an output processor whose ``process()`` returns ``(value, mask)`` per sample with mask 0 for steps without a label. Values at masked steps are ignored and may be NaN. For binary and multilabel labels, a weight set with :meth:`set_pos_weight` is passed as ``pos_weight``. Returns: Callable: The default loss function. """ assert ( len(self.label_keys) == 1 ), "Only one label key is supported if get_loss_function is called" label_key = self.label_keys[0] mode = self._resolve_mode(self.dataset.output_schema[label_key]) pos_weight = self.pos_weight if mode in ("binary", "multilabel") and pos_weight is not None: def weighted_bce(input, target, **kwargs): return F.binary_cross_entropy_with_logits( input, target, pos_weight=pos_weight.to(input.device), **kwargs ) return weighted_bce if mode == "binary": return F.binary_cross_entropy_with_logits elif mode == "multiclass": return F.cross_entropy elif mode == "multilabel": return F.binary_cross_entropy_with_logits elif mode == "regression": return F.mse_loss elif mode == "timeseries_binary": return functools.partial(_masked_step_loss, F.binary_cross_entropy_with_logits) elif mode == "timeseries_regression": return functools.partial(_masked_step_loss, F.mse_loss) else: raise ValueError(f"Invalid mode: {mode}")
[docs] def prepare_y_prob(self, logits: torch.Tensor) -> torch.Tensor: """ Prepares the predicted probabilities for model evaluation. This function converts the predicted logits to predicted probabilities depending on the mode. The default formats are: - binary: a tensor of shape (batch_size, 1) with values in [0, 1], which is obtained with `torch.sigmoid()` - multiclass: a tensor of shape (batch_size, num_classes) with values in [0, 1] and sum to 1, which is obtained with `torch.softmax()` - multilabel: a tensor of shape (batch_size, num_labels) with values in [0, 1], which is obtained with `torch.sigmoid()` - regression: a tensor of shape (batch_size, 1) with raw logits - timeseries_binary: a tensor of shape (batch_size, steps, 1) with values in [0, 1] per time step, from `torch.sigmoid()` - timeseries_regression: a tensor of shape (batch_size, steps, 1) with raw logits per time step Args: logits (torch.Tensor): The predicted logit tensor. Returns: torch.Tensor: The predicted probability tensor. """ assert ( len(self.label_keys) == 1 ), "Only one label key is supported if get_loss_function is called" label_key = self.label_keys[0] mode = self._resolve_mode(self.dataset.output_schema[label_key]) if mode in ["binary"]: y_prob = torch.sigmoid(logits) elif mode in ["multiclass"]: y_prob = F.softmax(logits, dim=-1) elif mode in ["multilabel"]: y_prob = torch.sigmoid(logits) elif mode in ["regression"]: y_prob = logits elif mode in ["timeseries_binary"]: y_prob = torch.sigmoid(logits) elif mode in ["timeseries_regression"]: y_prob = logits else: raise NotImplementedError return y_prob