Source code for pyhealth.datasets.sample_dataset

from collections.abc import Sequence
from pathlib import Path
import pickle
import shutil
import tempfile
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, Type
from typing import TYPE_CHECKING
import inspect
from bisect import bisect_right
import litdata
from ..utils import preserve_rng_state
from litdata.utilities.train_test_split import deepcopy_dataset
import copy
import logging

import numpy as np
import torch
from packaging.version import Version

from ..processors import get_processor, IgnoreProcessor
from ..processors.base_processor import FeatureProcessor

if TYPE_CHECKING:  # splitter imports this module
    from .splitter import Split

logger = logging.getLogger(__name__)

# How processed samples are laid out in the disk (litdata) cache. Bump when
# the layout changes; it is part of the processed-cache key, so caches written
# with an older layout are rebuilt instead of misread.
STORAGE_FORMAT = 2
# Record key holding one pickled payload with every non-tensor field.
_OBJECTS_KEY = "_pyhealth_objects"
_PICKLE_PROTOCOL = 5


def _decode_record(item: Any) -> Any:
    """Restores a sample written by :meth:`SampleBuilder.encode`.

    Records without the objects payload (caches written before
    ``STORAGE_FORMAT`` 2, or in-memory samples) are returned unchanged.
    """
    if isinstance(item, list):
        return [_decode_record(x) for x in item]
    if not isinstance(item, dict) or _OBJECTS_KEY not in item:
        return item
    payload = pickle.loads(item[_OBJECTS_KEY])
    values = payload["values"]
    return {k: values[k] if k in values else item[k] for k in payload["order"]}


def _draw_seed() -> int:
    """A shuffle seed drawn from torch's global generator.

    So ``torch.manual_seed(...)``, which users already call, makes shuffling
    reproducible, and different torch seeds give different orders.
    """
    return int(torch.randint(0, 2**31 - 1, ()).item())


class _MaskedSamples:
    """A re-iterable view of a sample stream that keeps only masked positions.

    Each iteration walks the underlying stream in order and yields the samples
    whose position is True in ``mask``. Nothing is collected, so fitting on a
    split costs no more memory than fitting on all samples.
    """

    def __init__(self, samples: Iterable[dict[str, Any]], mask: np.ndarray):
        self._samples = samples
        self._mask = mask

    def __iter__(self):
        for keep, sample in zip(self._mask, self._samples):
            if keep:
                yield sample


def _remap_index_mapping(
    mapping: dict[str, list[int]], indices: Sequence[int]
) -> dict[str, list[int]]:
    key_by_index = {
        index: key
        for key, mapped_indices in mapping.items()
        for index in mapped_indices
    }
    remapped: dict[str, list[int]] = {}
    for new_index, old_index in enumerate(indices):
        key = key_by_index.get(old_index)
        if key is not None:
            remapped.setdefault(key, []).append(new_index)
    return remapped


class SampleBuilder:
    """Fit feature processors and transform pickled samples without materializing a dataset.

    SampleBuilder is a lightweight helper used to:
        - Fit feature processors from provided `input_schema` and `output_schema` on an
            iterable of raw Python sample dictionaries.
        - Build mappings from patient IDs and record IDs to sample indices.
        - Transform pickled sample records into processed feature dictionaries using
            the fitted processors.

    Typical usage:
        builder = SampleBuilder(input_schema, output_schema)
        builder.fit(samples)
        builder.save(path)  # writes a schema.pkl metadata file

    After saving the schema, write ``builder.transform_for_storage`` of each
    pickled sample item with a litdata ``BinaryWriter`` (as ``set_task`` and
    ``create_sample_dataset`` do) to get a directory that can be loaded via
    SampleDataset.

    Pre-fitted processors passed as `input_processors` / `output_processors` are
    used as they are; every schema field without one gets a fitted processor.

    Examples:
        >>> from pyhealth.datasets.sample_dataset import SampleBuilder
        >>> from pyhealth.processors import SequenceProcessor
        >>> samples = [
        ...     {"patient_id": "p1", "codes": ["a", "b"], "age": [40.0], "label": 1},
        ...     {"patient_id": "p2", "codes": ["c"], "age": [55.0], "label": 0},
        ... ]
        >>> codes = SequenceProcessor()
        >>> codes.fit(samples, "codes")
        >>> builder = SampleBuilder(
        ...     input_schema={"codes": "sequence", "age": "tensor"},
        ...     output_schema={"label": "binary"},
        ...     input_processors={"codes": codes},
        ... )
        >>> builder.fit(samples)
        >>> builder.input_processors["codes"] is codes
        True
        >>> type(builder.input_processors["age"]).__name__
        'TensorProcessor'
    """

    def __init__(
        self,
        input_schema: Dict[str, Any],
        output_schema: Dict[str, Any],
        input_processors: Optional[Dict[str, FeatureProcessor]] = None,
        output_processors: Optional[Dict[str, FeatureProcessor]] = None,
    ) -> None:
        self.input_schema = input_schema
        self.output_schema = output_schema
        self._input_processors = (
            dict(input_processors) if input_processors is not None else {}
        )
        self._output_processors = (
            dict(output_processors) if output_processors is not None else {}
        )
        self._patient_to_index: Dict[str, List[int]] = {}
        self._record_to_index: Dict[str, List[int]] = {}
        self._fitted = False
        self._split = None
        self._split_indices: list[np.ndarray] | None = None

    @property
    def input_processors(self) -> Dict[str, FeatureProcessor]:
        if not self._fitted:
            raise RuntimeError(
                "SampleBuilder.fit must be called before accessing input_processors."
            )
        return self._input_processors

    @property
    def output_processors(self) -> Dict[str, FeatureProcessor]:
        if not self._fitted:
            raise RuntimeError(
                "SampleBuilder.fit must be called before accessing output_processors."
            )
        return self._output_processors

    @property
    def split_indices(self) -> list[np.ndarray] | None:
        """Sample indices of each split part (train first), or None without a split."""
        if not self._fitted:
            raise RuntimeError(
                "SampleBuilder.fit must be called before accessing split_indices."
            )
        return self._split_indices

    @property
    def patient_to_index(self) -> Dict[str, List[int]]:
        if not self._fitted:
            raise RuntimeError(
                "SampleBuilder.fit must be called before accessing patient_to_index."
            )
        return self._patient_to_index

    @property
    def record_to_index(self) -> Dict[str, List[int]]:
        if not self._fitted:
            raise RuntimeError(
                "SampleBuilder.fit must be called before accessing record_to_index."
            )
        return self._record_to_index

    def _get_processor_instance(self, processor_spec):
        """Instantiate a processor using the same resolution logic as SampleDataset."""
        if isinstance(processor_spec, tuple):
            spec, kwargs = processor_spec
            if isinstance(spec, str):
                return get_processor(spec)(**kwargs)
            if inspect.isclass(spec) and issubclass(spec, FeatureProcessor):
                return spec(**kwargs)
            raise ValueError(
                "Processor spec in tuple must be either a string alias or a "
                f"FeatureProcessor class, got {type(spec)}"
            )
        if isinstance(processor_spec, str):
            return get_processor(processor_spec)()
        if inspect.isclass(processor_spec) and issubclass(
            processor_spec, FeatureProcessor
        ):
            return processor_spec()
        if isinstance(processor_spec, FeatureProcessor):
            return processor_spec
        raise ValueError(
            "Processor spec must be either a string alias, a FeatureProcessor "
            f"class, or a tuple (spec, kwargs_dict), got {type(processor_spec)}"
        )

    def fit(
        self,
        samples: Iterable[Dict[str, Any]],
        split: "Split | None" = None,
    ) -> None:
        """Fit processors and build mapping indices from an iterator of samples.

        Args:
            samples: Iterable of sample dictionaries (e.g., python dicts). Each
                sample should contain keys covering both the configured
                `input_schema` and `output_schema`. These samples are not
                required to be pickled; `fit` operates on in-memory dicts.
                It is iterated several times and never collected, so a
                streaming dataset works without loading it into memory.
            split: Optional :class:`~pyhealth.datasets.Split`, such as
                :class:`~pyhealth.datasets.PatientSplit`. When given,
                processors are fitted on the first (training) part only, and
                the sample indices of every part are kept in
                ``split_indices``.

        Behavior:
            - Validates the samples contain all keys specified by the input
              and output schemas.
            - Builds `patient_to_index` and `record_to_index` mappings by
              recording the sample indices associated with `patient_id` and
              `record_id`/`visit_id` fields.
            - Instantiates and fits a processor for every schema field that
              has none. Pre-fitted processors supplied to the constructor are
              used as they are; any remaining fields still get fitted ones.
        """
        # Validate the samples
        input_keys = set(self.input_schema.keys())
        output_keys = set(self.output_schema.keys())
        for sample in samples:
            assert input_keys.issubset(
                sample.keys()
            ), "Input schema does not match samples."
            assert output_keys.issubset(
                sample.keys()
            ), "Output schema does not match samples."

        # Build index mappings
        self._patient_to_index = {}
        self._record_to_index = {}
        n_samples = 0
        for i, sample in enumerate(samples):
            n_samples = i + 1
            patient_id = sample.get("patient_id")
            if patient_id is not None:
                self._patient_to_index.setdefault(patient_id, []).append(i)
            record_id = sample.get("record_id", sample.get("visit_id"))
            if record_id is not None:
                self._record_to_index.setdefault(record_id, []).append(i)

        # With a split, fit on the training patients only: a masked view of
        # the same stream, so nothing is loaded into memory.
        self._split = split
        self._split_indices = None
        fit_samples = samples
        if split is not None:
            self._split_indices = split.split_indices(self._patient_to_index)
            train_mask = np.zeros(n_samples, dtype=bool)
            train_mask[self._split_indices[0]] = True
            fit_samples = _MaskedSamples(samples, train_mask)

        # Fit a processor for every schema field that has none. Supplied
        # (pre-fitted) processors are used as they are, never refitted.
        learned_on_all = []
        for schema, processors in (
            (self.input_schema, self._input_processors),
            (self.output_schema, self._output_processors),
        ):
            for key, spec in schema.items():
                if key not in processors:
                    processor = self._get_processor_instance(spec)
                    processor.fit(fit_samples, key)
                    processors[key] = processor
                    if split is None and getattr(processor, "learns_statistics", False):
                        learned_on_all.append(key)
        if learned_on_all:
            logger.warning(
                "Processors for %s learn statistics from the data and were fitted "
                "on all samples, so validation/test patients shaped them. Fit on "
                "the training patients with set_task(task, split=PatientSplit(...)).",
                ", ".join(learned_on_all),
            )

        self._fitted = True

    def transform(self, sample: dict[str, bytes]) -> Dict[str, Any]:
        """Transform a single serialized (pickled) sample using fitted processors.

        Args:
            sample: A mapping with a single key `"sample"` whose value is a
                pickled Python dictionary (produced by `pickle.dumps`). The
                pickled dictionary should mirror the schema that was used to
                fit this builder.

        Returns:
            A Python dictionary where each key is either an input or output
            feature name. Values for keys present in the corresponding fitted
            processors have been processed through their FeatureProcessor and
            are returned as the output of that processor. Keys not covered by
            the input/output processors are returned unchanged.
        """
        if not self._fitted:
            raise RuntimeError("SampleBuilder.fit must be called before transform().")

        transformed: Dict[str, Any] = {}
        for key, value in pickle.loads(sample["sample"]).items():
            if key in self._input_processors:
                # Skip ignored features
                if isinstance(self._input_processors[key], IgnoreProcessor):
                    continue
                transformed[key] = self._input_processors[key].process(value)
            elif key in self._output_processors:
                # Skip ignored features
                if isinstance(self._output_processors[key], IgnoreProcessor):
                    continue
                transformed[key] = self._output_processors[key].process(value)
            else:
                transformed[key] = value
        return transformed

    def encode(self, transformed: dict[str, Any]) -> dict[str, Any]:
        """Lays out one processed sample for the disk (litdata) cache.

        litdata infers one serializer per value from the first sample it
        writes and expects every later sample to have the same structure, so a
        list whose length varies between samples cannot be written as-is.
        Fields whose processor declares ``stores_tensor`` stay tensors; every
        other field (including keys not in the schema, such as ``patient_id``)
        goes into one pickled payload, which has the same structure in every
        sample whatever the values look like. :class:`SampleDataset` decodes it
        on read, so callers get back the original Python objects.

        Args:
            transformed: Output of :meth:`transform`.

        Returns:
            A record with the tensor fields plus one bytes payload.

        Raises:
            ValueError: If a field declared ``stores_tensor`` is not a tensor.
        """
        tensor_fields = self._tensor_fields()
        record: dict[str, Any] = {}
        objects: dict[str, Any] = {}
        for key, value in transformed.items():
            if key in tensor_fields:
                if not isinstance(value, torch.Tensor):
                    processor = self._input_processors.get(key) or self._output_processors.get(key)
                    raise ValueError(
                        f"Field '{key}': {type(processor).__name__} declares "
                        f"stores_tensor=True but process() returned "
                        f"{type(value).__name__}. Set stores_tensor = False on the "
                        "processor, or return a torch.Tensor."
                    )
                record[key] = value
            else:
                objects[key] = value
        record[_OBJECTS_KEY] = pickle.dumps(
            {"order": list(transformed), "values": objects}, protocol=_PICKLE_PROTOCOL
        )
        return record

    def transform_for_storage(self, sample: dict[str, bytes]) -> dict[str, Any]:
        """:meth:`transform` followed by :meth:`encode`, for writing to disk."""
        return self.encode(self.transform(sample))

    def _tensor_fields(self) -> set:
        processors = {**self._input_processors, **self._output_processors}
        return {
            key
            for key, processor in processors.items()
            if getattr(processor, "stores_tensor", False)
        }

    def save(self, path: str) -> None:
        """Save fitted metadata to the given path as a pickled file.

        Args:
            path: Location where the builder will write a pickled metadata file
                (commonly named `schema.pkl`). The saved metadata contains
                the fitted input/output schemas, processors, and index
                mappings. This file is read by `SampleDataset` during
                construction.
        """
        if not self._fitted:
            raise RuntimeError("SampleBuilder.fit must be called before save().")
        metadata = {
            "input_schema": self.input_schema,
            "output_schema": self.output_schema,
            "input_processors": self._input_processors,
            "output_processors": self._output_processors,
            "patient_to_index": self._patient_to_index,
            "record_to_index": self._record_to_index,
            "fit_split": self._split.to_dict() if self._split is not None else None,
            "storage_format": STORAGE_FORMAT,
        }
        with open(path, "wb") as f:
            pickle.dump(metadata, f)

    @staticmethod
    def load(path: str) -> "SampleBuilder":
        """Load a SampleBuilder from a pickled metadata file.

        Args:
            path: Location of the pickled metadata file (commonly named `schema.pkl`).

        Returns:
            A SampleBuilder instance with loaded metadata.
        """
        with open(path, "rb") as f:
            metadata = pickle.load(f)

        builder = SampleBuilder(
            input_schema=metadata["input_schema"],
            output_schema=metadata["output_schema"],
        )
        builder._input_processors = metadata["input_processors"]
        builder._output_processors = metadata["output_processors"]
        builder._patient_to_index = metadata["patient_to_index"]
        builder._record_to_index = metadata["record_to_index"]
        builder._fitted = True
        return builder


[docs]class SampleDataset(litdata.StreamingDataset): """A streaming dataset that loads sample metadata and processors from disk. SampleDataset expects the `path` directory to contain a `schema.pkl` file created by a `SampleBuilder.save(...)` call. The `schema.pkl` must include the fitted `input_schema`, `output_schema`, `input_processors`, `output_processors`, `patient_to_index` and `record_to_index` mappings. Attributes: input_schema: The configuration used to instantiate processors for input features (string aliases or processor specs). output_schema: The configuration used to instantiate processors for output features. input_processors: A mapping of input feature names to fitted FeatureProcessor instances. output_processors: A mapping of output feature names to fitted FeatureProcessor instances. patient_to_index: Dictionary mapping patient IDs to the list of sample indices associated with that patient. record_to_index: Dictionary mapping record/visit IDs to the list of sample indices associated with that record. dataset_name: Optional human friendly dataset name. task_name: Optional human friendly task name. Examples: >>> from pyhealth.datasets import create_sample_dataset >>> dataset = create_sample_dataset( # doctest: +SKIP ... samples=[ ... {"patient_id": "p1", "feature": 1, "label": 0}, ... {"patient_id": "p2", "feature": 2, "label": 1}, ... ], ... input_schema={"feature": "raw"}, ... output_schema={"label": "raw"}, ... in_memory=False, ... ) >>> dataset.subset([1]).patient_to_index # doctest: +SKIP {'p2': [0]} >>> dataset.close() # doctest: +SKIP """
[docs] def __init__( self, path: str, dataset_name: Optional[str] = None, task_name: Optional[str] = None, delete_on_close: bool = False, **kwargs, ) -> None: """Initialize a SampleDataset pointing at a directory created by SampleBuilder. Args: path: Path to a directory containing a `schema.pkl` produced by `SampleBuilder.save` and associated pickled sample files. dataset_name: Optional human-friendly dataset name. task_name: Optional human-friendly task name. delete_on_close: Whether ``close()`` should delete ``path``. This is reserved for temporary datasets owned by PyHealth. **kwargs: Extra keyword arguments forwarded to `litdata.StreamingDataset` (such as streaming options). """ super().__init__(path, **kwargs) self.path = path self.dataset_name = "" if dataset_name is None else dataset_name self.task_name = "" if task_name is None else task_name self._delete_on_close = delete_on_close with open(f"{path}/schema.pkl", "rb") as f: metadata = pickle.load(f) self.input_schema: dict[str, Any] = metadata["input_schema"] self.output_schema: dict[str, Any] = metadata["output_schema"] self.input_processors: dict[str, FeatureProcessor] = metadata["input_processors"] self.output_processors: dict[str, FeatureProcessor]= metadata["output_processors"] self._remove_ignored_processors() self.patient_to_index = metadata["patient_to_index"] self.record_to_index = metadata["record_to_index"] # The split the processors were fitted on, or None if fitted on all # samples (also None for caches written before this was recorded). self.fit_split: dict | None = metadata.get("fit_split") self.storage_format: int = metadata.get("storage_format", 1)
def __getitem__(self, index: Any) -> Any: """Returns the sample(s) at ``index`` as the original Python objects. Index access, iteration, ``subset()`` views and ``set_task`` split parts all read through here, so every path decodes the stored payload. """ return _decode_record(super().__getitem__(index))
[docs] def set_shuffle(self, shuffle: bool, seed: int | None = None) -> None: """Sets whether iterating the dataset shuffles, and the shuffle seed. The order of each pass is determined by ``seed`` and the epoch (see ``set_epoch``); litdata shuffles with its own generator, so the global random state is not used. Loaders from :func:`~pyhealth.datasets.get_dataloader` call this for you at the start of every pass. Args: shuffle: Whether ``iter(dataset)`` shuffles. seed: Seed of the shuffle order. Default: drawn from torch's random generator, so ``torch.manual_seed`` controls it. Examples: >>> dataset.set_shuffle(True, seed=0) # doctest: +SKIP """ super().set_shuffle(shuffle) if shuffle: # litdata rebuilds its shuffler from ``seed`` on every iteration. self.seed = _draw_seed() if seed is None else int(seed)
def _remove_ignored_processors(self): """Remove any processors that are IgnoreProcessor instances.""" for key in [ key for key, proc in self.input_processors.items() if isinstance(proc, IgnoreProcessor) ]: del self.input_processors[key] del self.input_schema[key] for key in [ key for key, proc in self.output_processors.items() if isinstance(proc, IgnoreProcessor) ]: del self.output_processors[key] del self.output_schema[key] def __str__(self) -> str: """Returns a string representation of the dataset. Returns: str: A string with the dataset and task names. """ return f"Sample dataset {self.dataset_name} {self.task_name}"
[docs] def subset(self, indices: Union[Sequence[int], slice]) -> "SampleDataset": """Create a StreamingDataset restricted to the provided indices. The subset reads with litdata's POSIX-fast mmap reads turned off (litdata >= 0.2.68). They cache per index run, not per chunk file, which makes a patient-level split ~20x slower to iterate. The full dataset keeps them. """ new_dataset = deepcopy_dataset(self) if len(new_dataset.subsampled_files) != len(new_dataset.region_of_interest): raise ValueError( "The provided dataset has mismatched subsampled_files and region_of_interest lengths." ) dataset_length = sum( end - start for start, end in new_dataset.region_of_interest ) if isinstance(indices, slice): indices = range(*indices.indices(dataset_length)) indices = list(indices) if any(idx < 0 or idx >= dataset_length for idx in indices): raise ValueError( f"Subset indices must be in [0, {dataset_length - 1}] for the provided dataset." ) # Build chunk boundaries so we can translate global indices into # chunk-local (start, end) pairs that litdata understands. chunk_starts: List[int] = [] chunk_boundaries: List[Tuple[str, int, int, int, int]] = [] cursor = 0 for filename, (roi_start, roi_end) in zip( new_dataset.subsampled_files, new_dataset.region_of_interest ): chunk_len = roi_end - roi_start if chunk_len <= 0: continue chunk_starts.append(cursor) chunk_boundaries.append( (filename, roi_start, roi_end, cursor, cursor + chunk_len) ) cursor += chunk_len new_subsampled_files: List[str] = [] new_roi: List[Tuple[int, int]] = [] prev_chunk_idx: Optional[int] = None for idx in indices: chunk_idx = bisect_right(chunk_starts, idx) - 1 if chunk_idx < 0 or idx >= chunk_boundaries[chunk_idx][4]: raise ValueError(f"Index {idx} is out of bounds for the dataset.") filename, roi_start, _, global_start, _ = chunk_boundaries[chunk_idx] offset_in_chunk = roi_start + (idx - global_start) if ( new_roi and prev_chunk_idx == chunk_idx and offset_in_chunk == new_roi[-1][1] ): new_roi[-1] = (new_roi[-1][0], new_roi[-1][1] + 1) else: new_subsampled_files.append(filename) new_roi.append((offset_in_chunk, offset_in_chunk + 1)) prev_chunk_idx = chunk_idx new_dataset.subsampled_files = new_subsampled_files new_dataset.region_of_interest = new_roi new_dataset.patient_to_index = _remap_index_mapping( self.patient_to_index, indices ) new_dataset.record_to_index = _remap_index_mapping( self.record_to_index, indices ) new_dataset._delete_on_close = False new_dataset.reset() # The subset lists one (file, region) entry per run of consecutive # indices, so a patient-level split has thousands of entries over a # few chunk files. POSIX-fast caches per entry, not per file: it # re-maps the file and re-parses its offset table for each entry. # Batched decoding stays on: turning it off as well gains nothing, # and with both off litdata <= 0.2.76 cannot random-access chunks # stored in its Arrow-footer layout (Lightning-AI/litData#954). if Version(litdata.__version__) >= Version("0.2.68"): new_dataset.posix_fast = None # type: ignore return new_dataset
[docs] def close(self) -> None: """Cleans up any temporary directories used by the dataset.""" if ( self._delete_on_close and self.input_dir.path is not None and Path(self.input_dir.path).exists() ): shutil.rmtree(self.input_dir.path)
# -------------------------------------------------------------- # Context manager support # -------------------------------------------------------------- def __enter__(self): return self def __exit__(self, exc_type, exc, tb): self.close()
class InMemorySampleDataset(SampleDataset): """A SampleDataset that loads all samples into memory for fast access. InMemorySampleDataset extends SampleDataset by eagerly loading and transforming all samples into memory during initialization. This allows for fast, repeated access to samples without disk I/O, at the cost of higher memory usage. Examples: >>> from pyhealth.datasets import create_sample_dataset >>> dataset = create_sample_dataset( ... samples=[{"patient_id": "p1", "feature": 1, "label": 0}], ... input_schema={"feature": "raw"}, ... output_schema={"label": "raw"}, ... ) >>> dataset[0]["patient_id"] 'p1' Note: This class is intended for testing and debugging purposes where dataset sizes are small enough to fit into memory. """ def __init__( self, samples: List[Dict[str, Any]], input_schema: Dict[str, Any], output_schema: Dict[str, Any], dataset_name: Optional[str] = None, task_name: Optional[str] = None, input_processors: Optional[Dict[str, FeatureProcessor]] = None, output_processors: Optional[Dict[str, FeatureProcessor]] = None, ) -> None: """Initialize an InMemorySampleDataset from in-memory samples. This constructor fits a SampleBuilder on the provided samples, transforms all samples into memory, and sets up the dataset attributes. Args: samples: A list of sample dictionaries (in-memory). input_schema: Schema describing how input keys should be handled. output_schema: Schema describing how output keys should be handled. dataset_name: Optional human-friendly dataset name. task_name: Optional human-friendly task name. input_processors: Optional pre-fitted input processors to use instead of creating new ones from the input_schema. output_processors: Optional pre-fitted output processors to use instead of creating new ones from the output_schema. """ builder = SampleBuilder( input_schema=input_schema, output_schema=output_schema, input_processors=input_processors, output_processors=output_processors, ) builder.fit(samples) self.dataset_name = "" if dataset_name is None else dataset_name self.task_name = "" if task_name is None else task_name self.input_schema = builder.input_schema self.output_schema = builder.output_schema self.input_processors = builder.input_processors self.output_processors = builder.output_processors self._remove_ignored_processors() self.patient_to_index = builder.patient_to_index self.record_to_index = builder.record_to_index self._data = [builder.transform({"sample": pickle.dumps(s)}) for s in samples] self._shuffle = False self._shuffle_seed = 0 self._epoch = 1 # counted from 1, as litdata does def set_shuffle(self, shuffle: bool, seed: int | None = None) -> None: """Sets whether iterating the dataset shuffles, and the shuffle seed. The order of each pass is determined by ``seed`` and the epoch (see ``set_epoch``), using a private generator, so the global random state is not used. Loaders from :func:`~pyhealth.datasets.get_dataloader` call this for you at the start of every pass. Args: shuffle: Whether ``iter(dataset)`` shuffles. seed: Seed of the shuffle order. Default: drawn from torch's random generator, so ``torch.manual_seed`` controls it. Examples: >>> dataset.set_shuffle(True, seed=0) # doctest: +SKIP """ self._shuffle = shuffle if shuffle: self._shuffle_seed = _draw_seed() if seed is None else int(seed) self._epoch = 1 def set_epoch(self, epoch: int) -> None: """Sets the epoch whose shuffle order the next pass uses. Iterating advances the epoch by one, so direct iteration gives a new order each pass. With worker processes, iteration happens on copies of the dataset, so the loader sets the epoch in the main process instead. Args: epoch: The epoch number (counted from 1, as litdata does). Examples: >>> dataset.set_epoch(2) # doctest: +SKIP """ self._epoch = int(epoch) def __len__(self) -> int: """Returns the number of samples in the dataset. Returns: int: The total number of samples. """ return len(self._data) def __getitem__(self, index: int) -> Dict[str, Any]: # type: ignore """Retrieve a processed sample by index. Args: index: The index of the sample to retrieve. Returns: A dictionary containing processed input and output features. """ return self._data[index] def __iter__(self) -> Iterable[Dict[str, Any]]: # type: ignore """Returns an iterator over all samples in the dataset. Returns: An iterator yielding processed sample dictionaries. """ if self._shuffle: rng = np.random.default_rng([self._shuffle_seed, self._epoch]) order = rng.permutation(len(self._data)) else: order = np.arange(len(self._data)) self._epoch += 1 # With DataLoader worker processes, each worker iterates its own copy # of the dataset; give each a disjoint share so no sample repeats. worker = torch.utils.data.get_worker_info() if worker is not None: order = order[worker.id :: worker.num_workers] # The order is fixed here, not lazily, so later changes to the # dataset's settings don't affect a pass already started. return (self._data[i] for i in order) def subset(self, indices: Union[Sequence[int], slice]) -> SampleDataset: if isinstance(indices, slice): subset_indices = list(range(*indices.indices(len(self)))) samples = self._data[indices] else: raw_indices = list(indices) samples = [self._data[i] for i in raw_indices] subset_indices = [ index if index >= 0 else len(self) + index for index in raw_indices ] new_dataset = copy.deepcopy(self) new_dataset._data = samples new_dataset.patient_to_index = _remap_index_mapping( self.patient_to_index, subset_indices ) new_dataset.record_to_index = _remap_index_mapping( self.record_to_index, subset_indices ) return new_dataset def close(self) -> None: pass # No temporary directories to clean up for in-memory dataset def _write_samples(builder: "SampleBuilder", samples: list[dict], path: Path) -> None: """Writes transformed samples to a litdata directory in this process. ``litdata.optimize`` starts helper processes even with ``num_workers=0``, and when the transform raises it leaves them running. They hold multiprocessing's resource-tracker pipe open, so a later tracker shutdown (another dataset build, or interpreter exit) waits forever on Linux. Writing with ``BinaryWriter`` here, as ``set_task`` does, starts no processes and lets errors propagate cleanly. """ from litdata.streaming.writer import BinaryWriter from .base_dataset import _litdata_merge writer = BinaryWriter(cache_dir=str(path), chunk_bytes="64MB") for i, sample in enumerate(samples): writer.add_item(i, builder.transform_for_storage({"sample": pickle.dumps(sample)})) writer.done() _litdata_merge(path) @preserve_rng_state() def create_sample_dataset( samples: List[Dict[str, Any]], input_schema: Dict[str, Any], output_schema: Dict[str, Any], dataset_name: Optional[str] = None, task_name: Optional[str] = None, input_processors: Optional[Dict[str, FeatureProcessor]] = None, output_processors: Optional[Dict[str, FeatureProcessor]] = None, in_memory: bool = True, ): """Convenience helper to create an on-disk SampleDataset from in-memory samples. This helper will: - Create a temporary directory for the dataset output. - Fit a `SampleBuilder` with the provided schemas and samples. - Save the fitted `schema.pkl` to the temporary directory. - Write the transformed samples into the directory with a litdata ``BinaryWriter``, in this process (no worker processes). - Return a `SampleDataset` instance pointed at the temporary directory. Args: samples: A list of Python dictionaries representing raw samples. input_schema: Schema describing how input keys should be handled. output_schema: Schema describing how output keys should be handled. dataset_name: Optional dataset name to attach to the returned SampleDataset instance. task_name: Optional task name to attach to the returned SampleDataset instance. input_processors: Optional pre-fitted input processors to use instead of creating new ones from the input_schema. output_processors: Optional pre-fitted output processors to use instead of creating new ones from the output_schema. in_memory: If True, returns an InMemorySampleDataset instead of a disk-backed SampleDataset. Returns: An instance of `SampleDataset` loaded from the temporary directory containing the optimized, chunked samples and `schema.pkl` metadata. Non-tensor fields (raw values, keys outside the schema) may vary in length and shape between samples; the disk-backed dataset returns them unchanged. Examples: >>> from pyhealth.datasets import create_sample_dataset >>> samples = [ ... {"patient_id": "p1", "codes": ["A", "B"], "times": [0.5, 1.0], "label": 1}, ... {"patient_id": "p2", "codes": ["C"], "times": [0.2], "label": 0}, ... ] >>> dataset = create_sample_dataset( ... samples, {"codes": "sequence", "times": "raw"}, {"label": "binary"} ... ) >>> dataset[1]["times"] [0.2] """ if in_memory: return InMemorySampleDataset( samples=samples, input_schema=input_schema, output_schema=output_schema, dataset_name=dataset_name, task_name=task_name, input_processors=input_processors, output_processors=output_processors, ) else: path = Path(tempfile.mkdtemp()) builder = SampleBuilder( input_schema=input_schema, # type: ignore output_schema=output_schema, # type: ignore input_processors=input_processors, output_processors=output_processors, ) builder.fit(samples) builder.save(str(path / "schema.pkl")) _write_samples(builder, samples, path) return SampleDataset( path=str(path), dataset_name=dataset_name, task_name=task_name, delete_on_close=True, )