Load exposure cache into memory
This commit is contained in:
26
dataset.py
26
dataset.py
@@ -41,6 +41,12 @@ def _monthly_exposure_columns() -> list[str]:
|
||||
return cols
|
||||
|
||||
|
||||
def _load_readonly_npy(path: Path) -> np.ndarray:
|
||||
arr = np.load(path)
|
||||
arr.setflags(write=False)
|
||||
return arr
|
||||
|
||||
|
||||
class ExposureCache:
|
||||
"""Eid-sequence-aligned exposure windows from prepare_exposure_cache.py."""
|
||||
|
||||
@@ -86,17 +92,17 @@ class ExposureCache:
|
||||
"Regenerate it with the current prepare_exposure_cache.py."
|
||||
)
|
||||
|
||||
self.eids = np.load(eid_path, mmap_mode="r")
|
||||
self.raw_tokens = np.load(token_path, mmap_mode="r")
|
||||
self.age_days = np.load(age_path, mmap_mode="r")
|
||||
self.onset_dates = np.load(onset_date_path, mmap_mode="r")
|
||||
self.row_index = np.load(row_index_path, mmap_mode="r")
|
||||
self.eid_index = np.load(eid_index_path, mmap_mode="r")
|
||||
self.eid_start = np.load(eid_start_path, mmap_mode="r")
|
||||
self.daily = np.load(daily_path, mmap_mode="r")
|
||||
self.monthly = np.load(monthly_path, mmap_mode="r")
|
||||
self.eids = _load_readonly_npy(eid_path)
|
||||
self.raw_tokens = _load_readonly_npy(token_path)
|
||||
self.age_days = _load_readonly_npy(age_path)
|
||||
self.onset_dates = _load_readonly_npy(onset_date_path)
|
||||
self.row_index = _load_readonly_npy(row_index_path)
|
||||
self.eid_index = _load_readonly_npy(eid_index_path)
|
||||
self.eid_start = _load_readonly_npy(eid_start_path)
|
||||
self.daily = _load_readonly_npy(daily_path)
|
||||
self.monthly = _load_readonly_npy(monthly_path)
|
||||
quality_path = cache_dir / "exposure_quality.npy"
|
||||
self.quality = np.load(quality_path, mmap_mode="r") if quality_path.is_file() else None
|
||||
self.quality = _load_readonly_npy(quality_path) if quality_path.is_file() else None
|
||||
|
||||
if self.daily.ndim != 3 or self.daily.shape[1:] != DAILY_EXPOSURE_SHAPE:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user