Load exposure cache into memory
This commit is contained in:
26
dataset.py
26
dataset.py
@@ -41,6 +41,12 @@ def _monthly_exposure_columns() -> list[str]:
|
|||||||
return cols
|
return cols
|
||||||
|
|
||||||
|
|
||||||
|
def _load_readonly_npy(path: Path) -> np.ndarray:
|
||||||
|
arr = np.load(path)
|
||||||
|
arr.setflags(write=False)
|
||||||
|
return arr
|
||||||
|
|
||||||
|
|
||||||
class ExposureCache:
|
class ExposureCache:
|
||||||
"""Eid-sequence-aligned exposure windows from prepare_exposure_cache.py."""
|
"""Eid-sequence-aligned exposure windows from prepare_exposure_cache.py."""
|
||||||
|
|
||||||
@@ -86,17 +92,17 @@ class ExposureCache:
|
|||||||
"Regenerate it with the current prepare_exposure_cache.py."
|
"Regenerate it with the current prepare_exposure_cache.py."
|
||||||
)
|
)
|
||||||
|
|
||||||
self.eids = np.load(eid_path, mmap_mode="r")
|
self.eids = _load_readonly_npy(eid_path)
|
||||||
self.raw_tokens = np.load(token_path, mmap_mode="r")
|
self.raw_tokens = _load_readonly_npy(token_path)
|
||||||
self.age_days = np.load(age_path, mmap_mode="r")
|
self.age_days = _load_readonly_npy(age_path)
|
||||||
self.onset_dates = np.load(onset_date_path, mmap_mode="r")
|
self.onset_dates = _load_readonly_npy(onset_date_path)
|
||||||
self.row_index = np.load(row_index_path, mmap_mode="r")
|
self.row_index = _load_readonly_npy(row_index_path)
|
||||||
self.eid_index = np.load(eid_index_path, mmap_mode="r")
|
self.eid_index = _load_readonly_npy(eid_index_path)
|
||||||
self.eid_start = np.load(eid_start_path, mmap_mode="r")
|
self.eid_start = _load_readonly_npy(eid_start_path)
|
||||||
self.daily = np.load(daily_path, mmap_mode="r")
|
self.daily = _load_readonly_npy(daily_path)
|
||||||
self.monthly = np.load(monthly_path, mmap_mode="r")
|
self.monthly = _load_readonly_npy(monthly_path)
|
||||||
quality_path = cache_dir / "exposure_quality.npy"
|
quality_path = cache_dir / "exposure_quality.npy"
|
||||||
self.quality = np.load(quality_path, mmap_mode="r") if quality_path.is_file() else None
|
self.quality = _load_readonly_npy(quality_path) if quality_path.is_file() else None
|
||||||
|
|
||||||
if self.daily.ndim != 3 or self.daily.shape[1:] != DAILY_EXPOSURE_SHAPE:
|
if self.daily.ndim != 3 or self.daily.shape[1:] != DAILY_EXPOSURE_SHAPE:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
|
|||||||
Reference in New Issue
Block a user