Remove legacy event and mixed distribution paths

This commit is contained in:
2026-08-01 14:23:18 +08:00
parent dfb22adf2d
commit de6f9b75b9
22 changed files with 370 additions and 463 deletions

View File

@@ -4,7 +4,7 @@ This script converts raw UK Biobank CSV exports into the artefacts consumed by
DeepHealth:
* ``ukb_event_data.npy``: ``(N, 3)`` uint32 array of ``(eid, days, label)``
disease/death/checkup events sorted by patient then time.
disease/death events sorted by patient then time.
* ``ukb_basic_info.csv``: basic patient table indexed by ``eid`` with ``sex``.
* ``ukb_other_info.npy``: ``(M, 5)`` float64 array of
``(eid, type, value, value_kind, time)`` rows. ``type=0`` is reserved for
@@ -219,7 +219,8 @@ with open(labels_file, encoding="utf-8") as f: # Open labels file
for idx, line in enumerate(f): # Enumerate to assign incremental label IDs
parts = line.strip().split(" ") # Split by space
if parts and parts[0]: # Guard against empty lines
# Start labels from 1 to reserve 0 for padding, 1 for checkup
# Keep raw disease ids at 2+ so existing prepared data and model
# vocabulary indices remain stable; raw id 1 is unused.
label_dict[parts[0]] = idx + 2
# Pre-build lookup: ICD/Death column name -> integer label for fast per-column extraction
@@ -327,19 +328,6 @@ for ukb_chunk in tqdm.tqdm(ukb_iterator, desc="Processing UK Biobank data"):
if cancer_frames:
event_list.append(np.vstack(cancer_frames))
# Add checkup events with label=1 using date_of_assessment (already in days from dob)
if "date_of_assessment" in ukb_chunk.columns:
doa_series = ukb_chunk["date_of_assessment"].dropna()
if not doa_series.empty:
checkup_data = np.column_stack(
(
doa_series.index.values,
doa_series.values.astype(int),
np.ones(len(doa_series), dtype=int),
)
)
event_list.append(checkup_data)
# Combine tabular chunks
final_tabular = pd.concat(tabular_list, axis=0, ignore_index=False)
final_tabular.index.name = "eid" # Ensure index named consistently