Remove legacy event and mixed distribution paths
This commit is contained in:
@@ -4,7 +4,7 @@ This script converts raw UK Biobank CSV exports into the artefacts consumed by
|
||||
DeepHealth:
|
||||
|
||||
* ``ukb_event_data.npy``: ``(N, 3)`` uint32 array of ``(eid, days, label)``
|
||||
disease/death/checkup events sorted by patient then time.
|
||||
disease/death events sorted by patient then time.
|
||||
* ``ukb_basic_info.csv``: basic patient table indexed by ``eid`` with ``sex``.
|
||||
* ``ukb_other_info.npy``: ``(M, 5)`` float64 array of
|
||||
``(eid, type, value, value_kind, time)`` rows. ``type=0`` is reserved for
|
||||
@@ -219,7 +219,8 @@ with open(labels_file, encoding="utf-8") as f: # Open labels file
|
||||
for idx, line in enumerate(f): # Enumerate to assign incremental label IDs
|
||||
parts = line.strip().split(" ") # Split by space
|
||||
if parts and parts[0]: # Guard against empty lines
|
||||
# Start labels from 1 to reserve 0 for padding, 1 for checkup
|
||||
# Keep raw disease ids at 2+ so existing prepared data and model
|
||||
# vocabulary indices remain stable; raw id 1 is unused.
|
||||
label_dict[parts[0]] = idx + 2
|
||||
|
||||
# Pre-build lookup: ICD/Death column name -> integer label for fast per-column extraction
|
||||
@@ -327,19 +328,6 @@ for ukb_chunk in tqdm.tqdm(ukb_iterator, desc="Processing UK Biobank data"):
|
||||
if cancer_frames:
|
||||
event_list.append(np.vstack(cancer_frames))
|
||||
|
||||
# Add checkup events with label=1 using date_of_assessment (already in days from dob)
|
||||
if "date_of_assessment" in ukb_chunk.columns:
|
||||
doa_series = ukb_chunk["date_of_assessment"].dropna()
|
||||
if not doa_series.empty:
|
||||
checkup_data = np.column_stack(
|
||||
(
|
||||
doa_series.index.values,
|
||||
doa_series.values.astype(int),
|
||||
np.ones(len(doa_series), dtype=int),
|
||||
)
|
||||
)
|
||||
event_list.append(checkup_data)
|
||||
|
||||
# Combine tabular chunks
|
||||
final_tabular = pd.concat(tabular_list, axis=0, ignore_index=False)
|
||||
final_tabular.index.name = "eid" # Ensure index named consistently
|
||||
|
||||
Reference in New Issue
Block a user