Remove extra info token pathway
This commit is contained in:
126
prepare_data.py
126
prepare_data.py
@@ -6,13 +6,6 @@ DeepHealth:
|
||||
* ``ukb_event_data.npy``: ``(N, 3)`` uint32 array of ``(eid, days, label)``
|
||||
disease/death/checkup events sorted by patient then time.
|
||||
* ``ukb_basic_info.csv``: basic patient table indexed by ``eid`` with ``sex``.
|
||||
* ``ukb_other_info.npy``: ``(M, 5)`` float64 array of
|
||||
``(eid, type, value, value_kind, time)`` rows. ``type=0`` is reserved for
|
||||
padding, ``value_kind=1`` means continuous, and ``value_kind=2`` means
|
||||
categorical.
|
||||
* ``cate_types.csv``: categorical-variable metadata with
|
||||
``type,name,n_categories``. Dataset code computes global category ids after
|
||||
experiment-specific variable selection.
|
||||
|
||||
Processing steps
|
||||
----------------
|
||||
@@ -21,10 +14,7 @@ Processing steps
|
||||
3. Extract ICD-10 date fields and cancer date/type fields into long-form
|
||||
events and map codes to integer labels via ``labels.csv``.
|
||||
4. De-duplicate events per ``(eid, label)`` keeping the earliest occurrence.
|
||||
5. Convert available non-sex tabular fields into unified other-information
|
||||
tokens timestamped by ``date_of_assessment``.
|
||||
6. Write event data, sex, unified other-information tokens, and categorical
|
||||
type metadata.
|
||||
5. Write event data and sex.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -45,108 +35,6 @@ import pandas as pd # Pandas for data manipulation
|
||||
import tqdm # Progress bar for chunk processing
|
||||
|
||||
|
||||
CONT_VALUE_KIND = 1
|
||||
CATE_VALUE_KIND = 2
|
||||
|
||||
|
||||
def _sort_values(values):
|
||||
"""Sort mixed pandas/numpy scalar values deterministically."""
|
||||
try:
|
||||
return sorted(values)
|
||||
except TypeError:
|
||||
return sorted(values, key=lambda x: str(x))
|
||||
|
||||
|
||||
def _build_other_info_tokens(
|
||||
table: pd.DataFrame,
|
||||
feature_fields: list[str],
|
||||
*,
|
||||
time_col: str = "date_of_assessment",
|
||||
) -> tuple[np.ndarray, pd.DataFrame]:
|
||||
"""Convert wide tabular features into (eid, type, value, kind, time) rows."""
|
||||
rows = []
|
||||
cate_meta = []
|
||||
present_features = [
|
||||
col for col in feature_fields
|
||||
if col in table.columns and col not in {time_col, "sex"}
|
||||
]
|
||||
|
||||
if time_col not in table.columns:
|
||||
raise ValueError(
|
||||
f"{time_col!r} is required to timestamp other-information tokens"
|
||||
)
|
||||
|
||||
token_times = pd.to_numeric(table[time_col], errors="coerce")
|
||||
|
||||
for type_idx, col in enumerate(present_features, start=1):
|
||||
series = table[col]
|
||||
n_unique = series.dropna().nunique()
|
||||
valid_time = token_times.notna()
|
||||
|
||||
if n_unique > 10:
|
||||
numeric = pd.to_numeric(series, errors="coerce")
|
||||
valid = numeric.notna() & valid_time
|
||||
if not valid.any():
|
||||
continue
|
||||
rows.append(
|
||||
np.column_stack(
|
||||
(
|
||||
table.index[valid].to_numpy(dtype=np.float64),
|
||||
np.full(valid.sum(), type_idx, dtype=np.float64),
|
||||
numeric[valid].to_numpy(dtype=np.float64),
|
||||
np.full(valid.sum(), CONT_VALUE_KIND, dtype=np.float64),
|
||||
token_times[valid].to_numpy(dtype=np.float64),
|
||||
)
|
||||
)
|
||||
)
|
||||
else:
|
||||
unique_vals = _sort_values(series.dropna().unique())
|
||||
value_map = {val: idx + 1 for idx, val in enumerate(unique_vals)}
|
||||
mapped = series.map(value_map)
|
||||
valid = mapped.notna() & valid_time
|
||||
n_categories = len(unique_vals)
|
||||
cate_meta.append(
|
||||
{
|
||||
"type": type_idx,
|
||||
"name": col,
|
||||
"n_categories": n_categories,
|
||||
}
|
||||
)
|
||||
if not valid.any():
|
||||
continue
|
||||
rows.append(
|
||||
np.column_stack(
|
||||
(
|
||||
table.index[valid].to_numpy(dtype=np.float64),
|
||||
np.full(valid.sum(), type_idx, dtype=np.float64),
|
||||
mapped[valid].to_numpy(dtype=np.float64),
|
||||
np.full(valid.sum(), CATE_VALUE_KIND, dtype=np.float64),
|
||||
token_times[valid].to_numpy(dtype=np.float64),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
cate_types = pd.DataFrame(
|
||||
cate_meta,
|
||||
columns=["type", "name", "n_categories"],
|
||||
)
|
||||
if not rows:
|
||||
return np.empty((0, 5), dtype=np.float64), cate_types
|
||||
out = np.vstack(rows)
|
||||
return out[np.lexsort((out[:, 3], out[:, 1], out[:, 0]))], cate_types
|
||||
|
||||
|
||||
def _unique_preserve_order(values):
|
||||
"""Return unique values while preserving first-seen order."""
|
||||
seen = set()
|
||||
out = []
|
||||
for value in values:
|
||||
if value not in seen:
|
||||
seen.add(value)
|
||||
out.append(value)
|
||||
return out
|
||||
|
||||
|
||||
# CSV mapping field IDs to human-readable names
|
||||
field_map_file = "field_ids_enriched.csv"
|
||||
|
||||
@@ -369,17 +257,5 @@ final_tabular = final_tabular.convert_dtypes()
|
||||
basic_info = final_tabular[["sex"]].copy()
|
||||
basic_info.to_csv("ukb_basic_info.csv")
|
||||
|
||||
# Save unified other-information tokens. Missing values simply produce no token.
|
||||
other_info_fields = _unique_preserve_order(
|
||||
basic_info_fields + assessment_fields + exposure_fields
|
||||
)
|
||||
other_info, cate_types = _build_other_info_tokens(
|
||||
final_tabular,
|
||||
other_info_fields,
|
||||
time_col="date_of_assessment",
|
||||
)
|
||||
np.save("ukb_other_info.npy", other_info)
|
||||
cate_types.to_csv("cate_types.csv", index=False)
|
||||
|
||||
# Save event data
|
||||
np.save("ukb_event_data.npy", data)
|
||||
|
||||
Reference in New Issue
Block a user