Parallelize extra-info summary reduction

run_missing_evaluations.sh

@@ -10,6 +10,7 @@ PYTHON_BIN="${PYTHON_BIN:-python}"
 DEVICE="${DEVICE:-cuda}"
 EVAL_SPLIT="${EVAL_SPLIT:-test}"
 NUM_WORKERS="${NUM_WORKERS:-4}"
+CPU_REDUCE_WORKERS="${CPU_REDUCE_WORKERS:-}"
 NUM_WORKERS_AUC="${NUM_WORKERS_AUC:-}"
 BATCH_SIZE="${BATCH_SIZE:-}"
 DATASET_SUBSET_SIZE="${DATASET_SUBSET_SIZE:-}"
@@ -41,6 +42,12 @@ auc_args() {
   fi
 }
 
+cpu_reduce_args() {
+  if [[ -n "${CPU_REDUCE_WORKERS}" ]]; then
+    printf '%s\n' --cpu_reduce_workers "${CPU_REDUCE_WORKERS}"
+  fi
+}
+
 has_completed_dir() {
   local dir="$1"
   shift
@@ -127,6 +134,9 @@ for run_path in runs/*; do
   auc_extra=()
   while IFS= read -r arg; do auc_extra+=("${arg}"); done < <(auc_args)
 
+  cpu_reduce_extra=()
+  while IFS= read -r arg; do cpu_reduce_extra+=("${arg}"); done < <(cpu_reduce_args)
+
   run_dir_result_if_missing \
     "evaluate_auc.py" \
     "${run_path}" \
@@ -153,7 +163,7 @@ for run_path in runs/*; do
         "${run_path}/extra_info_attribution_${EVAL_SPLIT}" \
         "manifest.json" \
         "summary_extra_info_disease_parameters.csv" \
-        "${PYTHON_BIN}" evaluate_extra_info_attribution.py "${common[@]}"
+        "${PYTHON_BIN}" evaluate_extra_info_attribution.py "${common[@]}" "${cpu_reduce_extra[@]}"
     else
       echo "    skip evaluate_extra_info_attribution.py: run has no extra-info types"
     fi