updating hyperoptplot for new hyperopt keys

Author: tgiani

validphys2/src/validphys/hyperoptplot.py

@@ -301,24 +301,18 @@ def parse_statistics(trial):
     testing loss
     status of the run
     """
+    hypr_keys = ["hyper_losses_chi2", "hyper_losses_phi2", "hyper_losses_logp", "trvl_losses_chi2exp"]
     dict_out = {}
     results = trial["result"]
     validation_loss = results[KEYWORDS["vl"]]
     testing_loss = results[KEYWORDS["tl"]]
-    # was this a ok run?
-    ok = bool(results["status"] == "ok")
 
-    dict_out[KEYWORDS["good"]] = ok
+    dict_out[KEYWORDS["good"]] = bool(results["status"] == "ok")
     dict_out[KEYWORDS["vl"]] = validation_loss
     dict_out[KEYWORDS["tl"]] = testing_loss
 
-    # Kfolding information
-    # average = results["kfold_meta"]["hyper_avg"]
-    # std = results["kfold_meta"]["hyper_std"]
-    # dict_out["avg"] = average
-    # dict_out["std"] = std
-    dict_out["hlosses"] = results["kfold_meta"]["hyper_losses"]
-    dict_out["vlosses"] = results["kfold_meta"]["validation_losses"]
+    for key in hypr_keys:  # K-folding information
+        dict_out[key] = results["kfold_meta"][key]
     return dict_out
 
 
@@ -344,27 +338,34 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe
     """
     Read a trial dictionary and compute the true loss and decide whether the run passes or not
     """
-    test_f = 1.0 - validation_multiplier
-    val_loss = float(trial_dict[KEYWORDS["vl"]])
+    hypr_metric_keys = ["chi2", "phi2", "logp"]
     if loss_target == "average":
-        test_loss = np.array(trial_dict["hlosses"]).mean()
+        for hypr_key in hypr_metric_keys:
+            trial_dict[f"hyper_loss_{hypr_key}"] = np.array(
+                trial_dict[f"hyper_losses_{hypr_key}"]
+            ).mean()
     elif loss_target == "best_worst":
-        test_loss = np.array(trial_dict["hlosses"]).max()
+        for hypr_key in hypr_metric_keys:
+            trial_dict[f"hyper_loss_{hypr_key}"] = np.array(
+                trial_dict[f"hyper_losses_{hypr_key}"]
+            ).max()
     elif loss_target == "std":
-        test_loss = np.array(trial_dict["hlosses"]).std()
-    loss = val_loss * validation_multiplier + test_loss * test_f
+        for hypr_key in hypr_metric_keys:
+            trial_dict[f"hyper_loss_{hypr_key}"] = np.array(
+                trial_dict[f"hyper_losses_{hypr_key}"]
+            ).std()
+    else:
+        raise ValueError(f"Loss target {loss_target} is not valid.")
 
-    if (
-        loss > fail_threshold
-        or val_loss > fail_threshold
-        or test_loss > fail_threshold
-        or np.isnan(loss)
-    ):
-        trial_dict["good"] = False
-        # Set the loss an order of magnitude above the result so it shows obviously on the plots
-        loss *= 10
+    for hypr_key in hypr_metric_keys:
+        if np.isnan(trial_dict[f"hyper_loss_{hypr_key}"]):
+            trial_dict[f"hyper_loss_{hypr_key}"] *= 100
+    
+    trial_dict["trvl_loss_chi2exp"] = np.array(
+                trial_dict["trvl_losses_chi2exp"]
+            ).mean()
 
-    trial_dict["loss"] = loss
+    return
 
 
 def generate_dictionary(
@@ -543,10 +544,6 @@ def hyperopt_dataframe(commandline_args):
     # Make into a dataframe and transpose or the plotting code will complain
     best_trial = best_trial_series.to_frame().T
 
-    log.info("Best setup:")
-    with pd.option_context("display.max_rows", None, "display.max_columns", None):
-        log.info(best_trial)
-
     return dataframe, best_trial
 
 
@@ -585,8 +582,13 @@ def hyperopt_table(hyperopt_dataframe):
     Generates a table containing complete information on all the tested setups that passed the
     filters set in the commandline arguments.
     """
+    drop_keys = ["hyper_losses_chi2", "hyper_losses_phi2", "hyper_losses_logp"]
+    drop_keys += [f"layer_{idx}" for idx in range(1, 5)]
+    drop_keys += ["activation_per_layer_0"]
+
     dataframe, _ = hyperopt_dataframe
-    dataframe = dataframe.sort_values(by=["loss"])
+    dataframe.drop(columns=drop_keys, inplace=True)
+    dataframe.sort_values(by=["hyper_loss_chi2"], inplace=True)
     return dataframe