| 1 |
import warnings |
| 2 |
from collections import OrderedDict |
| 3 |
|
| 4 |
import numpy as np |
| 5 |
import pandas as pd |
| 6 |
import xarray |
| 7 |
|
| 8 |
# these are convenience function used only for development |
| 9 |
|
| 10 |
|
| 11 |
# returns a dict with various error metrics for two arrays (predicted and reference) |
| 12 |
def compute_error_metrics(arr_pred, arr_ref): |
| 13 |
stats = OrderedDict() |
| 14 |
|
| 15 |
eps = 1e-30 |
| 16 |
|
| 17 |
# simple error |
| 18 |
err = arr_pred - arr_ref |
| 19 |
|
| 20 |
# squared and absolute errors |
| 21 |
se = err**2 |
| 22 |
ae = abs(err) |
| 23 |
|
| 24 |
# normalized absolute error (between 0 and 1) |
| 25 |
naerr = ae / (eps + abs(arr_pred) + abs(arr_ref)) |
| 26 |
|
| 27 |
# ---- Global statistics |
| 28 |
|
| 29 |
dm = np.nanmean(arr_pred) - np.nanmean(arr_ref) |
| 30 |
item = {"value": dm, "descr": "Difference of Means"} |
| 31 |
stats["dm"] = item |
| 32 |
|
| 33 |
rmse = np.sqrt(np.nanmean(se) + eps) |
| 34 |
item = {"value": rmse, "descr": "Root Mean Square Error"} |
| 35 |
stats["rmse"] = item |
| 36 |
|
| 37 |
bias = np.nanmean(err) |
| 38 |
item = {"value": bias, "descr": "Mean Error (bias)"} |
| 39 |
stats["bias"] = item |
| 40 |
|
| 41 |
mae = np.nanmean(ae) |
| 42 |
item = {"value": mae, "descr": "Mean Absolute Error"} |
| 43 |
stats["mae"] = item |
| 44 |
|
| 45 |
mnae = np.nanmean(naerr) |
| 46 |
item = {"value": mnae, "descr": "Mean Normalized Absolute Error"} |
| 47 |
stats["mnae"] = item |
| 48 |
|
| 49 |
p99 = np.nanpercentile(naerr, q=99) |
| 50 |
item = {"value": p99, "descr": "99% percentile of the Normalized Absolute Error"} |
| 51 |
stats["p99"] = item |
| 52 |
|
| 53 |
p90 = np.nanpercentile(naerr, q=90) |
| 54 |
item = {"value": p90, "descr": "90% percentile of the Normalized Absolute Error"} |
| 55 |
stats["p90"] = item |
| 56 |
|
| 57 |
p50 = np.nanpercentile(naerr, q=50) |
| 58 |
item = {"value": p50, "descr": "50% percentile of the Normalized Absolute Error"} |
| 59 |
stats["p50"] = item |
| 60 |
|
| 61 |
return stats |
| 62 |
|
| 63 |
|
| 64 |
# returns a dataframe that summarizes statistics for all variables |
| 65 |
def summarize_metrics( |
| 66 |
out_py: xarray.Dataset, |
| 67 |
out_c: dict, |
| 68 |
short_titles: bool = False, |
| 69 |
verbose: bool = True, |
| 70 |
): |
| 71 |
|
| 72 |
data_vars = list(out_py.data_vars) |
| 73 |
|
| 74 |
# dataframes used to collect results and to merge |
| 75 |
dfs = [] |
| 76 |
|
| 77 |
for var in data_vars: |
| 78 |
# check that python variable appears in C data |
| 79 |
if var not in out_c: |
| 80 |
warnings.warn(f"Skipping variable '{var}'! Not found in C outputs.") |
| 81 |
continue |
| 82 |
else: |
| 83 |
if verbose: |
| 84 |
print(f"Computing variable '{var}'.") |
| 85 |
stats = compute_error_metrics(out_py[var], out_c[var]) |
| 86 |
if short_titles: |
| 87 |
tmp = {k: [stats[k]["value"]] for k in stats} |
| 88 |
else: |
| 89 |
tmp = {stats[k]["descr"]: [stats[k]["value"]] for k in stats} |
| 90 |
df = pd.DataFrame.from_dict(tmp, orient="columns") |
| 91 |
df.index = [var] |
| 92 |
dfs.append(df) |
| 93 |
|
| 94 |
return pd.concat(dfs) |
| 95 |
|
| 96 |
|
| 97 |
# helper function to visualize errors |
| 98 |
def visualize_errors(out_py, out_c, clip=True, sub_az=8, sub_rg=1): |
| 99 |
import matplotlib.pyplot as plt |
| 100 |
|
| 101 |
for var in out_py.data_vars: |
| 102 |
if var not in out_c: |
| 103 |
Warning(f"Skipping variable '{var}'! Not found in C outputs.") |
| 104 |
if np.iscomplexobj(out_py[var]): |
| 105 |
img_c = np.abs(out_c[var]) |
| 106 |
img_py = np.abs(out_py[var]) |
| 107 |
err = abs(out_py[var] - out_c[var]) |
| 108 |
t_err = "Absolute difference: python - C" |
| 109 |
else: |
| 110 |
img_c = out_c[var] |
| 111 |
img_py = out_py[var] |
| 112 |
err = out_py[var] - out_c[var] |
| 113 |
t_err = "Difference: python - C" |
| 114 |
|
| 115 |
if clip: |
| 116 |
m = 0.5 * (np.nanmean(img_py) + np.nanmean(img_py)) |
| 117 |
img_c = img_c.clip(0, 2 * m) |
| 118 |
img_py = img_py.clip(0, 2 * m) |
| 119 |
|
| 120 |
plt.figure(figsize=(10, 6)) |
| 121 |
plt.suptitle(var) |
| 122 |
plt.subplot(131) |
| 123 |
plt.imshow(err[::sub_az, ::sub_rg], interpolation="none") |
| 124 |
plt.title(t_err) |
| 125 |
plt.axis("off") |
| 126 |
plt.colorbar(fraction=0.046, pad=0.04, location="bottom") |
| 127 |
plt.subplot(132) |
| 128 |
plt.imshow(img_c[::sub_az, ::sub_rg], interpolation="none") |
| 129 |
plt.title("C") |
| 130 |
plt.axis("off") |
| 131 |
plt.colorbar(fraction=0.046, pad=0.04, location="bottom") |
| 132 |
plt.subplot(133) |
| 133 |
plt.imshow(img_py[::sub_az, ::sub_rg], interpolation="none") |
| 134 |
plt.colorbar(fraction=0.046, pad=0.04, location="bottom") |
| 135 |
plt.title("python") |
| 136 |
plt.axis("off") |
| 137 |
plt.tight_layout() |