Debian Codegraph @

polsarpro/dev/metrics.py

raw
1 import warnings
2 from collections import OrderedDict
3
4 import numpy as np
5 import pandas as pd
6 import xarray
7
8 # these are convenience function used only for development
9
10
11 # returns a dict with various error metrics for two arrays (predicted and reference)
12 def compute_error_metrics(arr_pred, arr_ref):
13 stats = OrderedDict()
14
15 eps = 1e-30
16
17 # simple error
18 err = arr_pred - arr_ref
19
20 # squared and absolute errors
21 se = err**2
22 ae = abs(err)
23
24 # normalized absolute error (between 0 and 1)
25 naerr = ae / (eps + abs(arr_pred) + abs(arr_ref))
26
27 # ---- Global statistics
28
29 dm = np.nanmean(arr_pred) - np.nanmean(arr_ref)
30 item = {"value": dm, "descr": "Difference of Means"}
31 stats["dm"] = item
32
33 rmse = np.sqrt(np.nanmean(se) + eps)
34 item = {"value": rmse, "descr": "Root Mean Square Error"}
35 stats["rmse"] = item
36
37 bias = np.nanmean(err)
38 item = {"value": bias, "descr": "Mean Error (bias)"}
39 stats["bias"] = item
40
41 mae = np.nanmean(ae)
42 item = {"value": mae, "descr": "Mean Absolute Error"}
43 stats["mae"] = item
44
45 mnae = np.nanmean(naerr)
46 item = {"value": mnae, "descr": "Mean Normalized Absolute Error"}
47 stats["mnae"] = item
48
49 p99 = np.nanpercentile(naerr, q=99)
50 item = {"value": p99, "descr": "99% percentile of the Normalized Absolute Error"}
51 stats["p99"] = item
52
53 p90 = np.nanpercentile(naerr, q=90)
54 item = {"value": p90, "descr": "90% percentile of the Normalized Absolute Error"}
55 stats["p90"] = item
56
57 p50 = np.nanpercentile(naerr, q=50)
58 item = {"value": p50, "descr": "50% percentile of the Normalized Absolute Error"}
59 stats["p50"] = item
60
61 return stats
62
63
64 # returns a dataframe that summarizes statistics for all variables
65 def summarize_metrics(
66 out_py: xarray.Dataset,
67 out_c: dict,
68 short_titles: bool = False,
69 verbose: bool = True,
70 ):
71
72 data_vars = list(out_py.data_vars)
73
74 # dataframes used to collect results and to merge
75 dfs = []
76
77 for var in data_vars:
78 # check that python variable appears in C data
79 if var not in out_c:
80 warnings.warn(f"Skipping variable '{var}'! Not found in C outputs.")
81 continue
82 else:
83 if verbose:
84 print(f"Computing variable '{var}'.")
85 stats = compute_error_metrics(out_py[var], out_c[var])
86 if short_titles:
87 tmp = {k: [stats[k]["value"]] for k in stats}
88 else:
89 tmp = {stats[k]["descr"]: [stats[k]["value"]] for k in stats}
90 df = pd.DataFrame.from_dict(tmp, orient="columns")
91 df.index = [var]
92 dfs.append(df)
93
94 return pd.concat(dfs)
95
96
97 # helper function to visualize errors
98 def visualize_errors(out_py, out_c, clip=True, sub_az=8, sub_rg=1):
99 import matplotlib.pyplot as plt
100
101 for var in out_py.data_vars:
102 if var not in out_c:
103 Warning(f"Skipping variable '{var}'! Not found in C outputs.")
104 if np.iscomplexobj(out_py[var]):
105 img_c = np.abs(out_c[var])
106 img_py = np.abs(out_py[var])
107 err = abs(out_py[var] - out_c[var])
108 t_err = "Absolute difference: python - C"
109 else:
110 img_c = out_c[var]
111 img_py = out_py[var]
112 err = out_py[var] - out_c[var]
113 t_err = "Difference: python - C"
114
115 if clip:
116 m = 0.5 * (np.nanmean(img_py) + np.nanmean(img_py))
117 img_c = img_c.clip(0, 2 * m)
118 img_py = img_py.clip(0, 2 * m)
119
120 plt.figure(figsize=(10, 6))
121 plt.suptitle(var)
122 plt.subplot(131)
123 plt.imshow(err[::sub_az, ::sub_rg], interpolation="none")
124 plt.title(t_err)
125 plt.axis("off")
126 plt.colorbar(fraction=0.046, pad=0.04, location="bottom")
127 plt.subplot(132)
128 plt.imshow(img_c[::sub_az, ::sub_rg], interpolation="none")
129 plt.title("C")
130 plt.axis("off")
131 plt.colorbar(fraction=0.046, pad=0.04, location="bottom")
132 plt.subplot(133)
133 plt.imshow(img_py[::sub_az, ::sub_rg], interpolation="none")
134 plt.colorbar(fraction=0.046, pad=0.04, location="bottom")
135 plt.title("python")
136 plt.axis("off")
137 plt.tight_layout()