clarena.pipelines.cl_full_eval
The submodule in pipelines for continual learning full evaluation.
1r""" 2The submodule in `pipelines` for continual learning full evaluation. 3""" 4 5__all__ = ["CLFullEvaluation"] 6 7 8import logging 9import os 10 11import pandas as pd 12from matplotlib import pyplot as plt 13from omegaconf import DictConfig 14 15# always get logger for built-in logging in each module 16pylogger = logging.getLogger(__name__) 17 18 19class CLFullEvaluation: 20 r"""The base class for continual learning full evaluation.""" 21 22 def __init__(self, cfg: DictConfig) -> None: 23 r""" 24 **Args:** 25 - **cfg** (`DictConfig`): the config dict for the continual learning full evaluation. 26 """ 27 self.cfg: DictConfig = cfg 28 r"""The complete config dict.""" 29 30 CLFullEvaluation.sanity_check(self) 31 32 self.eval_tasks: list[int] = ( 33 cfg.eval_tasks 34 if isinstance(cfg.eval_tasks, list) 35 else list(range(1, cfg.eval_tasks + 1)) 36 ) 37 r"""The list of task IDs to evaluate.""" 38 39 self.main_acc_csv_path: str = cfg.main_acc_csv_path 40 r"""The path to the main experiment accuracy CSV file.""" 41 if cfg.get("refjoint_acc_csv_path"): 42 self.refjoint_acc_csv_path: str = cfg.refjoint_acc_csv_path 43 r"""The path to the reference joint learning experiment accuracy CSV file.""" 44 if cfg.get("refindependent_acc_csv_path"): 45 self.refindependent_acc_csv_path: str = cfg.refindependent_acc_csv_path 46 r"""The path to the reference independent learning experiment accuracy CSV file.""" 47 if cfg.get("refrandom_acc_csv_path"): 48 self.refrandom_acc_csv_path: str = cfg.refrandom_acc_csv_path 49 r"""The path to the reference random learning experiment accuracy CSV file.""" 50 51 self.output_dir: str = cfg.output_dir 52 r"""The folder for storing the evaluation results.""" 53 54 self.bwt_save_dir: str = cfg.bwt_save_dir 55 r"""The folder storing the BWT metric results.""" 56 if cfg.get("fwt_save_dir"): 57 self.fwt_save_dir: str | None = cfg.fwt_save_dir 58 r"""The folder storing the FWT metric results.""" 59 if cfg.get("fr_save_dir"): 60 self.fr_save_dir: str | None = cfg.fr_save_dir 61 r"""The folder storing the FR metric results.""" 62 63 self.bwt_csv_path: str = os.path.join(self.bwt_save_dir, cfg.bwt_csv_name) 64 r"""The file path to store the BWT metrics as CSV file.""" 65 if cfg.get("fwt_csv_name"): 66 self.fwt_csv_path: str | None = os.path.join( 67 self.fwt_save_dir, cfg.fwt_csv_name 68 ) 69 r"""The file path to store the FWT metrics as CSV file.""" 70 if cfg.get("fr_csv_name"): 71 self.fr_csv_path: str | None = os.path.join( 72 self.fr_save_dir, cfg.fr_csv_name 73 ) 74 r"""The file path to store the FR metrics as CSV file.""" 75 76 if cfg.get("bwt_plot_name"): 77 self.bwt_plot_path: str | None = os.path.join( 78 self.bwt_save_dir, cfg.bwt_plot_name 79 ) 80 r"""The file path to store the BWT metrics as plot figure.""" 81 if cfg.get("fwt_plot_name"): 82 self.fwt_plot_path: str | None = os.path.join( 83 self.fwt_save_dir, cfg.fwt_plot_name 84 ) 85 r"""The file path to store the FWT metrics as plot figure.""" 86 87 def sanity_check(self) -> None: 88 r"""Sanity check for config.""" 89 90 # check required config fields 91 required_config_fields = [ 92 "pipeline", 93 "eval_tasks", 94 "main_acc_csv_path", 95 "output_dir", 96 "bwt_save_dir", 97 "bwt_csv_name", 98 ] 99 for field in required_config_fields: 100 if not self.cfg.get(field): 101 raise KeyError( 102 f"Field `{field}` is required in the experiment index config." 103 ) 104 105 # warn if any reference experiment result is not provided 106 if not self.cfg.get("refindependent_acc_csv_path"): 107 pylogger.warning( 108 "`refindependent_acc_csv_path` not provided. Forward Transfer (FWT) cannot be evaluated." 109 ) 110 111 if not self.cfg.get("refjoint_acc_csv_path"): 112 pylogger.warning( 113 "`refjoint_acc_csv_path` not provided. Forgetting Rate (FR) cannot be evaluated." 114 ) 115 116 if not self.cfg.get("refrandom_acc_csv_path"): 117 pylogger.warning( 118 "`refrandom_acc_csv_path` not provided. Forgetting Rate (FR) cannot be evaluated." 119 ) 120 121 def run(self) -> None: 122 r"""The main method to run the continual learning full evaluation.""" 123 124 # BWT 125 self.evaluate_and_save_bwt_to_csv( 126 main_acc_csv_path=self.main_acc_csv_path, 127 eval_tasks=self.eval_tasks, 128 save_path=self.bwt_csv_path, 129 ) 130 if self.bwt_plot_path: 131 self.plot_bwt_curve_from_csv( 132 bwt_csv_path=self.bwt_csv_path, save_path=self.bwt_plot_path 133 ) 134 135 # FWT 136 self.evaluate_and_save_fwt_to_csv( 137 main_acc_csv_path=self.main_acc_csv_path, 138 refindependent_acc_csv_path=self.refindependent_acc_csv_path, 139 eval_tasks=self.eval_tasks, 140 save_path=self.fwt_csv_path, 141 ) 142 if self.fwt_plot_path: 143 self.plot_fwt_curve_from_csv( 144 fwt_csv_path=self.fwt_csv_path, save_path=self.fwt_plot_path 145 ) 146 147 # FR 148 self.evaluate_and_save_fr_to_csv( 149 main_acc_csv_path=self.main_acc_csv_path, 150 refjoint_acc_csv_path=self.refjoint_acc_csv_path, 151 refrandom_acc_csv_path=self.refrandom_acc_csv_path, 152 eval_tasks=self.eval_tasks, 153 save_path=self.fr_csv_path, 154 ) 155 156 def evaluate_and_save_bwt_to_csv( 157 self, main_acc_csv_path: str, eval_tasks: list[int], save_path: str 158 ) -> None: 159 """Evaluate the backward transfer (BWT) from the main experiment accuracy CSV file and save it to a CSV file. 160 161 **Args:** 162 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 163 - **eval_tasks** (`list[int]`): the list of tasks to evaluate BWT. 164 - **save_path** (`str`): the path to save the BWT CSV file. 165 """ 166 # delete task 1 which is the first task and has no BWT 167 eval_tasks = eval_tasks[2:] 168 169 acc_main_df = pd.read_csv(main_acc_csv_path) 170 171 bwt_data: list[dict[str, float | int]] = [] 172 for N in eval_tasks: # skip the first task. BWT cannot be defined for it 173 bwt_N = 0.0 174 for t in range(1, N): 175 a_t_N = float( 176 acc_main_df.loc[ 177 acc_main_df["after_training_task"] == N, 178 f"test_on_task_{t}", 179 ].iloc[0] 180 ) 181 a_t_t = float( 182 acc_main_df.loc[ 183 acc_main_df["after_training_task"] == t, 184 f"test_on_task_{t}", 185 ].iloc[0] 186 ) 187 bwt_N += a_t_N - a_t_t 188 bwt_N /= N - 1 189 bwt_data.append({"after_training_task": N, "BWT": float(bwt_N)}) 190 191 bwt_df = pd.DataFrame(bwt_data) 192 bwt_df.to_csv(save_path, index=False) 193 pylogger.info("Saved BWT to %s.", save_path) 194 195 def evaluate_and_save_fwt_to_csv( 196 self, 197 main_acc_csv_path: str, 198 refindependent_acc_csv_path: str, 199 eval_tasks: list[int], 200 save_path: str, 201 ) -> None: 202 """Evaluate the forward transfer (FWT) from the main experiment accuracy CSV file and reference independenet learning experiment accuracy CSV file, and save it to a CSV file. 203 204 **Args:** 205 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 206 - **refindependent_acc_csv_path** (`str`): the path to the reference independent learning accuracy CSV file. 207 - **eval_tasks** (`list[int]`): the list of tasks to evaluate FWT. 208 - **save_path** (`str`): the path to save the FWT CSV file. 209 """ 210 # delete task 1 which is the first task and has no FWT 211 eval_tasks = [task for task in eval_tasks if task != 1] 212 213 acc_main_df = pd.read_csv(main_acc_csv_path) 214 acc_refil_df = pd.read_csv(refindependent_acc_csv_path) 215 216 fwt_data: list[dict[str, float | int]] = [] 217 for N in eval_tasks: # skip the first task. FWT cannot be defined for it 218 fwt_N = 0.0 219 for t in range(2, N + 1): 220 a_t_I = float( 221 acc_refil_df.loc[ 222 acc_refil_df["after_training_task"] == t, 223 f"test_on_task_{t}", 224 ].iloc[0] 225 ) 226 a_t_t = float( 227 acc_main_df.loc[ 228 acc_main_df["after_training_task"] == t, 229 f"test_on_task_{t}", 230 ].iloc[0] 231 ) 232 fwt_N += a_t_t - a_t_I 233 fwt_N /= N - 1 234 fwt_data.append({"after_training_task": N, "FWT": float(fwt_N)}) 235 236 fwt_df = pd.DataFrame(fwt_data) 237 fwt_df.to_csv(save_path, index=False) 238 pylogger.info("Saved FWT to %s.", save_path) 239 240 def evaluate_and_save_fr_to_csv( 241 self, 242 main_acc_csv_path: str, 243 refjoint_acc_csv_path: str, 244 refrandom_acc_csv_path: str, 245 eval_tasks: list[int], 246 save_path: str, 247 ) -> dict[str, float]: 248 """evaluate the forgetting rate (FR) from the main experiment accuracy CSV file and save it to a CSV file. 249 250 **Args:** 251 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 252 - **refjoint_acc_csv_path** (`str`): the path to the reference joint learning accuracy CSV file. 253 - **refrandom_acc_csv_path** (`str`): the path to the reference random learning experiment accuracy CSV file. 254 - **eval_tasks** (`list[int]`): the list of tasks to evaluate FR. 255 - **save_path** (`str`): the path to save the FR CSV file. 256 """ 257 258 acc_main_df = pd.read_csv(main_acc_csv_path) 259 acc_refjl_df = pd.read_csv(refjoint_acc_csv_path) 260 acc_refrandom_df = pd.read_csv(refrandom_acc_csv_path) 261 262 fr_data = [] 263 N = eval_tasks[ 264 -1 265 ] # under our framework, we can only compute FR for the last task where the joint learning of all tasks is conducted for reference 266 fr_N = 0.0 267 for t in eval_tasks: 268 a_t_N = acc_main_df.loc[ 269 acc_main_df["after_training_task"] == N, f"test_on_task_{t}" 270 ] 271 a_t_N_J = acc_refjl_df.loc[ 272 0, f"test_on_task_{t}" 273 ] # joint learning of all tasks 274 a_t_R = acc_refrandom_df.loc[ 275 acc_main_df["after_training_task"] == N, f"test_on_task_{t}" 276 ] 277 fr_N += (a_t_N - a_t_R) / (a_t_N_J - a_t_R) 278 fr_N /= N 279 fr_N -= 1 280 fr_data.append({"after_training_task": N, "FR": float(fr_N)}) 281 282 fr_df = pd.DataFrame([{"after_training_task": N, "FR": float(fr_N)}]) 283 fr_df.to_csv(save_path, index=False) 284 pylogger.info("Saved FWT to %s.", save_path) 285 286 def plot_bwt_curve_from_csv(self, bwt_csv_path: str, save_path: str) -> None: 287 """Plot the backward transfer (BWT) barchart from saved CSV file and save the plot. 288 289 **Args:** 290 - **bwt_csv_path** (`str`): the path to the CSV file where the `evaluate_and_save_bwt_to_csv()` method saved the BWT metric. 291 - **save_path** (`str`): the path to save plot. 292 """ 293 data = pd.read_csv(bwt_csv_path) 294 fig, ax = plt.subplots(figsize=(10, 5)) 295 ax.plot(data["after_training_task"], data["BWT"], marker="o") 296 ax.set_xlabel("Task ID") 297 ax.set_ylabel("Backward Transfer (BWT)") 298 ax.set_title("Backward Transfer (BWT) Plot") 299 fig.savefig(save_path) 300 plt.close(fig) 301 302 def plot_fwt_curve_from_csv(self, fwt_csv_path: str, save_path: str) -> None: 303 """Plot the forward transfer (FWT) barchart from saved CSV file and save the plot. 304 305 **Args:** 306 - **bwt_csv_path** (`str`): the path to the CSV file where the `evaluate_and_save_fwt_to_csv()` method saved the FWT metric. 307 - **save_path** (`str`): the path to save plot. 308 """ 309 data = pd.read_csv(fwt_csv_path) 310 fig, ax = plt.subplots(figsize=(10, 5)) 311 ax.plot(data["after_training_task"], data["FWT"], marker="o") 312 ax.set_xlabel("Task ID") 313 ax.set_ylabel("Forward Transfer (FWT)") 314 ax.set_title("Forward Transfer (FWT) Plot") 315 fig.savefig(save_path) 316 plt.close(fig)
class
CLFullEvaluation:
20class CLFullEvaluation: 21 r"""The base class for continual learning full evaluation.""" 22 23 def __init__(self, cfg: DictConfig) -> None: 24 r""" 25 **Args:** 26 - **cfg** (`DictConfig`): the config dict for the continual learning full evaluation. 27 """ 28 self.cfg: DictConfig = cfg 29 r"""The complete config dict.""" 30 31 CLFullEvaluation.sanity_check(self) 32 33 self.eval_tasks: list[int] = ( 34 cfg.eval_tasks 35 if isinstance(cfg.eval_tasks, list) 36 else list(range(1, cfg.eval_tasks + 1)) 37 ) 38 r"""The list of task IDs to evaluate.""" 39 40 self.main_acc_csv_path: str = cfg.main_acc_csv_path 41 r"""The path to the main experiment accuracy CSV file.""" 42 if cfg.get("refjoint_acc_csv_path"): 43 self.refjoint_acc_csv_path: str = cfg.refjoint_acc_csv_path 44 r"""The path to the reference joint learning experiment accuracy CSV file.""" 45 if cfg.get("refindependent_acc_csv_path"): 46 self.refindependent_acc_csv_path: str = cfg.refindependent_acc_csv_path 47 r"""The path to the reference independent learning experiment accuracy CSV file.""" 48 if cfg.get("refrandom_acc_csv_path"): 49 self.refrandom_acc_csv_path: str = cfg.refrandom_acc_csv_path 50 r"""The path to the reference random learning experiment accuracy CSV file.""" 51 52 self.output_dir: str = cfg.output_dir 53 r"""The folder for storing the evaluation results.""" 54 55 self.bwt_save_dir: str = cfg.bwt_save_dir 56 r"""The folder storing the BWT metric results.""" 57 if cfg.get("fwt_save_dir"): 58 self.fwt_save_dir: str | None = cfg.fwt_save_dir 59 r"""The folder storing the FWT metric results.""" 60 if cfg.get("fr_save_dir"): 61 self.fr_save_dir: str | None = cfg.fr_save_dir 62 r"""The folder storing the FR metric results.""" 63 64 self.bwt_csv_path: str = os.path.join(self.bwt_save_dir, cfg.bwt_csv_name) 65 r"""The file path to store the BWT metrics as CSV file.""" 66 if cfg.get("fwt_csv_name"): 67 self.fwt_csv_path: str | None = os.path.join( 68 self.fwt_save_dir, cfg.fwt_csv_name 69 ) 70 r"""The file path to store the FWT metrics as CSV file.""" 71 if cfg.get("fr_csv_name"): 72 self.fr_csv_path: str | None = os.path.join( 73 self.fr_save_dir, cfg.fr_csv_name 74 ) 75 r"""The file path to store the FR metrics as CSV file.""" 76 77 if cfg.get("bwt_plot_name"): 78 self.bwt_plot_path: str | None = os.path.join( 79 self.bwt_save_dir, cfg.bwt_plot_name 80 ) 81 r"""The file path to store the BWT metrics as plot figure.""" 82 if cfg.get("fwt_plot_name"): 83 self.fwt_plot_path: str | None = os.path.join( 84 self.fwt_save_dir, cfg.fwt_plot_name 85 ) 86 r"""The file path to store the FWT metrics as plot figure.""" 87 88 def sanity_check(self) -> None: 89 r"""Sanity check for config.""" 90 91 # check required config fields 92 required_config_fields = [ 93 "pipeline", 94 "eval_tasks", 95 "main_acc_csv_path", 96 "output_dir", 97 "bwt_save_dir", 98 "bwt_csv_name", 99 ] 100 for field in required_config_fields: 101 if not self.cfg.get(field): 102 raise KeyError( 103 f"Field `{field}` is required in the experiment index config." 104 ) 105 106 # warn if any reference experiment result is not provided 107 if not self.cfg.get("refindependent_acc_csv_path"): 108 pylogger.warning( 109 "`refindependent_acc_csv_path` not provided. Forward Transfer (FWT) cannot be evaluated." 110 ) 111 112 if not self.cfg.get("refjoint_acc_csv_path"): 113 pylogger.warning( 114 "`refjoint_acc_csv_path` not provided. Forgetting Rate (FR) cannot be evaluated." 115 ) 116 117 if not self.cfg.get("refrandom_acc_csv_path"): 118 pylogger.warning( 119 "`refrandom_acc_csv_path` not provided. Forgetting Rate (FR) cannot be evaluated." 120 ) 121 122 def run(self) -> None: 123 r"""The main method to run the continual learning full evaluation.""" 124 125 # BWT 126 self.evaluate_and_save_bwt_to_csv( 127 main_acc_csv_path=self.main_acc_csv_path, 128 eval_tasks=self.eval_tasks, 129 save_path=self.bwt_csv_path, 130 ) 131 if self.bwt_plot_path: 132 self.plot_bwt_curve_from_csv( 133 bwt_csv_path=self.bwt_csv_path, save_path=self.bwt_plot_path 134 ) 135 136 # FWT 137 self.evaluate_and_save_fwt_to_csv( 138 main_acc_csv_path=self.main_acc_csv_path, 139 refindependent_acc_csv_path=self.refindependent_acc_csv_path, 140 eval_tasks=self.eval_tasks, 141 save_path=self.fwt_csv_path, 142 ) 143 if self.fwt_plot_path: 144 self.plot_fwt_curve_from_csv( 145 fwt_csv_path=self.fwt_csv_path, save_path=self.fwt_plot_path 146 ) 147 148 # FR 149 self.evaluate_and_save_fr_to_csv( 150 main_acc_csv_path=self.main_acc_csv_path, 151 refjoint_acc_csv_path=self.refjoint_acc_csv_path, 152 refrandom_acc_csv_path=self.refrandom_acc_csv_path, 153 eval_tasks=self.eval_tasks, 154 save_path=self.fr_csv_path, 155 ) 156 157 def evaluate_and_save_bwt_to_csv( 158 self, main_acc_csv_path: str, eval_tasks: list[int], save_path: str 159 ) -> None: 160 """Evaluate the backward transfer (BWT) from the main experiment accuracy CSV file and save it to a CSV file. 161 162 **Args:** 163 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 164 - **eval_tasks** (`list[int]`): the list of tasks to evaluate BWT. 165 - **save_path** (`str`): the path to save the BWT CSV file. 166 """ 167 # delete task 1 which is the first task and has no BWT 168 eval_tasks = eval_tasks[2:] 169 170 acc_main_df = pd.read_csv(main_acc_csv_path) 171 172 bwt_data: list[dict[str, float | int]] = [] 173 for N in eval_tasks: # skip the first task. BWT cannot be defined for it 174 bwt_N = 0.0 175 for t in range(1, N): 176 a_t_N = float( 177 acc_main_df.loc[ 178 acc_main_df["after_training_task"] == N, 179 f"test_on_task_{t}", 180 ].iloc[0] 181 ) 182 a_t_t = float( 183 acc_main_df.loc[ 184 acc_main_df["after_training_task"] == t, 185 f"test_on_task_{t}", 186 ].iloc[0] 187 ) 188 bwt_N += a_t_N - a_t_t 189 bwt_N /= N - 1 190 bwt_data.append({"after_training_task": N, "BWT": float(bwt_N)}) 191 192 bwt_df = pd.DataFrame(bwt_data) 193 bwt_df.to_csv(save_path, index=False) 194 pylogger.info("Saved BWT to %s.", save_path) 195 196 def evaluate_and_save_fwt_to_csv( 197 self, 198 main_acc_csv_path: str, 199 refindependent_acc_csv_path: str, 200 eval_tasks: list[int], 201 save_path: str, 202 ) -> None: 203 """Evaluate the forward transfer (FWT) from the main experiment accuracy CSV file and reference independenet learning experiment accuracy CSV file, and save it to a CSV file. 204 205 **Args:** 206 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 207 - **refindependent_acc_csv_path** (`str`): the path to the reference independent learning accuracy CSV file. 208 - **eval_tasks** (`list[int]`): the list of tasks to evaluate FWT. 209 - **save_path** (`str`): the path to save the FWT CSV file. 210 """ 211 # delete task 1 which is the first task and has no FWT 212 eval_tasks = [task for task in eval_tasks if task != 1] 213 214 acc_main_df = pd.read_csv(main_acc_csv_path) 215 acc_refil_df = pd.read_csv(refindependent_acc_csv_path) 216 217 fwt_data: list[dict[str, float | int]] = [] 218 for N in eval_tasks: # skip the first task. FWT cannot be defined for it 219 fwt_N = 0.0 220 for t in range(2, N + 1): 221 a_t_I = float( 222 acc_refil_df.loc[ 223 acc_refil_df["after_training_task"] == t, 224 f"test_on_task_{t}", 225 ].iloc[0] 226 ) 227 a_t_t = float( 228 acc_main_df.loc[ 229 acc_main_df["after_training_task"] == t, 230 f"test_on_task_{t}", 231 ].iloc[0] 232 ) 233 fwt_N += a_t_t - a_t_I 234 fwt_N /= N - 1 235 fwt_data.append({"after_training_task": N, "FWT": float(fwt_N)}) 236 237 fwt_df = pd.DataFrame(fwt_data) 238 fwt_df.to_csv(save_path, index=False) 239 pylogger.info("Saved FWT to %s.", save_path) 240 241 def evaluate_and_save_fr_to_csv( 242 self, 243 main_acc_csv_path: str, 244 refjoint_acc_csv_path: str, 245 refrandom_acc_csv_path: str, 246 eval_tasks: list[int], 247 save_path: str, 248 ) -> dict[str, float]: 249 """evaluate the forgetting rate (FR) from the main experiment accuracy CSV file and save it to a CSV file. 250 251 **Args:** 252 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 253 - **refjoint_acc_csv_path** (`str`): the path to the reference joint learning accuracy CSV file. 254 - **refrandom_acc_csv_path** (`str`): the path to the reference random learning experiment accuracy CSV file. 255 - **eval_tasks** (`list[int]`): the list of tasks to evaluate FR. 256 - **save_path** (`str`): the path to save the FR CSV file. 257 """ 258 259 acc_main_df = pd.read_csv(main_acc_csv_path) 260 acc_refjl_df = pd.read_csv(refjoint_acc_csv_path) 261 acc_refrandom_df = pd.read_csv(refrandom_acc_csv_path) 262 263 fr_data = [] 264 N = eval_tasks[ 265 -1 266 ] # under our framework, we can only compute FR for the last task where the joint learning of all tasks is conducted for reference 267 fr_N = 0.0 268 for t in eval_tasks: 269 a_t_N = acc_main_df.loc[ 270 acc_main_df["after_training_task"] == N, f"test_on_task_{t}" 271 ] 272 a_t_N_J = acc_refjl_df.loc[ 273 0, f"test_on_task_{t}" 274 ] # joint learning of all tasks 275 a_t_R = acc_refrandom_df.loc[ 276 acc_main_df["after_training_task"] == N, f"test_on_task_{t}" 277 ] 278 fr_N += (a_t_N - a_t_R) / (a_t_N_J - a_t_R) 279 fr_N /= N 280 fr_N -= 1 281 fr_data.append({"after_training_task": N, "FR": float(fr_N)}) 282 283 fr_df = pd.DataFrame([{"after_training_task": N, "FR": float(fr_N)}]) 284 fr_df.to_csv(save_path, index=False) 285 pylogger.info("Saved FWT to %s.", save_path) 286 287 def plot_bwt_curve_from_csv(self, bwt_csv_path: str, save_path: str) -> None: 288 """Plot the backward transfer (BWT) barchart from saved CSV file and save the plot. 289 290 **Args:** 291 - **bwt_csv_path** (`str`): the path to the CSV file where the `evaluate_and_save_bwt_to_csv()` method saved the BWT metric. 292 - **save_path** (`str`): the path to save plot. 293 """ 294 data = pd.read_csv(bwt_csv_path) 295 fig, ax = plt.subplots(figsize=(10, 5)) 296 ax.plot(data["after_training_task"], data["BWT"], marker="o") 297 ax.set_xlabel("Task ID") 298 ax.set_ylabel("Backward Transfer (BWT)") 299 ax.set_title("Backward Transfer (BWT) Plot") 300 fig.savefig(save_path) 301 plt.close(fig) 302 303 def plot_fwt_curve_from_csv(self, fwt_csv_path: str, save_path: str) -> None: 304 """Plot the forward transfer (FWT) barchart from saved CSV file and save the plot. 305 306 **Args:** 307 - **bwt_csv_path** (`str`): the path to the CSV file where the `evaluate_and_save_fwt_to_csv()` method saved the FWT metric. 308 - **save_path** (`str`): the path to save plot. 309 """ 310 data = pd.read_csv(fwt_csv_path) 311 fig, ax = plt.subplots(figsize=(10, 5)) 312 ax.plot(data["after_training_task"], data["FWT"], marker="o") 313 ax.set_xlabel("Task ID") 314 ax.set_ylabel("Forward Transfer (FWT)") 315 ax.set_title("Forward Transfer (FWT) Plot") 316 fig.savefig(save_path) 317 plt.close(fig)
The base class for continual learning full evaluation.
CLFullEvaluation(cfg: omegaconf.dictconfig.DictConfig)
23 def __init__(self, cfg: DictConfig) -> None: 24 r""" 25 **Args:** 26 - **cfg** (`DictConfig`): the config dict for the continual learning full evaluation. 27 """ 28 self.cfg: DictConfig = cfg 29 r"""The complete config dict.""" 30 31 CLFullEvaluation.sanity_check(self) 32 33 self.eval_tasks: list[int] = ( 34 cfg.eval_tasks 35 if isinstance(cfg.eval_tasks, list) 36 else list(range(1, cfg.eval_tasks + 1)) 37 ) 38 r"""The list of task IDs to evaluate.""" 39 40 self.main_acc_csv_path: str = cfg.main_acc_csv_path 41 r"""The path to the main experiment accuracy CSV file.""" 42 if cfg.get("refjoint_acc_csv_path"): 43 self.refjoint_acc_csv_path: str = cfg.refjoint_acc_csv_path 44 r"""The path to the reference joint learning experiment accuracy CSV file.""" 45 if cfg.get("refindependent_acc_csv_path"): 46 self.refindependent_acc_csv_path: str = cfg.refindependent_acc_csv_path 47 r"""The path to the reference independent learning experiment accuracy CSV file.""" 48 if cfg.get("refrandom_acc_csv_path"): 49 self.refrandom_acc_csv_path: str = cfg.refrandom_acc_csv_path 50 r"""The path to the reference random learning experiment accuracy CSV file.""" 51 52 self.output_dir: str = cfg.output_dir 53 r"""The folder for storing the evaluation results.""" 54 55 self.bwt_save_dir: str = cfg.bwt_save_dir 56 r"""The folder storing the BWT metric results.""" 57 if cfg.get("fwt_save_dir"): 58 self.fwt_save_dir: str | None = cfg.fwt_save_dir 59 r"""The folder storing the FWT metric results.""" 60 if cfg.get("fr_save_dir"): 61 self.fr_save_dir: str | None = cfg.fr_save_dir 62 r"""The folder storing the FR metric results.""" 63 64 self.bwt_csv_path: str = os.path.join(self.bwt_save_dir, cfg.bwt_csv_name) 65 r"""The file path to store the BWT metrics as CSV file.""" 66 if cfg.get("fwt_csv_name"): 67 self.fwt_csv_path: str | None = os.path.join( 68 self.fwt_save_dir, cfg.fwt_csv_name 69 ) 70 r"""The file path to store the FWT metrics as CSV file.""" 71 if cfg.get("fr_csv_name"): 72 self.fr_csv_path: str | None = os.path.join( 73 self.fr_save_dir, cfg.fr_csv_name 74 ) 75 r"""The file path to store the FR metrics as CSV file.""" 76 77 if cfg.get("bwt_plot_name"): 78 self.bwt_plot_path: str | None = os.path.join( 79 self.bwt_save_dir, cfg.bwt_plot_name 80 ) 81 r"""The file path to store the BWT metrics as plot figure.""" 82 if cfg.get("fwt_plot_name"): 83 self.fwt_plot_path: str | None = os.path.join( 84 self.fwt_save_dir, cfg.fwt_plot_name 85 ) 86 r"""The file path to store the FWT metrics as plot figure."""
Args:
- cfg (
DictConfig): the config dict for the continual learning full evaluation.
def
sanity_check(self) -> None:
88 def sanity_check(self) -> None: 89 r"""Sanity check for config.""" 90 91 # check required config fields 92 required_config_fields = [ 93 "pipeline", 94 "eval_tasks", 95 "main_acc_csv_path", 96 "output_dir", 97 "bwt_save_dir", 98 "bwt_csv_name", 99 ] 100 for field in required_config_fields: 101 if not self.cfg.get(field): 102 raise KeyError( 103 f"Field `{field}` is required in the experiment index config." 104 ) 105 106 # warn if any reference experiment result is not provided 107 if not self.cfg.get("refindependent_acc_csv_path"): 108 pylogger.warning( 109 "`refindependent_acc_csv_path` not provided. Forward Transfer (FWT) cannot be evaluated." 110 ) 111 112 if not self.cfg.get("refjoint_acc_csv_path"): 113 pylogger.warning( 114 "`refjoint_acc_csv_path` not provided. Forgetting Rate (FR) cannot be evaluated." 115 ) 116 117 if not self.cfg.get("refrandom_acc_csv_path"): 118 pylogger.warning( 119 "`refrandom_acc_csv_path` not provided. Forgetting Rate (FR) cannot be evaluated." 120 )
Sanity check for config.
def
run(self) -> None:
122 def run(self) -> None: 123 r"""The main method to run the continual learning full evaluation.""" 124 125 # BWT 126 self.evaluate_and_save_bwt_to_csv( 127 main_acc_csv_path=self.main_acc_csv_path, 128 eval_tasks=self.eval_tasks, 129 save_path=self.bwt_csv_path, 130 ) 131 if self.bwt_plot_path: 132 self.plot_bwt_curve_from_csv( 133 bwt_csv_path=self.bwt_csv_path, save_path=self.bwt_plot_path 134 ) 135 136 # FWT 137 self.evaluate_and_save_fwt_to_csv( 138 main_acc_csv_path=self.main_acc_csv_path, 139 refindependent_acc_csv_path=self.refindependent_acc_csv_path, 140 eval_tasks=self.eval_tasks, 141 save_path=self.fwt_csv_path, 142 ) 143 if self.fwt_plot_path: 144 self.plot_fwt_curve_from_csv( 145 fwt_csv_path=self.fwt_csv_path, save_path=self.fwt_plot_path 146 ) 147 148 # FR 149 self.evaluate_and_save_fr_to_csv( 150 main_acc_csv_path=self.main_acc_csv_path, 151 refjoint_acc_csv_path=self.refjoint_acc_csv_path, 152 refrandom_acc_csv_path=self.refrandom_acc_csv_path, 153 eval_tasks=self.eval_tasks, 154 save_path=self.fr_csv_path, 155 )
The main method to run the continual learning full evaluation.
def
evaluate_and_save_bwt_to_csv( self, main_acc_csv_path: str, eval_tasks: list[int], save_path: str) -> None:
157 def evaluate_and_save_bwt_to_csv( 158 self, main_acc_csv_path: str, eval_tasks: list[int], save_path: str 159 ) -> None: 160 """Evaluate the backward transfer (BWT) from the main experiment accuracy CSV file and save it to a CSV file. 161 162 **Args:** 163 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 164 - **eval_tasks** (`list[int]`): the list of tasks to evaluate BWT. 165 - **save_path** (`str`): the path to save the BWT CSV file. 166 """ 167 # delete task 1 which is the first task and has no BWT 168 eval_tasks = eval_tasks[2:] 169 170 acc_main_df = pd.read_csv(main_acc_csv_path) 171 172 bwt_data: list[dict[str, float | int]] = [] 173 for N in eval_tasks: # skip the first task. BWT cannot be defined for it 174 bwt_N = 0.0 175 for t in range(1, N): 176 a_t_N = float( 177 acc_main_df.loc[ 178 acc_main_df["after_training_task"] == N, 179 f"test_on_task_{t}", 180 ].iloc[0] 181 ) 182 a_t_t = float( 183 acc_main_df.loc[ 184 acc_main_df["after_training_task"] == t, 185 f"test_on_task_{t}", 186 ].iloc[0] 187 ) 188 bwt_N += a_t_N - a_t_t 189 bwt_N /= N - 1 190 bwt_data.append({"after_training_task": N, "BWT": float(bwt_N)}) 191 192 bwt_df = pd.DataFrame(bwt_data) 193 bwt_df.to_csv(save_path, index=False) 194 pylogger.info("Saved BWT to %s.", save_path)
Evaluate the backward transfer (BWT) from the main experiment accuracy CSV file and save it to a CSV file.
Args:
- main_acc_csv_path (
str): the path to the main experiment accuracy CSV file. - eval_tasks (
list[int]): the list of tasks to evaluate BWT. - save_path (
str): the path to save the BWT CSV file.
def
evaluate_and_save_fwt_to_csv( self, main_acc_csv_path: str, refindependent_acc_csv_path: str, eval_tasks: list[int], save_path: str) -> None:
196 def evaluate_and_save_fwt_to_csv( 197 self, 198 main_acc_csv_path: str, 199 refindependent_acc_csv_path: str, 200 eval_tasks: list[int], 201 save_path: str, 202 ) -> None: 203 """Evaluate the forward transfer (FWT) from the main experiment accuracy CSV file and reference independenet learning experiment accuracy CSV file, and save it to a CSV file. 204 205 **Args:** 206 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 207 - **refindependent_acc_csv_path** (`str`): the path to the reference independent learning accuracy CSV file. 208 - **eval_tasks** (`list[int]`): the list of tasks to evaluate FWT. 209 - **save_path** (`str`): the path to save the FWT CSV file. 210 """ 211 # delete task 1 which is the first task and has no FWT 212 eval_tasks = [task for task in eval_tasks if task != 1] 213 214 acc_main_df = pd.read_csv(main_acc_csv_path) 215 acc_refil_df = pd.read_csv(refindependent_acc_csv_path) 216 217 fwt_data: list[dict[str, float | int]] = [] 218 for N in eval_tasks: # skip the first task. FWT cannot be defined for it 219 fwt_N = 0.0 220 for t in range(2, N + 1): 221 a_t_I = float( 222 acc_refil_df.loc[ 223 acc_refil_df["after_training_task"] == t, 224 f"test_on_task_{t}", 225 ].iloc[0] 226 ) 227 a_t_t = float( 228 acc_main_df.loc[ 229 acc_main_df["after_training_task"] == t, 230 f"test_on_task_{t}", 231 ].iloc[0] 232 ) 233 fwt_N += a_t_t - a_t_I 234 fwt_N /= N - 1 235 fwt_data.append({"after_training_task": N, "FWT": float(fwt_N)}) 236 237 fwt_df = pd.DataFrame(fwt_data) 238 fwt_df.to_csv(save_path, index=False) 239 pylogger.info("Saved FWT to %s.", save_path)
Evaluate the forward transfer (FWT) from the main experiment accuracy CSV file and reference independenet learning experiment accuracy CSV file, and save it to a CSV file.
Args:
- main_acc_csv_path (
str): the path to the main experiment accuracy CSV file. - refindependent_acc_csv_path (
str): the path to the reference independent learning accuracy CSV file. - eval_tasks (
list[int]): the list of tasks to evaluate FWT. - save_path (
str): the path to save the FWT CSV file.
def
evaluate_and_save_fr_to_csv( self, main_acc_csv_path: str, refjoint_acc_csv_path: str, refrandom_acc_csv_path: str, eval_tasks: list[int], save_path: str) -> dict[str, float]:
241 def evaluate_and_save_fr_to_csv( 242 self, 243 main_acc_csv_path: str, 244 refjoint_acc_csv_path: str, 245 refrandom_acc_csv_path: str, 246 eval_tasks: list[int], 247 save_path: str, 248 ) -> dict[str, float]: 249 """evaluate the forgetting rate (FR) from the main experiment accuracy CSV file and save it to a CSV file. 250 251 **Args:** 252 - **main_acc_csv_path** (`str`): the path to the main experiment accuracy CSV file. 253 - **refjoint_acc_csv_path** (`str`): the path to the reference joint learning accuracy CSV file. 254 - **refrandom_acc_csv_path** (`str`): the path to the reference random learning experiment accuracy CSV file. 255 - **eval_tasks** (`list[int]`): the list of tasks to evaluate FR. 256 - **save_path** (`str`): the path to save the FR CSV file. 257 """ 258 259 acc_main_df = pd.read_csv(main_acc_csv_path) 260 acc_refjl_df = pd.read_csv(refjoint_acc_csv_path) 261 acc_refrandom_df = pd.read_csv(refrandom_acc_csv_path) 262 263 fr_data = [] 264 N = eval_tasks[ 265 -1 266 ] # under our framework, we can only compute FR for the last task where the joint learning of all tasks is conducted for reference 267 fr_N = 0.0 268 for t in eval_tasks: 269 a_t_N = acc_main_df.loc[ 270 acc_main_df["after_training_task"] == N, f"test_on_task_{t}" 271 ] 272 a_t_N_J = acc_refjl_df.loc[ 273 0, f"test_on_task_{t}" 274 ] # joint learning of all tasks 275 a_t_R = acc_refrandom_df.loc[ 276 acc_main_df["after_training_task"] == N, f"test_on_task_{t}" 277 ] 278 fr_N += (a_t_N - a_t_R) / (a_t_N_J - a_t_R) 279 fr_N /= N 280 fr_N -= 1 281 fr_data.append({"after_training_task": N, "FR": float(fr_N)}) 282 283 fr_df = pd.DataFrame([{"after_training_task": N, "FR": float(fr_N)}]) 284 fr_df.to_csv(save_path, index=False) 285 pylogger.info("Saved FWT to %s.", save_path)
evaluate the forgetting rate (FR) from the main experiment accuracy CSV file and save it to a CSV file.
Args:
- main_acc_csv_path (
str): the path to the main experiment accuracy CSV file. - refjoint_acc_csv_path (
str): the path to the reference joint learning accuracy CSV file. - refrandom_acc_csv_path (
str): the path to the reference random learning experiment accuracy CSV file. - eval_tasks (
list[int]): the list of tasks to evaluate FR. - save_path (
str): the path to save the FR CSV file.
def
plot_bwt_curve_from_csv(self, bwt_csv_path: str, save_path: str) -> None:
287 def plot_bwt_curve_from_csv(self, bwt_csv_path: str, save_path: str) -> None: 288 """Plot the backward transfer (BWT) barchart from saved CSV file and save the plot. 289 290 **Args:** 291 - **bwt_csv_path** (`str`): the path to the CSV file where the `evaluate_and_save_bwt_to_csv()` method saved the BWT metric. 292 - **save_path** (`str`): the path to save plot. 293 """ 294 data = pd.read_csv(bwt_csv_path) 295 fig, ax = plt.subplots(figsize=(10, 5)) 296 ax.plot(data["after_training_task"], data["BWT"], marker="o") 297 ax.set_xlabel("Task ID") 298 ax.set_ylabel("Backward Transfer (BWT)") 299 ax.set_title("Backward Transfer (BWT) Plot") 300 fig.savefig(save_path) 301 plt.close(fig)
Plot the backward transfer (BWT) barchart from saved CSV file and save the plot.
Args:
- bwt_csv_path (
str): the path to the CSV file where theevaluate_and_save_bwt_to_csv()method saved the BWT metric. - save_path (
str): the path to save plot.
def
plot_fwt_curve_from_csv(self, fwt_csv_path: str, save_path: str) -> None:
303 def plot_fwt_curve_from_csv(self, fwt_csv_path: str, save_path: str) -> None: 304 """Plot the forward transfer (FWT) barchart from saved CSV file and save the plot. 305 306 **Args:** 307 - **bwt_csv_path** (`str`): the path to the CSV file where the `evaluate_and_save_fwt_to_csv()` method saved the FWT metric. 308 - **save_path** (`str`): the path to save plot. 309 """ 310 data = pd.read_csv(fwt_csv_path) 311 fig, ax = plt.subplots(figsize=(10, 5)) 312 ax.plot(data["after_training_task"], data["FWT"], marker="o") 313 ax.set_xlabel("Task ID") 314 ax.set_ylabel("Forward Transfer (FWT)") 315 ax.set_title("Forward Transfer (FWT) Plot") 316 fig.savefig(save_path) 317 plt.close(fig)
Plot the forward transfer (FWT) barchart from saved CSV file and save the plot.
Args:
- bwt_csv_path (
str): the path to the CSV file where theevaluate_and_save_fwt_to_csv()method saved the FWT metric. - save_path (
str): the path to save plot.