clarena.pipelines.cl_main_eval

  1r"""
  2The submodule in `pipelines` for continual learning main evaluation.
  3"""
  4
  5__all__ = ["CLMainEvaluation"]
  6
  7import logging
  8
  9import hydra
 10import lightning as L
 11import torch
 12from lightning import Callback, Trainer
 13from omegaconf import DictConfig, ListConfig
 14
 15from clarena.cl_datasets import CLDataset
 16
 17# always get logger for built-in logging in each module
 18pylogger = logging.getLogger(__name__)
 19
 20
 21class CLMainEvaluation:
 22    r"""The base class for continual learning main evaluation."""
 23
 24    def __init__(self, cfg: DictConfig) -> None:
 25        r"""
 26        **Args:**
 27        - **cfg** (`DictConfig`): the config dict for the continual learning main evaluation.
 28        """
 29        self.cfg: DictConfig = cfg
 30        r"""The complete config dict."""
 31
 32        CLMainEvaluation.sanity_check(self)
 33
 34        # required config fields
 35        self.main_model_path: str = cfg.main_model_path
 36        r"""The file path of the model to evaluate."""
 37        self.cl_paradigm: str = cfg.clmlp
 38        r"""The continual learning paradigm."""
 39        self.eval_tasks: list[int] = (
 40            cfg.eval_tasks
 41            if isinstance(cfg.eval_tasks, ListConfig)
 42            else list(range(1, cfg.eval_tasks + 1))
 43        )
 44        r"""The list of task IDs to evaluate."""
 45        self.global_seed: int = cfg.global_seed
 46        r"""The global seed for the entire experiment."""
 47        self.output_dir: str = cfg.output_dir
 48        r"""The folder for storing the experiment results."""
 49
 50        # components
 51        self.cl_dataset: CLDataset
 52        r"""CL dataset object."""
 53        self.callbacks: list[Callback]
 54        r"""Callback objects."""
 55        self.trainer: Trainer
 56        r"""Trainer object."""
 57
 58    def sanity_check(self) -> None:
 59        r"""Sanity check for config."""
 60
 61        # check required config fields
 62        required_config_fields = [
 63            "pipeline",
 64            "main_model_path",
 65            "eval_tasks",
 66            "cl_paradigm",
 67            "global_seed",
 68            "cl_dataset",
 69            "trainer",
 70            "metrics",
 71            "callbacks",
 72            "output_dir",
 73            # "hydra" is excluded as it doesn't appear
 74            "misc",
 75        ]
 76        for field in required_config_fields:
 77            if not self.cfg.get(field):
 78                raise KeyError(
 79                    f"Field `{field}` is required in the experiment index config."
 80                )
 81
 82        # check cl_paradigm
 83        if self.cfg.cl_paradigm not in ["TIL", "CIL", "DIL"]:
 84            raise ValueError(
 85                f"Field `cl_paradigm` should be either 'TIL', 'CIL' or 'DIL' but got {self.cfg.cl_paradigm}!"
 86            )
 87
 88        # check eval_tasks
 89        if self.cfg.cl_dataset.get("num_tasks"):
 90            num_tasks = self.cfg.cl_dataset.get("num_tasks")
 91        elif self.cfg.cl_dataset.get("class_split"):
 92            num_tasks = len(self.cfg.cl_dataset.class_split)
 93        elif self.cfg.cl_dataset.get("datasets"):
 94            num_tasks = len(self.cfg.cl_dataset.datasets)
 95        else:
 96            raise KeyError(
 97                "`num_tasks` is required in cl_dataset config. Please specify `num_tasks` (for `CLPermutedDataset`) or `class_split` (for `CLSplitDataset`) or `datasets` (for `CLCombinedDataset`) in cl_dataset config."
 98            )
 99
100        eval_tasks = self.cfg.eval_tasks
101        if isinstance(eval_tasks, ListConfig):
102            if len(eval_tasks) < 1:
103                raise ValueError("`eval_tasks` must contain at least one task.")
104            if any(t < 1 or t > num_tasks for t in eval_tasks):
105                raise ValueError(
106                    f"All task IDs in `eval_tasks` must be between 1 and {num_tasks}."
107                )
108        elif isinstance(eval_tasks, int):
109            if eval_tasks < 0 or eval_tasks > num_tasks:
110                raise ValueError(
111                    f"`eval_tasks` as integer must be between 0 and {num_tasks}."
112                )
113        else:
114            raise TypeError(
115                "`eval_tasks` must be either a list of integers or an integer."
116            )
117
118    def instantiate_cl_dataset(self, cl_dataset_cfg: DictConfig) -> None:
119        r"""Instantiate the CL dataset object from `cl_dataset_cfg`."""
120        pylogger.debug(
121            "Instantiating CL dataset <%s> (clarena.cl_datasets.CLDataset)...",
122            cl_dataset_cfg.get("_target_"),
123        )
124        self.cl_dataset = hydra.utils.instantiate(cl_dataset_cfg)
125        pylogger.debug(
126            "CL dataset <%s> (clarena.cl_datasets.CLDataset) instantiated!",
127            cl_dataset_cfg.get("_target_"),
128        )
129
130    def instantiate_callbacks(
131        self, metrics_cfg: DictConfig, callbacks_cfg: DictConfig
132    ) -> None:
133        r"""Instantiate the list of callbacks objects from `metrics_cfg` and `callbacks_cfg`. Note that `metrics_cfg` is a list of metric callbacks and `callbacks_cfg` is a list of callbacks other the metric callbacks. The instantiated callbacks contain both metric callbacks and other callbacks."""
134        pylogger.debug("Instantiating callbacks (lightning.Callback)...")
135
136        # instantiate metric callbacks
137        metric_callbacks: list[Callback] = [
138            hydra.utils.instantiate(callback) for callback in metrics_cfg
139        ]
140
141        # instantiate other callbacks
142        other_callbacks: list[Callback] = [
143            hydra.utils.instantiate(callback) for callback in callbacks_cfg
144        ]
145
146        # add metric callbacks to the list of callbacks
147        self.callbacks = metric_callbacks + other_callbacks
148        pylogger.debug("Callbacks (lightning.Callback) instantiated!")
149
150    def instantiate_trainer(
151        self,
152        trainer_cfg: DictConfig,
153        callbacks: list[Callback],
154    ) -> None:
155        r"""Instantiate the trainer object from `trainer_cfg` and `callbacks`."""
156
157        pylogger.debug("Instantiating trainer (lightning.Trainer)...")
158        self.trainer = hydra.utils.instantiate(
159            trainer_cfg,
160            callbacks=callbacks,
161        )
162        pylogger.debug("Trainer (lightning.Trainer) instantiated!")
163
164    def set_global_seed(self, global_seed: int) -> None:
165        r"""Set the `global_seed` for the entire evaluation."""
166        L.seed_everything(self.global_seed, workers=True)
167        pylogger.debug("Global seed is set as %d.", global_seed)
168
169    def run(self) -> None:
170        r"""The main method to run the continual learning main evaluation."""
171
172        self.set_global_seed(self.global_seed)
173
174        # load the model from file
175        model = torch.load(self.main_model_path)
176
177        # components
178        self.instantiate_cl_dataset(cl_dataset_cfg=self.cfg.cl_dataset)
179        self.cl_dataset.set_cl_paradigm(cl_paradigm=self.cl_paradigm)
180        self.instantiate_callbacks(
181            metrics_cfg=self.cfg.metrics, callbacks_cfg=self.cfg.callbacks
182        )
183        self.instantiate_trainer(
184            trainer_cfg=self.cfg.trainer,
185            callbacks=self.callbacks,
186        )  # trainer should be instantiated after callbacks
187
188        # setup tasks for dataset
189        self.cl_dataset.setup_tasks_eval(eval_tasks=self.eval_tasks)
190
191        # evaluation skipping training and validation
192        self.trainer.test(
193            model=model,
194            datamodule=self.cl_dataset,
195        )