Merge pull request #88 from diningphil/minor-improvements

tqdm when debug, possibility to run multiple trainings for model sele…

Author: diningphil

.github/requirements.txt

@@ -1,4 +1,5 @@
 PyYAML
+tqdm
 tensorboard
 tqdm
 ogb

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [1.5.2] Some Improvements
+
+### Added
+
+- Implemented a convenient tqdm progress bar in debug mode to track speed of training and evaluation.
+- Created a new splitter class, `SameInnerSplitSplitter`, which allows you to average the validation scores of the 
+same model selection configuration over multiple runs without changing the inner data split. Cannot be combined with a
+double/nested CV approach, for which you should use the base `Splitter` class to generate different inner data splits.
+- Trying out a helper mechanism to print to terminal information about the experiment that broke (if any) 
+when you are not in debug mode.
+
 ## [1.5.1] New default behavior - more efficient training
 
 ### Changed - PLEASE READ

docs/conf.py

@@ -24,7 +24,7 @@
 author = "Federico Errica"
 
 # The full version, including alpha/beta/rc tags
-release = "1.5.1"
+release = "1.5.2"
 
 
 # -- General configuration ---------------------------------------------------

pydgn/data/splitter.py

@@ -437,6 +437,131 @@ def save(self, path: str):
         print("Done.")
 
 
+class SameInnerSplitSplitter(Splitter):
+    r"""
+    Splitter subclass that can be used to have multiple training runs of the
+    same configuration at model selection time. It is not meant to be combined
+    with a double-nested CV, for which the different inner splits are already
+    enough to gauge the training stability of each configuration.
+    """
+    def split(
+        self,
+        dataset: pydgn.data.dataset.DatasetInterface,
+        targets: np.ndarray = None,
+    ):
+        r"""
+        Computes the splits and stores them in the list fields
+        ``self.outer_folds`` and ``self.inner_folds``.
+        IMPORTANT: calling split() sets the seed of numpy, torch, and
+        random for reproducibility.
+
+        Args:
+            dataset (:class:`~pydgn.data.dataset.DatasetInterface`):
+                the Dataset object
+            targets (np.ndarray]): targets used for stratification.
+                Default is ``None``
+        """
+        np.random.seed(self.seed)
+        torch.manual_seed(self.seed)
+        torch.cuda.manual_seed(self.seed)
+        random.seed(self.seed)
+
+        idxs = range(len(dataset))
+
+        stratified = self.stratify
+        outer_idxs = np.array(idxs)
+
+        outer_splitter = self._get_splitter(
+            n_splits=self.n_outer_folds,
+            stratified=stratified,
+            eval_ratio=self.test_ratio,
+        )  # This is the true test (outer test)
+
+        for train_idxs, test_idxs in outer_splitter.split(
+            outer_idxs, y=targets if stratified else None
+        ):
+
+            assert set(train_idxs) == set(outer_idxs[train_idxs])
+            assert set(test_idxs) == set(outer_idxs[test_idxs])
+
+            inner_fold_splits = []
+            inner_idxs = outer_idxs[
+                train_idxs
+            ]  # equals train_idxs because outer_idxs was ordered
+            inner_targets = (
+                targets[train_idxs] if targets is not None else None
+            )
+
+            inner_splitter = self._get_splitter(
+                n_splits=self.n_inner_folds,
+                stratified=stratified,
+                eval_ratio=self.inner_val_ratio,
+            )  # The inner "test" is, instead, the validation set
+
+            for inner_train_idxs, inner_val_idxs in inner_splitter.split(
+                inner_idxs, y=inner_targets if stratified else None
+            ):
+                inner_fold = InnerFold(
+                    train_idxs=inner_idxs[inner_train_idxs].tolist(),
+                    val_idxs=inner_idxs[inner_val_idxs].tolist(),
+                )
+
+                # False if empty
+                assert not bool(
+                    set(inner_train_idxs)
+                    & set(inner_val_idxs)
+                    & set(test_idxs)
+                )
+                assert not bool(
+                    set(inner_idxs[inner_train_idxs])
+                    & set(inner_idxs[inner_val_idxs])
+                    & set(test_idxs)
+                )
+
+                # we ignore the different inner splits and use only the first
+                # one to be reused multiple times (effectively simulating
+                # multiple training runs of the same configuration on the same
+                # training/validation data split
+                for _ in range(self.n_inner_folds):
+                    inner_fold_splits.append(inner_fold)
+                break
+
+            self.inner_folds.append(inner_fold_splits)
+
+            # Obtain outer val from outer train in an holdout fashion
+            outer_val_splitter = self._get_splitter(
+                n_splits=1,
+                stratified=stratified,
+                eval_ratio=self.outer_val_ratio,
+            )
+            outer_train_idxs, outer_val_idxs = list(
+                outer_val_splitter.split(inner_idxs, y=inner_targets)
+            )[0]
+
+            # False if empty
+            assert not bool(
+                set(outer_train_idxs) & set(outer_val_idxs) & set(test_idxs)
+            )
+            assert not bool(
+                set(outer_train_idxs) & set(outer_val_idxs) & set(test_idxs)
+            )
+            assert not bool(
+                set(inner_idxs[outer_train_idxs])
+                & set(inner_idxs[outer_val_idxs])
+                & set(test_idxs)
+            )
+
+            np.random.shuffle(outer_train_idxs)
+            np.random.shuffle(outer_val_idxs)
+            np.random.shuffle(test_idxs)
+            outer_fold = OuterFold(
+                train_idxs=inner_idxs[outer_train_idxs].tolist(),
+                val_idxs=inner_idxs[outer_val_idxs].tolist(),
+                test_idxs=outer_idxs[test_idxs].tolist(),
+            )
+            self.outer_folds.append(outer_fold)
+
+
 class TemporalSplitter(Splitter):
     r"""
     Reads the entire dataset and returns the targets. In this case, each

pydgn/evaluation/evaluator.py

@@ -86,10 +86,16 @@ def run_valid(
     """
     if not osp.exists(fold_results_torch_path):
         start = time.time()
-        experiment = experiment_class(config, fold_exp_folder, exp_seed)
-        train_res, val_res = experiment.run_valid(dataset_getter, logger)
-        elapsed = time.time() - start
-        torch.save((train_res, val_res, elapsed), fold_results_torch_path)
+        try:
+            experiment = experiment_class(config, fold_exp_folder, exp_seed)
+            train_res, val_res = experiment.run_valid(dataset_getter, logger)
+            elapsed = time.time() - start
+            torch.save((train_res, val_res, elapsed), fold_results_torch_path)
+        except Exception as e:
+            print(f'There has been an issue with configuration '
+                  f'in {fold_exp_folder}!')
+            print(e)
+            elapsed = -1
     else:
         _, _, elapsed = torch.load(fold_results_torch_path)
     return dataset_getter.outer_k, dataset_getter.inner_k, config_id, elapsed
@@ -138,17 +144,21 @@ def run_test(
         a tuple with outer fold id, final run id, and time elapsed
     """
     if not osp.exists(final_run_torch_path):
-        start = time.time()
-        experiment = experiment_class(
-            best_config[CONFIG], final_run_exp_path, exp_seed
-        )
-        res = experiment.run_test(dataset_getter, logger)
-        elapsed = time.time() - start
-
-        train_res, val_res, test_res = res
-        torch.save(
-            (train_res, val_res, test_res, elapsed), final_run_torch_path
-        )
+        try:
+            start = time.time()
+            experiment = experiment_class(
+                best_config[CONFIG], final_run_exp_path, exp_seed
+            )
+            res = experiment.run_test(dataset_getter, logger)
+            elapsed = time.time() - start
+            train_res, val_res, test_res = res
+            torch.save(
+                (train_res, val_res, test_res, elapsed), final_run_torch_path
+            )
+        except Exception as e:
+            print(f'There has been an issue in {final_run_exp_path}!')
+            print(e)
+            elapse = -1
     else:
         res = torch.load(final_run_torch_path)
         elapsed = res[-1]

pydgn/training/engine.py

@@ -6,6 +6,7 @@
 from torch.utils.data import SequentialSampler
 from torch_geometric.data import Data
 from torch_geometric.loader import DataLoader
+from tqdm import tqdm
 
 import pydgn
 from pydgn.log.logger import Logger
@@ -146,6 +147,7 @@ def __init__(
         self.eval_training = eval_training
         self.store_last_checkpoint = store_last_checkpoint
         self.training = False
+        self.logger = None
 
         # For dynamic graph learning
         self.reset_eval_model_hidden_state = reset_eval_model_hidden_state
@@ -421,7 +423,11 @@ def _loop(self, loader: DataLoader):
         self.state.update(loader_iterable=iter(loader))
 
         # Loop over data
-        for id_batch in range(len(loader)):
+        for id_batch in tqdm(range(len(loader)),
+                             desc=f'Epoch {self.state.epoch+1}, {self.state.set} set',
+                             unit='batch',
+                             disable=not self.logger.debug,
+                             leave=False):
             self.state.update(id_batch=id_batch)
             # EngineCallback will store fetched data in state.batch_input
             self._dispatch(EventHandler.ON_FETCH_DATA, self.state)
@@ -544,6 +550,7 @@ def train(
              validation_loss, validation_score, validation_embeddings,
              test_loss, test_score, test_embeddings)
         """
+        self.logger = logger
 
         try:
             # Initialize variables

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pydgn"
-version = "1.5.1"
+version = "1.5.2"
 description = "A Python Package for Deep Graph Networks"
 authors = [ { name="Federico Errica", email="[email protected]" } ]
 readme = "README.md"
@@ -18,6 +18,7 @@ requires-python = ">=3.8"
 
 dependencies = [
     "PyYAML>=5.4",
+    "tqdm>=4.66.1",
     "Requests>=2.31.0",
     "scikit_learn>=1.3.0",
     "tensorboard>=2.11.0",