Merge pull request #97 from diningphil/store-metrics-over-time

added option to store metrics over time in a file

Author: diningphil

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [1.5.6] Storing metrics on disk
+
+### Added
+
+- You can now store metrics trend across epochs using `Plotter`. Just pass the argument `store_on_disk=True` in the configuration file of the experiment.
+
 ## [1.5.5] 
 
 ### Fixed

docs/conf.py

@@ -24,7 +24,7 @@
 author = "Federico Errica"
 
 # The full version, including alpha/beta/rc tags
-release = "1.5.5"
+release = "1.5.6"
 
 
 # -- General configuration ---------------------------------------------------

examples/MODEL_CONFIGS/config_SupToyDGN.yml

@@ -119,4 +119,7 @@ grid:
           checkpoint: True  # store the best checkpoint
 
     # Plotter of metrics
-    plotter: pydgn.training.callback.plotter.Plotter
+    plotter:
+      - class_name: pydgn.training.callback.plotter.Plotter
+        args:
+          store_on_disk: True

pydgn/training/callback/plotter.py

@@ -1,6 +1,7 @@
 import os
 from pathlib import Path
 
+import torch
 from torch.utils.tensorboard import SummaryWriter
 
 from pydgn.static import *
@@ -14,17 +15,29 @@ class Plotter(EventHandler):
 
     Args:
         exp_path (str): path where to store the Tensorboard logs
+        store_on_disk (bool): whether to store all metrics on disk.
+            Defaults to False
         kwargs (dict): additional arguments that may depend on the plotter
     """
 
-    def __init__(self, exp_path: str, **kwargs: dict):
+    def __init__(self,
+                 exp_path: str,
+                 store_on_disk: bool = False,
+                 **kwargs: dict):
         super().__init__()
         self.exp_path = exp_path
+        self.store_on_disk = store_on_disk
 
         if not os.path.exists(Path(self.exp_path, TENSORBOARD)):
             os.makedirs(Path(self.exp_path, TENSORBOARD))
         self.writer = SummaryWriter(log_dir=Path(self.exp_path, "tensorboard"))
 
+        self.stored_metrics = {"losses": {}, "scores": {}}
+        self.stored_metrics_path = Path(self.exp_path, "metrics_data.torch")
+        if os.path.exists(self.stored_metrics_path):
+            self.stored_metrics = torch.load(self.stored_metrics_path)
+
+
     def on_epoch_end(self, state: State):
         """
         Writes Training, Validation and (if any) Test metrics to Tensorboard
@@ -46,6 +59,13 @@ def on_epoch_end(self, state: State):
 
             self.writer.add_scalars(loss_name, loss_scalars, state.epoch)
 
+            if self.store_on_disk:
+                t = "losses"
+                if not k in self.stored_metrics[t]:
+                    self.stored_metrics[t][k] = [v.item()]
+                else:
+                    self.stored_metrics[t][k].append(v.item())
+
         for k, v in state.epoch_results[SCORES].items():
             score_scalars = {}
             # Remove training/validation/test prefix (coupling with Engine)
@@ -59,6 +79,20 @@ def on_epoch_end(self, state: State):
 
             self.writer.add_scalars(score_name, score_scalars, state.epoch)
 
+            if self.store_on_disk:
+                t = "scores"
+                if not k in self.stored_metrics[t]:
+                    self.stored_metrics[t][k] = [v.item()]
+                else:
+                    self.stored_metrics[t][k].append(v.item())
+
+        if self.store_on_disk:
+            try:
+                torch.save(self.stored_metrics, self.stored_metrics_path)
+            except RuntimeError as e:
+                print(e)
+
+
     def on_fit_end(self, state: State):
         """
         Frees resources by closing the Tensorboard writer

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pydgn"
-version = "1.5.5"
+version = "1.5.6"
 description = "A Python Package for Deep Graph Networks"
 authors = [ { name="Federico Errica", email="[email protected]" } ]
 readme = "README.md"