ci: speed up test runs (#85)

* ci: speed up test runs

* fix: rm max_epochs in evaluators, run log_metrics

* fix: replace jinja value with 5 in single

* Apply suggestions from code review [skip ci]

Co-authored-by: vfdev <vfdev.5@gmail.com>

* remove useless code, fix #87

* remove useless code, fix #87

* readd custom events in single

* This commit add the somehow missed evaluation

* fix: reuse eval_epoch_length

Co-authored-by: vfdev <vfdev.5@gmail.com>

Author: ydcjeff

.github/run_test.sh

@@ -20,7 +20,12 @@ elif [ $1 == "unittest" ]; then
 elif [ $1 == "default" ]; then
     for file in $(find ./tests/dist -iname "main.py" -not -path "./tests/dist/launch/*" -not -path "./tests/dist/spawn/*" -not -path "./tests/dist/single/*")
     do
-        python $file --verbose --log_every_iters 2 --num_workers 1 --epoch_length 10
+        python $file \
+            --verbose \
+            --log_every_iters 2 \
+            --num_workers 1 \
+            --train_epoch_length 10 \
+            --eval_epoch_length 10
     done
 elif [ $1 == "launch" ]; then
     for file in $(find ./tests/dist/launch -iname "main.py" -not -path "./tests/dist/launch/single/*")
@@ -31,7 +36,8 @@ elif [ $1 == "launch" ]; then
             --verbose \
             --backend gloo \
             --num_workers 1 \
-            --epoch_length 10 \
+            --eval_epoch_length 10 \
+            --train_epoch_length 10 \
             --log_every_iters 2
     done
 elif [ $1 == "spawn" ]; then
@@ -41,7 +47,8 @@ elif [ $1 == "spawn" ]; then
             --verbose \
             --backend gloo \
             --num_workers 1 \
-            --epoch_length 10 \
+            --eval_epoch_length 10 \
+            --train_epoch_length 10 \
             --nproc_per_node 2 \
             --log_every_iters 2
     done

templates/_base/_argparse.py

@@ -56,10 +56,15 @@
         "type": int,
         "help": "master node port for torch native backends. Default: %(default)s",
     },
-    "epoch_length": {
+    "train_epoch_length": {
         "default": None,
         "type": int,
-        "help": "epoch_length of Engine.run(). Default: %(default)s"
+        "help": "epoch_length of Engine.run() for training. Default: %(default)s"
+    },
+    "eval_epoch_length": {
+        "default": None,
+        "type": int,
+        "help": "epoch_length of Engine.run() for evaluation. Default: %(default)s"
     },
     # ignite handlers options
     "save_every_iters": {

templates/gan/main.py

@@ -207,7 +207,7 @@ def create_plots(engine):
     # setup if done. let's run the training
     # ------------------------------------------
 
-    trainer.run(train_dataloader, max_epochs=config.max_epochs, epoch_length=config.epoch_length)
+    trainer.run(train_dataloader, max_epochs=config.max_epochs, epoch_length=config.train_epoch_length)
 
     # ------------------------------------------------------------
     # close the logger after the training completed / terminated

templates/image_classification/main.py

@@ -12,7 +12,7 @@
 from ignite.metrics import Accuracy, Loss
 
 from datasets import get_datasets
-from trainers import create_trainers, TrainEvents
+from trainers import create_trainers
 from utils import setup_logging, log_metrics, log_basic_info, initialize, resume_from, get_handlers, get_logger
 from config import get_default_parser
 
@@ -140,30 +140,6 @@ def run(local_rank: int, config: Any, *args: Any, **kwargs: Any):
     if config.resume_from:
         resume_from(to_load=to_save, checkpoint_fp=config.resume_from)
 
-    # --------------------------------------------
-    # let's trigger custom events we registered
-    # we will use a `event_filter` to trigger that
-    # `event_filter` has to return boolean
-    # whether this event should be executed
-    # here will log the gradients on the 1st iteration
-    # and every 100 iterations
-    # --------------------------------------------
-
-    @trainer.on(TrainEvents.BACKWARD_COMPLETED(lambda _, ev: (ev % 100 == 0) or (ev == 1)))
-    def _():
-        # do something interesting
-        pass
-
-    # ----------------------------------------
-    # here we will use `every` to trigger
-    # every 100 iterations
-    # ----------------------------------------
-
-    @trainer.on(TrainEvents.OPTIM_STEP_COMPLETED(every=100))
-    def _():
-        # do something interesting
-        pass
-
     # --------------------------------
     # print metrics to the stderr
     # with `add_event_handler` API
@@ -182,23 +158,22 @@ def _():
 
     @trainer.on(Events.EPOCH_COMPLETED(every=1))
     def _():
-        evaluator.run(eval_dataloader, max_epochs=1)
-        evaluator.add_event_handler(Events.EPOCH_COMPLETED(every=1), log_metrics, tag="eval")
+        evaluator.run(eval_dataloader, epoch_length=config.eval_epoch_length)
+        log_metrics(evaluator, "eval")
 
     # --------------------------------------------------
     # let's try run evaluation first as a sanity check
     # --------------------------------------------------
 
     @trainer.on(Events.STARTED)
     def _():
-        evaluator.run(eval_dataloader, max_epochs=1, epoch_length=2)
-        evaluator.state.max_epochs = None
+        evaluator.run(eval_dataloader, epoch_length=config.eval_epoch_length)
 
     # ------------------------------------------
     # setup if done. let's run the training
     # ------------------------------------------
 
-    trainer.run(train_dataloader, max_epochs=config.max_epochs, epoch_length=config.epoch_length)
+    trainer.run(train_dataloader, max_epochs=config.max_epochs, epoch_length=config.train_epoch_length)
 
     # ------------------------------------------------------------
     # close the logger after the training completed / terminated

templates/single/config.py

@@ -1 +1,15 @@
-{% include "_argparse.py" %}
+{% extends "_argparse.py" %}
+{% block get_default_parser %}
+UPDATES = {
+    # training options
+    "max_epochs": {
+        "default": 5,
+        "type": int,
+        "help": "max_epochs of ignite.Engine.run() for training. Default: %(default)s",
+    }
+}
+
+DEFAULTS.update(UPDATES)
+
+{{ super() }}
+{% endblock %}

templates/single/main.py

@@ -160,24 +160,23 @@ def _():
 
     @trainer.on(Events.EPOCH_COMPLETED(every=1))
     def _():
-        evaluator.run(eval_dataloader, max_epochs=1)
-        evaluator.add_event_handler(Events.EPOCH_COMPLETED(every=1), log_metrics, tag="eval")
+        evaluator.run(eval_dataloader, epoch_length=config.eval_epoch_length)
+        log_metrics(evaluator, "eval")
 
     # --------------------------------------------------
     # let's try run evaluation first as a sanity check
     # --------------------------------------------------
 
     @trainer.on(Events.STARTED)
     def _():
-        evaluator.run(eval_dataloader, max_epochs=1, epoch_length=2)
-        evaluator.state.max_epochs = None
+        evaluator.run(eval_dataloader, epoch_length=config.eval_epoch_length)
 
     # ------------------------------------------
     # setup if done. let's run the training
     # ------------------------------------------
     # TODO : PLEASE provide `max_epochs` parameters
 
-    trainer.run(train_dataloader, epoch_length=config.epoch_length)
+    trainer.run(train_dataloader, max_epochs=config.max_epochs, epoch_length=config.train_epoch_length)
 
     # ------------------------------------------------------------
     # close the logger after the training completed / terminated