Tiling Module (#40)

* update tiling

* update det stage

* update

* remove redundant configure hyperparams

* remove configure_hyperparams

* update

* update

* update evaluation IoU to 0:5:0.95:0.05

* update

* update inferrer

* update inferrer

* add todo to exporter

* adapter detection inferrer for tiling

* fix bug

* revert unnecessary changes

* fix typo and add minor change

Author: eugene123tw

mpa/cls/inferrer.py

@@ -100,17 +100,17 @@ def _infer(self, cfg, dump_features=False, dump_saliency_map=False):
             outputs = data_infos
         else:
             with FeatureVectorHook(model.module.backbone) if dump_features else nullcontext() as fhook:
-              with SaliencyMapHook(model.module.backbone) if dump_saliency_map else nullcontext() as shook:
-                  for data in data_loader:
-                      with torch.no_grad():
-                          result = model(return_loss=False, **data)
-                      eval_predictions.extend(result)
-                  feature_vectors = fhook.records if dump_features else [None] * len(self.dataset)
-                  saliency_maps = shook.records if dump_saliency_map else [None] * len(self.dataset)
+                with SaliencyMapHook(model.module.backbone) if dump_saliency_map else nullcontext() as shook:
+                    for data in data_loader:
+                        with torch.no_grad():
+                            result = model(return_loss=False, **data)
+                        eval_predictions.extend(result)
+                    feature_vectors = fhook.records if dump_features else [None] * len(self.dataset)
+                    saliency_maps = shook.records if dump_saliency_map else [None] * len(self.dataset)
 
         assert len(eval_predictions) == len(feature_vectors) == len(saliency_maps), \
-                'Number of elements should be the same, however, number of outputs are ' \
-                f"{len(eval_predictions)}, {len(feature_vectors)}, and {len(saliency_maps)}"
+               'Number of elements should be the same, however, number of outputs are ' \
+               f"{len(eval_predictions)}, {len(feature_vectors)}, and {len(saliency_maps)}"
         outputs = dict(
             eval_predictions=eval_predictions,
             feature_vectors=feature_vectors,

mpa/cls/stage.py

@@ -84,10 +84,6 @@ def configure(self, model_cfg, model_ckpt, data_cfg, training=True, **kwargs):
             if cfg.model.get('multilabel', False) or cfg.model.get('hierarchical', False):
                 cfg.model.head.pop('topk', None)
 
-        # Other hyper-parameters
-        if cfg.get('hyperparams', False):
-            self.configure_hyperparams(cfg, training, **kwargs)
-
         return cfg
 
     @staticmethod
@@ -147,7 +143,7 @@ def configure_task(cfg, training, model_meta=None, **kwargs):
 
         model_tasks, dst_classes = None, None
         model_classes, data_classes = [], []
-        train_data_cfg = Stage.get_train_data_cfg(cfg)
+        train_data_cfg = Stage.get_data_cfg(cfg, "train")
         if isinstance(train_data_cfg, list):
             train_data_cfg = train_data_cfg[0]
 
@@ -268,18 +264,6 @@ def configure_task(cfg, training, model_meta=None, **kwargs):
                 cfg.model.head.num_old_classes = len(old_classes)
         return model_tasks, dst_classes
 
-    @staticmethod
-    def configure_hyperparams(cfg, training, **kwargs):
-        hyperparams = kwargs.get('hyperparams', None)
-        if hyperparams is not None:
-            bs = hyperparams.get('bs', None)
-            if bs is not None:
-                cfg.data.samples_per_gpu = bs
-
-            lr = hyperparams.get('lr', None)
-            if lr is not None:
-                cfg.optimizer.lr = lr
-
 
 def refine_tasks(train_cfg, meta, adapt_type):
     new_tasks = train_cfg['tasks']

mpa/cls/trainer.py

@@ -150,7 +150,7 @@ def train_worker(gpu, dataset, cfg, distributed, validate, timestamp, meta):
 
         # prepare data loaders
         dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset]
-        train_data_cfg = Stage.get_train_data_cfg(cfg)
+        train_data_cfg = Stage.get_data_cfg(cfg, "train")
         drop_last = train_data_cfg.drop_last if train_data_cfg.get('drop_last', False) else False
 
         # updated to adapt list of dataset for the 'train'

mpa/det/exporter.py

@@ -34,6 +34,7 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
 
         output_path = os.path.join(cfg.work_dir, 'export')
         os.makedirs(output_path, exist_ok=True)
+
         model = build_detector(cfg.model)
         if model_ckpt:
             load_checkpoint(model=model, filename=model_ckpt, map_location='cpu')
@@ -47,7 +48,7 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
                 model = model.cpu()
             precision = kwargs.pop('precision', 'FP32')
             logger.info(f'Model will be exported with precision {precision}')
-            
+
             export_model(model, cfg, output_path, target='openvino', precision=precision)
         except Exception as ex:
             # output_model.model_status = ModelStatus.FAILED

mpa/det/inferrer.py

@@ -7,15 +7,17 @@
 from mmcv.parallel import MMDataParallel, is_module_wrapper
 from mmcv.runner import load_checkpoint
 
-from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor
+from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor, ImageTilingDataset
 from mmdet.models import build_detector
 from mmdet.parallel import MMDataCPU
 from mmdet.utils.deployment import get_saliency_map, get_feature_vector
+from mmdet.apis import single_gpu_test
 
 from mpa.registry import STAGES
 from .stage import DetectionStage
 from mpa.utils.logger import get_logger
 
+
 logger = get_logger()
 
 
@@ -33,9 +35,9 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
         """
         self._init_logger()
         mode = kwargs.get('mode', 'train')
-        eval = kwargs.get('eval', False)
-        dump_features = kwargs.get('dump_features', False)
-        dump_saliency_map = kwargs.get('dump_saliency_map', False)
+        eval = kwargs.pop('eval', False)
+        dump_features = kwargs.pop('dump_features', False)
+        dump_saliency_map = kwargs.pop('dump_saliency_map', False)
         if mode not in self.mode:
             return {}
 
@@ -82,7 +84,7 @@ def infer(self, cfg, eval=False, dump_features=False, dump_saliency_map=False):
             input_source = cfg.get('input_source')
             logger.info(f'Inferring on input source: data.{input_source}')
             if input_source == 'train':
-                src_data_cfg = self.get_train_data_cfg(cfg)
+                src_data_cfg = self.get_data_cfg(cfg, input_source)
             else:
                 src_data_cfg = cfg.data[input_source]
             data_cfg.test_mode = src_data_cfg.get('test_mode', False)
@@ -180,10 +182,7 @@ def dummy_dump_saliency_hook(model, input, out):
             model = model.module
         with eval_model.module.backbone.register_forward_hook(feature_vector_hook):
             with eval_model.module.backbone.register_forward_hook(saliency_map_hook):
-                for data in data_loader:
-                    with torch.no_grad():
-                        result = eval_model(return_loss=False, rescale=True, **data)
-                    eval_predictions.extend(result)
+                eval_predictions = single_gpu_test(eval_model, data_loader)
 
         for key in [
                 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best',
@@ -193,7 +192,20 @@ def dummy_dump_saliency_hook(model, input, out):
 
         metric = None
         if eval:
-            metric = dataset.evaluate(eval_predictions, **cfg.evaluation)[cfg.evaluation.metric]
+            metric = dataset.evaluate(eval_predictions, **cfg.evaluation)
+            metric = metric['mAP'] if isinstance(cfg.evaluation.metric, list) else metric[cfg.evaluation.metric]
+
+        # Check and unwrap ImageTilingDataset object from TaskAdaptEvalDataset
+        while hasattr(dataset, 'dataset') and not isinstance(dataset, ImageTilingDataset):
+            dataset = dataset.dataset
+
+        if isinstance(dataset, ImageTilingDataset):
+            saliency_maps = [saliency_maps[i] for i in range(dataset.num_samples)]
+            feature_vectors = [feature_vectors[i] for i in range(dataset.num_samples)]
+            if not dataset.merged_results:
+                eval_predictions = dataset.merge(eval_predictions)
+            else:
+                eval_predictions = dataset.merged_results
 
         outputs = dict(
             classes=target_classes,

mpa/det/stage.py

@@ -62,10 +62,6 @@ def configure(self, model_cfg, model_ckpt, data_cfg, training=True, **kwargs):
         if training:
             self.configure_regularization(cfg)
 
-        # Other hyper-parameters
-        if 'hyperparams' in cfg:
-            self.configure_hyperparams(cfg, training, **kwargs)
-
         # Hooks
         self.configure_hook(cfg)
 
@@ -117,12 +113,15 @@ def configure_data(self, cfg, training, **kwargs):
                         seed=cfg.seed
                     )
                 )
-            if 'dataset' in cfg.data.train:
-                train_cfg = self.get_train_data_cfg(cfg)
-                train_cfg.ote_dataset = cfg.data.train.pop('ote_dataset', None)
-                train_cfg.labels = cfg.data.train.get('labels', None)
-                train_cfg.data_classes = cfg.data.train.pop('data_classes', None)
-                train_cfg.new_classes = cfg.data.train.pop('new_classes', None)
+        for subset in ("train", "val", "test"):
+            if 'dataset' in cfg.data[subset]:
+                subset_cfg = self.get_data_cfg(cfg, subset)
+                subset_cfg.ote_dataset = cfg.data[subset].pop('ote_dataset', None)
+                subset_cfg.labels = cfg.data[subset].get('labels', None)
+                if 'data_classes' in cfg.data[subset]:
+                    subset_cfg.data_classes = cfg.data[subset].pop('data_classes')
+                if 'new_classes' in cfg.data[subset]:
+                    subset_cfg.new_classes = cfg.data[subset].pop('new_classes')
 
     def configure_task(self, cfg, training, **kwargs):
         """Adjust settings for task adaptation
@@ -200,7 +199,7 @@ def configure_task_classes(self, cfg, task_adapt_type, task_adapt_op):
 
     def configure_task_data_pipeline(self, cfg, model_classes, data_classes):
         # Trying to alter class indices of training data according to model class order
-        tr_data_cfg = self.get_train_data_cfg(cfg)
+        tr_data_cfg = self.get_data_cfg(cfg, "train")
         class_adapt_cfg = dict(type='AdaptClassLabels', src_classes=data_classes, dst_classes=model_classes)
         pipeline_cfg = tr_data_cfg.pipeline
         for i, op in enumerate(pipeline_cfg):
@@ -240,7 +239,7 @@ def configure_task_cls_incr(self, cfg, task_adapt_type, org_model_classes, model
         else:
             bbox_head = cfg.model.roi_head.bbox_head
         if task_adapt_type == 'mpa':
-            tr_data_cfg = self.get_train_data_cfg(cfg)
+            tr_data_cfg = self.get_data_cfg(cfg, "train")
             if tr_data_cfg.type != 'MPADetDataset':
                 tr_data_cfg.img_ids_dict = self.get_img_ids_for_incr(cfg, org_model_classes, model_classes)
                 tr_data_cfg.org_type = tr_data_cfg.type
@@ -311,7 +310,7 @@ def configure_task_cls_incr(self, cfg, task_adapt_type, org_model_classes, model
                     ConfigDict(type='AdaptiveTrainSchedulingHook', **adaptive_validation_interval)
                 )
         else:
-            src_data_cfg = Stage.get_train_data_cfg(cfg)
+            src_data_cfg = Stage.get_data_cfg(cfg, "train")
             src_data_cfg.pop('old_new_indices', None)
 
     def configure_regularization(self, cfg):
@@ -338,7 +337,7 @@ def get_img_ids_for_incr(cfg, org_model_classes, model_classes):
         new_classes = np.setdiff1d(model_classes, org_model_classes).tolist()
         old_classes = np.intersect1d(org_model_classes, model_classes).tolist()
 
-        src_data_cfg = Stage.get_train_data_cfg(cfg)
+        src_data_cfg = Stage.get_data_cfg(cfg, "train")
 
         ids_old, ids_new = [], []
         data_cfg = cfg.data.test.copy()
@@ -366,17 +365,6 @@ def get_img_ids_for_incr(cfg, org_model_classes, model_classes):
         )
         return outputs
 
-    def configure_hyperparams(self, cfg, training, **kwargs):
-        hyperparams = kwargs.get('hyperparams', None)
-        if hyperparams is not None:
-            bs = hyperparams.get('bs', None)
-            if bs is not None:
-                cfg.data.samples_per_gpu = bs
-
-            lr = hyperparams.get('lr', None)
-            if lr is not None:
-                cfg.optimizer.lr = lr
-
     @staticmethod
     def add_yolox_hooks(cfg):
         update_or_add_custom_hook(

mpa/det/trainer.py

@@ -155,7 +155,7 @@ def train_worker(gpu, target_classes, datasets, cfg, distributed=False,
         # Do clustering for SSD model
         if hasattr(cfg.model, 'bbox_head') and hasattr(cfg.model.bbox_head, 'anchor_generator'):
             if getattr(cfg.model.bbox_head.anchor_generator, 'reclustering_anchors', False):
-                train_cfg = Stage.get_train_data_cfg(cfg)
+                train_cfg = Stage.get_data_cfg(cfg, "train")
                 train_dataset = train_cfg.get('ote_dataset', None)
                 cfg, model = cluster_anchors(cfg, train_dataset, model)
         train_detector(

mpa/seg/inferrer.py

@@ -62,7 +62,7 @@ def infer(self, cfg, dump_features=False):
         input_source = cfg.get('input_source', 'test')
         self.logger.info(f'Inferring on input source: data.{input_source}')
         if input_source == 'train':
-            src_data_cfg = Stage.get_train_data_cfg(cfg)
+            src_data_cfg = Stage.get_data_cfg(cfg, "train")
         else:
             src_data_cfg = cfg.data[input_source]
         data_cfg = cfg.data.test.copy()

mpa/seg/stage.py

@@ -62,10 +62,6 @@ def configure(self, model_cfg, model_ckpt, data_cfg, training=True, **kwargs):
         if 'task_adapt' in cfg:
             self.configure_task(cfg, training, **kwargs)
 
-        # Other hyper-parameters
-        if 'hyperparams' in cfg:
-            self.configure_hyperparams(cfg, training, **kwargs)
-
         return cfg
 
     def configure_model(self, cfg, training, **kwargs):
@@ -154,7 +150,7 @@ def configure_task_cls_incr(self, cfg, task_adapt_type, org_model_classes, model
                             head.loss_decode = [self.configure_am_softmax_loss_with_ignore(model_classes)]
 
         # Dataset
-        src_data_cfg = Stage.get_train_data_cfg(cfg)
+        src_data_cfg = Stage.get_data_cfg(cfg, "train")
         for mode in ['train', 'val', 'test']:
             if src_data_cfg.type == 'MPASegIncrDataset':
                 if cfg.data[mode]['type'] != 'MPASegIncrDataset':
@@ -200,7 +196,7 @@ def configure_task_classes(self, cfg, task_adapt_op):
 
         # Model classes
         if task_adapt_op == 'REPLACE':
-            if len(data_classes) == 1: # 'background'
+            if len(data_classes) == 1:  # 'background'
                 model_classes = org_model_classes.copy()
             else:
                 model_classes = data_classes.copy()
@@ -225,14 +221,3 @@ def configure_task_classes(self, cfg, task_adapt_op):
                     head.num_classes = len(model_classes)
 
         return org_model_classes, model_classes, data_classes
-
-    def configure_hyperparams(self, cfg, training, **kwargs):
-        hyperparams = kwargs.get('hyperparams', None)
-        if hyperparams is not None:
-            bs = hyperparams.get('bs', None)
-            if bs is not None:
-                cfg.data.samples_per_gpu = bs
-
-            lr = hyperparams.get('lr', None)
-            if lr is not None:
-                cfg.optimizer.lr = lr

mpa/stage.py

@@ -260,19 +260,20 @@ def get_model_meta(cfg):
         return meta
 
     @staticmethod
-    def get_train_data_cfg(cfg):
-        if 'dataset' in cfg.data.train:  # Concat|RepeatDataset
-            dataset = cfg.data.train.dataset
+    def get_data_cfg(cfg, subset):
+        assert subset in ["train", "val", "test"], f"Unknown subset:{subset}"
+        if 'dataset' in cfg.data[subset]:  # Concat|RepeatDataset
+            dataset = cfg.data[subset].dataset
             while hasattr(dataset, 'dataset'):
                 dataset = dataset.dataset
             return dataset
         else:
-            return cfg.data.train
+            return cfg.data[subset]
 
     @staticmethod
     def get_data_classes(cfg):
         data_classes = []
-        train_cfg = Stage.get_train_data_cfg(cfg)
+        train_cfg = Stage.get_data_cfg(cfg, "train")
         if 'data_classes' in train_cfg:
             data_classes = list(train_cfg.pop('data_classes', []))
         elif 'classes' in train_cfg: