Changed related to ONNX support for yolact++ added

README.md

@@ -54,14 +54,42 @@ Some examples from our YOLACT base model (33.5 fps on a Titan Xp and 29.8 mAP on
    ```Shell
    sh data/scripts/COCO_test.sh
    ```
- - If you want to use YOLACT++, compile deformable convolutional layers (from [DCNv2](https://github.com/CharlesShang/DCNv2/tree/pytorch_1.0)).
+ - If you want to use YOLACT++, compile deformable convolutional layers.
    Make sure you have the latest CUDA toolkit installed from [NVidia's Website](https://developer.nvidia.com/cuda-toolkit).
+
+   Case 1: Only need to use pytorch code
    ```Shell
-   cd external/DCNv2
-   python setup.py build develop
+   pip install mmcv-full
    ```
+   Case 2: If you'd like to inference using onnx models. Follow - https://mmcv.readthedocs.io/en/latest/deployment/onnxruntime_op.html
+   ```Shell
+   wget https://github.com/microsoft/onnxruntime/releases/download/v1.8.1/onnxruntime-linux-x64-1.8.1.tgz
 
+   tar -zxvf onnxruntime-linux-x64-1.8.1.tgz
+   cd onnxruntime-linux-x64-1.8.1
+   export ONNXRUNTIME_DIR=$(pwd)
+   export LD_LIBRARY_PATH=$ONNXRUNTIME_DIR/lib:$LD_LIBRARY_PATH
+   ```
+   ```Shell
+   cd ..
+   git clone https://github.com/open-mmlab/mmcv.git
+   cd mmcv ## to MMCV root directory
+   MMCV_WITH_OPS=1 MMCV_WITH_ORT=1 python setup.py develop
+   ```
+   ```Shell
+   pip install onnxruntime==1.8.1
+   ```
 
+## ONNX export and Inference (tested for yolact_plus for image with size 550, supports batch-size=1)
+For pytorch model to ONNX conversion 
+```Shell
+python3 yolact2onnx.py --config yolact_plus_base_config --ckpt_path yolact_plus_base_159_180000.pth --onnx_paths yolact_plus.onnx maskiou_net.onnx --score_threshold 0.5
+```
+
+For ONNX inference
+```Shell
+python3 onnx_inference.py --config yolact_plus_base_config --img_path 15387869.jpg --onnx_paths yolact_plus.onnx maskiou_net.onnx --score_threshold 0.5
+```
 # Evaluation
 Here are our YOLACT models (released on April 5th, 2019) along with their FPS on a Titan Xp and mAP on `test-dev`:
 

backbone.py

@@ -5,7 +5,7 @@
 from collections import OrderedDict
 
 try:
-    from dcn_v2 import DCN
+    from mmcv.ops import ModulatedDeformConv2dPack as DCN
 except ImportError:
     def DCN(*args, **kwdargs):
         raise Exception('DCN could not be imported. If you want to use YOLACT++ models, compile DCN. Check the README for instructions.')
@@ -21,9 +21,6 @@ def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=nn.Ba
         if use_dcn:
             self.conv2 = DCN(planes, planes, kernel_size=3, stride=stride,
                                 padding=dilation, dilation=dilation, deformable_groups=1)
-            self.conv2.bias.data.zero_()
-            self.conv2.conv_offset_mask.weight.data.zero_()
-            self.conv2.conv_offset_mask.bias.data.zero_()
         else:
             self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
                                 padding=dilation, bias=False, dilation=dilation)

data/config.py

@@ -625,6 +625,7 @@ def print(self):
 
     # Use command-line arguments to set this.
     'no_jit': False,
+    'export_onnx':False
 
     'backbone': None,
     'name': 'base_config',

layers/functions/detection.py

@@ -29,7 +29,7 @@ def __init__(self, num_classes, bkg_label, top_k, conf_thresh, nms_thresh):
         self.use_cross_class_nms = False
         self.use_fast_nms = False
 
-    def __call__(self, predictions, net):
+    def __call__(self, predictions, net=None):
         """
         Args:
              loc_data: (tensor) Loc preds from loc layers
@@ -72,8 +72,10 @@ def __call__(self, predictions, net):
 
                 if result is not None and proto_data is not None:
                     result['proto'] = proto_data[batch_idx]
-
-                out.append({'detection': result, 'net': net})
+                if net is not None:
+                    out.append({'detection': result, 'net': net})
+                else:
+                    out.append({'detection': result})
 
         return out
 

layers/output_utils.py

@@ -12,7 +12,7 @@
 from utils import timer
 from .box_utils import crop, sanitize_coordinates
 
-def postprocess(det_output, w, h, batch_idx=0, interpolation_mode='bilinear',
+def postprocess(det_output, w, h, maskiou_net=None, batch_idx=0, interpolation_mode='bilinear',
                 visualize_lincomb=False, crop_masks=True, score_threshold=0):
     """
     Postprocesses the output of Yolact on testing mode into a format that makes sense,
@@ -31,10 +31,12 @@ def postprocess(det_output, w, h, batch_idx=0, interpolation_mode='bilinear',
         - boxes   [num_det, 4]: The bounding box for each detection in absolute point form.
         - masks   [num_det, h, w]: Full image masks for each detection.
     """
-
     dets = det_output[batch_idx]
-    net = dets['net']
-    dets = dets['detection']
+    if not maskiou_net:
+        net = dets['net']
+        dets = dets['detection']
+    else:
+        dets = dets['detection']
 
     if dets is None:
         return [torch.Tensor()] * 4 # Warning, this is 4 copies of the same thing
@@ -79,7 +81,11 @@ def postprocess(det_output, w, h, batch_idx=0, interpolation_mode='bilinear',
         if cfg.use_maskiou:
             with timer.env('maskiou_net'):                
                 with torch.no_grad():
-                    maskiou_p = net.maskiou_net(masks.unsqueeze(1))
+                    if maskiou_net is not None:
+                        maskiou_p = maskiou_net.run(None, {"input": masks.unsqueeze(1).numpy()})
+                        maskiou_p = torch.from_numpy(maskiou_p[0])
+                    else:
+                        maskiou_p = net.maskiou_net(masks.unsqueeze(1))
                     maskiou_p = torch.gather(maskiou_p, dim=1, index=classes.unsqueeze(1)).squeeze(1)
                     if cfg.rescore_mask:
                         if cfg.rescore_bbox: