Add DaSiamRPN for object tracking (#15)

* impl wrapper & demo

* add data for object tracking benchmark

* impl benchmark for DaSiamRPN

* update benchmark results for DaSiamRPN

Author: fengyuentau

README.md

@@ -36,6 +36,7 @@ Hardware Setup:
 | [PP-ResNet](./models/image_classification_ppresnet) | 224x224 | 56.05 | 602.58 | 98.64 |
 | [PP-HumanSeg](./models/human_segmentation_pphumanseg) | 192x192 | 19.92 | 105.32 | 67.97 |
 | [WeChatQRCode](./models/qrcode_wechatqrcode) | 100x100 | 7.04 | 37.68 | --- |
+| [DaSiamRPN](./models/object_tracking_dasiamrpn) | 1280x720 | 36.15 | 705.48 | 76.82 |
 
 ## License
 

benchmark/benchmark.py

@@ -85,14 +85,14 @@ def run(self, model):
         model.setBackend(self._backend)
         model.setTarget(self._target)
 
-        if 'video' in self._dataloader.name.lower():
-            model.init(self._dataloader.getROI())
-
-        for data in self._dataloader:
-            filename, img = data[:2]
-            size = [img.shape[1], img.shape[0]]
+        for idx, data in enumerate(self._dataloader):
+            filename, input_data = data[:2]
             if filename not in self._benchmark_results:
                 self._benchmark_results[filename] = dict()
+            if isinstance(input_data, np.ndarray):
+                size = [input_data.shape[1], input_data.shape[0]]
+            else:
+                size = input_data.getFrameSize()
             self._benchmark_results[filename][str(size)] = self._metric.forward(model, *data[1:])
 
     def printResults(self):

benchmark/config/object_tracking_dasiamrpn.yaml

@@ -0,0 +1,18 @@
+Benchmark:
+  name: "Object Tracking Benchmark"
+  type: "Tracking"
+  data:
+    type: "TrackingVideoLoader"
+    path: "benchmark/data/object_tracking"
+    files: ["throw_cup.mp4"]
+  metric:
+    type: "Tracking"
+    reduction: "gmean"
+  backend: "default"
+  target: "cpu"
+
+Model:
+  name: "DaSiamRPN"
+  model_path: "models/object_tracking_dasiamrpn/object_tracking_dasiamrpn_model_2021nov.onnx"
+  kernel_cls1_path: "models/object_tracking_dasiamrpn/object_tracking_dasiamrpn_kernel_cls1_2021nov.onnx"
+  kernel_r1_path: "models/object_tracking_dasiamrpn/object_tracking_dasiamrpn_kernel_r1_2021nov.onnx"

benchmark/download_data.py

@@ -184,7 +184,11 @@ def get_confirm_token(response):  # in case of large files
     qrcode=Downloader(name='qrcode',
         url='https://drive.google.com/u/0/uc?id=1_OXB7eiCIYO335ewkT6EdAeXyriFlq_H&export=download',
         sha='ac01c098934a353ca1545b5266de8bb4f176d1b3',
-        filename='qrcode.zip')
+        filename='qrcode.zip'),
+    object_tracking=Downloader(name='object_tracking',
+        url='https://drive.google.com/u/0/uc?id=1_cw5pUmTF-XmQVcQAI8fIp-Ewi2oMYIn&export=download',
+        sha='0bdb042632a245270013713bc48ad35e9221f3bb',
+        filename='object_tracking.zip')
 )
 
 if __name__ == '__main__':

benchmark/utils/dataloaders/base_dataloader.py

@@ -34,7 +34,7 @@ def __iter__(self):
 class _VideoStream:
     def __init__(self, filepath):
         self._filepath = filepath
-        self._video = cv.VideoCapture(filepath)
+        self._video = cv.VideoCapture(self._filepath)
 
     def __iter__(self):
         while True:
@@ -44,8 +44,21 @@ def __iter__(self):
             else:
                 break
 
+    def __next__(self):
+        while True:
+            has_frame, frame = self._video.read()
+            if has_frame:
+                return frame
+            else:
+                break
+
     def reload(self):
-        self._video = cv.VideoCapture(filepath)
+        self._video = cv.VideoCapture(self._filepath)
+
+    def getFrameSize(self):
+        w = int(self._video.get(cv.CAP_PROP_FRAME_WIDTH))
+        h = int(self._video.get(cv.CAP_PROP_FRAME_HEIGHT))
+        return [w, h]
 
 
 class _BaseVideoLoader:
@@ -56,6 +69,10 @@ def __init__(self, **kwargs):
         self._files = kwargs.pop('files', None)
         assert self._files,'Benchmark[\'data\'][\'files\'] cannot be empty.'
 
+        self._streams = dict()
+        for filename in self._files:
+            self._streams[filename] = _VideoStream(os.path.join(self._path, filename))
+
     @property
     def name(self):
         return self.__class__.__name__
@@ -64,4 +81,4 @@ def __len__(self):
         return len(self._files)
 
     def __getitem__(self, idx):
-        return self._files[idx], _VideoStream(os.path.join(self._path, self._files[idx]))
+        return self._files[idx], self._streams[idx]

benchmark/utils/dataloaders/tracking.py

@@ -1,3 +1,4 @@
+import os
 import numpy as np
 
 from .base_dataloader import _BaseVideoLoader
@@ -8,18 +9,19 @@ class TrackingVideoLoader(_BaseVideoLoader):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self._rois = self._load_roi()
+        self._first_frames = dict()
+        for filename in self._files:
+            stream = self._streams[filename]
+            self._first_frames[filename] = next(stream)
 
-        unsupported_keys = []
-        for k, _ in kwargs.items():
-            unsupported_keys.append(k)
-        print('Keys ({}) are not supported in Benchmark[\'data\'].'.format(str(unsupported_keys)))
+        self._rois = self._load_roi()
 
     def _load_roi(self):
         rois = dict.fromkeys(self._files, None)
         for filename in self._files:
-            rois[filename] = np.loadtxt(os.path.join(self._path, '{}.txt'.format(filename[:-4])), ndmin=2)
+            rois[filename] = np.loadtxt(os.path.join(self._path, '{}.txt'.format(filename[:-4])), dtype=np.int32, ndmin=2)
         return rois
 
-    def getROI(self):
-        return self._rois
+    def __getitem__(self, idx):
+        filename = self._files[idx]
+        return filename, self._streams[filename], self._first_frames[filename], self._rois[filename]

benchmark/utils/metrics/__init__.py

@@ -1,5 +1,6 @@
 from .base import Base
 from .detection import Detection
 from .recognition import Recognition
+from .tracking import Tracking
 
-__all__ = ['Base', 'Detection', 'Recognition']
+__all__ = ['Base', 'Detection', 'Recognition', 'Tracking']

benchmark/utils/metrics/tracking.py

@@ -0,0 +1,26 @@
+import cv2 as cv
+
+from .base_metric import BaseMetric
+from ..factory import METRICS
+
+@METRICS.register
+class Tracking(BaseMetric):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        if self._warmup or self._repeat:
+            print('warmup and repeat in metric for tracking do not function.')
+
+    def forward(self, model, *args, **kwargs):
+        stream, first_frame, rois = args
+
+        for roi in rois:
+            stream.reload()
+            model.init(first_frame, tuple(roi))
+            self._timer.reset()
+            for frame in stream:
+                self._timer.start()
+                model.infer(frame)
+                self._timer.stop()
+
+        return self._getResult()

models/__init__.py

@@ -5,6 +5,7 @@
 from .image_classification_ppresnet.ppresnet import PPResNet
 from .human_segmentation_pphumanseg.pphumanseg import PPHumanSeg
 from .qrcode_wechatqrcode.wechatqrcode import WeChatQRCode
+from .object_tracking_dasiamrpn.dasiamrpn import DaSiamRPN
 
 class Registery:
     def __init__(self, name):
@@ -24,4 +25,5 @@ def register(self, item):
 MODELS.register(SFace)
 MODELS.register(PPResNet)
 MODELS.register(PPHumanSeg)
-MODELS.register(WeChatQRCode)
+MODELS.register(WeChatQRCode)
+MODELS.register(DaSiamRPN)