new feature_homography.py sample

samples/python2/common.py

@@ -151,12 +151,9 @@ class RectSelector:
             self.drag_start = (x, y)
         if self.drag_start: 
             if flags & cv2.EVENT_FLAG_LBUTTON:
-                #h, w = self.frame.shape[:2]
                 xo, yo = self.drag_start
                 x0, y0 = np.minimum([xo, yo], [x, y])
                 x1, y1 = np.maximum([xo, yo], [x, y])
-                #x0, y0 = np.maximum(0, np.minimum([xo, yo], [x, y]))
-                #x1, y1 = np.minimum([w, h], np.maximum([xo, yo], [x, y]))
                 self.drag_rect = None
                 if x1-x0 > 0 and y1-y0 > 0:
                     self.drag_rect = (x0, y0, x1, y1)
@@ -168,9 +165,13 @@ class RectSelector:
                     self.callback(rect)
     def draw(self, vis):
         if not self.drag_rect:
-            return
+            return False
         x0, y0, x1, y1 = self.drag_rect
         cv2.rectangle(vis, (x0, y0), (x1, y1), (0, 255, 0), 2)
+        return True
+    @property
+    def dragging(self):
+        return self.drag_rect is not None
 
 
 def grouper(n, iterable, fillvalue=None):

samples/python2/feature_homography.py

@@ -3,94 +3,135 @@ Feature homography
 ==================
 
 Example of using features2d framework for interactive video homography matching.
+ORB features and FLANN matcher are used.
+
+Inspired by http://www.youtube.com/watch?v=-ZNYoL8rzPY
 
 Usage
 -----
 feature_homography.py [<video source>]
 
-Keys
-----
-SPACE - set reference frame
-ESC   - exit
-'''
+Select a textured planar object to track by drawing a box with a mouse.
 
+'''
 
 import numpy as np
 import cv2
 import video
-from common import draw_str, clock
-import sys
+import common
+from operator import attrgetter
 
+def get_size(a):
+    h, w = a.shape[:2]
+    return w, h
 
-detector = cv2.FastFeatureDetector(16, True)
-detector = cv2.GridAdaptedFeatureDetector(detector)
-extractor = cv2.DescriptorExtractor_create('ORB')
-
+    
 FLANN_INDEX_KDTREE = 1
 FLANN_INDEX_LSH    = 6
 flann_params= dict(algorithm = FLANN_INDEX_LSH,
                    table_number = 6, # 12
                    key_size = 12,     # 20
                    multi_probe_level = 1) #2
-matcher = cv2.FlannBasedMatcher(flann_params, {})  # bug : need to pass empty dict (#1329)
 
-green, red = (0, 255, 0), (0, 0, 255)
 
+MIN_MATCH_COUNT = 10
 
-if __name__ == '__main__':
-    print __doc__
+class App:
+    def __init__(self, src):
+        self.cap = video.create_capture(src)
+        self.ref_frame  = None
 
-    try: src = sys.argv[1]
-    except: src = 0
-    cap = video.create_capture(src)
+        self.detector = cv2.ORB( nfeatures = 1000 )
+        self.matcher = cv2.FlannBasedMatcher(flann_params, {})  # bug : need to pass empty dict (#1329)
 
-    ref_kp = None
+        cv2.namedWindow('plane')
+        self.rect_sel = common.RectSelector('plane', self.on_rect)
 
-    while True:
-        ret, img = cap.read()
-        vis = img.copy()
-        kp = detector.detect(img)
-        kp, desc = extractor.compute(img, kp)
+        self.frame = None
 
-        for p in kp:
-            x, y = np.int32(p.pt)
-            r = int(0.5*p.size)
-            cv2.circle(vis, (x, y), r, (0, 255, 0))
-        draw_str(vis, (20, 20), 'feature_n: %d' % len(kp))
+    def match_frames(self):
+        if len(self.frame_desc) < MIN_MATCH_COUNT or len(self.frame_desc) < MIN_MATCH_COUNT:
+            return
         
-        if ref_kp is not None:
-            raw_matches = matcher.knnMatch(desc, 2)
-            matches = []
-            for m in raw_matches:
-                if len(m) == 2:
-                    m1, m2 = m
-                    if m1.distance < m2.distance * 0.7:
-                        matches.append((m1.trainIdx, m1.queryIdx))
-            match_n = len(matches)
-
-            inlier_n = 0
-            if match_n > 10:
-                p0 = np.float32( [ref_kp[i].pt for i, j in matches] )
-                p1 = np.float32( [kp[j].pt for i, j in matches] )
-
-                H, status = cv2.findHomography(p0, p1, cv2.RANSAC, 10.0)
-                inlier_n = sum(status)
-                if inlier_n > 10:
-                    for (x1, y1), (x2, y2), inlier in zip(np.int32(p0), np.int32(p1), status):
-                        cv2.line(vis, (x1, y1), (x2, y2), (red, green)[inlier])
-
-                    h, w = img.shape[:2]
-                    overlay = cv2.warpPerspective(ref_img, H, (w, h))
-                    vis = cv2.addWeighted(vis, 0.5, overlay, 0.5, 0.0)
-            draw_str(vis, (20, 40), 'matched: %d ( %d outliers )' % (match_n, match_n-inlier_n))
+        raw_matches = self.matcher.knnMatch(self.ref_descs, trainDescriptors = self.frame_desc, k = 2)
+        p0, p1 = [], []
+        for m in raw_matches:
+            if len(m) == 2 and m[0].distance < m[1].distance * 0.75:
+                m = m[0]
+                p0.append( self.ref_points[m.queryIdx].pt )
+                p1.append( self.frame_points[m.trainIdx].pt )
+        p0, p1 = np.float32((p0, p1))
+        if len(p0) < MIN_MATCH_COUNT:
+            return
+
+        H, status = cv2.findHomography(p0, p1, cv2.RANSAC, 4.0)
+        status = status.ravel() != 0
+        if status.sum() < MIN_MATCH_COUNT:
+            return
+        p0, p1 = p0[status], p1[status]
+        return p0, p1, H
+
+
+    def on_frame(self, frame):
+        if self.frame is None or not self.rect_sel.dragging:
+            self.frame = frame = np.fliplr(frame).copy()
+            self.frame_points, self.frame_desc = self.detector.detectAndCompute(self.frame, None)
+            if self.frame_desc is None:  # detectAndCompute returns descs=None if not keypoints found
+                self.frame_desc = []
+        else:
+            self.ref_frame = None
+
+        w, h = get_size(self.frame)
+        vis = np.zeros((h, w*2, 3), np.uint8)
+        vis[:h,:w] = self.frame
+        self.rect_sel.draw(vis)
         
-        cv2.imshow('img', vis)
-        ch = 0xFF & cv2.waitKey(1)
-        if ch == ord(' '):
-            matcher.clear()
-            matcher.add([desc])
-            ref_kp = kp
-            ref_img = img.copy()
-        if ch == 27:
-            break
-    cv2.destroyAllWindows() 			
+        if self.ref_frame is not None:
+            vis[:h,w:] = self.ref_frame
+            x0, y0, x1, y1 = self.ref_rect
+            cv2.rectangle(vis, (x0+w, y0), (x1+w, y1), (0, 255, 0), 2)
+
+            for kp in self.ref_points:
+                x, y = kp.pt
+                cv2.circle(vis, (int(x+w), int(y)), 2, (0, 255, 255))
+
+
+            match = self.match_frames()
+            if match is not None:
+                p0, p1, H = match
+                for (x0, y0), (x1, y1) in zip(np.int32(p0), np.int32(p1)):
+                    cv2.line(vis, (x0+w, y0), (x1, y1), (0, 255, 0))
+                x0, y0, x1, y1 = self.ref_rect
+                corners = np.float32([[x0, y0], [x1, y0], [x1, y1], [x0, y1]])
+                corners = np.int32( cv2.perspectiveTransform(corners.reshape(1, -1, 2), H) )
+                cv2.polylines(vis, [corners], True, (255, 255, 255), 2)
+        
+        cv2.imshow('plane', vis)
+
+    def on_rect(self, rect):
+        x0, y0, x1, y1 = rect
+        self.ref_frame = self.frame.copy()
+        self.ref_rect = rect
+        points, descs = [], []
+        for kp, desc in zip(self.frame_points, self.frame_desc):
+            x, y = kp.pt
+            if x0 <= x <= x1 and y0 <= y <= y1:
+                points.append(kp)
+                descs.append(desc)
+        self.ref_points, self.ref_descs = points, np.uint8(descs)
+
+    def run(self):
+        while True:
+            ret, frame = self.cap.read()
+            self.on_frame(frame)
+            ch = cv2.waitKey(1)
+            if ch == 27:
+                break
+
+if __name__ == '__main__':
+    print __doc__
+
+    import sys
+    try: video_src = sys.argv[1]
+    except: video_src = '0'
+    App(video_src).run()