{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.434586,"end_time":"2022-02-12T05:26:34.367225","exception":false,"start_time":"2022-02-12T05:26:33.932639","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:42.450855Z","iopub.execute_input":"2022-02-17T06:21:42.451550Z","iopub.status.idle":"2022-02-17T06:21:42.456385Z","shell.execute_reply.started":"2022-02-17T06:21:42.451509Z","shell.execute_reply":"2022-02-17T06:21:42.455230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\nbbox.head()","metadata":{"papermill":{"duration":0.114618,"end_time":"2022-02-12T05:26:34.520378","exception":false,"start_time":"2022-02-12T05:26:34.40576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:42.458397Z","iopub.execute_input":"2022-02-17T06:21:42.458975Z","iopub.status.idle":"2022-02-17T06:21:42.502845Z","shell.execute_reply.started":"2022-02-17T06:21:42.458931Z","shell.execute_reply":"2022-02-17T06:21:42.502258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox.iloc[50]","metadata":{"papermill":{"duration":0.045979,"end_time":"2022-02-12T05:26:34.603198","exception":false,"start_time":"2022-02-12T05:26:34.557219","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:42.504181Z","iopub.execute_input":"2022-02-17T06:21:42.504561Z","iopub.status.idle":"2022-02-17T06:21:42.511788Z","shell.execute_reply.started":"2022-02-17T06:21:42.504517Z","shell.execute_reply":"2022-02-17T06:21:42.510931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 78\nframe1 = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i].video_id}/{bbox.iloc[i].video_frame}.jpg')\nframe2 = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i+1].video_id}/{bbox.iloc[i+1].video_frame}.jpg')\nframe3 = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i+2].video_id}/{bbox.iloc[i+2].video_frame}.jpg')\nframe1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB)\nframe2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2RGB) \nframe3 = cv2.cvtColor(frame3, cv2.COLOR_BGR2RGB)","metadata":{"papermill":{"duration":0.190349,"end_time":"2022-02-12T05:26:34.829161","exception":false,"start_time":"2022-02-12T05:26:34.638812","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:42.513368Z","iopub.execute_input":"2022-02-17T06:21:42.513926Z","iopub.status.idle":"2022-02-17T06:21:42.600308Z","shell.execute_reply.started":"2022-02-17T06:21:42.513879Z","shell.execute_reply":"2022-02-17T06:21:42.599275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(frame1)\nbox1 = eval(bbox.iloc[i].annotations)\nbox1 = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box1]\nprint(box1)","metadata":{"papermill":{"duration":0.614573,"end_time":"2022-02-12T05:26:35.480735","exception":false,"start_time":"2022-02-12T05:26:34.866162","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:42.602691Z","iopub.execute_input":"2022-02-17T06:21:42.603026Z","iopub.status.idle":"2022-02-17T06:21:42.990640Z","shell.execute_reply.started":"2022-02-17T06:21:42.602981Z","shell.execute_reply":"2022-02-17T06:21:42.989760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(frame2)\nbox2 = eval(bbox.iloc[i+1].annotations)\nbox2 = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box2]\nprint(box2)","metadata":{"papermill":{"duration":0.471719,"end_time":"2022-02-12T05:26:35.993346","exception":false,"start_time":"2022-02-12T05:26:35.521627","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:42.992109Z","iopub.execute_input":"2022-02-17T06:21:42.992599Z","iopub.status.idle":"2022-02-17T06:21:43.382030Z","shell.execute_reply.started":"2022-02-17T06:21:42.992567Z","shell.execute_reply":"2022-02-17T06:21:43.380988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(frame3)\nbox3 = eval(bbox.iloc[i+2].annotations)\nbox3 = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box3]\nprint(box3)","metadata":{"papermill":{"duration":0.476849,"end_time":"2022-02-12T05:26:36.516543","exception":false,"start_time":"2022-02-12T05:26:36.039694","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:43.383609Z","iopub.execute_input":"2022-02-17T06:21:43.384022Z","iopub.status.idle":"2022-02-17T06:21:43.772924Z","shell.execute_reply.started":"2022-02-17T06:21:43.383987Z","shell.execute_reply":"2022-02-17T06:21:43.772068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_matches(img1, keypoints1, img2, keypoints2, matches):\n    r, c = img1.shape[:2]\n    r1, c1 = img2.shape[:2]\n\n    # Create a blank image with the size of the first image + second image\n    output_img = np.zeros((max([r, r1]), c+c1, 3), dtype='uint8')\n    output_img[:r, :c, :] = np.dstack([img1, img1, img1])\n    output_img[:r1, c:c+c1, :] = np.dstack([img2, img2, img2])\n\n    # Go over all of the matching points and extract them\n    for match in matches:\n        img1_idx = match.queryIdx\n        img2_idx = match.trainIdx\n        (x1, y1) = keypoints1[img1_idx].pt\n        (x2, y2) = keypoints2[img2_idx].pt\n\n        # Draw circles on the keypoints\n        cv2.circle(output_img, (int(x1),int(y1)), 4, (0, 255, 255), 1)\n        cv2.circle(output_img, (int(x2)+c,int(y2)), 4, (0, 255, 255), 1)\n\n        # Connect the same keypoints\n        cv2.line(output_img, (int(x1),int(y1)), (int(x2)+c,int(y2)), (0, 255, 255), 1)\n    \n    return output_img","metadata":{"papermill":{"duration":0.065954,"end_time":"2022-02-12T05:26:36.636146","exception":false,"start_time":"2022-02-12T05:26:36.570192","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:43.774125Z","iopub.execute_input":"2022-02-17T06:21:43.774468Z","iopub.status.idle":"2022-02-17T06:21:43.786909Z","shell.execute_reply.started":"2022-02-17T06:21:43.774435Z","shell.execute_reply":"2022-02-17T06:21:43.786295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orb = cv2.ORB_create(nfeatures=2000)\n\nkeypoints1, descriptors1 = orb.detectAndCompute(frame1, None)\nkeypoints2, descriptors2 = orb.detectAndCompute(frame2, None)\n\nk_frame1 = cv2.drawKeypoints(frame1, keypoints1, (255, 0, 0))\nk_frame2 = cv2.drawKeypoints(frame2, keypoints2, (255, 0, 0))\n\nplt.figure(figsize=(15,20))\nplt.title(\"Key points on Frame1\")\nplt.imshow(k_frame1)\nplt.show()","metadata":{"papermill":{"duration":0.795352,"end_time":"2022-02-12T05:26:37.481937","exception":false,"start_time":"2022-02-12T05:26:36.686585","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:43.789371Z","iopub.execute_input":"2022-02-17T06:21:43.789618Z","iopub.status.idle":"2022-02-17T06:21:44.476645Z","shell.execute_reply.started":"2022-02-17T06:21:43.789589Z","shell.execute_reply":"2022-02-17T06:21:44.475842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,20))\nplt.title(\"Key points on Frame2\")\nplt.imshow(k_frame2)\nplt.show()","metadata":{"papermill":{"duration":0.662725,"end_time":"2022-02-12T05:26:38.217988","exception":false,"start_time":"2022-02-12T05:26:37.555263","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:44.478099Z","iopub.execute_input":"2022-02-17T06:21:44.478553Z","iopub.status.idle":"2022-02-17T06:21:45.051258Z","shell.execute_reply.started":"2022-02-17T06:21:44.478514Z","shell.execute_reply":"2022-02-17T06:21:45.050465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bf = cv2.BFMatcher_create(cv2.NORM_HAMMING)\nmatches = bf.knnMatch(descriptors1, descriptors2,k=2)","metadata":{"papermill":{"duration":0.13895,"end_time":"2022-02-12T05:26:38.452409","exception":false,"start_time":"2022-02-12T05:26:38.313459","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.052428Z","iopub.execute_input":"2022-02-17T06:21:45.052776Z","iopub.status.idle":"2022-02-17T06:21:45.094719Z","shell.execute_reply.started":"2022-02-17T06:21:45.052739Z","shell.execute_reply":"2022-02-17T06:21:45.093719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame1g = cv2.cvtColor(frame1, cv2.COLOR_RGB2GRAY)\nframe2g = cv2.cvtColor(frame2, cv2.COLOR_RGB2GRAY)\nframe3g = cv2.cvtColor(frame3, cv2.COLOR_RGB2GRAY)\ngood = []\nfor m, n in matches:\n    if m.distance < 0.6 * n.distance:\n        good.append(m)\n\nimg3 = draw_matches(frame1g, keypoints1, frame2g, keypoints2, good[:10])\nplt.figure(figsize=(20,25))\nplt.title(\"Good matching points btw F1 & F2\")\nplt.imshow(img3)\nplt.show()","metadata":{"papermill":{"duration":0.739107,"end_time":"2022-02-12T05:26:39.286765","exception":false,"start_time":"2022-02-12T05:26:38.547658","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.095947Z","iopub.execute_input":"2022-02-17T06:21:45.096163Z","iopub.status.idle":"2022-02-17T06:21:45.701831Z","shell.execute_reply.started":"2022-02-17T06:21:45.096137Z","shell.execute_reply":"2022-02-17T06:21:45.700944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(good))","metadata":{"papermill":{"duration":0.120366,"end_time":"2022-02-12T05:26:39.518996","exception":false,"start_time":"2022-02-12T05:26:39.39863","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.703486Z","iopub.execute_input":"2022-02-17T06:21:45.703967Z","iopub.status.idle":"2022-02-17T06:21:45.709114Z","shell.execute_reply.started":"2022-02-17T06:21:45.703920Z","shell.execute_reply":"2022-02-17T06:21:45.708279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MIN_MATCH_COUNT = 30\nif len(good)>MIN_MATCH_COUNT:\n    src_pts = np.float32([ keypoints1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)\n    dst_pts = np.float32([ keypoints2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n    H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC)#,5.0)\nelse:\n    print( \"Not enough matches are found - {}/{}\".format(len(good), MIN_MATCH_COUNT) )\n    matchesMask = None","metadata":{"papermill":{"duration":0.131009,"end_time":"2022-02-12T05:26:39.759518","exception":false,"start_time":"2022-02-12T05:26:39.628509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.710646Z","iopub.execute_input":"2022-02-17T06:21:45.711086Z","iopub.status.idle":"2022-02-17T06:21:45.725858Z","shell.execute_reply.started":"2022-02-17T06:21:45.711047Z","shell.execute_reply":"2022-02-17T06:21:45.725186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"H","metadata":{"papermill":{"duration":0.120673,"end_time":"2022-02-12T05:26:39.992074","exception":false,"start_time":"2022-02-12T05:26:39.871401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.726847Z","iopub.execute_input":"2022-02-17T06:21:45.727675Z","iopub.status.idle":"2022-02-17T06:21:45.739824Z","shell.execute_reply.started":"2022-02-17T06:21:45.727622Z","shell.execute_reply":"2022-02-17T06:21:45.738393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def traform_point(point,H):\n    x,y = point\n    X = np.array([[x],[y],[1]])\n    Y = H @ X\n    Y = Y[:2]/Y[2]\n    return float(Y[0]),float(Y[1])\n\ntraform_point([0,0],H)","metadata":{"papermill":{"duration":0.123249,"end_time":"2022-02-12T05:26:40.225387","exception":false,"start_time":"2022-02-12T05:26:40.102138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.741242Z","iopub.execute_input":"2022-02-17T06:21:45.742398Z","iopub.status.idle":"2022-02-17T06:21:45.752259Z","shell.execute_reply.started":"2022-02-17T06:21:45.742350Z","shell.execute_reply":"2022-02-17T06:21:45.751296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ensemble-boxes","metadata":{"papermill":{"duration":12.75879,"end_time":"2022-02-12T05:26:53.096502","exception":false,"start_time":"2022-02-12T05:26:40.337712","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:45.753760Z","iopub.execute_input":"2022-02-17T06:21:45.754853Z","iopub.status.idle":"2022-02-17T06:21:54.354265Z","shell.execute_reply.started":"2022-02-17T06:21:45.754807Z","shell.execute_reply":"2022-02-17T06:21:54.353313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import nms\n\n\nclass Tracker_custom:\n    def __init__(self,debug=False):\n        self.crurrent_frame = -1\n        self.transf_H = []\n        self.last_frame = None\n        self.n_frame_bbox = []\n        self.n_frame_conf = []\n        self.debug = debug\n    \n    def new_frame(self,new_frame,bbox=[],confs=[]):\n        self.crurrent_frame += 1\n        \n        if self.last_frame is None:\n            self.last_frame = new_frame\n            self.n_frame_bbox.append(bbox)\n            self.n_frame_conf.append(confs)\n\n        else : \n            # if not first frame :\n            H = self.get_transfor(new_frame)\n            if H is None:\n                #new video => no prediction traking\n                #no bbox => no prediction traking\n                if self.debug :\n                    print(\"no H\")\n                self.n_frame_bbox.append(bbox)\n                self.n_frame_conf.append(confs)\n                self.last_frame = new_frame\n                return\n                \n            self.transf_H.append(H)\n            last_boxes,last_confs = self.n_frame_bbox[-1],self.n_frame_conf[-1]\n            pred_boxes,pred_confs = self.pred_box_H(last_boxes,last_confs,H)\n            \n            # fusion predbox && deteted box\n            f_box,f_score = [],[]\n            for box,conf in zip(self.valid_box(pred_boxes),pred_confs):\n                f_box.append(box)\n                f_score.append(conf)\n                \n            for box,conf in zip(bbox,confs):\n                f_box.append(box)\n                f_score.append(conf)\n            \n            if len(f_box)==0:\n                # 0 bbox\n                if self.debug :\n                    print(\"no bbox\")\n                self.n_frame_bbox.append([])\n                self.n_frame_conf.append([])\n                self.last_frame = new_frame\n                return\n            labels = [1 for conf in f_score]\n            \n            image_size_x=self.last_frame.shape[1]\n            image_size_y=self.last_frame.shape[0]\n            f_box =  [[b[0]/image_size_x,b[1]/image_size_y,(b[0]+b[2])/image_size_x,(b[1]+b[3])/image_size_y] for b in f_box]\n            f_box =  [[min(max(x,0),1) for x in b] for b in f_box]\n            boxes, scores, labels = nms([f_box], [f_score], [labels], weights=None, iou_thr=.2)\n            #boxes, scores, labels = nms([pred_boxes,bbox], [pred_confs,confs], [labels[:len(pred_boxes)],labels[len(pred_boxes):]], weights=[1,1], iou_thr=.2)\n            fboxes = [[b[0]*image_size_x,b[1]*image_size_y,(b[2]-b[0])*image_size_x,(b[3]-b[1])*image_size_y] for b in boxes]\n            \n            \n            self.n_frame_bbox.append(self.valid_box(fboxes))\n            self.n_frame_conf.append(scores)\n            self.last_frame = new_frame\n    \n    def get_transfor(self,new_frame,MIN_MATCH_COUNT = 30):\n        if self.last_frame is None:\n            return None\n        \n        orb = cv2.ORB_create(nfeatures=2000)\n\n        keypoints1, descriptors1 = orb.detectAndCompute(self.last_frame, None)\n        keypoints2, descriptors2 = orb.detectAndCompute(new_frame, None)\n        \n        bf = cv2.BFMatcher_create(cv2.NORM_HAMMING)\n        matches = bf.knnMatch(descriptors1, descriptors2,k=2)\n        \n        good = []\n        for m, n in matches:\n            if m.distance < 0.5 * n.distance:\n                good.append(m)\n        \n        if len(good)<MIN_MATCH_COUNT:\n            #not enough matches\n            # maybe new video ??\n            return None\n        \n        src_pts = np.float32([ keypoints1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)\n        dst_pts = np.float32([ keypoints2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n        H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)\n        return H\n    \n    def pred_box_H(self,boxes,confs,H):\n        if H is None:\n            # new video ?? H not found:\n            return [],[] # no predicition form traking\n        new_boxes, new_confs = [],[]\n        for i in range(len(boxes)):\n            x,y,w,h = boxes[i]\n            conf = confs[i]\n            mx,my = x+w/2, y+h/2\n            X = np.array([[mx],[my],[1]])\n            Y = H @ X\n            Y = Y[:2]/Y[2]\n            new_mx,new_my = float(Y[0]),float(Y[1])\n            new_conf = .75 * conf # momentum in conf\n            start = np.array([mx,my])\n            end = np.array([new_mx,new_my])\n            dist = np.linalg.norm(end-start)\n            if new_conf > .1 and dist<50: # threshold conf && dit old/new < -xx\n                new_boxes.append([int(new_mx-w/2),int(new_my-h/2),int(w),int(h)])\n                new_confs.append(new_conf)\n                if self.debug :\n                    print(f' from {mx,my} to {new_mx,new_my} dist {dist}')  \n        return new_boxes,new_confs\n    \n    \n    def valid_box(self,boxes):\n        # keep the box in the img\n        new_boxes = []\n        for box in boxes:\n            x,y,w,h = box\n            x = int(max(1,min(x,self.last_frame.shape[1]-1)))\n            y = int(max(1,min(y,self.last_frame.shape[0]-1)))\n            w = int(max(5,min(w,self.last_frame.shape[1]-x-1)))\n            h = int(max(5,min(h,self.last_frame.shape[0]-y-1)))\n            area = w*h\n            if area >500:\n                new_boxes.append([x,y,w,h])\n        return new_boxes            \n","metadata":{"papermill":{"duration":0.977088,"end_time":"2022-02-12T05:26:54.191217","exception":false,"start_time":"2022-02-12T05:26:53.214129","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:54.355985Z","iopub.execute_input":"2022-02-17T06:21:54.356258Z","iopub.status.idle":"2022-02-17T06:21:54.394274Z","shell.execute_reply.started":"2022-02-17T06:21:54.356229Z","shell.execute_reply":"2022-02-17T06:21:54.393130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker = Tracker_custom(debug=True)\ntracker.new_frame(frame1,box1,[1,1])\ntracker.n_frame_bbox","metadata":{"papermill":{"duration":0.128931,"end_time":"2022-02-12T05:26:54.436351","exception":false,"start_time":"2022-02-12T05:26:54.30742","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:54.396492Z","iopub.execute_input":"2022-02-17T06:21:54.396955Z","iopub.status.idle":"2022-02-17T06:21:54.411445Z","shell.execute_reply.started":"2022-02-17T06:21:54.396903Z","shell.execute_reply":"2022-02-17T06:21:54.410832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# white = bbox @ frame n-1\n# red = bbox @ frame n (prediction)\n# blue = bbox grundtruth @ frame n","metadata":{"papermill":{"duration":0.123595,"end_time":"2022-02-12T05:26:54.675618","exception":false,"start_time":"2022-02-12T05:26:54.552023","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:54.414518Z","iopub.execute_input":"2022-02-17T06:21:54.415107Z","iopub.status.idle":"2022-02-17T06:21:54.422002Z","shell.execute_reply.started":"2022-02-17T06:21:54.415075Z","shell.execute_reply":"2022-02-17T06:21:54.421259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = frame1.copy()\nfor box in tracker.n_frame_bbox[-1]:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)\n    \nfor box in box1:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,255), 2)\n    \nplt.figure(figsize=(20,25))\nplt.title(\"frame 1 : detector gives bbox to the tracker\")\nplt.imshow(img)\nplt.show()\n# white = bbox @ frame n-1\n# red = bbox @ frame n (prediction)\n# pink = bbox grundtruth @ frame n","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.950678,"end_time":"2022-02-12T05:26:55.740952","exception":false,"start_time":"2022-02-12T05:26:54.790274","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:54.423175Z","iopub.execute_input":"2022-02-17T06:21:54.423563Z","iopub.status.idle":"2022-02-17T06:21:55.246436Z","shell.execute_reply.started":"2022-02-17T06:21:54.423533Z","shell.execute_reply":"2022-02-17T06:21:55.244768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# white = bbox @ frame n-1\n# red = bbox @ frame n (prediction)\n# blue = bbox grundtruth @ frame n","metadata":{"papermill":{"duration":0.189038,"end_time":"2022-02-12T05:26:56.08478","exception":false,"start_time":"2022-02-12T05:26:55.895742","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:55.247553Z","iopub.execute_input":"2022-02-17T06:21:55.247961Z","iopub.status.idle":"2022-02-17T06:21:55.251489Z","shell.execute_reply.started":"2022-02-17T06:21:55.247930Z","shell.execute_reply":"2022-02-17T06:21:55.250839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker.new_frame(frame2)\n\n\nimg = frame2.copy()\nfor box in tracker.n_frame_bbox[-1]:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)\n    \nfor box in box1:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (255,255,255), 2)\n    \nfor box in box2:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (0,0,255), 2)\n    \nplt.figure(figsize=(20,25))\nplt.title(\"frame 2 : no detection but tracker say there are still a bbox\")\nplt.imshow(img)\nplt.show()\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":5.051986,"end_time":"2022-02-12T05:27:01.283947","exception":false,"start_time":"2022-02-12T05:26:56.231961","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:55.252726Z","iopub.execute_input":"2022-02-17T06:21:55.253175Z","iopub.status.idle":"2022-02-17T06:21:56.208858Z","shell.execute_reply.started":"2022-02-17T06:21:55.253141Z","shell.execute_reply":"2022-02-17T06:21:56.207941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# white = bbox @ frame n-1\n# red = bbox @ frame n (prediction)\n# blue = bbox grundtruth @ frame n","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.1961,"end_time":"2022-02-12T05:27:01.675618","exception":false,"start_time":"2022-02-12T05:27:01.479518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:56.210466Z","iopub.execute_input":"2022-02-17T06:21:56.210753Z","iopub.status.idle":"2022-02-17T06:21:56.214857Z","shell.execute_reply.started":"2022-02-17T06:21:56.210710Z","shell.execute_reply":"2022-02-17T06:21:56.213566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker.new_frame(frame3)\nimg = frame3.copy()\nfor box in tracker.n_frame_bbox[-1]:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)\n    \nfor box in box2:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (255,255,255), 2)\n    \nfor box in box3:\n    x,y,w,h = box\n    x,y,w,h = int(x),int(y),int(w),int(h)\n    cv2.rectangle(img, (x,y), (x+w,y+h), (0,0,255), 2)\n    \nplt.figure(figsize=(20,25))\nplt.title(\"frame 3 : no detection but tracker say there are still a bbox\")\nplt.imshow(img)\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.148837,"end_time":"2022-02-12T05:27:03.017082","exception":false,"start_time":"2022-02-12T05:27:01.868245","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:56.217076Z","iopub.execute_input":"2022-02-17T06:21:56.217475Z","iopub.status.idle":"2022-02-17T06:21:57.166395Z","shell.execute_reply.started":"2022-02-17T06:21:56.217428Z","shell.execute_reply":"2022-02-17T06:21:57.165725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker.n_frame_conf","metadata":{"papermill":{"duration":0.244436,"end_time":"2022-02-12T05:27:03.495561","exception":false,"start_time":"2022-02-12T05:27:03.251125","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:57.167404Z","iopub.execute_input":"2022-02-17T06:21:57.168089Z","iopub.status.idle":"2022-02-17T06:21:57.174001Z","shell.execute_reply.started":"2022-02-17T06:21:57.168054Z","shell.execute_reply":"2022-02-17T06:21:57.172889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If i dont detect i predict with the tracker with conf = x * last_conf (.5 momentum)","metadata":{"papermill":{"duration":0.231786,"end_time":"2022-02-12T05:27:03.958719","exception":false,"start_time":"2022-02-12T05:27:03.726933","status":"completed"},"tags":[]}},{"cell_type":"code","source":"i = 48\nframe = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i].video_id}/{bbox.iloc[i].video_frame}.jpg')\nframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\nbox = eval(bbox.iloc[i].annotations)\nbox = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box]\ntracker = Tracker_custom(debug=True)\ntracker.new_frame(frame,box,[1,1])\nbox_prev = box\nfor j in range(1,4):\n    frame = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i+j].video_id}/{bbox.iloc[i+j].video_frame}.jpg')\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    box = eval(bbox.iloc[i+j].annotations)\n    box = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box]\n    \n    \n    tracker.new_frame(frame)\n    img = frame.copy()\n    for b in tracker.n_frame_bbox[-1]:\n        x,y,w,h = b\n        x,y,w,h = int(x),int(y),int(w),int(h)\n        cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)\n\n    for b in box_prev:\n        x,y,w,h = b\n        x,y,w,h = int(x),int(y),int(w),int(h)\n        cv2.rectangle(img, (x,y), (x+w,y+h), (255,255,255), 2)\n\n    for b in box:\n        x,y,w,h = b\n        x,y,w,h = int(x),int(y),int(w),int(h)\n        cv2.rectangle(img, (x,y), (x+w,y+h), (0,0,255), 2)\n        \n    box_prev = box[:][:]\n\n    plt.figure(figsize=(20,25))\n    plt.imshow(img)\n    plt.show()\n    ","metadata":{"papermill":{"duration":3.230566,"end_time":"2022-02-12T05:27:07.421994","exception":false,"start_time":"2022-02-12T05:27:04.191428","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:21:57.175320Z","iopub.execute_input":"2022-02-17T06:21:57.175553Z","iopub.status.idle":"2022-02-17T06:22:00.110607Z","shell.execute_reply.started":"2022-02-17T06:21:57.175525Z","shell.execute_reply":"2022-02-17T06:22:00.109426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# re- detection simulation\ni += j +5\nframe = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i].video_id}/{bbox.iloc[i].video_frame}.jpg')\nframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\nbox = eval(bbox.iloc[i].annotations)\nbox = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box]\ntracker.new_frame(frame,box,[1,1,1,1])\nbox_prev = box\nfor j in range(1,4):\n    frame = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{bbox.iloc[i+j].video_id}/{bbox.iloc[i+j].video_frame}.jpg')\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    box = eval(bbox.iloc[i+j].annotations)\n    box = [[x[\"x\"],x[\"y\"],x[\"width\"],x[\"height\"]] for x in box]\n    \n    \n    tracker.new_frame(frame)\n    img = frame.copy()\n    for b in tracker.n_frame_bbox[-1]:\n        x,y,w,h = b\n        x,y,w,h = int(x),int(y),int(w),int(h)\n        cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)\n\n    for b in box_prev:\n        x,y,w,h = b\n        x,y,w,h = int(x),int(y),int(w),int(h)\n        cv2.rectangle(img, (x,y), (x+w,y+h), (255,255,255), 2)\n\n    for b in box:\n        x,y,w,h = b\n        x,y,w,h = int(x),int(y),int(w),int(h)\n        cv2.rectangle(img, (x,y), (x+w,y+h), (0,0,255), 2)\n        \n    box_prev = box[:][:]\n\n    plt.figure(figsize=(20,25))\n    plt.imshow(img)\n    plt.show()\n    ","metadata":{"papermill":{"duration":3.592151,"end_time":"2022-02-12T05:27:11.371079","exception":false,"start_time":"2022-02-12T05:27:07.778928","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:22:00.111933Z","iopub.execute_input":"2022-02-17T06:22:00.112235Z","iopub.status.idle":"2022-02-17T06:22:03.215618Z","shell.execute_reply.started":"2022-02-17T06:22:00.112185Z","shell.execute_reply":"2022-02-17T06:22:03.214718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker.n_frame_conf","metadata":{"papermill":{"duration":0.503481,"end_time":"2022-02-12T05:27:12.36963","exception":false,"start_time":"2022-02-12T05:27:11.866149","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-17T06:22:03.216776Z","iopub.execute_input":"2022-02-17T06:22:03.217005Z","iopub.status.idle":"2022-02-17T06:22:03.224176Z","shell.execute_reply.started":"2022-02-17T06:22:03.216978Z","shell.execute_reply":"2022-02-17T06:22:03.223232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.48258,"end_time":"2022-02-12T05:27:13.330061","exception":false,"start_time":"2022-02-12T05:27:12.847481","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}