{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* Our team has created a scene detection using OpticalFlow to reset the tracker when the scene changes in this competition\n* This code was created based on a great notebook here[https://www.kaggle.com/daigohirooka/optical-flow-estimation-using-raft]","metadata":{"execution":{"iopub.status.busy":"2022-02-15T04:56:04.495377Z","iopub.execute_input":"2022-02-15T04:56:04.496218Z","iopub.status.idle":"2022-02-15T04:56:04.516520Z","shell.execute_reply.started":"2022-02-15T04:56:04.496095Z","shell.execute_reply":"2022-02-15T04:56:04.515600Z"}}},{"cell_type":"markdown","source":"# Scene Change Detection using Optical Flow","metadata":{}},{"cell_type":"markdown","source":"# What's Optical flow?\n\n> Optical flow or optic flow is the pattern of apparent motion of objects, surfaces, and edges in a visual scene caused by the relative motion between an observer and a scene. - https://en.wikipedia.org/wiki/Optical_flow","metadata":{"papermill":{"duration":0.018045,"end_time":"2022-02-12T05:45:42.578274","exception":false,"start_time":"2022-02-12T05:45:42.560229","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport sys\nsys.path.append('/kaggle/input/raft-pytorch')\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport torch\n\nfrom glob import glob\nfrom PIL import Image\nfrom tqdm import tqdm","metadata":{"papermill":{"duration":1.529022,"end_time":"2022-02-12T05:45:44.124433","exception":false,"start_time":"2022-02-12T05:45:42.595411","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.612462Z","iopub.execute_input":"2022-02-12T09:40:46.613296Z","iopub.status.idle":"2022-02-12T09:40:46.619108Z","shell.execute_reply.started":"2022-02-12T09:40:46.61324Z","shell.execute_reply":"2022-02-12T09:40:46.618052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\n\n#small_rate = 0.5 # image size to (360, 640, 3) estimated elapsed time total = 0.386 [h]\nsmall_rate = 0.25 # image size to (180, 320, 3) estimated elapsed time total = 0.244 [h]\n#small_rate = 0.2 # image size to (144, 256, 3) estimated elapsed time total = 0.753 [h]\n","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:40:46.621037Z","iopub.execute_input":"2022-02-12T09:40:46.621581Z","iopub.status.idle":"2022-02-12T09:40:46.632668Z","shell.execute_reply.started":"2022-02-12T09:40:46.621542Z","shell.execute_reply":"2022-02-12T09:40:46.631848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RAFT introduction\n\nI introduce the model: **RAFT: Recurrent All-Pairs Field Transforms for Optical Flow** which is originally introduced in ECCV2020 by Teed et. al. in Princeton University and prized Best Paper Award!.\n* https://arxiv.org/abs/2003.12039\n* https://github.com/princeton-vl/RAFT (licensed under the BSD 3-Clause License)\n\nBriefly, RAFT has below features\n* Recurrent optical flow estimation\n* Compute pixel-wise correlation between pair-wise input images and reuse it in the following recurrent step\n* Lightweight, rapid inference, and high accuracy\n\n![RAFT architecture image from https://github.com/princeton-vl/RAFT](https://github.com/princeton-vl/RAFT/raw/master/RAFT.png)\n\nThis is [my explanation slide](https://speakerdeck.com/daigo0927/raft-recurrent-all-pairs-field-transforms-for-optical-flow) in Japanese.","metadata":{"papermill":{"duration":0.016591,"end_time":"2022-02-12T05:45:44.157887","exception":false,"start_time":"2022-02-12T05:45:44.141296","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Run RAFT on sample images","metadata":{"papermill":{"duration":0.016414,"end_time":"2022-02-12T05:45:44.191425","exception":false,"start_time":"2022-02-12T05:45:44.175011","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from raft.core.raft import RAFT\nfrom raft.core.utils import flow_viz\nfrom raft.core.utils.utils import InputPadder\nfrom raft.config import RAFTConfig","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.380343,"end_time":"2022-02-12T05:45:44.588395","exception":false,"start_time":"2022-02-12T05:45:44.208052","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.634104Z","iopub.execute_input":"2022-02-12T09:40:46.634397Z","iopub.status.idle":"2022-02-12T09:40:46.641994Z","shell.execute_reply.started":"2022-02-12T09:40:46.634361Z","shell.execute_reply":"2022-02-12T09:40:46.64123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = RAFTConfig(\n    dropout=0,\n    alternate_corr=False,\n    small=False,\n    mixed_precision=False\n)\n\nmodel = RAFT(config)\nmodel","metadata":{"papermill":{"duration":0.140404,"end_time":"2022-02-12T05:45:44.746264","exception":false,"start_time":"2022-02-12T05:45:44.60586","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.643414Z","iopub.execute_input":"2022-02-12T09:40:46.643721Z","iopub.status.idle":"2022-02-12T09:40:46.721412Z","shell.execute_reply.started":"2022-02-12T09:40:46.643646Z","shell.execute_reply":"2022-02-12T09:40:46.720763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'device: {device}')\n\nweights_path = '/kaggle/input/raft-pytorch/raft-sintel.pth'\n# weights_path = '/kaggle/input/raft-pytorch/raft-things.pth'\n\nckpt = torch.load(weights_path, map_location=device)\nmodel.to(device)\nmodel.load_state_dict(ckpt)","metadata":{"papermill":{"duration":4.952278,"end_time":"2022-02-12T05:45:49.716041","exception":false,"start_time":"2022-02-12T05:45:44.763763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.723433Z","iopub.execute_input":"2022-02-12T09:40:46.723876Z","iopub.status.idle":"2022-02-12T09:40:46.790837Z","shell.execute_reply.started":"2022-02-12T09:40:46.72384Z","shell.execute_reply":"2022-02-12T09:40:46.790143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_files = glob('/kaggle/input/raft-pytorch/raft/demo-frames/*.png')\nimage_files = sorted(image_files)\n\nprint(f'Found {len(image_files)} images')\nprint(sorted(image_files))","metadata":{"papermill":{"duration":0.037142,"end_time":"2022-02-12T05:45:49.771432","exception":false,"start_time":"2022-02-12T05:45:49.73429","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.791939Z","iopub.execute_input":"2022-02-12T09:40:46.792551Z","iopub.status.idle":"2022-02-12T09:40:46.800096Z","shell.execute_reply.started":"2022-02-12T09:40:46.79251Z","shell.execute_reply":"2022-02-12T09:40:46.799384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(imfile, device):\n    img = np.array(Image.open(imfile)).astype(np.uint8)\n    img = torch.from_numpy(img).permute(2, 0, 1).float()\n    return img[None].to(device)\n\n\ndef viz(img1, img2, flo):\n    img1 = img1[0].permute(1,2,0).cpu().numpy()\n    img2 = img2[0].permute(1,2,0).cpu().numpy()\n    \n    flo_mean = np.mean(flo.cpu().numpy())\n    flo_std = np.std(flo.cpu().numpy())\n    \n    flo = flo[0].permute(1,2,0).cpu().numpy()\n    \n    # map flow to rgb image\n    flo = flow_viz.flow_to_image(flo)\n    \n    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20, 4))\n    ax1.set_title('input image1')\n    ax1.imshow(img1.astype(int))\n    ax2.set_title('input image2')\n    ax2.imshow(img2.astype(int))\n    ax3.set_title(f'optical flow: mean{flo_mean:.1f}, std{flo_std:.1f}')\n    cm = ax3.imshow(flo)\n    fig.colorbar(cm)\n#    plt.colorbar()\n    plt.show()","metadata":{"papermill":{"duration":0.031807,"end_time":"2022-02-12T05:45:49.821665","exception":false,"start_time":"2022-02-12T05:45:49.789858","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.801255Z","iopub.execute_input":"2022-02-12T09:40:46.801464Z","iopub.status.idle":"2022-02-12T09:40:46.814476Z","shell.execute_reply.started":"2022-02-12T09:40:46.801439Z","shell.execute_reply":"2022-02-12T09:40:46.813813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nn_vis = 3\n\nfor file1, file2 in tqdm(zip(image_files[:n_vis], image_files[1:1+n_vis])):\n    image1 = load_image(file1, device)\n    image2 = load_image(file2, device)\n\n    padder = InputPadder(image1.shape)\n    image1, image2 = padder.pad(image1, image2)\n    \n    with torch.no_grad():\n        flow_low, flow_up = model(image1, image2, iters=20, test_mode=True)\n        \n    viz(image1, image2, flow_up)","metadata":{"papermill":{"duration":3.707122,"end_time":"2022-02-12T05:45:53.54705","exception":false,"start_time":"2022-02-12T05:45:49.839928","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:46.815688Z","iopub.execute_input":"2022-02-12T09:40:46.816005Z","iopub.status.idle":"2022-02-12T09:40:49.500722Z","shell.execute_reply.started":"2022-02-12T09:40:46.815969Z","shell.execute_reply":"2022-02-12T09:40:49.500008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The first and second columns are input paired images and right column is the predicted optical flow.","metadata":{"papermill":{"duration":0.034899,"end_time":"2022-02-12T05:45:53.617894","exception":false,"start_time":"2022-02-12T05:45:53.582995","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport cv2 \n\nTRAIN_PATH_IMG = '/kaggle/input/tensorflow-great-barrier-reef/'\n\ndef get_path(row):\n#    row['image_path'] = f'{TRAIN_PATH_IMG}/clahe_img/video_{row.video_id}_{row.video_frame}.jpg'\n    row['image_path'] = f'{TRAIN_PATH_IMG}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\n\ndf = pd.read_csv(f'/kaggle/input/tensorflow-great-barrier-reef/train.csv')\n                 \n#Path of images\ndf = df.apply(get_path, axis=1)\ndf.head(5)\n","metadata":{"papermill":{"duration":20.359782,"end_time":"2022-02-12T05:46:14.161713","exception":false,"start_time":"2022-02-12T05:45:53.801931","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:40:49.509934Z","iopub.execute_input":"2022-02-12T09:40:49.510264Z","iopub.status.idle":"2022-02-12T09:41:03.952048Z","shell.execute_reply.started":"2022-02-12T09:40:49.510224Z","shell.execute_reply":"2022-02-12T09:41:03.951249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.video_id.value_counts()","metadata":{"papermill":{"duration":0.052885,"end_time":"2022-02-12T05:46:14.255488","exception":false,"start_time":"2022-02-12T05:46:14.202603","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:03.953531Z","iopub.execute_input":"2022-02-12T09:41:03.953793Z","iopub.status.idle":"2022-02-12T09:41:03.961418Z","shell.execute_reply.started":"2022-02-12T09:41:03.953751Z","shell.execute_reply":"2022-02-12T09:41:03.960722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sequence.value_counts()","metadata":{"papermill":{"duration":0.050263,"end_time":"2022-02-12T05:46:14.34595","exception":false,"start_time":"2022-02-12T05:46:14.295687","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:03.962961Z","iopub.execute_input":"2022-02-12T09:41:03.963528Z","iopub.status.idle":"2022-02-12T09:41:03.974678Z","shell.execute_reply.started":"2022-02-12T09:41:03.963488Z","shell.execute_reply":"2022-02-12T09:41:03.973823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = [] # same sequence\n\nfor i in range(20):\n    img = cv2.imread(df.image_path[i])[:,:,::-1]\n    img = cv2.resize(img, dsize=None, fx=small_rate, fy=small_rate)\n    frames.append(img)\nframes = np.stack(frames, axis=0)\nprint (img.shape)","metadata":{"papermill":{"duration":0.740432,"end_time":"2022-02-12T05:46:15.12361","exception":false,"start_time":"2022-02-12T05:46:14.383178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:03.975917Z","iopub.execute_input":"2022-02-12T09:41:03.976793Z","iopub.status.idle":"2022-02-12T09:41:04.513744Z","shell.execute_reply.started":"2022-02-12T09:41:03.976742Z","shell.execute_reply":"2022-02-12T09:41:04.512866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames2 = [] # include sequence change\nfor i in range(20):\n    img = cv2.imread(df.image_path[i + 475])[:,:,::-1]\n    img = cv2.resize(img, dsize=None, fx=small_rate, fy=small_rate)\n    frames2.append(img)\n\nframes2 = np.stack(frames2, axis=0)","metadata":{"papermill":{"duration":0.798504,"end_time":"2022-02-12T05:46:15.963545","exception":false,"start_time":"2022-02-12T05:46:15.165041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:04.514895Z","iopub.execute_input":"2022-02-12T09:41:04.515142Z","iopub.status.idle":"2022-02-12T09:41:05.035664Z","shell.execute_reply.started":"2022-02-12T09:41:04.515108Z","shell.execute_reply":"2022-02-12T09:41:05.034929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames3 = [] # include video change\nfor i in range(20):\n    img = cv2.imread(df.image_path[i + 6705])[:,:,::-1]\n    img = cv2.resize(img, dsize=None, fx=small_rate, fy=small_rate)\n    frames3.append(img)\n\nframes3 = np.stack(frames3, axis=0)","metadata":{"papermill":{"duration":0.608488,"end_time":"2022-02-12T05:46:16.611566","exception":false,"start_time":"2022-02-12T05:46:16.003078","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:05.036854Z","iopub.execute_input":"2022-02-12T09:41:05.037114Z","iopub.status.idle":"2022-02-12T09:41:05.468298Z","shell.execute_reply.started":"2022-02-12T09:41:05.037081Z","shell.execute_reply":"2022-02-12T09:41:05.467561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Same sequence","metadata":{"papermill":{"duration":0.037439,"end_time":"2022-02-12T05:46:16.71975","exception":false,"start_time":"2022-02-12T05:46:16.682311","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#%%time\nimport time\n\nstart = time.time()\n\nn_vis = 10\nfor i in range(n_vis):\n    image1 = torch.from_numpy(frames[i]).permute(2, 0, 1).float().to(device)\n    image2 = torch.from_numpy(frames[i+1]).permute(2, 0, 1).float().to(device)\n    \n    image1 = image1[None].to(device)\n    image2 = image2[None].to(device)\n\n    padder = InputPadder(image1.shape)\n    image1, image2 = padder.pad(image1, image2)\n    \n    with torch.no_grad():\n        flow_low, flow_up = model(image1, image2, iters=20, test_mode=True)\n        \n    viz(image1, image2, flow_up)\n    \n\nend = time.time()\nprint (f'elapsed time= {(end - start) / n_vis:.3f} [sec/frame]')","metadata":{"papermill":{"duration":7.287055,"end_time":"2022-02-12T05:46:24.045453","exception":false,"start_time":"2022-02-12T05:46:16.758398","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:05.469679Z","iopub.execute_input":"2022-02-12T09:41:05.469943Z","iopub.status.idle":"2022-02-12T09:41:13.228484Z","shell.execute_reply.started":"2022-02-12T09:41:05.469909Z","shell.execute_reply":"2022-02-12T09:41:13.227845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# sequence change","metadata":{"papermill":{"duration":0.123857,"end_time":"2022-02-12T05:46:24.292021","exception":false,"start_time":"2022-02-12T05:46:24.168164","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#%%time\n\nstart = time.time()\n\nfor i in range(n_vis):\n    image1 = torch.from_numpy(frames2[i]).permute(2, 0, 1).float().to(device)\n    image2 = torch.from_numpy(frames2[i+1]).permute(2, 0, 1).float().to(device)\n    \n    image1 = image1[None].to(device)\n    image2 = image2[None].to(device)\n\n    padder = InputPadder(image1.shape)\n    image1, image2 = padder.pad(image1, image2)\n    \n    with torch.no_grad():\n        flow_low, flow_up = model(image1, image2, iters=10, test_mode=True)\n        \n    viz(image1, image2, flow_up)\n    \nend = time.time()\n\nprint (f'elapsed time= {(end - start) / n_vis:.3f} [sec/frame]')","metadata":{"papermill":{"duration":6.003668,"end_time":"2022-02-12T05:46:30.418134","exception":false,"start_time":"2022-02-12T05:46:24.414466","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:13.229928Z","iopub.execute_input":"2022-02-12T09:41:13.230398Z","iopub.status.idle":"2022-02-12T09:41:20.33646Z","shell.execute_reply.started":"2022-02-12T09:41:13.230362Z","shell.execute_reply":"2022-02-12T09:41:20.33583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# video change","metadata":{"papermill":{"duration":0.200635,"end_time":"2022-02-12T05:46:30.824126","exception":false,"start_time":"2022-02-12T05:46:30.623491","status":"completed"},"tags":[]}},{"cell_type":"code","source":"start = time.time()\n\nfor i in range(n_vis):\n    image1 = torch.from_numpy(frames3[i]).permute(2, 0, 1).float().to(device)\n    image2 = torch.from_numpy(frames3[i+1]).permute(2, 0, 1).float().to(device)\n    \n    image1 = image1[None].to(device)\n    image2 = image2[None].to(device)\n\n    padder = InputPadder(image1.shape)\n    image1, image2 = padder.pad(image1, image2)\n    \n    with torch.no_grad():\n        flow_low, flow_up = model(image1, image2, iters=10, test_mode=True)\n        \n    viz(image1, image2, flow_up)\n    \nend = time.time()\n\nprint (f'elapsed time= {(end - start) / n_vis:.3f} [sec/frame]')","metadata":{"papermill":{"duration":6.591144,"end_time":"2022-02-12T05:46:37.624338","exception":false,"start_time":"2022-02-12T05:46:31.033194","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:20.337717Z","iopub.execute_input":"2022-02-12T09:41:20.33934Z","iopub.status.idle":"2022-02-12T09:41:28.160462Z","shell.execute_reply.started":"2022-02-12T09:41:20.339298Z","shell.execute_reply":"2022-02-12T09:41:28.159819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# elapsed time without display","metadata":{"papermill":{"duration":0.260403,"end_time":"2022-02-12T05:46:38.16809","exception":false,"start_time":"2022-02-12T05:46:37.907687","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#%%time\n\nstart = time.time()\n\nfor i in range(n_vis):\n    image1 = torch.from_numpy(frames2[i]).permute(2, 0, 1).float().to(device)\n    image2 = torch.from_numpy(frames2[i+1]).permute(2, 0, 1).float().to(device)\n    \n    image1 = image1[None].to(device)\n    image2 = image2[None].to(device)\n\n    padder = InputPadder(image1.shape)\n    image1, image2 = padder.pad(image1, image2)\n    \n    with torch.no_grad():\n        flow_low, flow_up = model(image1, image2, iters=10, test_mode=True)\n        \n    #viz(image1, image2, flow_up)\n    \nend = time.time()\n\nprint (f'elapsed time= {(end - start) / n_vis:.3f} [sec/frame]')","metadata":{"papermill":{"duration":0.78392,"end_time":"2022-02-12T05:46:39.218492","exception":false,"start_time":"2022-02-12T05:46:38.434572","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:28.161915Z","iopub.execute_input":"2022-02-12T09:41:28.162426Z","iopub.status.idle":"2022-02-12T09:41:29.199727Z","shell.execute_reply.started":"2022-02-12T09:41:28.162387Z","shell.execute_reply":"2022-02-12T09:41:29.198985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (f'estimated elapsed time total = {((end - start) / n_vis) * 13499 / 60 / 60 :.3f} [h]')","metadata":{"papermill":{"duration":0.281398,"end_time":"2022-02-12T05:46:39.787524","exception":false,"start_time":"2022-02-12T05:46:39.506126","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-12T09:41:29.201016Z","iopub.execute_input":"2022-02-12T09:41:29.201446Z","iopub.status.idle":"2022-02-12T09:41:29.206916Z","shell.execute_reply.started":"2022-02-12T09:41:29.201408Z","shell.execute_reply":"2022-02-12T09:41:29.206234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# predict all images","metadata":{"execution":{"iopub.status.busy":"2022-02-12T06:02:25.10791Z","iopub.execute_input":"2022-02-12T06:02:25.108232Z","iopub.status.idle":"2022-02-12T06:02:25.126931Z","shell.execute_reply.started":"2022-02-12T06:02:25.108152Z","shell.execute_reply":"2022-02-12T06:02:25.126253Z"}}},{"cell_type":"code","source":"if DEBUG:\n    df = df[:1000]","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:41:29.210282Z","iopub.execute_input":"2022-02-12T09:41:29.210997Z","iopub.status.idle":"2022-02-12T09:41:29.215982Z","shell.execute_reply.started":"2022-02-12T09:41:29.210957Z","shell.execute_reply":"2022-02-12T09:41:29.215284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor i in range(len(df.image_path) -1):\n    \n    img1 = cv2.imread(df.image_path[i])[:,:,::-1]\n    img1 = cv2.resize(img1, dsize=None, fx=small_rate, fy=small_rate)\n    \n    img2 = cv2.imread(df.image_path[i+1])[:,:,::-1]\n    img2 = cv2.resize(img2, dsize=None, fx=small_rate, fy=small_rate)\n    \n    image1 = torch.from_numpy(img1).permute(2, 0, 1).float().to(device)\n    image2 = torch.from_numpy(img2).permute(2, 0, 1).float().to(device)\n    \n    image1 = image1[None].to(device)\n    image2 = image2[None].to(device)\n\n    padder = InputPadder(image1.shape)\n    image1, image2 = padder.pad(image1, image2)\n    \n    with torch.no_grad():\n        flow_low, flow_up = model(image1, image2, iters=10, test_mode=True)\n        \n    flow_up = flow_up.cpu().numpy()\n    df.loc[i, 'flow_mean'] = np.mean(flow_up)\n    df.loc[i, 'flow_std'] = np.std(flow_up)\n    df.loc[i, 'flow_med'] = np.median(flow_up)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:41:29.217508Z","iopub.execute_input":"2022-02-12T09:41:29.217765Z","iopub.status.idle":"2022-02-12T09:44:16.181945Z","shell.execute_reply.started":"2022-02-12T09:41:29.21773Z","shell.execute_reply":"2022-02-12T09:44:16.180457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# scene change detection from stats of optical flow","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:44:16.183297Z","iopub.execute_input":"2022-02-12T09:44:16.183558Z","iopub.status.idle":"2022-02-12T09:44:16.189189Z","shell.execute_reply.started":"2022-02-12T09:44:16.183523Z","shell.execute_reply":"2022-02-12T09:44:16.188525Z"}}},{"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(20,5))\nax2 = ax1.twinx()\nax1.plot(df.sequence, label='seq_id', c='g',linewidth=0.5)\nax2.plot(df.flow_mean, 'o', label='mean')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:44:16.190882Z","iopub.execute_input":"2022-02-12T09:44:16.191375Z","iopub.status.idle":"2022-02-12T09:44:16.476699Z","shell.execute_reply.started":"2022-02-12T09:44:16.191337Z","shell.execute_reply":"2022-02-12T09:44:16.476025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(20,5))\nax2 = ax1.twinx()\n\nax1.plot(df.sequence, label='seq_id', c='g',linewidth=0.5)\nax2.plot(df.flow_med, 'o', label='median')\n\nax1.set_ylabel(r'sequence id')\nax2.set_ylabel(r'flow value')\n\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:44:16.477862Z","iopub.execute_input":"2022-02-12T09:44:16.47821Z","iopub.status.idle":"2022-02-12T09:44:16.78107Z","shell.execute_reply.started":"2022-02-12T09:44:16.478158Z","shell.execute_reply":"2022-02-12T09:44:16.780401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(20,5))\nax2 = ax1.twinx()\nax1.plot(df.sequence, label='seq_id', c='g',linewidth=0.5)\nax2.plot(df.flow_std, 'o', label='std')\nplt.legend()\n\nax1.set_ylabel(r'sequence id')\nax2.set_ylabel(r'flow value')","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:44:16.78226Z","iopub.execute_input":"2022-02-12T09:44:16.782509Z","iopub.status.idle":"2022-02-12T09:44:17.071634Z","shell.execute_reply.started":"2022-02-12T09:44:16.782474Z","shell.execute_reply":"2022-02-12T09:44:17.070964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nplt.hist(df.flow_std, bins=100);","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:44:17.073021Z","iopub.execute_input":"2022-02-12T09:44:17.073481Z","iopub.status.idle":"2022-02-12T09:44:17.421271Z","shell.execute_reply.started":"2022-02-12T09:44:17.073444Z","shell.execute_reply":"2022-02-12T09:44:17.420612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('opt_flow.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-12T09:44:17.422537Z","iopub.execute_input":"2022-02-12T09:44:17.422782Z","iopub.status.idle":"2022-02-12T09:44:17.44479Z","shell.execute_reply.started":"2022-02-12T09:44:17.422738Z","shell.execute_reply":"2022-02-12T09:44:17.444127Z"},"trusted":true},"execution_count":null,"outputs":[]}]}