{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Seq-Nms + Yolov5**","metadata":{}},{"cell_type":"markdown","source":"# https://github.com/lrghust/Seq-NMS/blob/master/seqnms.py\n# \n# https://github.com/amusi/Non-Maximum-Suppression/blob/master/nms.py","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nfrom tqdm import tqdm\nimport sys\n\nsys.path.append('../input/tensorflow-great-barrier-reef')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-13T21:49:46.061861Z","iopub.execute_input":"2022-02-13T21:49:46.062441Z","iopub.status.idle":"2022-02-13T21:49:47.520524Z","shell.execute_reply.started":"2022-02-13T21:49:46.062342Z","shell.execute_reply":"2022-02-13T21:49:47.51967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:47.525674Z","iopub.execute_input":"2022-02-13T21:49:47.527703Z","iopub.status.idle":"2022-02-13T21:49:48.936938Z","shell.execute_reply.started":"2022-02-13T21:49:47.527662Z","shell.execute_reply":"2022-02-13T21:49:48.935992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:48.938939Z","iopub.execute_input":"2022-02-13T21:49:48.939201Z","iopub.status.idle":"2022-02-13T21:49:48.964455Z","shell.execute_reply.started":"2022-02-13T21:49:48.939165Z","shell.execute_reply":"2022-02-13T21:49:48.963806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = torch.hub.load('../input/yolov5-lib-ds', \n                       'custom', \n#                        path='/kaggle/input/reef-baseline-fold12/l6_3600_uflip_vm5_f12_up/f1/best.pt',\n                       path='../input/1920-yolov5m6-19-cots/best.pt',\n                       source='local',\n                       force_reload=True)  # local repo\nmodel.conf = 0.00\nmodel.iou = 1\nmodel.max_det = 1000","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:48.966278Z","iopub.execute_input":"2022-02-13T21:49:48.96646Z","iopub.status.idle":"2022-02-13T21:49:55.81965Z","shell.execute_reply.started":"2022-02-13T21:49:48.966437Z","shell.execute_reply":"2022-02-13T21:49:55.818882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = ('background', 'starfish')\nimport time\nIOU_THRESH = 0.4\nimport copy","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:55.821124Z","iopub.execute_input":"2022-02-13T21:49:55.821944Z","iopub.status.idle":"2022-02-13T21:49:55.82621Z","shell.execute_reply.started":"2022-02-13T21:49:55.821905Z","shell.execute_reply":"2022-02-13T21:49:55.825343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def nms(bounding_boxes, confidence_score, threshold):\n    # Copied from https://github.com/amusi/Non-Maximum-Suppression/blob/master/nms.py\n    # If no bounding boxes, return empty list\n    if len(bounding_boxes) == 0:\n        return [], []\n\n    # Bounding boxes\n    boxes = np.array(bounding_boxes)\n\n    # coordinates of bounding boxes\n    start_x = boxes[:, 0]\n    start_y = boxes[:, 1]\n    end_x = boxes[:, 2]\n    end_y = boxes[:, 3]\n\n    # Confidence scores of bounding boxes\n    score = np.array(confidence_score)\n\n    # Picked bounding boxes\n    picked_boxes = []\n    picked_score = []\n\n    # Compute areas of bounding boxes\n    areas = (end_x - start_x + 1) * (end_y - start_y + 1)\n\n    # Sort by confidence score of bounding boxes\n    order = np.argsort(score)\n\n    # Iterate bounding boxes\n    while order.size > 0:\n        # The index of largest confidence score\n        index = order[-1]\n\n        # Pick the bounding box with largest confidence score\n        picked_boxes.append(bounding_boxes[index])\n        picked_score.append(confidence_score[index])\n\n        # Compute ordinates of intersection-over-union(IOU)\n        x1 = np.maximum(start_x[index], start_x[order[:-1]])\n        x2 = np.minimum(end_x[index], end_x[order[:-1]])\n        y1 = np.maximum(start_y[index], start_y[order[:-1]])\n        y2 = np.minimum(end_y[index], end_y[order[:-1]])\n\n        # Compute areas of intersection-over-union\n        w = np.maximum(0.0, x2 - x1 + 1)\n        h = np.maximum(0.0, y2 - y1 + 1)\n        intersection = w * h\n\n        # Compute the ratio between intersection and union\n        ratio = intersection / (areas[index] + areas[order[:-1]] - intersection)\n\n        left = np.where(ratio < threshold)\n        order = order[left]\n\n    return picked_boxes, picked_score","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:55.827412Z","iopub.execute_input":"2022-02-13T21:49:55.827901Z","iopub.status.idle":"2022-02-13T21:49:55.842507Z","shell.execute_reply.started":"2022-02-13T21:49:55.827866Z","shell.execute_reply":"2022-02-13T21:49:55.841739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Copied from https://github.com/lrghust/Seq-NMS/blob/master/seqnms.py\ndef createLinks(dets_all):\n    links_all=[]\n    frame_num=len(dets_all[0])\n    cls_num=len(CLASSES)-1\n    for cls_ind in range(cls_num): \n        links_cls=[] \n#         link_begin=time.time()\n        for frame_ind in range(frame_num-1): \n            dets1=dets_all[cls_ind][frame_ind]\n            dets2=dets_all[cls_ind][frame_ind+1]\n            box1_num=len(dets1)\n            box2_num=len(dets2)\n            if frame_ind==0:\n                areas1=np.empty(box1_num)\n                for box1_ind,box1 in enumerate(dets1):\n                    areas1[box1_ind]=(box1[2]-box1[0]+1)*(box1[3]-box1[1]+1)\n            else: \n                areas1=areas2\n            areas2=np.empty(box2_num)\n            for box2_ind,box2 in enumerate(dets2):\n                areas2[box2_ind]=(box2[2]-box2[0]+1)*(box2[3]-box2[1]+1)\n            links_frame=[] \n            for box1_ind,box1 in enumerate(dets1):\n                area1=areas1[box1_ind]\n                x1=np.maximum(box1[0],dets2[:,0])\n                y1=np.maximum(box1[1],dets2[:,1])\n                x2=np.minimum(box1[2],dets2[:,2])\n                y2=np.minimum(box1[3],dets2[:,3])\n                w =np.maximum(0.0, x2 - x1 + 1)\n                h =np.maximum(0.0, y2 - y1 + 1)\n                inter = w * h\n                ovrs = inter / (area1 + areas2 - inter)\n                links_box=[ovr_ind for ovr_ind,ovr in enumerate(ovrs) if ovr >= IOU_THRESH]\n                links_frame.append(links_box)\n            links_cls.append(links_frame)\n#         link_end=time.time()\n#         print('link: {:.4f}s'.format(link_end - link_begin))\n        links_all.append(links_cls)\n    return links_all\n\ndef maxPath(dets_all,links_all):\n    for cls_ind,links_cls in enumerate(links_all):\n#         max_begin=time.time()\n        dets_cls=dets_all[cls_ind]\n        while True:\n            rootindex,maxpath,maxsum=findMaxPath(links_cls,dets_cls)\n            if len(maxpath) <= 1:\n                break\n            rescore(dets_cls,rootindex,maxpath,maxsum)\n            deleteLink(dets_cls,links_cls,rootindex,maxpath,IOU_THRESH)\n#         max_end=time.time()\n#         print ('max path: {:.4f}s'.format(max_end - max_begin))\n\ndef NMS(dets_all):\n    for cls_ind,dets_cls in enumerate(dets_all):\n        for frame_ind,dets in enumerate(dets_cls):\n            keep=nms(dets, NMS_THRESH)\n            dets_all[cls_ind][frame_ind]=dets[keep, :]\n\ndef findMaxPath(links,dets):\n    maxpaths=[]\n    roots=[] \n    maxpaths.append([ (box[4],[ind]) for ind,box in enumerate(dets[-1])])\n    for link_ind,link in enumerate(links[::-1]): \n        curmaxpaths=[]\n        linkflags=np.zeros(len(maxpaths[0]),int)\n        det_ind=len(links)-link_ind-1\n        for ind,linkboxes in enumerate(link):\n            if linkboxes == []:\n                curmaxpaths.append((dets[det_ind][ind][4],[ind]))\n                continue\n            linkflags[linkboxes]=1\n            prev_ind=np.argmax([maxpaths[0][linkbox][0] for linkbox in linkboxes])\n            prev_score=maxpaths[0][linkboxes[prev_ind]][0]\n            prev_path=copy.copy(maxpaths[0][linkboxes[prev_ind]][1])\n            prev_path.insert(0,ind)\n            curmaxpaths.append((dets[det_ind][ind][4]+prev_score,prev_path))\n        root=[maxpaths[0][ind] for ind,flag in enumerate(linkflags) if flag == 0]\n        roots.insert(0,root)\n        maxpaths.insert(0,curmaxpaths)\n    roots.insert(0,maxpaths[0])\n    maxscore=0\n    maxpath=[]\n    for index,paths in enumerate(roots):\n        if paths==[]:\n            continue\n        maxindex=np.argmax([path[0] for path in paths])\n        if paths[maxindex][0]>maxscore:\n            maxscore=paths[maxindex][0]\n            maxpath=paths[maxindex][1]\n            rootindex=index\n    return rootindex,maxpath,maxscore\n\ndef rescore(dets, rootindex, maxpath, maxsum):\n    newscore=maxsum/len(maxpath)\n    for i,box_ind in enumerate(maxpath):\n        dets[rootindex+i][box_ind][4]=newscore\n\ndef deleteLink(dets,links, rootindex, maxpath,thesh):\n    for i,box_ind in enumerate(maxpath):\n        areas=[(box[2]-box[0]+1)*(box[3]-box[1]+1) for box in dets[rootindex+i]]\n        area1=areas[box_ind]\n        box1=dets[rootindex+i][box_ind]\n        x1=np.maximum(box1[0],dets[rootindex+i][:,0])\n        y1=np.maximum(box1[1],dets[rootindex+i][:,1])\n        x2=np.minimum(box1[2],dets[rootindex+i][:,2])\n        y2=np.minimum(box1[3],dets[rootindex+i][:,3])\n        w =np.maximum(0.0, x2 - x1 + 1)\n        h =np.maximum(0.0, y2 - y1 + 1)\n        inter = w * h\n        ovrs = inter / (area1 + areas - inter)\n        deletes=[ovr_ind for ovr_ind,ovr in enumerate(ovrs) if ovr >= IOU_THRESH]\n        if rootindex+i<len(links):\n            for delete_ind in deletes:\n                links[rootindex+i][delete_ind]=[]\n        if i > 0 or rootindex>0:\n            for priorbox in links[rootindex+i-1]:\n                for delete_ind in deletes:\n                    if delete_ind in priorbox:\n                        priorbox.remove(delete_ind)\n                        \n\ndef dsnms(preds):\n    dets = [[]]\n    for detection in preds:\n        fid = []\n        for idx, row in detection.pandas().xyxy[0].iterrows():\n            fid.append([row.xmin, row.ymin, row.xmax, row.ymax, row.confidence])\n        dets[0].append(fid)\n    dets = np.array(dets)\n    links=createLinks(dets)\n    maxPath(dets,links)\n    return dets","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:55.844153Z","iopub.execute_input":"2022-02-13T21:49:55.844517Z","iopub.status.idle":"2022-02-13T21:49:55.884903Z","shell.execute_reply.started":"2022-02-13T21:49:55.844479Z","shell.execute_reply":"2022-02-13T21:49:55.884088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:55.886338Z","iopub.execute_input":"2022-02-13T21:49:55.886634Z","iopub.status.idle":"2022-02-13T21:49:55.896116Z","shell.execute_reply.started":"2022-02-13T21:49:55.886597Z","shell.execute_reply":"2022-02-13T21:49:55.895336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n    anno = ''\n    r = model(img, size=3600, augment=True)\n    preds.append(r)\n    preds = preds[-2:]\n    if r.pandas().xyxy[0].shape[0] == 0:\n        anno = ''\n    elif idx == 0:\n        detecs = []\n        scores = []\n        for idx, row in r.pandas().xyxy[0].iterrows():\n            detecs.append([row.xmin, row.ymin, row.xmax, row.ymax])\n            scores.append(row.confidence)\n        detecs, scores = nms(detecs, scores, IOU_THRESH)\n        for i, det in enumerate(detecs):\n            if scores[i] > 0.25:\n                anno += '{} {} {} {} {} '.format(scores[i], int(det[0]), int(det[1]), int(det[2] - det[0]), int(det[3] - det[1]))\n    \n    else:\n        pre_nms_dets = dsnms(preds)\n        correct_frame = pre_nms_dets[0][1]\n        detecs, scores = nms(correct_frame[:, :4], correct_frame[:, 4], IOU_THRESH)\n        for i, det in enumerate(detecs):\n            if scores[i] > 0.25:\n                anno += '{} {} {} {} {} '.format(scores[i], int(det[0]), int(det[1]), int(det[2] - det[0]), int(det[3] - det[1]))\n    pred_df['annotations'] = anno.strip(' ')\n    env.predict(pred_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:49:55.897769Z","iopub.execute_input":"2022-02-13T21:49:55.898076Z","iopub.status.idle":"2022-02-13T21:50:03.93492Z","shell.execute_reply.started":"2022-02-13T21:49:55.898032Z","shell.execute_reply":"2022-02-13T21:50:03.934239Z"},"trusted":true},"execution_count":null,"outputs":[]}]}