{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The purpose of this Notebook is a to test a homemade Yolo like model.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport random\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport ast\nimport os\nfrom tqdm import tqdm\ntqdm.pandas()\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport keras.backend as K\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-13T20:55:54.474655Z","iopub.execute_input":"2022-02-13T20:55:54.475154Z","iopub.status.idle":"2022-02-13T20:55:54.484005Z","shell.execute_reply.started":"2022-02-13T20:55:54.475116Z","shell.execute_reply":"2022-02-13T20:55:54.483261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Exploration","metadata":{}},{"cell_type":"code","source":"trainpath = '../input/tensorflow-great-barrier-reef/train.csv'\ntrainDF=pd.read_csv(trainpath)\ntrainDF.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:54.486713Z","iopub.execute_input":"2022-02-13T20:55:54.487184Z","iopub.status.idle":"2022-02-13T20:55:54.533478Z","shell.execute_reply.started":"2022-02-13T20:55:54.487146Z","shell.execute_reply":"2022-02-13T20:55:54.532806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:54.535012Z","iopub.execute_input":"2022-02-13T20:55:54.535734Z","iopub.status.idle":"2022-02-13T20:55:54.553897Z","shell.execute_reply.started":"2022-02-13T20:55:54.535696Z","shell.execute_reply":"2022-02-13T20:55:54.553202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF['annotations'] = trainDF['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ntrainDF['nb_bbox']=trainDF['annotations'].apply(lambda x:len(x))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:54.555052Z","iopub.execute_input":"2022-02-13T20:55:54.556024Z","iopub.status.idle":"2022-02-13T20:55:54.942396Z","shell.execute_reply.started":"2022-02-13T20:55:54.555983Z","shell.execute_reply":"2022-02-13T20:55:54.941585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bboxes(list_dict):\n    result = []\n    for dict in list_dict:\n        bbox = (dict['x'],dict['y'],dict['width'],dict['height'])\n        result.append(bbox)\n    return result\n\ntrainDF['bboxes']=trainDF['annotations'].progress_apply(get_bboxes)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:54.944646Z","iopub.execute_input":"2022-02-13T20:55:54.94491Z","iopub.status.idle":"2022-02-13T20:55:54.999323Z","shell.execute_reply.started":"2022-02-13T20:55:54.944872Z","shell.execute_reply":"2022-02-13T20:55:54.998535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF['path']='../input/tensorflow-great-barrier-reef/train_images/video_'+trainDF['video_id'].map(str)+'/'+trainDF['video_frame'].map(str)+'.jpg'","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:55.00081Z","iopub.execute_input":"2022-02-13T20:55:55.001088Z","iopub.status.idle":"2022-02-13T20:55:55.03555Z","shell.execute_reply.started":"2022-02-13T20:55:55.001052Z","shell.execute_reply":"2022-02-13T20:55:55.034793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF.head(50)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:55.037026Z","iopub.execute_input":"2022-02-13T20:55:55.037306Z","iopub.status.idle":"2022-02-13T20:55:55.114957Z","shell.execute_reply.started":"2022-02-13T20:55:55.037269Z","shell.execute_reply":"2022-02-13T20:55:55.11428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(trainDF,x='nb_bbox')","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:55.115998Z","iopub.execute_input":"2022-02-13T20:55:55.116355Z","iopub.status.idle":"2022-02-13T20:55:55.37098Z","shell.execute_reply.started":"2022-02-13T20:55:55.116322Z","shell.execute_reply":"2022-02-13T20:55:55.370204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plotbbox(video,imageid):\n    path = '../input/tensorflow-great-barrier-reef/train_images/video_'+str(video)+'/'+str(imageid)+'.jpg'\n    img = Image.open(path)\n    bboxes = trainDF[(trainDF['video_id']==video)&(trainDF['video_frame']==imageid)].iloc[0]['bboxes']\n    plt.figure(figsize=(20,20))\n    plt.imshow(img)\n    ax = plt.gca()\n    for bbox in bboxes:\n        rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='r',facecolor='none')\n        ax.add_patch(rect)\n    print(img.size)\n    \nplotbbox(0,49)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:55.372274Z","iopub.execute_input":"2022-02-13T20:55:55.372539Z","iopub.status.idle":"2022-02-13T20:55:56.215386Z","shell.execute_reply.started":"2022-02-13T20:55:55.372492Z","shell.execute_reply":"2022-02-13T20:55:56.214603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def coco2yolo(bbox):\n    x = bbox[0]\n    y = bbox[1]\n    w = bbox[2]\n    h = bbox[3]\n    xm = x+w/2\n    ym = y+h/2\n    return (xm,ym,w,h)\n\ndef yolo2coco(bbox):\n    xm = bbox[0]\n    ym = bbox[1]\n    w = bbox[2]\n    h = bbox[3]\n    x = xm-w/2\n    y = ym-h/2\n    return (x,y,w,h)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.219622Z","iopub.execute_input":"2022-02-13T20:55:56.220007Z","iopub.status.idle":"2022-02-13T20:55:56.228327Z","shell.execute_reply.started":"2022-02-13T20:55:56.219968Z","shell.execute_reply":"2022-02-13T20:55:56.227598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Create dataset","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = (720,1280)\nIMG_SHAPE = (720,1280,3)\nNB_I = 7\nNB_J = 7\nNB_CELLS = 7\nNB_CLASS = 0 # ==> not used \nNB_BOX = 1\nTENSOR_DEPTH = NB_BOX*5+NB_CLASS\nBATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.232215Z","iopub.execute_input":"2022-02-13T20:55:56.232843Z","iopub.status.idle":"2022-02-13T20:55:56.242119Z","shell.execute_reply.started":"2022-02-13T20:55:56.232804Z","shell.execute_reply":"2022-02-13T20:55:56.241385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF=pd.read_csv(trainpath)\ntrainDF['path']='../input/tensorflow-great-barrier-reef/train_images/video_'+trainDF['video_id'].map(str)+'/'+trainDF['video_frame'].map(str)+'.jpg'","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.244355Z","iopub.execute_input":"2022-02-13T20:55:56.244964Z","iopub.status.idle":"2022-02-13T20:55:56.318413Z","shell.execute_reply.started":"2022-02-13T20:55:56.244928Z","shell.execute_reply":"2022-02-13T20:55:56.317699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF['listbbox']=trainDF['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ntrainDF['nbboxes']=trainDF['listbbox'].progress_apply(lambda x: len(x))\ntrainDF=trainDF[trainDF['nbboxes']>0]\ntrainDF.describe()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.319843Z","iopub.execute_input":"2022-02-13T20:55:56.320094Z","iopub.status.idle":"2022-02-13T20:55:56.75228Z","shell.execute_reply.started":"2022-02-13T20:55:56.32006Z","shell.execute_reply":"2022-02-13T20:55:56.751464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X,y = trainDF['path'],trainDF['annotations']\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.753603Z","iopub.execute_input":"2022-02-13T20:55:56.753913Z","iopub.status.idle":"2022-02-13T20:55:56.764111Z","shell.execute_reply.started":"2022-02-13T20:55:56.753877Z","shell.execute_reply":"2022-02-13T20:55:56.76335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train = trainDF[trainDF['video_id'].isin([0,1])]['path']\n# y_train = trainDF[trainDF['video_id'].isin([0,1])]['annotations']\n\n# X_test = trainDF[trainDF['video_id']==2]['path']\n# y_test = trainDF[trainDF['video_id']==2]['annotations']","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.76553Z","iopub.execute_input":"2022-02-13T20:55:56.765782Z","iopub.status.idle":"2022-02-13T20:55:56.772355Z","shell.execute_reply.started":"2022-02-13T20:55:56.765749Z","shell.execute_reply":"2022-02-13T20:55:56.771537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.773738Z","iopub.execute_input":"2022-02-13T20:55:56.77403Z","iopub.status.idle":"2022-02-13T20:55:56.786433Z","shell.execute_reply.started":"2022-02-13T20:55:56.773992Z","shell.execute_reply":"2022-02-13T20:55:56.785767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_init = y_test","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.789003Z","iopub.execute_input":"2022-02-13T20:55:56.789726Z","iopub.status.idle":"2022-02-13T20:55:56.794183Z","shell.execute_reply.started":"2022-02-13T20:55:56.789504Z","shell.execute_reply":"2022-02-13T20:55:56.793403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_encoder(bboxes):\n    label_matrix = np.zeros([NB_I,NB_J,TENSOR_DEPTH])\n    xsize = IMG_SIZE[0]/NB_I\n    ysize = IMG_SIZE[1]/NB_J\n    for bbox in bboxes:\n        bbox = coco2yolo(bbox)\n        xidx = int(bbox[0]/xsize)\n        xidx = min(NB_I-1,xidx)\n        yidx = int(bbox[1]/ysize)\n        yidx = min(NB_J-1,yidx)\n        x = bbox[0]/xsize-xidx\n        y = bbox[1]/ysize-yidx\n        w = bbox[2]/IMG_SIZE[0]\n        w = np.sqrt(w)\n        h = bbox[3]/IMG_SIZE[1]\n        h = np.sqrt(h)\n  #      print('indexes: ',xidx,yidx)\n  #      print('bbox: ',bbox)\n        if label_matrix[xidx,yidx,0]==0:\n            label_matrix[xidx,yidx,0]=1\n            label_matrix[xidx,yidx,1:5]=[x,y,w,h]\n    label_tensor = label_matrix # tf.convert_to_tensor(label_matrix, np.float32)\n#    print(\"label encode\")\n    return label_tensor","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.79532Z","iopub.execute_input":"2022-02-13T20:55:56.79589Z","iopub.status.idle":"2022-02-13T20:55:56.806657Z","shell.execute_reply.started":"2022-02-13T20:55:56.795849Z","shell.execute_reply":"2022-02-13T20:55:56.805901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_y(annotations):\n    annotations = ast.literal_eval(annotations)\n    bboxes = get_bboxes(annotations)\n    label_tensor = label_encoder(bboxes)\n    return label_tensor","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.808034Z","iopub.execute_input":"2022-02-13T20:55:56.808408Z","iopub.status.idle":"2022-02-13T20:55:56.818404Z","shell.execute_reply.started":"2022-02-13T20:55:56.808371Z","shell.execute_reply":"2022-02-13T20:55:56.817526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = y_train.progress_apply(prepare_y)\ny_test = y_test.progress_apply(prepare_y)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:56.819875Z","iopub.execute_input":"2022-02-13T20:55:56.820225Z","iopub.status.idle":"2022-02-13T20:55:57.276031Z","shell.execute_reply.started":"2022-02-13T20:55:56.820095Z","shell.execute_reply":"2022-02-13T20:55:57.275342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = np.stack(y_train.to_numpy())\ny_test = np.stack(y_test.to_numpy())\nprint('y_train.shape: ',y_train.shape)\nprint('y_test.shape: ',y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:57.277008Z","iopub.execute_input":"2022-02-13T20:55:57.277626Z","iopub.status.idle":"2022-02-13T20:55:57.298177Z","shell.execute_reply.started":"2022-02-13T20:55:57.277593Z","shell.execute_reply":"2022-02-13T20:55:57.297536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef image_loader(path):\n    Image_string = tf.io.read_file(path)\n    Image = tf.image.decode_jpeg(Image_string, channels=3)\n    return Image\n\n@tf.function\ndef data_flipper(img,label_matrix):\n    reverse_img = tf.image.flip_left_right(img)\n    #reverse_matrix = np.zeros([NB_I,NB_J,TENSOR_DEPTH])\n    C = label_matrix[...,0]\n    X = label_matrix[...,1]\n    Y = label_matrix[...,2]\n    W = label_matrix[...,3]\n    H = label_matrix[...,4]\n    X = 1-X\n    X = X*C\n    temp = tf.stack([C,X,Y,W,H],axis=-1)\n    reverse_matrix = tf.reverse(temp,[0])\n    return reverse_img, reverse_matrix\n    \n@tf.function\ndef prepare_data(path,label_matrix):\n    img = image_loader(path)\n    if random.random()<0.5: img, label_matrix = data_flipper(img, label_matrix)\n    return img,label_matrix","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:10:51.712252Z","iopub.execute_input":"2022-02-13T21:10:51.71258Z","iopub.status.idle":"2022-02-13T21:10:51.724958Z","shell.execute_reply.started":"2022-02-13T21:10:51.712539Z","shell.execute_reply":"2022-02-13T21:10:51.724015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = tf.data.Dataset.from_tensor_slices((X_train,y_train)).map(prepare_data).shuffle(100).batch(BATCH_SIZE).prefetch(2) #Test si ça marche en cache\nvalidation_ds = tf.data.Dataset.from_tensor_slices((X_test,y_test)).map(prepare_data).batch(BATCH_SIZE).prefetch(2)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:57.31535Z","iopub.execute_input":"2022-02-13T20:55:57.315587Z","iopub.status.idle":"2022-02-13T20:55:57.392307Z","shell.execute_reply.started":"2022-02-13T20:55:57.315558Z","shell.execute_reply":"2022-02-13T20:55:57.391632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_decoder(label_matrix, threshold = 0.5):\n    \n    xsize = IMG_SIZE[0]/NB_I\n    ysize = IMG_SIZE[1]/NB_J\n    bboxes = []\n    \n    for i in range(NB_I):\n        for j in range(NB_J):\n            if label_matrix[i,j,0]>threshold:\n                x = label_matrix[i,j,1]\n                y = label_matrix[i,j,2]\n                x = (i+x)*xsize\n                y = (j+y)*ysize\n                w = label_matrix[i,j,3]\n                w = w*w*IMG_SIZE[0]\n                h = label_matrix[i,j,4]\n                h = h*h*IMG_SIZE[1]\n                bbox=(x,y,w,h)\n                bbox = yolo2coco(bbox)\n                bboxes.append(bbox)\n    \n    return bboxes\n\n\ndef label_submitter(label_matrix, threshold = 0.5):\n    \n    xsize = IMG_SIZE[0]/NB_I\n    ysize = IMG_SIZE[1]/NB_J\n    bboxes = []\n    \n    for i in range(NB_I):\n        for j in range(NB_J):\n            if label_matrix[i,j,0]>threshold:\n                x = label_matrix[i,j,1]\n                y = label_matrix[i,j,2]\n                x = (i+x)*xsize\n                y = (j+y)*ysize\n                w = label_matrix[i,j,3]\n                w = w*w*IMG_SIZE[0]\n                h = label_matrix[i,j,4]\n                h = h*h*IMG_SIZE[1]\n                c = label_matrix[i,j,0]\n                bbox=(x,y,w,h)\n                bbox = yolo2coco(bbox)\n                wc = min(IMG_SIZE[0]-bbox[0]-1,bbox[2])\n                hc = min(IMG_SIZE[1]-bbox[1]-1,bbox[3])\n                bbox = (c,int(bbox[0]),int(bbox[1]),int(wc),int(hc))\n                bboxes.append(bbox)\n    \n    return bboxes","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:57.393626Z","iopub.execute_input":"2022-02-13T20:55:57.393868Z","iopub.status.idle":"2022-02-13T20:55:57.407578Z","shell.execute_reply.started":"2022-02-13T20:55:57.393836Z","shell.execute_reply":"2022-02-13T20:55:57.406793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img,lbl = next(iter(train_ds))\nprint(img.shape)\nprint(lbl.shape)\n# print('X: ',img[17])\n# print('------------------------------------------------------------------------------------')\n# print('y: ',lbl[2])\npred_bboxes = label_decoder(lbl[2,:,:,:],threshold=0.1)\nplt.figure(figsize=(20,20))\nplt.imshow(img.numpy()[31,:,:,:].astype('uint8'))\nax = plt.gca()\nfor i,bbox in enumerate(pred_bboxes):\n    rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='r',facecolor='none')\n    ax.add_patch(rect)\n    ax.text(bbox[0],bbox[1],i)\n    print('bbox ',i,' at {:.2f} {:.2f}'.format(bbox[0].numpy(),bbox[1].numpy()))\n# plt.imshow(img.numpy()[31,:,:,:].astype('uint8'))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:55:57.409031Z","iopub.execute_input":"2022-02-13T20:55:57.409454Z","iopub.status.idle":"2022-02-13T20:56:00.850433Z","shell.execute_reply.started":"2022-02-13T20:55:57.409417Z","shell.execute_reply":"2022-02-13T20:56:00.849615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testbb=[(10,20,30,40)]\nlabel_submitter(label_encoder(testbb))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.853563Z","iopub.execute_input":"2022-02-13T20:56:00.853903Z","iopub.status.idle":"2022-02-13T20:56:00.860511Z","shell.execute_reply.started":"2022-02-13T20:56:00.853872Z","shell.execute_reply":"2022-02-13T20:56:00.859593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bboxes = label_decoder(lbl[2,:,:,:].numpy())\nbboxes","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.861937Z","iopub.execute_input":"2022-02-13T20:56:00.86224Z","iopub.status.idle":"2022-02-13T20:56:00.871827Z","shell.execute_reply.started":"2022-02-13T20:56:00.862206Z","shell.execute_reply":"2022-02-13T20:56:00.870978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ploter(img,lbl):\n    #img,lbl=prepare_data(path,label_matrix)\n    bboxes=label_decoder(lbl)\n    print(bboxes)\n    plt.figure(figsize=(20,20))\n    plt.imshow(img.numpy().astype('uint8'))\n    ax=plt.gca()\n    for i,bbox in enumerate(bboxes):\n        rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='r',facecolor='none')\n        ax.add_patch(rect)\n        ax.text(bbox[0],bbox[1],i)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:08:32.282564Z","iopub.execute_input":"2022-02-13T21:08:32.283582Z","iopub.status.idle":"2022-02-13T21:08:32.291981Z","shell.execute_reply.started":"2022-02-13T21:08:32.283531Z","shell.execute_reply":"2022-02-13T21:08:32.291176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ploter(image_loader(X_train[21]),y_train[21])","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:08:36.85538Z","iopub.execute_input":"2022-02-13T21:08:36.856092Z","iopub.status.idle":"2022-02-13T21:08:37.640075Z","shell.execute_reply.started":"2022-02-13T21:08:36.856054Z","shell.execute_reply":"2022-02-13T21:08:37.639361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#a,b=data_flipper(image_loader(X_train[21]),y_train[21])\n#ploter(a,b.numpy())","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:11:01.605539Z","iopub.execute_input":"2022-02-13T21:11:01.606134Z","iopub.status.idle":"2022-02-13T21:11:02.769243Z","shell.execute_reply.started":"2022-02-13T21:11:01.606091Z","shell.execute_reply":"2022-02-13T21:11:02.768558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#b","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:07:11.6296Z","iopub.execute_input":"2022-02-13T21:07:11.629867Z","iopub.status.idle":"2022-02-13T21:07:11.642375Z","shell.execute_reply.started":"2022-02-13T21:07:11.629838Z","shell.execute_reply":"2022-02-13T21:07:11.641581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Create a Yolo model\nWill be based on Yolo_v1  \n\nLabel encoder and decoder is done  \nModel is done  \nLoss function is done  \nTo do = dataset preparation  ","metadata":{}},{"cell_type":"code","source":"strategy = 'no re-use'","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.968257Z","iopub.status.idle":"2022-02-13T20:56:00.968703Z","shell.execute_reply.started":"2022-02-13T20:56:00.968455Z","shell.execute_reply":"2022-02-13T20:56:00.968488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(input_shape):\n    inputs = keras.Input(input_shape)\n\n    x = layers.Rescaling(scale = 1/127.5, offset=-1)(inputs)\n    \n    x = layers.Conv2D(filters=32, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool2D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Conv2D(filters=32, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool2D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Conv2D(filters=64, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool2D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.01)(x)\n    \n    x = layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool2D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.02)(x)\n\n    x = layers.Conv2D(filters=256, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool2D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.03)(x)\n\n    x = layers.Conv2D(filters=512, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool2D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.04)(x)\n\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dense(units=1024, activation=\"relu\")(x)\n    x = layers.Dropout(0.2)(x)\n    \n    x = layers.Dense(units=NB_I*NB_J*TENSOR_DEPTH, activation=\"sigmoid\")(x) # We get the number of output we need\n    outputs = layers.Reshape((NB_I,NB_J,TENSOR_DEPTH))(x) # We reshape to fit with label shape\n\n    model = keras.Model(inputs, outputs, name=\"yolo_JB\")\n    model.summary()\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.970025Z","iopub.status.idle":"2022-02-13T20:56:00.970429Z","shell.execute_reply.started":"2022-02-13T20:56:00.97021Z","shell.execute_reply":"2022-02-13T20:56:00.970233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model_transfer_learning(input_shape):\n    \n    base_model = keras.applications.Xception(\n        weights='imagenet',\n        input_shape = input_shape,\n        include_top = False)\n    \n    base_model.trainable = False\n    \n    inputs = keras.Input(input_shape)\n    x = layers.Rescaling(scale = 1/127.5, offset=-1)(inputs)\n    \n    x = base_model(x, training = False)\n    \n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dense(units=1024, activation=\"relu\")(x)\n    x = layers.Dropout(0.2)(x)\n    \n    x = layers.Dense(units=NB_I*NB_J*TENSOR_DEPTH, activation=\"sigmoid\")(x) # We get the number of output we need\n    outputs = layers.Reshape((NB_I,NB_J,TENSOR_DEPTH))(x) # We reshape to fit with label shape\n\n    model = keras.Model(inputs, outputs, name=\"yolo_JB\")\n    model.summary()\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.971776Z","iopub.status.idle":"2022-02-13T20:56:00.972186Z","shell.execute_reply.started":"2022-02-13T20:56:00.971966Z","shell.execute_reply":"2022-02-13T20:56:00.971989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# yolo_JB = get_model(IMG_SHAPE)\nyolo_JB = get_model_transfer_learning(IMG_SHAPE)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.973647Z","iopub.status.idle":"2022-02-13T20:56:00.974058Z","shell.execute_reply.started":"2022-02-13T20:56:00.973835Z","shell.execute_reply":"2022-02-13T20:56:00.973859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yolo_JB.output_shape","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.975655Z","iopub.status.idle":"2022-02-13T20:56:00.976065Z","shell.execute_reply.started":"2022-02-13T20:56:00.975842Z","shell.execute_reply":"2022-02-13T20:56:00.975865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yoloLoss(y_true,y_pred):\n    \n    # responseMask is our 1ij object\n    responseMask = y_true[...,:NB_BOX]\n    noResponseMask = 1-responseMask\n    \n    # position loss\n    posLoss = K.sum(responseMask*K.square(y_true[...,NB_BOX:]-y_pred[...,NB_BOX:]))\n    \n    # confidence loss\n    confLossObj = K.sum(responseMask*K.square(y_true[...,:NB_BOX]-y_pred[...,:NB_BOX]))\n    confLossNoObj = K.sum(noResponseMask*K.square(y_true[...,:NB_BOX]-y_pred[...,:NB_BOX]))\n    \n    # lbda parameters\n    lbda_coord = 5\n    lbda_noobj = 0.5\n    \n    # Total loss\n    loss = lbda_coord * posLoss + confLossObj + lbda_noobj * confLossNoObj\n    \n    return loss\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.977369Z","iopub.status.idle":"2022-02-13T20:56:00.977814Z","shell.execute_reply.started":"2022-02-13T20:56:00.977592Z","shell.execute_reply":"2022-02-13T20:56:00.977616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def myMetrics(y_true,y_pred):\n    \n    xc_true = y_true[:,:,:,1]\n    xc_pred = y_pred[:,:,:,1]\n    yc_true = y_true[:,:,:,2]\n    yc_pred = y_pred[:,:,:,2]\n    \n    w_true = y_true[:,:,:,3]\n    w_pred = y_pred[:,:,:,3]\n    h_true = y_true[:,:,:,4]\n    h_pred = y_pred[:,:,:,4]\n    \n    # Yolo 2 Coco\n    \n    xa_true = xc_true - w_true/2\n    xb_true = xc_true + w_true/2\n    ya_true = yc_true - h_true/2\n    yb_true = yc_true + h_true/2\n\n    xa_pred = xc_pred - w_pred/2\n    xb_pred = xc_pred + w_pred/2\n    ya_pred = yc_pred - h_pred/2\n    yb_pred = yc_pred + h_pred/2\n    \n    # Calculate intersection\n    \n    xa_inter = K.maximum(xa_true,xa_pred)\n    xb_inter = K.minimum(xb_true,xb_pred)\n    ya_inter = K.maximum(ya_true,ya_pred)\n    yb_inter = K.minimum(yb_true,yb_pred)\n    \n    w_inter = xb_inter - xa_inter\n    w_inter = K.maximum(w_inter,0.)\n    \n    h_inter = yb_inter - ya_inter\n    h_inter = K.maximum(h_inter,0.)\n    \n    # Calculate areas\n    \n    A_inter = w_inter * h_inter\n    A_true = w_true * h_true\n    A_pred = w_pred * h_pred\n    \n    # Get result\n    \n    IoU = A_inter/(A_true + A_pred - A_inter)\n    \n    result = K.sum(IoU)\n    \n    return result/BATCH_SIZE\n    \n  \n\n \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.979132Z","iopub.status.idle":"2022-02-13T20:56:00.979562Z","shell.execute_reply.started":"2022-02-13T20:56:00.979316Z","shell.execute_reply":"2022-02-13T20:56:00.979338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img1,lbl1 = next(iter(train_ds))\nimg2,lbl2 = next(iter(validation_ds))\nyoloLoss(lbl1,lbl2)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.980853Z","iopub.status.idle":"2022-02-13T20:56:00.981256Z","shell.execute_reply.started":"2022-02-13T20:56:00.981034Z","shell.execute_reply":"2022-02-13T20:56:00.981058Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yolo_JB.compile(\n    loss=yoloLoss,\n    optimizer=tf.keras.optimizers.Adam(), metrics=myMetrics)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.982602Z","iopub.status.idle":"2022-02-13T20:56:00.98301Z","shell.execute_reply.started":"2022-02-13T20:56:00.982788Z","shell.execute_reply":"2022-02-13T20:56:00.982811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if strategy == 're-use':\n    yolo_JB = keras.models.load_model('../input/starfishes-2/my_h5_model.h5', compile=False)\nelse:\n    history = yolo_JB.fit(train_ds, validation_data=validation_ds, epochs=40)\n    history_df = pd.DataFrame(history.history)\n    history_df.head(2)\n    history_df.loc[:, ['loss', 'val_loss']].plot(title=\"yoloLoss\")\n    history_df.loc[:, ['myMetrics', 'val_myMetrics']].plot(title=\"IoU aggregation\")\n    yolo_JB.save(\"my_h5_model.h5\")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.984302Z","iopub.status.idle":"2022-02-13T20:56:00.984742Z","shell.execute_reply.started":"2022-02-13T20:56:00.984505Z","shell.execute_reply":"2022-02-13T20:56:00.984542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_results(imagepath, label):\n    image = image_loader(imagepath)\n    img = image.numpy().astype('uint8')\n    image = tf.expand_dims(image, axis=0)\n    annotations = ast.literal_eval(label)\n    bboxes = get_bboxes(annotations)\n    prediction = yolo_JB.predict(image)\n    pred_bboxes = label_decoder(prediction[0,:,:,:],threshold=0.1)\n    plt.figure(figsize=(20,20))\n    plt.imshow(img)\n    ax = plt.gca()\n    for bbox in bboxes:\n        rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='r',facecolor='none')\n        ax.add_patch(rect)\n    for bbox in pred_bboxes:\n        rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='g',facecolor='none')\n        ax.add_patch(rect)\n\ncompare_results(trainDF.path[49],trainDF.annotations[49])","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.986051Z","iopub.status.idle":"2022-02-13T20:56:00.986457Z","shell.execute_reply.started":"2022-02-13T20:56:00.986236Z","shell.execute_reply":"2022-02-13T20:56:00.986259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Submission","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef \nenv=greatbarrierreef.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.987915Z","iopub.status.idle":"2022-02-13T20:56:00.988322Z","shell.execute_reply.started":"2022-02-13T20:56:00.988104Z","shell.execute_reply":"2022-02-13T20:56:00.988127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def compare_results(imagepath, label):\n#     image = image_loader(imagepath)\n#     img = image.numpy().astype('uint8')\n#     image = tf.expand_dims(image, axis=0)\n#     annotations = ast.literal_eval(label)\n#     bboxes = get_bboxes(annotations)\n#     prediction = yolo_JB.predict(image)\n#     pred_bboxes = label_decoder(prediction[0,:,:,:],threshold=0.1)\n#     plt.figure(figsize=(20,20))\n#     plt.imshow(img)\n#     ax = plt.gca()\n#     for bbox in bboxes:\n#         rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='r',facecolor='none')\n#         ax.add_patch(rect)\n#     for bbox in pred_bboxes:\n#         rect = Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=5,edgecolor='g',facecolor='none')\n#         ax.add_patch(rect)\n\n# fig, axes = plt.subplots(5,5,figsize=(15,15))\n#     axes = axes.flatten()\n#     for ax,idx in zip(axes,idxlist):\n#         ax.imshow(X[:,:,idx])","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.989867Z","iopub.status.idle":"2022-02-13T20:56:00.990274Z","shell.execute_reply.started":"2022-02-13T20:56:00.990053Z","shell.execute_reply":"2022-02-13T20:56:00.990076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\n\nfig, axes = plt.subplots(3,1,figsize=(45,45))\naxes = axes.flatten()\ni = 0\n\nfor (pixel_array, sample_prediction_df) in iter_test:\n    print(pixel_array.shape)\n    pixel_array = np.expand_dims(pixel_array, axis=0)\n    prediction = yolo_JB.predict(pixel_array)\n    #print(prediction)\n    bboxes = label_submitter(prediction[0,:,:,:], threshold=0.1)\n    axes[i].imshow(pixel_array[0,:,:,:])\n    predictions = []\n    for bbox in bboxes:\n        predictions.append('{:.2f} {} {} {} {}'.format(bbox[0], bbox[1], bbox[2], bbox[3], bbox[4]))\n        rect = Rectangle((bbox[1],bbox[2]),bbox[3],bbox[4],linewidth=5,edgecolor='r',facecolor='none')\n        axes[i].add_patch(rect)\n        axes[i].text(bbox[1],bbox[2],'{:.2f}'.format(bbox[0]))\n    i = i+1\n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)   # register your predictions\n    print('Prediction:', prediction_str)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.992259Z","iopub.status.idle":"2022-02-13T20:56:00.993083Z","shell.execute_reply.started":"2022-02-13T20:56:00.992839Z","shell.execute_reply":"2022-02-13T20:56:00.992865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'{:.2f} {} {} {} {}'.format(0.356, int(1.57), 2, 12, 0.1)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T20:56:00.994428Z","iopub.status.idle":"2022-02-13T20:56:00.994863Z","shell.execute_reply.started":"2022-02-13T20:56:00.994642Z","shell.execute_reply":"2022-02-13T20:56:00.994665Z"},"trusted":true},"execution_count":null,"outputs":[]}]}