{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Introduction to Deep Learning Lab 3**\n\"\"\" In this Project our team will implement Unet For estimate the Position of Vehicle\"\"\"\n+ **Reference**\n1. Implementation Unet paper\nhttps://arxiv.org/pdf/1505.04597.pdf \n2. Reference Notebook from Kaggle Competition \nCompetition_Centernet_U_net_model\nhttps://www.kaggle.com/khaledmgamal/pku-competition-centernet-u-net \n3. In this project we also implement Custom Unet backbone MobileNetV2 Base Style","metadata":{}},{"cell_type":"markdown","source":"## **1 Prepare & Processing Data**\n\"\"\"Here is The Following Steps for Data Processing\"\"\"\n1. **Reading data** -- extract Pose Information {Yaw, Pitch, Roll, X, Y, Z}\n2. **Projection Coordinate** of [3D posiotn in 2D image Dimension] and Rotate {X, Y, Z} value by Euler angles\n3. **Image reading-- Resize** [Image, mask_target, Scale_Coordinate]\n4. **Image Visualization** Training data [Images, ground truth]\n5. **Processing data with Data Augmentation Pipeline**\n","metadata":{}},{"cell_type":"markdown","source":" ## 2 **Construct Unet-Like Model Architecture**\n1. **Writing Custom Data-Generator** \n2. **Model Construct Encoder BASE Efficienet-B0 Backbone**\n3. **Model Construct Encoder Efficienet-B1 Backbone**","metadata":{}},{"cell_type":"markdown","source":"## **3 Model Evaluate** \n1. Evaluate Model on Testing Set\n2. Making Prediction and Submit","metadata":{}},{"cell_type":"markdown","source":"### 1. **Reading data** -- extract Pose Information {Yaw, Pitch, Roll, X, Y, Z}","metadata":{}},{"cell_type":"code","source":"\"\"\"Importing DATA from Kaggle Dataset\"\"\"\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n        #print(os.path.join(dirname, filename))\n### Data Path\nPATH = '../input/pku-autonomous-driving/'\nprint(os.listdir(PATH))\n\n## Loading the Dataset Images IDs\ntrain = pd.read_csv(PATH + 'train.csv')\ntrain.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-03T15:34:45.974513Z","iopub.execute_input":"2021-06-03T15:34:45.974825Z","iopub.status.idle":"2021-06-03T15:34:46.074466Z","shell.execute_reply.started":"2021-06-03T15:34:45.974796Z","shell.execute_reply":"2021-06-03T15:34:46.073682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"Helper Function \"\"\"\n#Convert yaw, pitch, roll, x, y, z string corressponding values by Dicts type\ndef str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords\n\n# Will use this model later\npoints_df = pd.DataFrame()\nfor col in ['x', 'y', 'z', 'yaw', 'pitch', 'roll']:\n    arr = []\n    for ps in train['PredictionString']:\n        coords = str2coords(ps)\n        arr += [c[col] for c in coords]\n    points_df[col] = arr\nxzy_slope = LinearRegression()\nX = points_df[['x', 'z']]\ny = points_df['y']\nxzy_slope.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:34:47.710117Z","iopub.execute_input":"2021-06-03T15:34:47.710471Z","iopub.status.idle":"2021-06-03T15:34:51.389818Z","shell.execute_reply.started":"2021-06-03T15:34:47.710441Z","shell.execute_reply":"2021-06-03T15:34:51.388946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. **Projection Coordinate** of [3D posiotn in 2D image Dimension] and Rotate {X, Y, Z} value by Euler angles","metadata":{}},{"cell_type":"code","source":"\"\"\"Function to Project 3D Position on 2D image\"\"\"\nimport cv2\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n\n\ndef get_img_coords(s):\n    '''\n    Args\n    s is the string X, Y, Z, yall..\n    Return is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    #convert string to Dicts\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\n\n\"\"\"3D visualization and Rotation matrix\"\"\"\n\nfrom math import sin, cos\n## Convert Eluer angle to Roation matrix \n# https://phas.ubc.ca/~berciu/TEACHING/PHYS206/LECTURES/FILES/euler.pdf\ndef euler_to_Rot(yaw, pitch, roll): \n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))\n## Reading image function \ndef imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n\n## Anotation on the Image \ndef draw_line(image, points): \n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n## Draw dot on the Image \ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n    return image\n\n\n## Visualization Checking the Image 3D project on 2D and Annotation on the image\ndef visualize(img, coords):\n    # You will also need functions from the previous cells\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        center_point = np.array([x, y, z]).reshape([1,3])\n        Rotation_matrix = euler_to_Rot(yaw, pitch, roll).T#Rotation matrix to transform from car coordinate frame to camera coordinate frame\n        bounding_box = np.array([[x_l, -y_l, -z_l],\n                      [x_l, -y_l, z_l],\n                      [-x_l, -y_l, z_l],\n                      [-x_l, -y_l, -z_l],\n                     ]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rotation_matrix,bounding_box)+center_point.T)\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img_point=np.dot(camera_matrix, center_point.T).T\n        img_point[:, 0] /= img_point[:, 2]\n        img_point[:, 1] /=img_point[:, 2]\n        img = draw_points(img,img_point.astype(int))\n    \n    return img\n\n\nn_rows = 2\nimport matplotlib.pyplot as plt\nfor idx in range(n_rows):\n    fig, axes = plt.subplots(1, 2, figsize=(12,8))\n    img = imread(PATH + 'train_images/' + train['ImageId'].iloc[20+idx] + '.jpg')\n    axes[0].imshow(img)\n    img_vis = visualize(img, str2coords(train['PredictionString'].iloc[20+idx]))\n    axes[1].imshow(img_vis)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:34:53.863522Z","iopub.execute_input":"2021-06-03T15:34:53.863825Z","iopub.status.idle":"2021-06-03T15:34:57.634444Z","shell.execute_reply.started":"2021-06-03T15:34:53.863796Z","shell.execute_reply":"2021-06-03T15:34:57.633649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. **Image reading-- Resize** [Image, mask_target, Scale_Coordinate]\n1. Regr_Process: process x, y, z, yaw, pitch, roll. \n2. Regr-Back undo the process x=x*100 get the prediction from the network\n3. Extract_coords: extract infromation from network \n4. x= (x-img.shape[0]//2)*IMGHEIGHT / (img,shape[0]//2) /MODELSCALE-[y will be the same only changex]\n5. Creating Dataset and Dataloader for the dataset\n","metadata":{}},{"cell_type":"code","source":"IMG_WIDTH = 1024\nIMG_HEIGHT = IMG_WIDTH // 16 * 5\nMODEL_SCALE = 8\n#rotate the image object\ndef rotate(x, angle): \n    x= x+angle\n    x= x- (x+np.pi) //(2*np.pi) *2 *np.pi\n    return x\n\n#regr preprocess image \ndef _regr_preprocess(regr_dict, flip=False):\n    if flip:\n        for k in ['x', 'pitch', 'roll']:\n            regr_dict[k] = -regr_dict[k]\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    \n    return regr_dict\n\n\n## Rescale image \ndef preprocess_image(img, flip=False):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 6]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    if flip:\n        img = img[:,::-1]\n    return (img / 255).astype('float32')\n\n\n## Rescale mask and project the object coordinate \ndef get_mask_and_regr(img, labels, flip=False):\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    coords = str2coords(labels)\n    xs, ys = get_img_coords(labels)\n    \n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        #print(x,img.shape[0] // 2,y, img.shape[1] // 6)\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        #x=(x*1/2)*(IMG_HEIGHT / MODEL_SCALE)/(img.shape[0] // 2)\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // 6) * IMG_WIDTH / (img.shape[1] * 4/3) / MODEL_SCALE\n        #y=(y* 4/3)*(IMG_WIDTH / MODEL_SCALE)/((img.shape[1] * 3/4) )\n\n        y = np.round(y).astype('int')\n        #print(x,y)\n\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict, flip)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n            \n    if flip:\n        mask = np.array(mask[:,::-1])\n        regr = np.array(regr[:,::-1])\n        \n    return mask, regr\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:02.722801Z","iopub.execute_input":"2021-06-03T15:35:02.723178Z","iopub.status.idle":"2021-06-03T15:35:02.754357Z","shell.execute_reply.started":"2021-06-03T15:35:02.723146Z","shell.execute_reply":"2021-06-03T15:35:02.753268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Helper Functions To Convert back 2D dimention to 3D dimension COORDINATE & ANGLE**\n1. Taking mask and regression mask image from prediction result -> predict new coorrdinate --> project new coordinate and visualization the result","metadata":{}},{"cell_type":"code","source":"from scipy.optimize import minimize\nfrom sklearn.linear_model import LinearRegression\n\n# Reference from https://www.kaggle.com/theshockwaverider/eda-visualization-baseline\ndef convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):\n    return x * fx / z + cx, y * fy / z + cy\n\n## Function to return New extract coordinate \ndef optimize_xy(r,c,x0,y0, z0, flipped=False):\n    def distance_fn(xyz):\n        x, y,z = xyz\n        xx= -x if flipped else x\n        slope_err = (xzy_slope.predict([[xx,z]])[0] - y)**2\n        x, y = convert_3d_to_2d(x, y, z)\n        y, x = x, y\n        x = (x - IMG_SHAPE[0] // 2) * IMG_HEIGHT / (IMG_SHAPE[0] // 2) / MODEL_SCALE\n        y = (y + IMG_SHAPE[1] // 6) * IMG_WIDTH / (IMG_SHAPE[1] * 4 / 3) / MODEL_SCALE\n        return max(0.2, (x-r)**2 + (y-c)**2) + max(0.4, slope_err)\n    \n    res = minimize(distance_fn, [x0, y0, z0], method='Powell')\n    x_new, y_new, z_new = res.x\n    return x_new, y_new, z_new\n\n## Setting Thresshold for low prediction accuracy coordinate  \ndef clear_duplicates(coords):\n    for c1 in coords:\n        xyz1 = np.array([c1['x'], c1['y'], c1['z']])\n        for c2 in coords:\n            xyz2 = np.array([c2['x'], c2['y'], c2['z']])\n            distance = np.sqrt(((xyz1 - xyz2)**2).sum())\n            if distance < DISTANCE_THRESH_CLEAR:\n                if c1['confidence'] < c2['confidence']:\n                    c1['confidence'] = -1\n    return [c for c in coords if c['confidence'] > 0]\n\n\n## Extract coordinate from result \ndef extract_coords(prediction, flipped=False):\n    logits = prediction[0]\n    \n    regr_output = prediction[1:]\n    points = np.argwhere(logits > 0)\n    col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])\n    coords = []\n    for r, c in points:\n        regr_dict = dict(zip(col_names, regr_output[:, r, c]))\n        coords.append(_regr_back(regr_dict))\n        coords[-1]['confidence'] = 1 / (1 + np.exp(-logits[r, c]))\n        coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = \\\n                optimize_xy(r, c,\n                            coords[-1]['x'],\n                            coords[-1]['y'],\n                            coords[-1]['z'], flipped)\n    coords = clear_duplicates(coords)\n    return coords\n\n## Function mapping name and values\ndef coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    s = []\n    for c in coords:\n        for n in names:\n            s.append(str(c.get(n, 0)))\n    return ' '.join(s)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:05.709839Z","iopub.execute_input":"2021-06-03T15:35:05.710179Z","iopub.status.idle":"2021-06-03T15:35:05.726036Z","shell.execute_reply.started":"2021-06-03T15:35:05.710148Z","shell.execute_reply":"2021-06-03T15:35:05.725148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **4. Image Visualization Training data [Images, ground truth]**","metadata":{}},{"cell_type":"code","source":"\"\"\"1 Viuslaization data Processing as input input \n   2 Image coordinate project form 3D to 2D\n   3. Create the Mask as the Output Base on \n\n\"\"\"\n\nimg0 = imread(PATH + 'train_images/' + train['ImageId'][300] + '.jpg')\nimg = preprocess_image(img0,flip=True)\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][300],flip=True)\n\nprint('img.shape', img.shape, 'std:', np.std(img))\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\nprint('regr.shape', regr.shape, 'std:', np.std(regr))\nprint(img[:,::-1].shape)\n\nfig=plt.figure( figsize=(25, 20), facecolor='w', edgecolor='k')\nfig.subplots_adjust(hspace = .4, wspace=.0001)\nrow= 8\ncolumns=1\ni=1\nall_data=[img,img[:,::-1], mask, regr[:,:,-2] ]\nall_label=[\"rescale image\", \"rescale flip image\",\"Grouth-truth Mask\", \"Yaw values\" ]\nfor id in range(4): \n      #print(id1, id2)\n    fig.add_subplot(row, columns, i)\n    img=all_data[id]\n    plt.imshow(img)\n    plt.gca().set_title(all_label[id] )\n    i+=1\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:08.653510Z","iopub.execute_input":"2021-06-03T15:35:08.653833Z","iopub.status.idle":"2021-06-03T15:35:09.435079Z","shell.execute_reply.started":"2021-06-03T15:35:08.653796Z","shell.execute_reply":"2021-06-03T15:35:09.434288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.1 Example of Given Prediction Mask -- Regression Mask Image then model output","metadata":{}},{"cell_type":"code","source":"IMG_SHAPE=img.shape\nDISTANCE_THRESH_CLEAR = 2\nidx=2\nimg0 = imread(PATH + 'train_images/' + train['ImageId'].iloc[20+idx] + '.jpg')\n\nprint(img0.shape)\n\nimg1 = img0[:,::-1]\nprint(img1.shape)\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][20+idx])\nprint(regr.shape)\nregr= np.rollaxis(regr, 2, 0)\n##get back coordinate \ncoords= extract_coords(np.concatenate([mask[None], regr], 0))\nprint(f'_____coordinate PREDICTION example______:')\nprint(coords)\n\nfig=plt.figure( figsize=(15, 12), facecolor='w', edgecolor='k')\nfig.subplots_adjust(hspace = .4, wspace=.0001)\n\ni=1\nall_data=[img1, mask,regr[4, :, :] ]\nall_label=['image_input', 'Example Mask Prediction', 'Example Regr Mask Prediciton']\nrow=3\ncolumns=1\n\nfor id in range(3):\n    fig.add_subplot(row, columns, i)\n    img=all_data[id]\n    plt.imshow(img)\n    plt.gca().set_title(all_label[id] )\n    i+=1\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:12.722943Z","iopub.execute_input":"2021-06-03T15:35:12.723311Z","iopub.status.idle":"2021-06-03T15:35:14.603635Z","shell.execute_reply.started":"2021-06-03T15:35:12.723280Z","shell.execute_reply":"2021-06-03T15:35:14.602849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **5. Buidling Data Augmentation Pipeline**\n+ '''Data Augmentation Pipeline Inlcude'''\n1. **Contrast Enhancement**\n2. **BrightNess Enhancement**\n3. **Fliping L-R position**\n4. **Adding Noise to Images**\n","metadata":{}},{"cell_type":"code","source":"'''1. Contrast Enhancement Augmentation'''\ndef Contrast_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    df = pd.DataFrame(columns = ['ImageId', 'PredictionString'])\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'), img_number)\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        img = Image.open(fname)\n        ##add contrast here\n        enh = ImageEnhance.Contrast(img)\n        img_enh = enh.enhance(np.random.uniform(1.5, 2))#PIL.Image\n        ##save image \n        img_enh.save('Contrast_'+str(i)+\".jpg\")\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        a={'ImageId':('Contrast_'+str(i)+'.jpg'),'PredictionString':pstring}\n        df=df.append(a,ignore_index=True)\n        #show example of augmentation after and before\n        if i==0:\n            print('Original\\n')\n            plt.imshow(img)\n            plt.show()\n            print('Contrast_enhance\\n')\n            plt.imshow(img_enh)\n            plt.show()\n       \n      \n    train.reset_index(inplace=True)    \n    return df\n\n'''2. Brightness Ehancement'''\n\ndef Brightness_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    df = pd.DataFrame(columns = ['ImageId', 'PredictionString'])\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'), img_number)\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        img = Image.open(fname)\n        ##add Brightness here\n        enh = ImageEnhance.Brightness(img)\n        img_enh = enh.enhance(np.random.uniform(0.5, 1.0))#PIL.Image\n        ##save image\n        img_enh.save('Brightness_'+ str(i)+\".jpg\")\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        a={'ImageId':('Brightness_'+str(i)+'.jpg'),'PredictionString':pstring}\n        df=df.append(a,ignore_index=True)\n        #show example of augmentation after and before\n        if i==0:\n            print('Original\\n')\n            plt.imshow(img)\n            plt.show()\n            print('Brightness_enhance\\n')\n            plt.imshow(img_enh)\n            plt.show()\n    train.reset_index(inplace=True)    \n    return df\n\n'''3. Fliping Image Position'''\n\ndef FlipLR_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    df = pd.DataFrame(columns = ['ImageId', 'PredictionString'])\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'), img_number)\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        img = Image.open(fname)\n        ##add Brightness here\n        enh =np.fliplr(img)\n        img_enh = Image.fromarray(enh)\n        ##save image\n        img_enh.save('FlipLR_'+str(i)+\".jpg\")\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        x=pstring.split()\n        s=''\n        for r in range(len(x)):\n            if r%7==1 or r%7==3 or r%7==5:\n                x[r]=str(-float(x[r]))\n        for r in x:\n            s=s+r+' '\n        s=s[:-1]\n        a={'ImageId':('FlipLR_'+str(i)+'.jpg'),'PredictionString':s}\n        df=df.append(a,ignore_index=True)\n        #show example of augmentation after and before\n        if i==0:\n            print('Original\\n')\n            plt.imshow(img)\n            plt.show()\n            print('FlipLR_enhance\\n')\n            plt.imshow(img_enh)\n            plt.show()\n    train.reset_index(inplace=True)    \n    return df\n\ndef add_noise(image):\n    \"\"\"gauss noise\"\"\"\n    row,col,ch= image.shape\n    mean = 0\n    var = np.random.random()*0.01 #0.001~0.01\n    sigma = var**0.5\n    gauss = np.random.normal(mean,sigma,(row,col,ch))\n    gauss = gauss.reshape(row,col,ch)\n    noisy = image + gauss\n    noisy = np.clip(noisy, 0, 1)\n    return noisy\n\ndef Noise_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    df = pd.DataFrame(columns = ['ImageId', 'PredictionString'])\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'),img_number)\n\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        ori_img = plt.imread(fname)\n        img = (ori_img/255).astype('float32')\n        ##add noise here\n        enh = add_noise(img)#gauss\n        img_enh = Image.fromarray((enh * 255).astype(np.uint8)).resize((3384, 2710)).convert('RGB')\n        #save image\n        img_enh.save('Noise_'+str(i)+\".jpg\")\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        a={'ImageId':('Noisep_'+str(i)+'.jpg'),'PredictionString':pstring}\n        df=df.append(a,ignore_index=True)\n        #show example of augmentation after and before\n        if i==0:\n            print('Original\\n')\n            plt.imshow(ori_img)\n            plt.show()\n            print('Noise_enhance\\n')\n            plt.imshow(img_enh)\n            plt.show()\n    train.reset_index(inplace=True)    \n    return df\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T14:59:32.755569Z","iopub.execute_input":"2021-06-03T14:59:32.755909Z","iopub.status.idle":"2021-06-03T14:59:32.788163Z","shell.execute_reply.started":"2021-06-03T14:59:32.755878Z","shell.execute_reply":"2021-06-03T14:59:32.787092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**VISUALIZE DATA AUGMENTAITON TECHNIQUE APPLY**","metadata":{}},{"cell_type":"code","source":"# create images and dataframe\nimport glob\nfrom PIL import Image, ImageEnhance\nfrom matplotlib import pyplot as plt\nfrom glob import glob\nContrast_df=Contrast_enhance(5)\nBrightness_df=Brightness_enhance(5)\nFlipLR_df=FlipLR_enhance(5)\nNoise_df=Noise_enhance(5)\n\n\n#merge data frame with train dataframe\ntrain = pd.read_csv(PATH + 'train.csv')\ntrain=train.append(Contrast_df,ignore_index=True)\ntrain=train.append(Brightness_df,ignore_index=True)\ntrain=train.append(FlipLR_df,ignore_index=True)\ntrain=train.append(Noise_df,ignore_index=True)\nprint(train)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T14:59:38.417478Z","iopub.execute_input":"2021-06-03T14:59:38.417824Z","iopub.status.idle":"2021-06-03T15:00:02.443693Z","shell.execute_reply.started":"2021-06-03T14:59:38.417779Z","shell.execute_reply":"2021-06-03T15:00:02.442762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **6. Creating Dataset and Dataloader for the Dataset**","metadata":{}},{"cell_type":"code","source":"'''Data Generator Optional  for Keras Model'''\n\n# from tensorflow import keras\n# from tensorflow.keras.preprocessing.image import load_img\n# import torch\n\n# class CarDataset(keras.utils.Sequence):\n#     \"\"\"Helper to iterate over the data (as Numpy arrays).\"\"\"\n\n#     def __init__(self,batch_size, df, root_dir, training= True ):\n        \n#         self.batch_size = batch_size\n#         self.img_id = df\n#         self.root_dir = root_dir\n#         self.training = training\n        \n\n#     def __len__(self):\n#         print(len(self.img_id) // self.batch_size)\n#         return len(self.img_id) // self.batch_size\n    \n\n#     def __getitem__(self, idx):\n        \n#         \"\"\"Returns tuple (input, target) correspond to batch #idx.\"\"\"\n#         i = idx * self.batch_size\n        \n#         print(i)\n#          # Get index name\n#         idx, labels = self.img_id.values[idx]\n#         # Get image name\n#         img_name = self.root_dir.format(idx)\n        \n#         # Augmentation\n#         flip = False\n#         if self.training:\n#             flip = np.random.randint(2) == 1\n#         ## reading image\n#         img0 = imread(img_name, True)\n#         img = preprocess_image(img0, flip=flip)\n#         img = np.rollaxis(img, 2, 0)\n        \n#         # Get mask and regression maps\n#         mask, regr = get_mask_and_regr(img0, labels,flip=flip )\n#         regr = np.rollaxis(regr, 2, 0)\n        \n#         return [img, mask, regr]\n\n\n'''Testing Data Loader'''\n# from sklearn.model_selection import train_test_split\n\n# train_images_dir = PATH + 'train_images/{}.jpg'\n# train_masks_dir = PATH + 'train_masks/{}.jpg'\n# test_images_dir = PATH + 'test_images/{}.jpg'\n# test_masks_dir = PATH + 'test_masks/{}.jpg'\n\n# df_train, df_dev = train_test_split(train, test_size=0.1, random_state=42)\n# # class CarDataset(keras.utils.Sequence):\n# #     \"\"\"Helper to iterate over the data (as Numpy arrays).\"\"\"\n\n# #     def __init__(self,batch_size, df, root_dir, training= True ):\n# ds_data= CarDataset(batch_size=32, df=df_train,root_dir= train_images_dir)\n# image, test, test1= ds_data.__getitem__(1)\n        \n       ","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:00:29.94709Z","iopub.execute_input":"2021-06-02T15:00:29.947445Z","iopub.status.idle":"2021-06-02T15:00:29.953479Z","shell.execute_reply.started":"2021-06-02T15:00:29.947415Z","shell.execute_reply":"2021-06-02T15:00:29.952629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###","metadata":{}},{"cell_type":"code","source":"'''____Pytorch Data Loader____ '''\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nclass Custome_Generator(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        \n        self.training = training\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n          \n        # Augmentation\n        flip = False\n        if self.training:\n            flip = np.random.randint(2) == 1\n        \n        # Read image\n        img0 = imread(img_name, fast_mode=True)\n\n        img = preprocess_image(img0, flip=False)\n\n        img = np.rollaxis(img, 2, 0)\n        #print(img.shape)\n        \n        # Get mask and regression maps\n        mask, regr = get_mask_and_regr(img0, labels, flip=False)\n        regr = np.rollaxis(regr, 2, 0)\n        return [img, mask, regr]\n\n    \n    \n    \n    \n'''_______Testing Data Generator Pytorch______'''\n\ntrain_images_dir = PATH + 'train_images/{}.jpg'\ntest_images_dir = PATH + 'test_images/{}.jpg'\ntest= pd.read_csv(PATH+ 'sample_submission.csv')\ndf_train, df_val = train_test_split(train, test_size=0.01, random_state=42)\ndf_test = test\n\n# Create dataset objects\ntrain_dataset = Custome_Generator(df_train, train_images_dir, training=True,)\nimage, mask, regr=train_dataset.__getitem__(0)\n\nval_dataset = Custome_Generator(df_val, train_images_dir,)\ntest_dataset = Custome_Generator(df_test, test_images_dir,)\n\n\n'''________Creating the Pytorch Data Loader_______'''\nBATCH_SIZE=4\n#train, val, test Data Generator \ntrain_datagen= DataLoader(dataset= train_dataset, batch_size= BATCH_SIZE, shuffle=True, num_workers=4)\n\nval_datagen=DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\ntest_datagen = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n\n'''________Visualize the Ouput of DataGenerator_______'''\nfig=plt.figure( figsize=(12, 8), facecolor='w', edgecolor='k')\nfig.subplots_adjust(hspace = .4, wspace=.0001)\n\ni=1\nall_data=[np.rollaxis(image, 0, 3), mask,regr[5, : , :] ]\nall_label=['image', 'mask', 'regression map']\nrow=3\ncolumns=1\nfor id in range(3):\n    fig.add_subplot(row, columns, i)\n    img=all_data[id]\n    plt.imshow(img)\n    plt.gca().set_title(all_label[id] )\n    i+=1\nplt.show()\n#plt.imshow(image.reshape(320, 1024,3))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:24.625841Z","iopub.execute_input":"2021-06-03T15:35:24.626564Z","iopub.status.idle":"2021-06-03T15:35:25.549995Z","shell.execute_reply.started":"2021-06-03T15:35:24.626511Z","shell.execute_reply":"2021-06-03T15:35:25.549216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### **Building Unet-Like model** \n1. Model with Encoder part will using MobileNetV2, and Efficient Net \n2. Construct model and Training model","metadata":{}},{"cell_type":"code","source":"## Import EfficinetModel \n!pip install efficientnet-pytorch\n!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:28.110779Z","iopub.execute_input":"2021-06-03T15:35:28.111106Z","iopub.status.idle":"2021-06-03T15:35:39.282914Z","shell.execute_reply.started":"2021-06-03T15:35:28.111073Z","shell.execute_reply":"2021-06-03T15:35:39.281957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n## Import EfficientNet Base\nfrom efficientnet_pytorch import EfficientNet \n\nEfficientNet_base = EfficientNet.from_pretrained('efficientnet-b0')\n#print(EfficientNet_base.eval())\nmobileNetV2_base = torch.hub.load('pytorch/vision:v0.9.0', 'mobilenet_v2', pretrained=True,)\n# mobileNetV2_base.eval()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:44.684668Z","iopub.execute_input":"2021-06-03T15:35:44.685009Z","iopub.status.idle":"2021-06-03T15:35:44.906852Z","shell.execute_reply.started":"2021-06-03T15:35:44.684975Z","shell.execute_reply":"2021-06-03T15:35:44.906007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building -- Traning Custom Unet Model","metadata":{}},{"cell_type":"code","source":"'''In this section we will Construct Unet Architecture with Encoder \n\n1.Encoder EfficientB0\n2.Encoder Efficienet B1\n\n'''\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom tqdm import tqdm#_notebook as tqdm\n\nfrom functools import reduce\nimport os\nfrom scipy.optimize import minimize\nimport plotly.express as px\nimport torch.nn.functional as F\n\nclass Unet_EfficienetB0(nn.Module):\n    '''Mixture of previous classes'''\n    def __init__(self, n_classes):\n        super(Unet_EfficienetB0, self).__init__()\n        ## EfficientNet B0-- B1 backbone\n        self.base_model = EfficientNet.from_pretrained('efficientnet-b0')\n        \n        self.conv0 = double_conv(5, 64)\n        self.conv1 = double_conv(64, 128)\n        self.conv2 = double_conv(128, 512)\n        self.conv3 = double_conv(512, 1024)\n        \n        self.mp = nn.MaxPool2d(2)\n        \n        self.up1 = up_sample(1282 + 1024, 512)\n        self.up2 = up_sample(512 + 512, 256)\n        self.outc = nn.Conv2d(256, n_classes, 1)\n\n    def forward(self, x):\n        batch_size = x.shape[0]\n        mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])\n        x0 = torch.cat([x, mesh1], 1)\n        x1 = self.mp(self.conv0(x0))\n        x2 = self.mp(self.conv1(x1))\n        x3 = self.mp(self.conv2(x2))\n        x4 = self.mp(self.conv3(x3))\n        \n        x_center = x[:, :, :, IMG_WIDTH // 8: -IMG_WIDTH // 8]\n        feats = self.base_model.extract_features(x_center)\n        bg = torch.zeros([feats.shape[0], feats.shape[1], feats.shape[2], feats.shape[3] // 8]).cuda()\n        feats = torch.cat([bg, feats, bg], 3)\n        \n        # Add positional info\n        mesh2 = get_mesh(batch_size, feats.shape[2], feats.shape[3])\n        feats = torch.cat([feats, mesh2], 1)\n        \n        x = self.up1(feats, x4)\n        x = self.up2(x, x3)\n        x = self.outc(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:48.767508Z","iopub.execute_input":"2021-06-03T15:35:48.767843Z","iopub.status.idle":"2021-06-03T15:35:48.783072Z","shell.execute_reply.started":"2021-06-03T15:35:48.767811Z","shell.execute_reply":"2021-06-03T15:35:48.781978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass double_conv(nn.Module):\n    '''(conv => GN => ReLU) * 2\n    \n    Uing GroupNorm help stable training in small batch_size\n    \n    '''\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            #nn.GroupNorm(out_ch, 3, 6),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n#             nn.BatchNorm2d(out_ch),\n            #nn.GroupNorm(out_ch, 3, 6),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass up_sample(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up_sample, self).__init__()\n\n        #  would be a nice idea if the upsampling could be learned too,\n        #  but my machine do not have enough memory to handle all those weights\n        if bilinear:\n            self.up_sample = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up_sample = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2=None):\n        x1 = self.up_sample(x1)\n        \n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, (diffX // 2, diffX - diffX//2,\n                        diffY // 2, diffY - diffY//2))\n        \n        # for padding issues, see \n        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a\n        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd\n        \n        if x2 is not None:\n            x = torch.cat([x2, x1], dim=1)\n        else:\n            x = x1\n        x = self.conv(x)\n        return x\n\ndef get_mesh(batch_size, shape_x, shape_y):\n    mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))\n    mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mesh = torch.cat([torch.tensor(mg_x).cuda(), torch.tensor(mg_y).cuda()], 1)\n    return mesh","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:51.625831Z","iopub.execute_input":"2021-06-03T15:35:51.626202Z","iopub.status.idle":"2021-06-03T15:35:51.645454Z","shell.execute_reply.started":"2021-06-03T15:35:51.626157Z","shell.execute_reply":"2021-06-03T15:35:51.644534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##This Code for Training On Single GPU\n#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# print(device)\nn_epochs = 50\ntorch.cuda.set_device('cuda:0')\nmodel = Unet_EfficienetB0(8).cuda()\n\n## Model Train on Multiple GPU \n# device_ids=[1, 2, 3]\n# model= torch.nn.DataParallel(Unet_EfficienetB0(8), device_ids=device_ids)\n\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10) * len(train_datagen) // 3, gamma=0.1)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:54.590752Z","iopub.execute_input":"2021-06-03T15:35:54.591107Z","iopub.status.idle":"2021-06-03T15:35:54.954827Z","shell.execute_reply.started":"2021-06-03T15:35:54.591071Z","shell.execute_reply":"2021-06-03T15:35:54.953857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Here Summary of Our model Parameter**","metadata":{}},{"cell_type":"code","source":"from torchsummary import summary\nmodel= Unet_EfficienetB0(8)\nsummary(model.cuda(), ( 3, 320, 1024))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:35:57.205512Z","iopub.execute_input":"2021-06-03T15:35:57.205842Z","iopub.status.idle":"2021-06-03T15:35:57.782371Z","shell.execute_reply.started":"2021-06-03T15:35:57.205812Z","shell.execute_reply":"2021-06-03T15:35:57.781481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''###Define The LOSS Function '''\ndef criterion(prediction, mask, regr, size_average=True):\n    # Binary mask loss\n    pred_mask = torch.sigmoid(prediction[:, 0])\n    mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n    \n    # Sum\n    loss = mask_loss + regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:36:02.880814Z","iopub.execute_input":"2021-06-03T15:36:02.881240Z","iopub.status.idle":"2021-06-03T15:36:02.890357Z","shell.execute_reply.started":"2021-06-03T15:36:02.881197Z","shell.execute_reply":"2021-06-03T15:36:02.888583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(epoch, history=None):\n    model.train()\n\n    for batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(tqdm(train_datagen)):\n        #img_batch = torch.nn.DataParallel(img_batch,device_ids=device_ids)#img_batch.to(device_ids[0])\n        #mask_batch = torch.nn.DataParallel(mask_batch,device_ids=device_ids)#mask_batch.to(device_ids[0])\n        #regr_batch = torch.nn.DataParallel(regr_batch,device_ids=device_ids)#regr_batch.to(device_ids[0])\n        img_batch =  img_batch.cuda()\n        mask_batch =  mask_batch.cuda()\n        regr_batch =  regr_batch.cuda()\n        optimizer.zero_grad()\n        output = model(img_batch)\n        loss = criterion(output, mask_batch, regr_batch)\n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_datagen), 'train_loss'] = loss.data.cpu().numpy()\n        \n        loss.backward()\n        \n        optimizer.step()\n        exp_lr_scheduler.step()\n    \n    print('Train Epoch: {} \\tLR: {:.6f}\\tLoss: {:.6f}'.format(\n        epoch,\n        optimizer.state_dict()['param_groups'][0]['lr'],\n        loss.data))\n\ndef evaluate_model(epoch, history=None):\n    model.eval()\n    loss = 0\n    \n    with torch.no_grad():\n        for img_batch, mask_batch, regr_batch in val_datagen:\n#             img_batch =  torch.nn.DataParallel(img_batch,device_ids=device_ids)#.to(device_ids[0])\n#             mask_batch =  torch.nn.DataParallel(mask_batch,device_ids=device_ids)#.to(device_ids[0])\n#             regr_batch =  torch.nn.DataParallel(regr_batch,device_ids=device_ids)#.to(device_ids[0])\n            img_batch =  img_batch.cuda()\n            mask_batch =  mask_batch.cuda()\n            regr_batch =  regr_batch.cuda()\n\n            output = model(img_batch)\n\n            loss += criterion(output, mask_batch, regr_batch, size_average=False).data\n    \n    loss /= len(dev_loader.dataset)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()\n    \n    print('Dev loss: {:.4f}'.format(loss))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:36:04.906261Z","iopub.execute_input":"2021-06-03T15:36:04.906591Z","iopub.status.idle":"2021-06-03T15:36:04.915855Z","shell.execute_reply.started":"2021-06-03T15:36:04.906562Z","shell.execute_reply":"2021-06-03T15:36:04.914959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## The model Training in colab run more than 25 hours to complete at 60 epochs\n## This model suitable run on Several with multiple GPUS \nimport gc\nn_epochs=1\nhistory = pd.DataFrame()\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train_model(epoch, history)\n    evaluate_model(epoch, history)\n    state = {\n    'epoch': epoch,\n    'state_dict': model.state_dict(),\n    'optimizer': optimizer.state_dict(),\n    'lr_scheduler':exp_lr_scheduler.state_dict()\n    }\n    torch.save(state, '/Unetlike_model.pth')","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:36:07.312838Z","iopub.execute_input":"2021-06-03T15:36:07.313203Z","iopub.status.idle":"2021-06-03T15:36:13.937916Z","shell.execute_reply.started":"2021-06-03T15:36:07.313170Z","shell.execute_reply":"2021-06-03T15:36:13.936247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checkpoint = torch.load('/kaggle/input/model-06/model_06.pth')\ncheckpoint = torch.load('/kaggle/input/model-unet/unet_like_45epochs.pth',map_location='cuda:0')\nmodel.load_state_dict(checkpoint['state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer'])\nexp_lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])\nepoch = checkpoint['epoch']","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:36:21.759586Z","iopub.execute_input":"2021-06-03T15:36:21.759942Z","iopub.status.idle":"2021-06-03T15:36:22.250430Z","shell.execute_reply.started":"2021-06-03T15:36:21.759908Z","shell.execute_reply":"2021-06-03T15:36:22.249561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#img, mask, regr = val_dataset[10]\nimg, mask, regr = test_dataset[10]\nplt.figure(figsize=(16,16))\nplt.title('Input image')\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Ground truth mask')\nplt.imshow(mask)\nplt.show()\n\n#output = model(torch.tensor(img[None]).to(device))\noutput = model(torch.tensor(img[None]).cuda())\nlogits = output[0,0].data.cpu().numpy()\nplt.figure(figsize=(16,16))\nplt.title('Model predictions')\nplt.imshow(logits)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Model predictions thresholded')\nplt.imshow(logits > 0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:36:27.048424Z","iopub.execute_input":"2021-06-03T15:36:27.048740Z","iopub.status.idle":"2021-06-03T15:36:28.113685Z","shell.execute_reply.started":"2021-06-03T15:36:27.048710Z","shell.execute_reply":"2021-06-03T15:36:28.112744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfor i in range(len(df_test)):\n    s=''\n    img, mask, regr = test_dataset[10]\n    #print(img.shape)\n    img1 = img[:,::-1]\n    img1=img1.swapaxes(0,1)\n    img1=img1.swapaxes(1,2) \n    coords= extract_coords(np.concatenate([mask[None], regr], 0))\n    s=str(coords[0]['yaw'])+' '+str(coords[0]['pitch'])+' '+str(coords[0]['roll'])+' '+str(coords[0]['x'])+' '+str(coords[0]['y'])+' '+str(coords[0]['z'])+' '+str(coords[0]['confidence'])\n    df_test.iloc[i]['PredictionString']=s\n'''   ","metadata":{"execution":{"iopub.status.busy":"2021-06-03T14:23:47.913742Z","iopub.execute_input":"2021-06-03T14:23:47.914089Z","iopub.status.idle":"2021-06-03T14:23:51.117621Z","shell.execute_reply.started":"2021-06-03T14:23:47.914055Z","shell.execute_reply":"2021-06-03T14:23:51.115433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\n\ntest_loader = DataLoader(dataset=test_dataset, batch_size=4, shuffle=False, num_workers=4)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(test_loader):\n    with torch.no_grad():\n        output = model(img.cuda())\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        predictions.append(s)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(PATH + 'sample_submission.csv')\ntest['PredictionString'] = predictions\ntest.to_csv('predictions.csv', index=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:34:25.207465Z","iopub.execute_input":"2021-06-03T15:34:25.207789Z","iopub.status.idle":"2021-06-03T15:34:25.244352Z","shell.execute_reply.started":"2021-06-03T15:34:25.207757Z","shell.execute_reply":"2021-06-03T15:34:25.24252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}