{"cells":[{"metadata":{"colab_type":"text","id":"9fh2jZ_Hdr-X"},"cell_type":"markdown","source":"## Introduction\n\n**This notebook is modification of [Ruslan Baynazarov's](https://www.kaggle.com/hocop1) baseline kernel ver.14 [CenterNet Baseline](https://www.kaggle.com/hocop1/centernet-baseline/notebook) with PL score = 0.038,  \nThe uploaded model has been trained with the code below for 6 epochs (appx.6 hours) without image augmentations. Here it is to be trained for another 3 epochs.\n**\n#### The changes I made in the order of the perceived significance to the CV/PL score:\n\n1) changed dropout rate in the EfficientNet B0 base model from 0.2 to 0.02\n\n2) select threshold = -1.5 to filter logits so that the ratio of number of predicted cars to number of images approximates the one of the training set  \n\n3) use test_masks to remove cars from test predictions\n\n4) cars appearing outside of image boundaries are dropped (removed 997 cars out of 49,626 total )\n\n5) Removed broken images thanks to https://www.kaggle.com/c/pku-autonomous-driving/discussion/117621\n\n6) changed StepLR learning rate scheduler to MultiStepLR (gamma=0.5)\n\n7) pixel wise augmentations (albumentations lib)\n\n8) normalized images with subtracting total means per channel and dividing by standard deviation\n\nAll other building blocks laid out in CenterNet Baseline kernel ver.14 are intact.\n(preprocessing, efficientnet-B0 backbone, Adam optimizer)\n\n### What did not work. \n- using efficientnet B4, B5. The reason might be that the batch size of only 1 image could be fit to Tesla P100 GPU\n- gradient accumulation did not improve score. Batch Normalization layers are quoted to be the possible reason\n- was not able to change BatchNorm to GroupNorm, and I am still not sure if it would help (would be delighted if you could comment with lines of such code)\n\n### References:\n\nTook 3D visualization code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car\nCenterNet paper https://arxiv.org/pdf/1904.07850.pdf\nCenterNet repository https://github.com/xingyizhou/CenterNet\n\nWill be grateful for pointing to the bugs and advising corrections\n\n"},{"metadata":{"colab_type":"code","id":"JWvXTpZpSs6I","trusted":true,"colab":{}},"cell_type":"code","source":"!pip -q install efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","colab_type":"code","id":"1P8tLrYuSs6d","trusted":true,"colab":{}},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, copy, random, cv2, gc, time\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom functools import reduce\n\nfrom sklearn.model_selection import KFold #train_test_split\nfrom scipy.optimize import minimize\n\nimport math\nfrom math import sin, cos\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils import clip_grad_norm_, clip_grad_value_\nfrom torchvision import models\nfrom torchvision import transforms, utils\n\nfrom efficientnet_pytorch import EfficientNet\n\nfrom albumentations import ( Compose, OneOf, RandomBrightnessContrast,\n           RandomGamma, HueSaturationValue, RGBShift, MotionBlur, Blur,\n           GaussNoise, ChannelShuffle    \n                            )\n# not in Colab: MultiplicativeNoise, ISONoise \nosj = os.path.join\nosdir = os.listdir\n\nbase_seed = 1289\ndef torch_seed_everything(seed=base_seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\ntorch_seed_everything()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"9toXwemOSs6q","outputId":"112969c5-856f-4204-c792-88ba12b94fe4","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"colab":{"base_uri":"https://localhost:8080/","height":345}},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\nif device == 'cuda':\n    torch.cuda.empty_cache()\n    model, optimizer, loss =0,0,0\n    gc.collect()\n    \n#!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"HV2lM1SJSs7K","outputId":"923b5145-a87a-4393-f107-c5a6bf17a3d1","trusted":true,"colab":{"base_uri":"https://localhost:8080/","height":35}},"cell_type":"code","source":"platform =  'Kaggle'  #'Kaggle' # 'Colab' # 'gcp'\n\nif platform == 'Colab':\n    from google.colab import drive\n    drive.mount(\"/content/drive\", force_remount=False)\n","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"6gAWlNhA4d7j","outputId":"42ceee6d-f384-4d0f-8f68-8c97ca38e9ca","trusted":true,"colab":{"base_uri":"https://localhost:8080/","height":183}},"cell_type":"code","source":"debug = False\n#debug = True\n\neffnet_ver = 'b0' #'b4'\ndropout_rate = 0.02\n\nis_resume_train = True #False #True\nn_samples = 100_000 if not debug else 20\n\nimg_h, img_w =  512, 2048\n\nimg_orig_h, img_orig_w = 2710, 3384\nmodel_scale = 8\n\nbatch_size = 2\nnum_epochs = 6 if not debug else 2\nnum_epochs_resume = 3 if not debug else 2\nstart_epoch = 0\ngrad_accu_steps = 1 #32 // batch_size if not debug else 1\nif debug: \n  grad_accu_steps = 1 #2    #4 // batch_size\n  \nnum_splits = 8\nfold_to_train = 3\n\ndate_time_str = time.strftime(\"%d_%m_time_%H_%M\", time.localtime())\n\npath_dict =     {'Kaggle': '../input/pku-autonomous-driving/',\n                'Colab': './drive/My Drive/Colab/Peking/Data/',\n                 'gcp': 'gs://peking_12/data/raw_data',\n                'local': '/home/polk/Kaggle/Peking/data/raw_data'}\ntrain_test_path_dict = {'Kaggle': '../input/pku-autonomous-driving/',\n                        'Colab': './drive/My Drive/Colab/Peking/Data/', #Data_512_2048',\n                        'gcp': 'gs://peking_12/data/raw_data',\n                         'local': '/home/polk/Kaggle/Peking/data'}\npath_out_dict = {'Kaggle': './',\n                 'Colab': f\"./drive/My Drive/Colab/Peking/Output/Output_kaggle_eff_B0_{date_time_str}\",\n                 #'Colab': \"./drive/My Drive/Colab/Peking/Output/Out_eff_B2_28_12\",\n                 'gcp': f\"/home/ik/peking/logs/output_{date_time_str}\",\n                 'local': f\"/home/polk/Kaggle/Peking/Output_eff_{date_time_str}\"}\n\npath = path_dict[platform]\ntrain_test_path = train_test_path_dict[platform]\nout_path = path_out_dict[platform]\n\n\n#prev_out_path = \"./drive/My Drive/Colab/Peking/Output/Keep_outputs/PL_047_cv_3045_eff_b0_dropout_change_resume_KGL\"\n#prev_model_fn_path = osj(prev_out_path, \"model_fld_3_islastepoch_0_date_02_01_time_13_59.pth\")\nprev_model_fn_path = '../input/dropout-change-eff-b0-2-01-no-aug-epochs-6/model_fld_3_epoch_end_date_02_01_time_06_25.pth'\n\nif platform != 'Kaggle':\n    if not os.path.exists(out_path): os.mkdir(out_path)\n\nprint(\"\\ntrain and test images dir:\", train_test_path)\nprint(\"\\ntrain.csv, samples_submission.csv, ... directory:\", path)\nprint(\"\\nlogs, output dir:\", out_path)\nprint(\"\\npath directory list:\", os.listdir(path))   ","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"vV3_XSN3Ss7Y","outputId":"7fedb2c0-1e3d-4c3b-f059-1a168c7d21a2","trusted":true,"colab":{"base_uri":"https://localhost:8080/","height":126}},"cell_type":"code","source":"train = pd.read_csv(osj(path, 'train.csv'), nrows = n_samples)\n# drop images found in discussions https://www.kaggle.com/c/pku-autonomous-driving/discussion/117621\ndrop_images = ['ID_1a5a10365', 'ID_4d238ae90.jpg', 'ID_408f58e9f', 'ID_bb1d991f6', 'ID_c44983aeb'] \n\ntrain = train[~train['ImageId'].isin(drop_images)]\n\ntest = pd.read_csv(osj(path, 'sample_submission.csv'), nrows = n_samples)\n\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\n\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\nprint(\"train.shape:\", train.shape, \"\\ttest.shape:\",  test.shape)\ntrain.head(2)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"colab_type":"code","id":"R5C1vZnESs7y","trusted":true,"colab":{}},"cell_type":"code","source":"def imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n\ndef str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords\n\ndef rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\n\ndef add_number_of_cars(df):\n    \"\"\"df - train or test\"\"\"\n    df['numcars'] = [int((x.count(' ')+1)/7) for x in df['PredictionString']]\n    return df","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"_BENB35_Ss8A","trusted":true,"colab":{}},"cell_type":"code","source":"DISTANCE_THRESH_CLEAR = 2\n\ndef convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):\n    # taken from https://www.kaggle.com/theshockwaverider/eda-visualization-baseline\n    return x * fx / z + cx, y * fy / z + cy\n\ndef optimize_xy(r, c, x0, y0, z0):\n    def distance_fn(xyz):\n        x, y, z = xyz\n        x, y = convert_3d_to_2d(x, y, z0)\n        y, x = x, y\n        x = (x - img_orig_h // 2) * img_h / (img_orig_h // 2) / model_scale\n        x = np.round(x).astype('int')\n        y = (y + img_orig_w // 4) * img_w / (img_orig_w * 1.5) / model_scale\n        y = np.round(y).astype('int')\n        return (x-r)**2 + (y-c)**2\n    \n    res = minimize(distance_fn, [x0, y0, z0], method='Powell')\n    x_new, y_new, z_new = res.x\n    return x_new, y_new, z0\n\ndef clear_duplicates(coords):\n    for c1 in coords:\n        xyz1 = np.array([c1['x'], c1['y'], c1['z']])\n        for c2 in coords:\n            xyz2 = np.array([c2['x'], c2['y'], c2['z']])\n            distance = np.sqrt(((xyz1 - xyz2)**2).sum())\n            if distance < DISTANCE_THRESH_CLEAR:\n                if c1['confidence'] < c2['confidence']:\n                    c1['confidence'] = -1\n    return [c for c in coords if c['confidence'] > 0]\n\ndef extract_coords(prediction, threshold=0):\n    logits = prediction[0]\n    regr_output = prediction[1:]\n    points = np.argwhere(logits > threshold)  #>0\n    col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])\n    coords = []\n    for r, c in points:\n        regr_dict = dict(zip(col_names, regr_output[:, r, c]))\n        coords.append(_regr_back(regr_dict))\n        coords[-1]['confidence'] = 1 / (1 + np.exp(-logits[r, c]))\n        coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])\n    coords = clear_duplicates(coords)\n    return coords\n\ndef coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    s = []\n    for c in coords:\n        for n in names:\n            s.append(str(c.get(n, 0)))\n    return ' '.join(s)\n\n\ndef get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"EXOwKSZ9StBh","trusted":true,"colab":{}},"cell_type":"code","source":"def _regr_preprocess(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef preprocess_image(img):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 4]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (img_w, img_h))\n    return img\n\ndef get_mask_and_regr(img_orig_height, img_orig_width, labels):\n    mask = np.zeros([img_h // model_scale, img_w // model_scale], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([img_h // model_scale, img_w // model_scale, 7], dtype='float32')\n    coords = str2coords(labels)\n    xs, ys = get_img_coords(labels)\n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        x = (x - img_orig_height // 2) * img_h / (img_orig_height // 2) / model_scale\n        x = np.round(x).astype('int')\n        y = (y + img_orig_width // 4) * img_w / (img_orig_width * 1.5) / model_scale\n        y = np.round(y).astype('int')\n        if x >= 0 and x < img_h // model_scale and y >= 0 and y < img_w // model_scale:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n    return mask, regr","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"mQzX6ThWdsCE"},"cell_type":"markdown","source":"## Remove cars with coordinates outside the image boundaries\nThere are many cars coordinates that fall outside of image as noted in CenterNet Baseline kernel\nI remove them here from train"},{"metadata":{"colab_type":"code","id":"bOZqK-1vSs98","outputId":"27e3bd56-2d52-4769-f4de-bb29b92cf6b3","trusted":true,"colab":{"base_uri":"https://localhost:8080/","height":35}},"cell_type":"code","source":"def remove_out_image_cars(df):\n\n        def isnot_out(x,y):\n            # are x,y coordinates within boundaries of the image\n            return (x>=0)&(x<=img_orig_w)&(y>=0)&(y<=img_orig_h)\n\n        df = add_number_of_cars(df)\n\n        new_str_coords = []\n        counter_all_ls = []\n        for idx,str_coords in enumerate(df['PredictionString']):\n            coords = str2coords(str_coords, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z'])\n            xs, ys = get_img_coords(str_coords)\n            counter = 0\n            coords_new = []\n            \n            for (item,x,y) in zip(coords, xs, ys):\n                if isnot_out(x, y):\n                    coords_new.append(item)\n                    counter += 1\n                                \n            new_str_coords.append(coords2str(coords_new,  names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']))\n            counter_all_ls.append(counter)\n            \n        df['new_pred_string']  = new_str_coords \n        df['new_numcars'] = counter_all_ls\n\n        print(\"num of cars outside image bounds:\", df['numcars'].sum()-df['new_numcars'].sum(), \n            \"out of all\", df['numcars'].sum(), \" cars in train\")\n\n        del df['PredictionString'], train['numcars']\n        df.rename(columns={'new_pred_string': 'PredictionString'}, inplace=True)\n\n        return df\n    \ntrain = remove_out_image_cars(train)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"KnXxNBxKWquJ"},"cell_type":"markdown","source":"### Assert that new car coordinates are within image boundaries"},{"metadata":{"colab_type":"text","id":"fcU_cuBPSs_g","trusted":true},"cell_type":"code","source":"xs, ys = [], []\ntrain_image_idx =6\n\nfor ps in train['PredictionString']:\n    x, y = get_img_coords(ps)\n    xs += list(x)\n    ys += list(y)\n\nplt.figure(figsize=(12,6))\nplt.imshow(imread(osj(train_test_path,'train_images', train['ImageId'][train_image_idx] + '.jpg')), alpha=0.7)\nplt.scatter(xs, ys, color='red', s=20, alpha=1);","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"Fk-Rv_L_StAj"},"cell_type":"markdown","source":"Used code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car, but made it one function"},{"metadata":{"_kg_hide-input":true,"colab_type":"code","id":"yRypp57tStAn","trusted":true,"colab":{}},"cell_type":"code","source":"# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))\n\n\ndef draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n    return image\n\n\ndef visualize(img, coords):\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"vSY9RxxsBm1t"},"cell_type":"markdown","source":"### Albumentations augmentation\nAs noted in Discussions, test images have been augmented pixel-wise and with horizontal flip.\nOnly pixel augmentations are applied here"},{"metadata":{"colab_type":"code","id":"5t9OhkoSBnw7","trusted":true,"colab":{}},"cell_type":"code","source":"albu_list = [RandomBrightnessContrast(brightness_limit=(-0.3, 0.3), contrast_limit=(-0.3, 0.3), p=0.3),\n           RandomGamma(p=0.2), HueSaturationValue(p=0.3), RGBShift(p=0.3), MotionBlur(p=0.1), Blur(p=0.1), \n           GaussNoise(var_limit=(20,100), p=0.2), \n           ChannelShuffle(p=0.2)]\n\n# NOT in colab version: MultiplicativeNoise(multiplier=(0.7, 1.2), p=0.2), ISONoise(p=0.2),               \n# GaussNoise(var_limit=(10.0, 50.0), mean=0, always_apply=False, p=0.5)\n\np_transform_train = 0.1\nalbu_transform_train = Compose(albu_list, p=p_transform_train)\n\np_transform_val = 0.05\nalbu_transform_valid = Compose(albu_list, p=p_transform_val)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"Ta4fJliV-ucu","trusted":true,"colab":{}},"cell_type":"code","source":"def aug_visualize(original_imgs: list, augmented_images: list):\n    fig_height = 8*len(original_imgs)\n    fig_width = 20\n    f, ax = plt.subplots(len(original_imgs), 2, figsize=(fig_width, fig_height))\n    \n    for i, (img, aug_img) in enumerate(zip(original_imgs, augmented_images)):\n        assert len(img.shape)==3 & img.shape[2]==3\n        ax[i,0].imshow(img)\n        ax[i,0].set_title(f\"Original image \", fontsize=12)\n        ax[i,1].imshow(aug_img)\n        ax[i,1].set_title('Augmented image', fontsize=12)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"e5hxYcSKdsEO"},"cell_type":"markdown","source":"### Random augmentations demo"},{"metadata":{"id":"xQeMvhNedsET","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":55},"outputId":"3eafe36b-f71d-45eb-f056-1660287173d8","trusted":true},"cell_type":"markdown","source":"\"\"\"\nnum_imgs_to_show  = 4\np_transform_demo = 1\nalbu_transform_demo = Compose(albu_list, p=p_transform_train)\n\nassert num_imgs_to_show < 10\npaths = random.sample(train['ImageId'].tolist(), num_imgs_to_show)\noriginal_imgs = [cv2.imread(osj(train_test_path,'train_images',path+'.jpg' )) for path in paths]\naugmented_imgs = [albu_transform_demo(image=img)['image'] for img in original_imgs]\naug_visualize(original_imgs, augmented_imgs)\n\ndel augmented_imgs, albu_transform_demo, paths\n_ = gc.collect()\n\"\"\""},{"metadata":{"colab_type":"text","id":"HeuwMpjbStB2"},"cell_type":"markdown","source":"# PyTorch Dataset"},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"colab_type":"code","id":"rPeaz3NhStB6","trusted":true,"colab":{}},"cell_type":"code","source":"means = np.array([80.302637, 73.818344, 69.698726]).reshape(1,1,3)\nstds = np.array([52.455102, 52.848600, 50.285960]).reshape(1,1,3)\n# for original image sizes:\n            #means = np.array([145.375023, 136.948063, 122.689194]).reshape(1,1,3)\n            #stds = np.array([95.254658, 94.717646, 85.960257]).reshape(1,1,3)\n\ndef normalize(img, means, stds, tensor=False):\n    return (img - means)/stds\n\ndef denormalize(img, means, stds, resize_to_original=False):  \n    \"\"\" \n    input is numpy\n    convert back to (0,255) and moveaxis from 3,x,x to x,x,3 after denormalizing - multiply by stds and add means\n    img is a torch tensor\"\"\"\n\n    img = np.moveaxis(img, 0, 2)\n    img = img*stds + means\n    img = np.clip(img, 0, 255).astype('uint8')\n\n    if resize_to_original:\n        # revert def preprocess_image()\n        img = img[:,(img_w//4): (img_w - img_w//4),:]\n        img = cv2.copyMakeBorder( img, img.shape[0], 0,0,0, cv2.BORDER_CONSTANT) #, borderType)\n        img = cv2.resize(img, (img_orig_w, img_orig_h))\n            \n    return img\n\n\nclass CarDataset(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, root_dir_dropmasks=osj(path,'test_masks/{}.jpg'), training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.root_dir_dropmasks = root_dir_dropmasks\n        self.transform = transform\n        self.training = training\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        #import pdb; pdb.set_trace()\n        \n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        \n        # Read image\n        img = imread(img_name, True)\n        img = preprocess_image(img)\n        \n        if self.transform:\n            img = self.transform(image=img)['image']\n        \n        img = normalize(img, means, stds)\n        img = np.rollaxis(img, 2, 0)             \n        \n        # Get mask and regression maps\n        if self.training:\n            mask, regr = get_mask_and_regr(img_orig_h, img_orig_w, labels)\n            regr = np.rollaxis(regr, 2, 0)\n            dropmask = 0\n        else:\n            mask, regr = 0, 0 \n            dropmask_name = self.root_dir_dropmasks.format(idx)\n            if os.path.isfile(dropmask_name):\n                dropmask = imread(dropmask_name, True) #, 'test_masks', idx+'.jpg'))\n                dropmask = preprocess_image(dropmask)\n            else:\n                dropmask = np.zeros((img_h, img_w, 3))\n            #mask = preprocess_image(mask)\n\n        img = torch.as_tensor(img, dtype=torch.float32) #/255)\n        mask = torch.as_tensor(mask, dtype=torch.float32)\n        regr = torch.as_tensor(regr, dtype=torch.float32)\n        dropmask = torch.as_tensor(dropmask, dtype=torch.float32)\n        \n        return [img, mask, regr, dropmask]\n\nclass backup_CarDataset(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, training=True, transform=None, means = means, stds = stds):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n        self.training = training\n        self.means = means\n        self.stds = stds\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n         \n        \n        \n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        \n        # Read image\n        img = imread(img_name, True)\n        img = preprocess_image(img)\n        \n        if self.transform:\n            img = self.transform(image=img)['image']\n        \n        img = normalize(img, self.means, self.stds)\n        img = np.rollaxis(img, 2, 0)\n                        \n        # Get mask and regression maps\n        if self.training:\n            mask, regr = get_mask_and_regr(labels)\n            regr = np.rollaxis(regr, 2, 0)\n        else:\n            mask, regr = 0, 0\n        \n        img = torch.Tensor(img);    mask = torch.Tensor(mask);    regr = torch.Tensor(regr)\n        \n        return [img, mask, regr]","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"zZktBGqGStCE"},"cell_type":"markdown","source":"## Build Pytorch Model"},{"metadata":{"_kg_hide-input":false,"colab_type":"code","id":"MPQk8ybhStCr","trusted":true,"colab":{}},"cell_type":"code","source":"class double_conv(nn.Module):\n    '''(conv => BN => ReLU) * 2'''\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass up(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up, self).__init__()\n\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2=None):\n        x1 = self.up(x1)\n        \n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, (diffX // 2, diffX - diffX//2,\n                        diffY // 2, diffY - diffY//2))\n\n        if x2 is not None:\n            x = torch.cat([x2, x1], dim=1)\n        else:\n            x = x1\n        x = self.conv(x)\n        return x\n\ndef get_mesh(batch_size, shape_x, shape_y):\n    mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))\n    mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mesh = torch.cat([torch.tensor(mg_x).to(device), torch.tensor(mg_y).to(device)], 1)\n    return mesh","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"colab_type":"code","id":"wTUWKK8QStC3","trusted":true,"colab":{}},"cell_type":"code","source":"def set_dropout(model, drop_rate):\n    # source: https://discuss.pytorch.org/t/how-to-increase-dropout-rate-during-training/58107/4\n    for name, child in model.named_children():\n        if isinstance(child, torch.nn.Dropout):\n            child.p = drop_rate\n            print(\"name:\", name)\n            print(\"children:\\n\", child)\n\n\ndef effnet_dropout(drop_rate):\n    base_model0 = EfficientNet.from_pretrained(f\"efficientnet-{effnet_ver}\")\n    set_dropout(base_model0, drop_rate)\n    return base_model0\n\n\nclass MyUNet(nn.Module):\n    '''Mixture of previous classes'''\n    def __init__(self, n_classes, drop_rate):\n        super(MyUNet, self).__init__()\n        \n        self.drop_rate = dropout_rate\n        self.base_model = effnet_dropout(drop_rate = self.drop_rate)\n        self.conv0 = double_conv(5, 64)\n        self.conv1 = double_conv(64, 128)\n        self.conv2 = double_conv(128, 512)\n        self.conv3 = double_conv(512, 1024)\n        \n        self.mp = nn.MaxPool2d(2)\n        \n        if effnet_ver == 'b0': self.up1 = up(1282 + 1024, 512)\n        elif effnet_ver == 'b2': self.up1 = up(1410 + 1024, 512)\n        elif effnet_ver == 'b4': self.up1 = up(1794 + 1024, 512)\n        elif effnet_ver == 'b5': self.up1 = up(2050 + 1024, 512)\n        \n        self.up2 = up(512 + 512, 256)\n        self.outc = nn.Conv2d(256, n_classes, 1)\n\n    def forward(self, x):\n        batch_size = x.shape[0]\n        mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])\n        x0 = torch.cat([x, mesh1], 1)\n        \n        #import pdb; pdb.set_trace()\n        \n        x1 = self.mp(self.conv0(x0))\n        x2 = self.mp(self.conv1(x1))\n        x3 = self.mp(self.conv2(x2))\n        x4 = self.mp(self.conv3(x3))\n        \n        feats = self.base_model.extract_features(x)\n        \n        # Add positional info\n        mesh2 = get_mesh(batch_size, feats.shape[2], feats.shape[3])\n        feats = torch.cat([feats, mesh2], 1)\n        \n        x = self.up1(feats, x4)\n        x = self.up2(x, x3)\n        x = self.outc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"colab_type":"code","id":"F7FpWip8StDA","trusted":true,"colab":{}},"cell_type":"code","source":"def build_model(drop_rate):\n    # Gets the GPU if there is one, otherwise the cpu\n    print(\"device:\", device)\n    model = MyUNet(8, drop_rate).to(device)\n    #set_dropout(model, drop_rate=drop_rate)    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"colab_type":"code","id":"fIlWPnRUStDJ","trusted":true,"colab":{}},"cell_type":"code","source":"def criter_metric(prediction, mask, regr, size_average=True):\n    # Binary mask loss\n    pred_mask = torch.sigmoid(prediction[:, 0])\n        \n    mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n    \n    # Sum\n    loss = mask_loss + regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss, mask_loss, regr_loss\n\n\nw_mask = 0.1; w_regr = 0.9\nw_mask = torch.tensor( w_mask, device=device); w_regr = torch.tensor( w_regr, device=device)\ngamma=torch.tensor(5, device=device)\n\ndef criter_objective(prediction, mask, regr, size_average=True):\n            \n    pred_mask = torch.sigmoid(prediction[:, 0])\n        \n    #mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = mask * ((1-pred_mask)**gamma)* torch.log(pred_mask+1e-12)\\\n               + (1 - mask)*torch.log(1-pred_mask+1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n    \n    # Sum\n    loss = w_mask*mask_loss + w_regr*regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"colab_type":"code","id":"p1VzOrJ3StDR","trusted":true,"colab":{}},"cell_type":"code","source":"\ndef evaluate(model, epoch, history=None):\n    model.eval()\n    val_loss, mask_loss, regr_loss = torch.tensor(0., requires_grad=False), torch.tensor(0., requires_grad=False), torch.tensor(0., requires_grad=False)\n    \n    with torch.no_grad():\n        for img_batch, mask_batch, regr_batch, _ in dev_loader:\n            img_batch = img_batch.to(device)\n            mask_batch = mask_batch.to(device)\n            regr_batch = regr_batch.to(device)\n\n            output = model(img_batch)\n            \n            val_loss_batch, mask_loss_batch, regr_loss_batch = criter_metric(output, mask_batch, regr_batch, size_average=False)\n            val_loss += val_loss_batch\n            mask_loss += mask_loss_batch\n            regr_loss += regr_loss_batch\n    \n    val_loss /= len(dev_loader.dataset)\n    mask_loss /= len(dev_loader.dataset)\n    regr_loss /= len(dev_loader.dataset)\n            \n    return val_loss.cpu().numpy(), mask_loss.cpu().numpy(), regr_loss.cpu().numpy()\n\n\ndef save_checkpoint(state, model_name=None):\n        \"\"\"Save checkpoint if a new best is achieved\"\"\"\n        \n        if not model_name: model_name =  f\"model_date_{date_time_str}.pth\"\n        torch.save(state, osj(out_path, model_name))\n            \n\ndef train_eval_model(model, epoch, best_val_loss, history=None):\n    \n    model.train()\n    freq_eval = len(train_loader.dataset)//4\n\n    for batch_idx, (img_batch, mask_batch, regr_batch, _) in enumerate(tqdm(train_loader)):\n        img_batch = img_batch.to(device)\n        mask_batch = mask_batch.to(device)\n        regr_batch = regr_batch.to(device)\n                \n        output = model(img_batch)\n        loss = criter_objective(output, mask_batch, regr_batch)\n        train_loss = loss.data.cpu().numpy()\n        \n        loss = loss / grad_accu_steps \n        loss.backward()\n\n        if (batch_idx+1) % grad_accu_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n\n        lr_value = optimizer.state_dict()['param_groups'][0]['lr']\n\n        cond1 = ((batch_idx+1) % freq_eval)==0   # every freq_eval number of batches\n        cond2 = abs(batch_idx-len(train_loader))*10 > len(train_loader)  # if not too close to the end or start of epoch\n        is_end_epoch =  ((batch_idx+1) % len(train_loader) ) == 0\n        \n        if is_end_epoch | (cond1 & cond2):\n            t_eval0 = time.time()\n            val_loss, mask_val_loss, regr_val_loss = evaluate(model, dev_loader)\n            \n            #scheduler.step(val_loss)  for LRONPlateu scheduler\n            print(\"\\n=========================\")\n            print(f\"epoch={epoch}, batch={batch_idx} => lr={lr_value:.7f}, trn loss={loss:4f}\")\n            print(f\"VAL LOSS: {val_loss:.2f}, \\tBest Val Loss: {best_val_loss:.2f} \\teval time={(time.time()-t_eval0)/60:.2f} min.\")\n            print(\"mask_val_loss:\", mask_val_loss, \"\\tregr_val_loss\", regr_val_loss)\n            if is_end_epoch: print(f\"\\n===END of EPOCH {epoch}================\")\n            \n            history.loc[epoch*len(train_loader)+batch_idx, ['train_loss','val_loss', 'lr','mask_val_loss','regr_val_loss']] =\\\n                            [train_loss, val_loss, lr_value, mask_val_loss, regr_val_loss ]\n            history.to_csv(osj(out_path, f'history_{fld}.csv'))\n            \n            is_best = val_loss < best_val_loss\n            best_val_loss = min(val_loss, best_val_loss)\n                   \n        history.loc[epoch*len(train_loader)+batch_idx, ['train_loss', 'lr']] = [train_loss, lr_value ]\n    \n    return model, optimizer, history, best_val_loss, val_loss","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"colab_type":"code","id":"4nrA-vJiStDd","outputId":"bd23d3e8-d408-41ae-e20d-f5aee31d09cd","trusted":true,"colab":{"base_uri":"https://localhost:8080/","height":213,"referenced_widgets":["0020b8122b7f4efeb8699c1a47423ef4","27beeaae686c4f71bd44b46a4e5d3b95","4655bd286e3f48f3985b52f2c7a51495","9ce6c7aece884961a97a644e8a443b2b","17609aec87874afead62d382e513b21b","fb5c0565d0ac45ed83ad9b85e122c6e3","c50c94f686254caf9768444c4d3844be","4db469cc1f8b4393a22e4cae1732311a"]}},"cell_type":"code","source":"\ntrain_images_dir = osj(train_test_path,'train_images/{}.jpg')\ntest_images_dir = osj(train_test_path,'test_images/{}.jpg')\n\ndf_test = test\nkfold = KFold(n_splits=num_splits, shuffle=True, random_state= base_seed)\n\nfld = fold_to_train\ntrn_idx, val_idx = list(kfold.split(train))[fld]\n\ntrain_dataset = CarDataset(train[['ImageId', 'PredictionString']].iloc[trn_idx], train_images_dir,\n                           transform = albu_transform_train)\ndev_dataset = CarDataset(train[['ImageId', 'PredictionString']].iloc[val_idx], train_images_dir,\n                         transform = albu_transform_valid)\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n\nmodel = build_model(drop_rate=dropout_rate)\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n#optimizer = optim.AdamW(model.parameters(), lr=0.0015, weight_decay=0.00001)\n\nscheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[7,10,14,17,20], gamma=0.5) #milestones=[5,7,8], gamma=0.1)\n#scheduler = lr_scheduler.StepLR(optimizer, step_size=max(num_epochs, 10) * len(train_loader) // 8, gamma=0.5)\n#scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, \n#                                         verbose=True)\nif is_resume_train:\n    checkpoint = torch.load(prev_model_fn_path, map_location=device)            \n    model.load_state_dict(checkpoint['state_dict'])\n    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n    epoch_start = checkpoint['epoch']\n    n_epochs = num_epochs_resume\n    best_val_loss = prev_epoch_val_loss = checkpoint['best_val_loss']\n    prev_epoch_val_loss = copy.copy(best_val_loss)\n    scheduler.load_state_dict(checkpoint['scheduler'])\n    \n    del checkpoint; _ = gc.collect()\n\nelse: # train from epoch=0 if not resuming training\n    epoch_start = 0     \n    n_epochs = num_epochs\n    best_val_loss, prev_epoch_val_loss = 10e+6, 10e+6\n\nhistory = pd.DataFrame(np.full((len(train_loader)*n_epochs, 5), fill_value=-1, dtype=np.float32),\n                           columns=['train_loss','val_loss','lr','mask_val_loss','regr_val_loss'])\n\nprint(f\"\\n======================\\nTrain start. Output directory {out_path}\")\nprint(\"efficientnet-\", effnet_ver, \"n_epochs:\", n_epochs, \"len(train_loader):\", len(train_loader))\nfor epoch in range(epoch_start, n_epochs+epoch_start):\n    torch.cuda.empty_cache()\n    gc.collect()\n    model, optimizer, history, best_val_loss, val_loss = train_eval_model(model, epoch, best_val_loss, history)\n    scheduler.step()\n    \n    is_last_epoch = ( epoch==(n_epochs+epoch_start-1) )\n    if (val_loss < prev_epoch_val_loss)|(is_last_epoch):\n        params_save_model = { 'epoch': start_epoch + epoch, 'loss': history['train_loss'].iat[-1],\n                            'best_val_loss': best_val_loss,\n                            'state_dict': model.state_dict(),\n                            'optimizer_state_dict': optimizer.state_dict(),\n                            'scheduler': scheduler.state_dict() }\n        print(f\"Val_loss improved or end training => Saving at epoch {epoch} end w val_loss={val_loss:.2f}, best_val_loss = {best_val_loss:.2f}\")\n        save_checkpoint(params_save_model, model_name = f\"model_fld_{fld}_islastepoch_{is_last_epoch:d}_date_{date_time_str}.pth\")\n    \n    prev_epoch_val_loss = val_loss\n\nhistory.to_csv(osj(out_path, f\"history_final_fold_{fld}.csv\"), index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Join initially uploaded model's history log"},{"metadata":{"trusted":true},"cell_type":"code","source":"history_old = pd.read_csv('../input/dropout-change-eff-b0-2-01-no-aug-epochs-6/history_final_fold_3.csv')\nhistory_old = history_old[history_old['val_loss'] > 0]\nhistory_new = history[history['val_loss']>0]\nhistory_new.index = history_new.index + history_old.index.max()+1\nhistory_new = pd.concat([history_old, history_new], axis=0)\nhistory_new.tail()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"XlxOCV3fStE4","trusted":true,"colab":{}},"cell_type":"code","source":"if not debug: history['train_loss'].iloc[100:].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### History of validation loss (including previously trained model)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"jszJA9O-StE_","trusted":true,"colab":{}},"cell_type":"code","source":"series = history_new.dropna()['val_loss']\nplt.scatter(series.index, series);","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"ncCRhFt3StFH"},"cell_type":"markdown","source":"# Visualize predictions"},{"metadata":{"_kg_hide-input":true,"colab_type":"code","id":"7qbYoKOqStFK","trusted":true,"colab":{}},"cell_type":"code","source":"def plot_pred_mask(model, idx, threshold):\n    h,w = 12,6\n    img, mask, regr, _ = train_dataset[idx] #dev_dataset[0]\n    output = model(torch.tensor(img[None]).to(device))\n    logits = output[0,0].data.cpu().numpy()\n\n    img = img.data.cpu().numpy()  #.clone().detach()\n    img = denormalize(img, means, stds)\n    mask = mask.data.cpu().numpy()\n    regr = regr.data.cpu().numpy()\n     \n    plt.figure(figsize=(h,w))\n    plt.title('Input image')\n    plt.imshow(img)\n    plt.show()\n    \n    plt.figure(figsize=(h,w))\n    plt.title('Ground truth mask')\n    plt.imshow(mask)\n    plt.show()\n\n    plt.figure(figsize=(h,w))\n    plt.title('Model predictions')\n    plt.imshow(logits)\n    plt.show()\n\n    plt.figure(figsize=(h,w))\n    plt.title('Model predictions thresholded')\n    plt.imshow(logits > threshold0) #0)\n    plt.show()\n  \nthreshold0 = -1.5\nidx = 0\nplot_pred_mask(model, idx, threshold0)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"jmSerdPbStFS","trusted":true,"colab":{}},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()\nthreshold0 = -1.6\nn_imgs = 6 if not debug else 2\nfor idx in random.sample(range(len(dev_dataset)), n_imgs):\n    img, mask, regr, _ = dev_dataset[idx]\n        \n    output = model(img[None].to(device)).data.cpu().numpy()\n    img = img.data.cpu().numpy()\n    mask = mask.data.cpu().numpy() #\n    regr = regr.data.cpu().numpy()\n    \n    coords_pred = extract_coords(output[0], threshold=threshold0)\n    coords_true = extract_coords(np.concatenate([mask[None], regr], 0))\n    \n    img = denormalize(img, means, stds, resize_to_original=True)\n    \n    fig, axes = plt.subplots(1, 2, figsize=(16,12))\n    axes[0].set_title('Ground truth')\n    axes[0].imshow(visualize(img, coords_true)[(img.shape[0]//2):])\n    axes[1].set_title('Prediction')\n    axes[1].imshow(visualize(img, coords_pred)[(img.shape[0]//2):])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"36rNMJ5hP3TR","colab_type":"text"},"cell_type":"markdown","source":"## Filter Predictions"},{"metadata":{"trusted":true,"id":"KbxXiQ0bP3TU","colab_type":"code","colab":{}},"cell_type":"code","source":"# load best epoch end model or else initially loaded model\n\nis_last_epoch = False\nmodel_name = f\"model_fld_{fld}_islastepoch_{is_last_epoch:d}_date_{date_time_str}.pth\"\ntry:\n    checkpoint = torch.load(osj(out_path, model_name), map_location=device)\n    print(\"best model of the current session loaded.\")\nexcept: \n    checkpoint = torch.load(prev_model_fn_path, map_location=device) \n    print(\"NO best model in the current session. Previously trained model loaded\")\n\nmodel.load_state_dict(checkpoint['state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"G4NzMjEiP3Tp","colab_type":"code","colab":{}},"cell_type":"code","source":"def predict_test(model, threshold = 0): #-1.2  # -1.0) #-0.5)\n    preds, preds_before_drop_mask = [], []\n    test_loader = DataLoader(dataset=test_dataset, batch_size=4, shuffle=False, num_workers=4)\n    model.eval()\n    for img_batch, _, _, test_mask_batch in tqdm(test_loader):\n        with torch.no_grad():\n            output = model(img_batch.to(device)).data.cpu().numpy()\n            test_mask_batch = test_mask_batch.data.cpu().numpy()\n            for out, test_mask in zip(output, test_mask_batch):\n                coords = extract_coords(out, threshold= threshold) \n                s = coords2str(coords)\n                preds_before_drop_mask.append(s)\n                # use test_mask\n                #import pdb; pdb.set_trace()\n                if isinstance(test_mask, np.ndarray):\n                    test_mask = cv2.resize(test_mask, (256,64))[...,0]\n                    test_mask = np.where(test_mask>255//2, 100, 0)  # subtract from logits\n                        \n                    out[0,...] =  out[0,...] - test_mask\n                    \n                coords = extract_coords(out, threshold= threshold) \n                s = coords2str(coords)\n                preds.append(s)\n            \n    return preds, preds_before_drop_mask\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"7gM8Ey1vP3Ty","colab_type":"code","colab":{}},"cell_type":"code","source":"threshold0 = -1.5\ntest_thresh, test_thresh_before = {}, {}\ntest_batch_size = 4\ntest_images_dir = osj(train_test_path,'test_images/{}.jpg')\ntest_masks_dir = osj(path,'test_masks/{}.jpg')\n\ntest_dataset = CarDataset(test, test_images_dir, test_masks_dir, training=False)\n\ntest_loader = DataLoader(dataset=test_dataset, batch_size=test_batch_size, shuffle=False, num_workers=4)\n\nfor thresh in tqdm([threshold0]): #[-2, -1.5, -1]):\n    preds_test = pd.read_csv(osj(path,'sample_submission.csv'), nrows = n_samples)\n    preds_test_before = preds_test.copy()\n    #test_thresh['thresh_'+str(thresh)] = pd.read_csv(osj(path,'sample_submission.csv'))\n    temp_preds, temp_preds_before_drop_mask = predict_test(model, thresh)\n    preds_test['PredictionString']  = temp_preds[:n_samples]\n    preds_test_before['PredictionString'] = temp_preds_before_drop_mask[:n_samples]\n    test_thresh['thresh_'+str(thresh)] = preds_test\n    test_thresh_before['thresh_'+str(thresh)] = preds_test_before\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"6vhwbO6MP3T9","colab_type":"code","colab":{}},"cell_type":"code","source":"sub = test_thresh['thresh_'+str(thresh)]\nsub_before = test_thresh_before['thresh_'+str(thresh)]\nnumcars = pd.DataFrame({'ImageId': sub['ImageId'].values, 'numcars': [int((str(x).count(' ')+1)/7) for x in sub['PredictionString']]})\nnumcars_before = pd.DataFrame({'ImageId': sub_before['ImageId'].values, 'numcars': [int((str(x).count(' ')+1)/7) for x in sub_before['PredictionString']]})\nnumcars_compare = pd.concat([numcars[numcars_before['numcars']!=numcars['numcars']],\n          numcars_before[numcars_before['numcars']!=numcars['numcars']]], axis=1)\nnumcars_compare.columns = ['ImageId_new','numcars_new', 'ImageId_old','numcars_old']\nnumcars_compare = numcars_compare[['ImageId_old','ImageId_new','numcars_old','numcars_new']]\nprint(\"\\nNumber of images affected by test_masks reduction\", numcars_compare.shape[0])\nprint()\nnumcars_compare.head()\n              ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"4aLEAEYBP3UP","colab_type":"code","colab":{}},"cell_type":"code","source":"def print_numcar_stats(df, prefix):\n    print(f\"\\n{prefix}num cars: {df['numcars'].sum():,d}, \\tRatio of num cars to num images: {(df['numcars'].sum()/df.shape[0]):.2f}\")\n    \n\nprefix = f\"AFTER TEST MASKS, threshold = {threshold0}:\\n\"\nprint_numcar_stats(numcars, prefix)\nprefix = f\"BEFORE TEST MASKS, threshold = {threshold0}: \\n\"\nprint_numcar_stats(numcars_before, prefix)\nprint(f\"\\nnum cars in train: {train['new_numcars'].sum():,d}, \\tratio cars to images in train: {(train['new_numcars'].sum()/train.shape[0]):.2f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"vFHCl67TP3Ud","colab_type":"code","colab":{}},"cell_type":"code","source":"sub.to_csv(osj(out_path, f'sub_thresh_{threshold0:.2f}_cv_3045_eff_b0_dropout_change_resume_KGL.csv'), index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"accelerator":"GPU","colab":{"name":"CenterNet Baseline Pytorch with Dropout Reduced.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"0020b8122b7f4efeb8699c1a47423ef4":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_27beeaae686c4f71bd44b46a4e5d3b95","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_4655bd286e3f48f3985b52f2c7a51495","IPY_MODEL_9ce6c7aece884961a97a644e8a443b2b"]}},"27beeaae686c4f71bd44b46a4e5d3b95":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4655bd286e3f48f3985b52f2c7a51495":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_17609aec87874afead62d382e513b21b","_dom_classes":[],"description":"","_model_name":"IntProgressModel","bar_style":"","max":1863,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":312,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_fb5c0565d0ac45ed83ad9b85e122c6e3"}},"9ce6c7aece884961a97a644e8a443b2b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c50c94f686254caf9768444c4d3844be","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 17% 312/1863 [07:53&lt;38:46,  1.50s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4db469cc1f8b4393a22e4cae1732311a"}},"17609aec87874afead62d382e513b21b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"fb5c0565d0ac45ed83ad9b85e122c6e3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c50c94f686254caf9768444c4d3844be":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"4db469cc1f8b4393a22e4cae1732311a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"nbformat":4,"nbformat_minor":1}