{"cells":[{"metadata":{"_uuid":"500763b1284172336dde2c8d853884c79c78b9d5","_cell_guid":"35b0ba4b-23a3-488e-bf8a-ce835ac39b0f"},"cell_type":"markdown","source":"# Overview\nThe notebook uses the pretrained PSPNet to segment the scenes in the CVPR dataset. It then matches the relevant labels together so predictions can be made."},{"metadata":{"_uuid":"e4bb3b5eb3c34acdba95015fdc3693c24a079ef2","_cell_guid":"b015595e-bb96-40c2-8e56-4beee08e0dda"},"cell_type":"markdown","source":"# Load Pretrained Network\nHere we setup and load the Pretrained Network"},{"metadata":{"collapsed":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport sys\nfrom glob import glob\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom skimage.io import imread\nfrom skimage.segmentation import mark_boundaries\npsp_base_dir = os.path.join('..', 'input', 'modeldepotio-pspnet-pretrained')\npsp_model_dir = os.path.join(psp_base_dir, 'model', 'model')\ncityscape_weights = os.path.join(psp_base_dir, 'model', 'model', 'pspnet101-cityscapes')\npsp_code_dir = os.path.join(psp_base_dir, 'pspnet-tensorflow-master', 'PSPNet-tensorflow-master')\nDATA_DIR = os.path.join('..', 'input', 'cvpr-2018-autonomous-driving')","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"d8415749f100eb0ff73e881806305d0f6c55cf0e","_cell_guid":"58097ab2-ad37-42eb-ac58-f15f2f8815b6","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nfrom scipy import misc\nimport matplotlib.pyplot as plt\nsys.path.append(psp_code_dir)\nfrom model import PSPNet101, PSPNet50\nfrom tools import *","execution_count":2,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"2e1cbd39807cd53d0cc01eedac5feccb266d7a7e","_cell_guid":"ffabed1c-3e44-4053-a723-d06512786c50","trusted":true},"cell_type":"code","source":"# TODO: Change these values to where your model files are\nADE20k_param = {'crop_size': [473, 473],\n                'num_classes': 150, \n                'model': PSPNet50,\n                'weights_path': os.path.join(psp_model_dir, 'pspnet50-ade20k/model.ckpt-0')}\n\ncityscapes_param = {'crop_size': [720, 720],\n                    'num_classes': 19,\n                    'model': PSPNet101,\n                    'weights_path': os.path.join(psp_model_dir,'pspnet101-cityscapes/model.ckpt-0')}\n\nIMAGE_MEAN = np.array((103.939, 116.779, 123.68), dtype=np.float32)\n# TODO: If you want to inference on indoor data, change this value to `ADE20k_param`\nparam = cityscapes_param ","execution_count":3,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"089e46f7b0978555a6e8a7a92d7760fa068472ea","_cell_guid":"1ff5b404-8d51-4b11-8e2b-9e5abddc71e4","trusted":true},"cell_type":"code","source":"# make a placeholder for reading images\n#TODO: switch to batch loader to improve performance\npc_img_path = tf.placeholder('string')\nimg_np = tf.image.decode_jpeg(tf.read_file(pc_img_path), channels=3)\nimg_shape = tf.shape(img_np)\nh, w = (tf.maximum(param['crop_size'][0], img_shape[0]), tf.maximum(param['crop_size'][1], img_shape[1]))\nimg = preprocess(img_np, h, w)","execution_count":4,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"ff8e32b16c576be33c019cf0e3185819cc7910a7","_cell_guid":"68716a10-4c16-4a2d-9103-1d7a1dfec972","trusted":true},"cell_type":"code","source":"# Create network.\nPSPNet = param['model']\nnet = PSPNet({'data': img}, is_training=False, num_classes=param['num_classes'])","execution_count":5,"outputs":[]},{"metadata":{"_uuid":"2364ac9860fd175551429fef400c583695fe6471","_cell_guid":"68959a71-5d2c-49d1-8277-30ffd1f7c8b6","trusted":true},"cell_type":"code","source":"raw_output = net.layers['conv6']\n\n# Predictions.\nraw_output_up = tf.image.resize_bilinear(raw_output, size=[h, w], align_corners=True)\nraw_output_up = tf.image.crop_to_bounding_box(raw_output_up, 0, 0, img_shape[0], img_shape[1])\nraw_output_up = tf.argmax(raw_output_up, dimension=3)\npred = decode_labels(raw_output_up, img_shape, param['num_classes'])\n\n# Init tf Session\nconfig = tf.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsess = tf.Session(config=config)\ninit = tf.global_variables_initializer()\n\nsess.run(init)\n\nckpt_path = param['weights_path']\nloader = tf.train.Saver(var_list=tf.global_variables())\nloader.restore(sess, ckpt_path)\nprint(\"Restored model parameters from {}\".format(ckpt_path))","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"e748038aa3dd16afee8dd91a672c7488f37ccb64","_cell_guid":"b4f950bc-8520-477f-a8f1-0589ac41530f"},"cell_type":"markdown","source":"# Process the CVPR Data\nHere we load the CVPR Data and see how the model performs"},{"metadata":{"collapsed":true,"_uuid":"65a3cc82f7f03e67207114b8ff6d0ce031f66ff4","_cell_guid":"8321f67f-cc15-466c-a00a-837c75fe2fec","trusted":true},"cell_type":"code","source":"class_str = \"\"\"car, 33\nmotorbicycle, 34\nbicycle, 35\nperson, 36\nrider, 37\ntruck, 38\nbus, 39\ntricycle, 40\nothers, 0\nrover, 1\nsky, 17\ncar_groups, 161\nmotorbicycle_group, 162\nbicycle_group, 163\nperson_group, 164\nrider_group, 165\ntruck_group, 166\nbus_group, 167\ntricycle_group, 168\nroad, 49\nsiderwalk, 50\ntraffic_cone, 65\nroad_pile, 66\nfence, 67\ntraffic_light, 81\npole, 82\ntraffic_sign, 83\nwall, 84\ndustbin, 85\nbillboard, 86\nbuilding, 97\nbridge, 98\ntunnel, 99\noverpass, 100\nvegatation, 113\nunlabeled, 255\"\"\"\nclass_dict = {v.split(', ')[0]: int(v.split(', ')[-1]) for v in class_str.split('\\n')}","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"a9eab0d5e34127cbe4ccc9cbf3b3ab1a99209214","_cell_guid":"b212855f-b4ec-47b9-8224-cfb78bb21722","trusted":true},"cell_type":"code","source":"import pandas as pd\nall_paths = pd.DataFrame(dict(path = glob(os.path.join(DATA_DIR, '*', '*.*p*g'))))\nall_paths['split'] = all_paths['path'].map(lambda x: x.split('/')[-2].split('_')[0])\nall_paths['group'] = all_paths['path'].map(lambda x: x.split('/')[-2].split('_')[-1])\nall_paths['group'] = all_paths['group'].map(lambda x: 'color' if x == 'test' else x)\nall_paths['id'] = all_paths['path'].map(lambda x: '_'.join(os.path.splitext(os.path.basename(x))[0].split('_')[:4]))\ngroup_df = all_paths.pivot_table(values = 'path', columns = 'group', aggfunc = 'first', index = ['id', 'split']).reset_index()\ngroup_df.sample(5)","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"7d080089be173da75d86d26f93c6bdb6d884940a","_cell_guid":"f66bc515-fe63-4da1-9024-02400a9310a0","trusted":true},"cell_type":"code","source":"train_df = group_df.query('split==\"train\"')\nprint(train_df.shape[0], 'rows')\nsample_rows = 10\nfig, m_axs = plt.subplots(sample_rows, 5, figsize = (20, 6*sample_rows))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nout_rows = []\nfor (ax1, ax2, ax4, ax3, ax_c_crop), (_, c_row) in zip(m_axs, train_df.sample(sample_rows, random_state = 2018).iterrows()):\n    c_img = imread(c_row['color'])\n    l_img = imread(c_row['label'])//1000\n    seg_img = sess.run(raw_output_up, feed_dict = {pc_img_path: c_row['color']})[0]\n    ax1.imshow(c_img)\n    ax1.set_title('Color')\n    ax2.imshow(l_img, cmap = 'nipy_spectral')\n    ax2.set_title('Segments')\n    xd, yd = np.where(l_img>0)\n    bound_img = mark_boundaries(image = c_img, label_img = l_img, color = (1,0,0), background_label = 255, mode = 'thick')\n    ax3.imshow(bound_img[xd.min():xd.max(), yd.min():yd.max(),:])\n    ax3.set_title('Cropped Overlay')\n    ax4.imshow(seg_img)\n    ax4.set_title('PSP Image %d objects' % (np.max(seg_img) % 1000))\n    ax_c_crop.imshow(seg_img[xd.min():xd.max(), yd.min():yd.max()])\n    ax_c_crop.set_title('Cropped PSP')\nfig.savefig('sample_overview.png')","execution_count":9,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"f3073758cf8a766f025d323429635cc57bd10a5c","_cell_guid":"86c1b86d-48d0-4907-8851-11de83258bc4","trusted":true},"cell_type":"code","source":"# Decode the Labels from PSP\nrev_class_dict = {v: k for k,v in class_dict.items()}\nlabel_names = 'road,siderwalk,building,wall,fence,pole,traffic_light,traffic_sign,vegatation,terrain,sky,person,rider,car,truck,bus,train,motorbicycle,bicycle'.split(',')","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"cb4e2ee69c26ad0b2a4bd0379a67fa197446c756","_cell_guid":"e3feb7de-de8d-479c-bf04-31dbd4654515","trusted":true},"cell_type":"code","source":"idx_to_class = {}\nfor c_color_idx, c_label in enumerate(label_names):\n    if c_label in ['vegatation', 'building', 'sky']:\n        print('\\t Skipping', c_label)\n    if c_label in class_dict:\n        print(c_label, class_dict[c_label])\n        idx_to_class[c_color_idx] = class_dict[c_label]\n    else:\n        print('\\t', c_label, 'missing')","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"4dcd444ca7fae6455e1b2fd1fb90668d9f7da07a","_cell_guid":"0a8b10df-e123-480d-80fa-bf57b5e61c25","trusted":true},"cell_type":"code","source":"class_to_idx = {v:k for k,v in idx_to_class.items()}\nfig, m_axs = plt.subplots(2,3, figsize = (12,20))\nx_bins = np.arange(seg_img.max()+1)\nfor i, ax1 in zip(np.unique(l_img[l_img>0]), m_axs.flatten()):\n    un_ids = np.unique(seg_img[l_img==i].ravel())\n    ax1.hist(seg_img[l_img==i].ravel(), \n             x_bins, label = '{}'.format(i), normed = True, alpha = 0.25)\n    ax1.legend()\n    \n    ax1.set_title('CVPR {}->{}\\nPSP: {}'.format(rev_class_dict.get(i, ''), class_to_idx.get(i, ''), ', '.join(\n        ['{}-{}'.format(label_names[int(k)], int(k)) for k in un_ids])))\n    ax1.set_xticks(x_bins+0.5)\n    ax1.set_xticklabels(label_names, rotation = 60)","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a60a170d6eec19e6009a965dcc6dbb88e975d86b"},"cell_type":"code","source":"from skimage.measure import label\ndef rgb_seg_to_instimg(in_img):\n    out_img = np.zeros(in_img.shape, dtype = np.int64)\n    for i in np.unique(in_img[in_img>0]):\n        if i in idx_to_class:\n            j = idx_to_class[i]\n            inst_ids = label(in_img==i)[in_img==i]\n            out_img[in_img==i] = inst_ids+j*1000\n    return out_img","execution_count":22,"outputs":[]},{"metadata":{"_uuid":"9699aa964f397b5a115dd599bd50c0e95fb2e695","_cell_guid":"9e41ceb6-1f6d-4c04-b88b-97b62ce42376","trusted":true},"cell_type":"code","source":"%%time\nsample_rows = 4\nfig, m_axs = plt.subplots(sample_rows, 5, figsize = (20, 6*sample_rows))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nout_rows = []\nfor (ax1, ax2, ax4, ax3, ax_c_crop), (_, c_row) in zip(m_axs, train_df.sample(sample_rows, random_state = 2012).iterrows()):\n    c_img = imread(c_row['color'])\n    l_img = imread(c_row['label'])//1000\n    seg_img = sess.run(raw_output_up, feed_dict = {pc_img_path: c_row['color']})[0]\n    c_lab_img = rgb_seg_to_instimg(seg_img)\n    ax1.imshow(c_img)\n    ax1.set_title('Color')\n    ax2.imshow(l_img, cmap = 'nipy_spectral')\n    ax2.set_title('Segments')\n    xd, yd = np.where(l_img>0)\n    bound_img = mark_boundaries(image = c_img, label_img = l_img, color = (1,0,0), background_label = 255, mode = 'thick')\n    ax3.imshow(bound_img[xd.min():xd.max(), yd.min():yd.max(),:])\n    ax3.set_title('Cropped Overlay')\n    ax4.imshow(c_lab_img)\n    ax4.set_title('PSP Image %d objects' % (len(np.unique(c_lab_img))))\n    psp_bound_img = mark_boundaries(image = c_img, label_img = c_lab_img, color = (1,0,0), background_label = 255, mode = 'thick')\n    ax_c_crop.imshow(psp_bound_img[xd.min():xd.max(), yd.min():yd.max()])\n    ax_c_crop.set_title('Cropped PSP')\nfig.savefig('full_overview.png')","execution_count":23,"outputs":[]},{"metadata":{"_uuid":"530a2e34d49d2dcbe1986efc573409323f900c20","_cell_guid":"e07c7d42-b691-401f-9d2e-3ecae721caaf","trusted":true},"cell_type":"code","source":"test_df = group_df.query('split==\"test\"').drop(['label'], axis = 1)\nprint(test_df.shape[0], 'rows')\ntest_df.sample(3)","execution_count":18,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"e180d269e151e5209ea419bbcf3fc740c86f4317","_cell_guid":"d1c99c12-f959-4aa1-8bdf-b4a8edb0dc8c","trusted":true},"cell_type":"code","source":"def rle_encoding(x):\n    \"\"\" Run-length encoding based on\n    https://www.kaggle.com/rakhlin/fast-run-length-encoding-python\n    Modified by Konstantin, https://www.kaggle.com/lopuhin\n    \"\"\"\n    assert x.dtype == np.bool\n    dots = np.where(x.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.append([b, 0])\n        run_lengths[-1][1] += 1\n        prev = b\n    return '|'.join('{} {}'.format(*pair) for pair in run_lengths)\n\ndef segs_to_rle_rows(lab_img, **kwargs):\n    out_rows = []\n    for i in np.unique(lab_img[lab_img>0]):\n        c_dict = dict(**kwargs)\n        c_dict['LabelId'] = i//1000\n        c_dict['PixelCount'] = np.sum(lab_img==i)\n        c_dict['Confidence'] = 0.5 # our classifier isnt very good so lets not put the confidence too high\n        c_dict['EncodedPixels'] = rle_encoding(lab_img==i)\n        out_rows += [c_dict]\n    return out_rows","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"e797a93506a14402f09b3684eb17af40ea0b08cc","_cell_guid":"3b881f03-45d0-437b-845d-2db88f09ff10","trusted":true},"cell_type":"code","source":"# make sure it works on a simple case\nexp_df = pd.DataFrame(segs_to_rle_rows(c_lab_img, ImageId = -1))\n\nexp_df['LabelName'] = exp_df['LabelId'].map(rev_class_dict.get)\nprint(exp_df.shape[0], 'rows')\nexp_df.sample(5)","execution_count":24,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"142696987096fd999a35ed44de6f02b4ed37e45e","_cell_guid":"2e8d9508-d0f0-4acc-999e-cc41eedcc93b","trusted":true},"cell_type":"code","source":"def read_row(in_row):\n    cur_seg_img = sess.run(raw_output_up, feed_dict = {pc_img_path: in_row['color']})[0]\n    inst_img = rgb_seg_to_instimg(cur_seg_img)\n    return segs_to_rle_rows(inst_img, ImageId = in_row['id'])","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"afdfa399407f18a5ec7a9fac18937e8b89e5ed78","_cell_guid":"4206702b-5392-4785-a86d-fe1e684dc067","trusted":true},"cell_type":"code","source":"%%time\nfrom tqdm import tqdm_notebook\nall_rows = []\nfor _, c_row in tqdm_notebook(list(test_df.sample(5).iterrows())):\n    all_rows += read_row(c_row.to_dict())","execution_count":26,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"fc725eabf5b7a973cc452a947cea847b167710f5","_cell_guid":"3fad7dca-7a70-4bfa-b45d-df8f4ee36231","trusted":true},"cell_type":"code","source":"%%time\nall_rows = []\nfor _, c_row in tqdm_notebook(list(test_df.sample(80).iterrows())):\n    all_rows += read_row(c_row.to_dict())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2d9e03fe5ea79a1aeb907aa650f1a46f48e10e6","_cell_guid":"38f9e41f-8671-40e2-82b4-5b921df7307e","trusted":true},"cell_type":"code","source":"all_rows_df = pd.DataFrame(all_rows)\nprint('Total Output Rows', all_rows_df.shape[0])\nall_rows_df = all_rows_df[['ImageId', 'LabelId', 'PixelCount', 'Confidence', 'EncodedPixels']]\nall_rows_df.to_csv('psp_full_submission.csv', index = False)\nall_rows_df.sample(5)","execution_count":27,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"ce3fd5e8887e34fad283372006d4552a10e8c831","_cell_guid":"416ef92b-ce64-4983-8635-6d0e01bc013d","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}