{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/carvana-image-masking-challenge\" directory.\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"daf85ad7da2f53bedc0b98935f9a310d599301b2"},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","scrolled":true,"trusted":true},"cell_type":"code","source":"from subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce3a6d62d6736b0fde0dc8fefa1985732361e2f0","trusted":true},"cell_type":"code","source":"# Load all the necessary libraries\nimport numpy as np \nimport gzip\nimport os \nfrom os.path import basename\nimport glob\nimport time \nimport cv2\nimport pandas as pd \nimport random\nfrom PIL import Image\nfrom scipy import ndimage\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom scipy.misc import imresize\nfrom skimage.transform import resize\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Model \nfrom keras.layers import Input, merge, Conv2D, MaxPooling2D, UpSampling2D, Concatenate\nfrom keras.optimizers import Adam, SGD \nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping\nfrom keras.preprocessing.image import array_to_img, img_to_array, load_img, ImageDataGenerator\nfrom keras import backend as K\n\nK.set_image_dim_ordering('th') # Theano dimension ordering in this code\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"051a416d00a7ac812f2532797b6f217ba19a713b","trusted":true},"cell_type":"code","source":"INPUT_PATH='../input/'\nprint(os.getcwd())\ndims=[128,128]\nimg_rows=dims[0]\nimg_cols=dims[1]\ntrain=sorted(glob.glob(INPUT_PATH+'train/*.jpg'))\nmasks=sorted(glob.glob(INPUT_PATH+'train_masks/*.gif'))\ntest=sorted(glob.glob(INPUT_PATH+'test/*.jpg'))\nprint('Number of training images: ', len(train), 'Number of corresponding masks: ', len(masks), 'Number of test images: ', len(test))\n\nmeta=pd.read_csv(INPUT_PATH+'metadata.csv')\nmask_df=pd.read_csv(INPUT_PATH+'train_masks.csv')\nids_train=mask_df['img'].map(lambda s: s.split('_')[0]).unique()\nprint('Length of ids_train ', len(ids_train))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46604081c04a99f5a15e4e18e837174879948433","trusted":true},"cell_type":"code","source":"mask_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa82a0cb224c6649bc2e76fe8e2d7c0434cbdd82","trusted":true},"cell_type":"code","source":"image = cv2.imread(INPUT_PATH+\"train/00087a6bd4dc_01.jpg\")\nplt.imshow(image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b884c575c05584a3c03e936cc926ec1c5ce8b23","trusted":true},"cell_type":"code","source":"img = Image.open(INPUT_PATH+'train_masks/00087a6bd4dc_01_mask.gif').convert('RGB')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2ac07acba23cbbff5860e44c269104b6d2909cf"},"cell_type":"code","source":"img2mask=np.array(img)\nprint(img2mask.shape)\nprint(image.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e0595ca67319a4b98be248e6d19e4abdaf257d5"},"cell_type":"code","source":"masked_img=cv2.bitwise_and(image,img2mask)\nplt.imshow(masked_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6c910be8be8deb9fd188649988485c17a2ffe05"},"cell_type":"code","source":"masked_gray=cv2.cvtColor(masked_img,cv2.COLOR_BGR2GRAY)\nplt.imshow(masked_gray, cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0275d1dddf7e576ebd533bd83a72d3fea0cc441b"},"cell_type":"code","source":"masked_gray.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"03b8a96adae418bbdf47e9cf957e7a03e59759f1"},"cell_type":"code","source":"#cv2.imshow('masked_gray',masked_gray)\n#cv2.waitKey(0)\n#cv2.destroyAllWindows()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da7c139ba6f7969e12321fdbf01cccdd6c378677","trusted":true},"cell_type":"code","source":"smooth = 1.\ndef dice_coef(y_true, y_pred):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection=K.sum(y_true_f * y_pred_f)\n    return(2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef dice_coef_mp(y_true, y_pred):\n    y_true_f=y_true.flatten()\n    y_pred_f=y_pred.flatten()\n    intersection = np.sum(y_true_f*y_pred_f)\n    return(2. * intersection + smooth)/(np.sum(y_pred_f) + mp.sum(y_pred_f) + smooth)\n\ndef dice_coef_loss(y_true, y_pred):\n    return -dice_coef(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b7fe6993d004ae47dbace0dfd8a7cc7f5ef84ae","trusted":true},"cell_type":"code","source":"def get_layer(inputs, pixel, pool=True):\n    conv = Conv2D(pixel, (3, 3), padding=\"same\", activation='relu')(inputs)\n    conv = Conv2D(pixel, (3 ,3), padding=\"same\", activation='relu')(conv)\n    if pool:\n        pool = MaxPooling2D(pool_size=(2, 2))(conv)\n    else:\n        pool = None\n    return conv, pool","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"389523ea8a3e08b1c5dcd6a82d022132436eca07","trusted":true},"cell_type":"code","source":"def get_unet():\n    inputs = Input((3,img_rows, img_cols))\n    conv1 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(inputs)    \n    conv1 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(conv1)\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n\n    conv2 = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(pool1)\n    conv2 = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(conv2)    \n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n\n    conv3 = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(pool2)\n    conv3 = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(conv3)\n    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)\n\n    conv4 = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(pool3)\n    conv4 = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(conv4)\n    pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)\n\n    conv5 = Conv2D(512, (3, 3), padding=\"same\", activation='relu')(pool4)\n    conv5 = Conv2D(512, (3, 3), padding=\"same\", activation='relu')(conv5)\n\n    up6 = Concatenate(axis=1)([UpSampling2D(size=(2, 2))(conv5), conv4])\n    #      Concatenate(axis=3)([residual, upconv])\n    conv6 = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(up6)\n    conv6 = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(conv6)\n\n    up7 = Concatenate(axis=1)([UpSampling2D(size=(2, 2))(conv6), conv3])\n    conv7 = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(up7)\n    conv7 = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(conv7)\n\n    up8 = Concatenate(axis=1)([UpSampling2D(size=(2, 2))(conv7), conv2])\n    conv8 = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(up8)\n    conv8 = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(conv8)\n\n    up9 = Concatenate(axis=1)([UpSampling2D(size=(2, 2))(conv8), conv1])\n    conv9 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(up9)\n    conv9 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(conv9)\n\n    conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conv9)   #9\n\n    model = Model(inputs=inputs, outputs=conv10)\n    #      `Model(inputs=/input_19, outputs=sigmoid.0)`\n\n    #model.compile(optimizer=Adam(lr=1.0e-5), loss=dice_coef_loss, metrics=[dice_coef])  #LUNA16\n    model.compile(optimizer=Adam(5e-4), loss='binary_crossentropy', metrics=[dice_coef]) #ecobill\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"804edf217adf930e85d764ca048f7a900524733b","trusted":true},"cell_type":"code","source":"#split the train se into train and validation\ntrain_images, validation_images = train_test_split(train, train_size=0.8, test_size=0.2)\nprint('Split into training set with ', len(train_images), ' images anf validation set with ', len(validation_images), ' images')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1dfc04c866f53ab7d62c35b89531b2d3b017b4dd","trusted":true},"cell_type":"code","source":"#utility function to convert greyscale inages to rgb\ndef grey2rgb(img):\n    new_img = []\n    for i in range(img.shape[0]):\n        for j in range(img.shape[1]):\n            new_img.append(list(img[i][j])*3)\n    new_img = np.array(new_img).reshape(img.shape[0], img.shape[1], 3)\n    return new_img\n\n#generator that we will use to read data from the directory\ndef data_gen_small(data_dir, masks, images, batch_size, dims):\n    \"\"\" \n    data_dir: where the actual images are kept\n    mask_dir: where the actual masks are kept \n    images: the filenames of the images wi want to generate batches from \n    batch_size: self explanatory\n    dims: the dimensions in which wi want to rescale our images\n    \n    Image.resize(size, resample=0)\n    \n    Returns a resized copy of this image.\n    Parameters: \n    \n    size - The requested size in pixels, as a 2-tuple: (width, height).\n    resample - An optional resampling filter. This can be one of PIL.Image.NEAREST,\n    PIL.Image.BOX, PIL.Image.HAMMING, PIL.Image.BICUBIC or PIL.Image.LANCZOS\n    If omitted, or if the image has mode \"1\" or \"P\", it is set PIL.Image.NEAREST\n    \"\"\"\n    while True:\n        if batch_size==1:\n            ix=np.array([0])\n        else:\n            ix=np.random.choice(np.arange(len(images)), batch_size)\n        \n        imgs = []\n        labels = []\n        for i in ix:\n            # images\n            #print(images[i])\n            if batch_size==1:\n                original_img = cv2.imread(images)\n            else:\n                original_img = cv2.imread(images[i])\n            \n            resized_img = imresize(original_img, dims + [3]) #this looks like TensorFlow ordering\n            array_img = resized_img/255\n            array_img = array_img.swapaxes(0, 2)\n            imgs.append(array_img)\n            #imgs is a numpy array with dim: (batch size X 128 X 128 3)\n            #print('shape of imgs ', array_img.shape)\n            # masks\n            try:\n                mask_filename = basename(images[i])\n                no_extension = os.path.splitext(mask_filename)[0]\n                correct_mask = INPUT_PATH + 'train_masks/' + no_extension + '_mask.gif'\n                original_mask = Image.open(correct_mask).convert('L')\n                data = np.asarray(original_mask, dtype=\"int32\")\n                resized_mask = imresize(original_mask, dims+[3])\n                array_mask = resized_mask / 255\n                labels.append(array_mask)\n            except Exception as e:\n                labels=None\n            \n        imgs = np.array(imgs)\n        labels = np.array(labels)\n        try:\n            relabel = labels.reshape(-1, dims[0], dims[1], 1)\n            relabel = relabel.swapaxes(1, 3)\n        except Exception as e:\n            relabel=labels\n        yield imgs, relabel","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e215144d2883e07b345964e08aa3d08b31928209","trusted":true},"cell_type":"code","source":"train_gen = data_gen_small(INPUT_PATH + 'train/', masks, train_images, 2, dims) \nimg, msk = next(train_gen)\nprint('Size of batch: ', len(img))\nprint('shape of img ', img.shape, 'number dimensions: ', img[0].ndim)\nprint('shape of msk ', msk.shape, 'number dimensions: ', msk[0].ndim)\nnewshape = img[0].swapaxes(0,2)\nplt.imshow(newshape)\nplt.show()\n\n#try resize up \n\nresized_img = imresize(img[0], [1280, 1918]+[3])\nprint('resized up: ', resized_img.shape)\nnewshape = resized_img.swapaxes(0,1)\nprint('resized swapaxes: ', newshape.shape)\nprint('resized swapaxes shape[-1]: ', newshape.shape[-1])\n\nplt.imshow(newshape)\nplt.show()\n\nnewshape = msk.swapaxes(1,3)\nprint(newshape.shape)\nplt.imshow(grey2rgb(newshape[0]), alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6a78477ae8ad7bfd742248b20d85b3e2916f319","trusted":true},"cell_type":"code","source":"# create an instance of a validation generator:\nvalidation_gen = data_gen_small(INPUT_PATH + 'train/', masks, validation_images, 4, dims) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a13b64e7cc00f8e9f776c0d0dd8a34f2d326874","trusted":true,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"# define and compile the model\nmodel = get_unet()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4713785e5d00b235a9de1d8a171424430caf2d61","scrolled":false,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# fit the model and check dice_coef on validation data as end of each epoch\nmodel.fit_generator(train_gen, steps_per_epoch=50, epochs=35, validation_data=validation_gen, validation_steps=50)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf6e53cf66594b61c75a5ade87fc02b5db4d26b8","trusted":true},"cell_type":"code","source":"img, msk = next(validation_gen)\nprint(img.shape)\npredicted_mask = model.predict(img)\npredicted_mask.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fffc3156ef13dc2975d6f6adea0ac7f40830dc0"},"cell_type":"code","source":"newshape = predicted_mask.swapaxes(1,3)\nprint('newshape shape ', newshape.shape)\ngrey = grey2rgb(newshape[3])\nprint('grey shape ', grey.shape)\nplt.imshow(grey, alpha = 0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ae787bfe3752028e159caafdc141f259e3143ff","trusted":true},"cell_type":"code","source":"newshape = img[3].swapaxes(0,2)\nplt.imshow(newshape)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"230e87042ee774172d00f3546652b548e751933e"},"cell_type":"markdown","source":"**Prediction on the unseen data set**"},{"metadata":{"trusted":true,"_uuid":"0eeca4484f18649d8717d12d239f2f3ddb571ad2"},"cell_type":"code","source":"validation_test = data_gen_small(INPUT_PATH + 'test/', masks, test, 4, dims) \nimg_tst, msk_tst = next(validation_test)\nprint(img_tst.shape)\npredicted_mask_tst = model.predict(img_tst)\npredicted_mask_tst.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd1c6fddb9e009bbca79f7a09be08dc74b87a18f"},"cell_type":"code","source":"newshape_tst = predicted_mask_tst.swapaxes(1,3)\nprint('newshape shape ', newshape_tst.shape)\ngrey_tst = grey2rgb(newshape_tst[3])\nprint('grey shape ', grey_tst.shape)\nplt.imshow(grey_tst, alpha = 0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9bf96bba3719e12605bab069bad34390c995278"},"cell_type":"code","source":"newshape_tst = img_tst[3].swapaxes(0,2)\nprint(newshape_tst.shape)\nplt.imshow(newshape_tst)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91df30b8e2dea1f31a446c2cc7b5bd0860e699b2"},"cell_type":"code","source":"newshape_tst = predicted_mask_tst.swapaxes(1,3)\nprint('newshape shape ', newshape_tst.shape)\ngrey_tst = grey2rgb(newshape_tst[3])\nprint('grey shape ', grey_tst.shape)\nplt.imshow(grey_tst, alpha = 0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d45aab0127abd7c224dba2fc37bbdb293c3c65b1"},"cell_type":"code","source":"'''def rle_encode(mask_image):\n    pixels = mask_image.flatten()\n    # We avoid issues with '1' at the start or end (at the corners of \n    # the original image) by setting those pixels to '0' explicitly.\n    # We do not expect these to be non-zero for an accurate mask, \n    # so this should not harm the score.\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] = runs[1::2] - runs[:-1:2]\n    return runs'''\ndef rle (img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    bytes = np.where(img.flatten()==1)[0]\n    runs = []\n    prev = -2\n    for b in bytes:\n        if (b>prev+1): runs.extend((b+1, 0))\n        runs[-1] += 1\n        prev = b\n    \n    return ' '.join([str(i) for i in runs])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3961389bcd42977eeaa8d3de31a9a30f135f929"},"cell_type":"code","source":"from tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52f93b2119fab88fbfd0e1b923e3f16eb2fe0a2c"},"cell_type":"code","source":"df=pd.DataFrame(columns=['img','rle_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cb4f701bd474a497671586d1e6bc6d43ac8b220"},"cell_type":"code","source":"'''def par_predict(tst_img):\n    validation_test = data_gen_small(INPUT_PATH + 'test/', 'masks', tst_img, 1, dims)\n    img_tst, msk_tst = next(validation_test)\n    predicted_mask_tst = model.predict(img_tst)\n    \n    newshape_tst = predicted_mask_tst.swapaxes(1,3)\n    grey_tst = grey2rgb(newshape_tst[0])\n    gray_tst = np.array(grey_tst*255, dtype='uint8')\n    gray_tst[gray_tst>128]=1\n    gray_tst[gray_tst<=128]=0\n    \n    mask=rle_encode(gray_tst)\n    file=tst_img.split('/')[-1]\n    df.loc[len(df)]=[file, mask]\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"094cc6c9cf99a3e1838612e1148fdd35ade8cb53"},"cell_type":"code","source":"'''def par_predict(img_num):\n    validation_test = data_gen_small(INPUT_PATH + 'test/', 'masks', test[img_num], 1, dims)\n    img_tst, msk_tst = next(validation_test)\n    predicted_mask_tst = model.predict(img_tst)\n    \n    newshape_tst = predicted_mask_tst.swapaxes(1,3)\n    grey_tst = grey2rgb(newshape_tst[0])\n    gray_tst = np.array(grey_tst*255, dtype='uint8')\n    gray_tst[gray_tst<=128]=0\n    gray_tst[gray_tst>128]=1\n    \n    #mask=rle_encode(gray_tst)\n    #print(gray_tst[gray_tst==1])\n    mask=rle(gray_tst)\n    file=test[img_num].split('/')[-1]\n    return [file, mask]'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4194b0fd6aee265fa3e769f69dea08c8a522bb0b"},"cell_type":"code","source":"'''for i in tqdm(test, total=len(test)):\n    validation_test = data_gen_small(INPUT_PATH + 'test/', masks, i, 1, dims)\n    img_tst, msk_tst = next(validation_test)\n    predicted_mask_tst = model.predict(img_tst)\n    \n    newshape_tst = predicted_mask_tst.swapaxes(1,3)\n    grey_tst = grey2rgb(newshape_tst[0])\n    gray_tst = np.array(grey_tst*255, dtype='uint8')\n    gray_tst[gray_tst>128]=1\n    gray_tst[gray_tst<=128]=0\n    \n    mask=rle_encode(gray_tst)\n    file=i.split('/')[-1]\n    df.loc[len(df)]=[file, mask]'''\n\n'''#Parallel(n_jobs=num_cores)(par_predict(i) for i in tqdm(test, total=len(test)))\n#b=0\nln=len(test)\n#ln=int(ln/4)\n#ln=10 #testing the rle file\nst=len(df)\nprint('Number images going to be processed: {}'.format(ln))\n#pbar = tqdm(total = ln+1)\nfor i in tqdm(range(st,ln)):\n    df.loc[len(df)]=par_predict(i)\n    if i % 1000 == 0:\n        df.to_csv('csv_to_submit.csv', index = False)\n#pbar.close()\ndf.to_csv('csv_to_submit.csv', index = False)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e41b09845931d8c5bc58be86d1bfaf20ead3387"},"cell_type":"code","source":"'''# import the modules we'll need\nfrom IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport base64\n\n# function that takes in a dataframe and creates a text link to  \n# download it (will only work for files < 2MB or so)\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv()\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n# create a random sample dataframe\n#df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))\n\n# create a link to download the dataframe\ncreate_download_link(df)\n\n# ↓ ↓ ↓  Yay, download link! ↓ ↓ ↓ '''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f30dddf4fd8fc85445d706a5b37a486e5220ec3"},"cell_type":"code","source":"'''\nFast inplementation of Run-Length Encoding algorithm\nTakes only 200 seconds to process 5635 mask files\n'''\n\nimport numpy as np\nfrom PIL import Image\nimport os\n\n\ndef rle_encoding(x):\n    '''\n    x: numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns run length as list\n    '''\n    dots = np.where(x.T.flatten()==1)[0] # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b+1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6898a5b0fafa004f5f63d005ecf44b0685868bf"},"cell_type":"code","source":"conv = lambda l: ' '.join(map(str, l)) # list -> string\n#N = 100     # process first N masks\nN = len(test)\nfor i in tqdm(range(N)):\n    m=test[i]\n    validation_test = data_gen_small(INPUT_PATH + 'test/', 'masks', m, 1, dims)\n    img_tst, msk_tst = next(validation_test)\n    predicted_mask_tst = model.predict(img_tst)\n    newshape_tst = predicted_mask_tst.swapaxes(1,3)\n    grey_tst = grey2rgb(newshape_tst[0])\n    encoding=conv(rle_encoding(grey_tst))\n    file=m.split('/')[-1]\n    df.loc[len(df)]=[file,encoding]\n    \n#check output\n'''conv = lambda l: ' '.join(map(str, l)) # list -> string\nsubject, img = 1, 1\nprint('\\n{},{},{}'.format(subject, img, conv(encodings)))\n'''\ndf.to_csv('csv_to_submit.csv', index = False)\n'''print(len(df))\nprint(df)\nprint('The End')'''\n\n# train_masks.csv:\n#print('1,1,168153 9 168570 15 168984 22 169401 26 169818 30 170236 34 170654 36 171072 39 171489 42 171907 44 172325 46 172742 50 173159 53 173578 54 173997 55 174416 56 174834 58 175252 60 175670 62 176088 64 176507 65 176926 66 177345 66 177764 67 178183 67 178601 69 179020 70 179438 71 179857 71 180276 71 180694 73 181113 73 181532 73 181945 2 181950 75 182365 79 182785 79 183205 78 183625 78 184045 77 184465 76 184885 75 185305 75 185725 74 186145 73 186565 72 186985 71 187405 71 187825 70 188245 69 188665 68 189085 68 189506 66 189926 65 190346 63 190766 63 191186 62 191606 62 192026 61 192446 60 192866 59 193286 59 193706 58 194126 57 194546 56 194966 55 195387 53 195807 53 196227 51 196647 50 197067 50 197487 48 197907 47 198328 45 198749 42 199169 40 199589 39 200010 35 200431 33 200853 29 201274 27 201697 20 202120 15 202544 6')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dedbb6cc639d50d9ba9c4e1de9e96ff8c1c3074"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}