{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import pandas as pd\nsample_submission = pd.read_csv(\"../input/understanding_cloud_organization/sample_submission.csv\")\ntrain = pd.read_csv(\"../input/understanding_cloud_organization/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install segmentation-models\nimport segmentation_models as sm\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport os\nimport random\nfrom tqdm import tqdm_notebook\nimport cv2\nimport gc\n\nimport albumentations as albu\nimport keras\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.layers import Input, UpSampling2D, Conv2D, Activation\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.losses import binary_crossentropy\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras import optimizers\n\nfrom sklearn.model_selection import train_test_split\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/understanding_cloud_organization/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(path + 'train.csv')\nprint(len(tr))\ntr.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2mask(rle, imgshape):\n    width = imgshape[0]\n    height= imgshape[1]\n    \n    mask= np.zeros( width*height ).astype(np.uint8)\n    \n    array = np.asarray([int(x) for x in rle.split()])\n    starts = array[0::2]\n    lengths = array[1::2]\n\n    current_position = 0\n    for index, start in enumerate(starts):\n        mask[int(start):int(start+lengths[index])] = 1\n        current_position += lengths[index]\n        \n    return np.flipud( np.rot90( mask.reshape(height, width), k=1 ) )\n\ndef mask2rle(img):\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_loss(y_true, y_pred):\n    smooth = 1.\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = y_true_f * y_pred_f\n    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n    return 1. - score\n\ndef bce_dice_loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_names_all = tr['Image_Label'].apply(lambda x: x.split('_')[0]).unique()\nlen(img_names_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_ep = False\ndef keras_generator(batch_size):  \n    global new_ep\n    while True:   \n        \n        x_batch = []\n        y_batch = []        \n        for _ in range(batch_size):                         \n            if new_ep == True:\n                img_names =  img_names_all\n                new_ep = False\n            \n            fn = img_names[random.randrange(0, len(img_names))]                                       \n\n            img = cv2.imread(path + 'train_images/'+ fn)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)                       \n            masks = []\n            for rle in tr[tr['Image_Label'].apply(lambda x: x.split('_')[0]) == fn]['EncodedPixels']:                \n                if pd.isnull(rle):\n                    mask = np.zeros((img_size, img_size))\n                else:\n                    mask = rle2mask(rle, img.shape)\n                    mask = cv2.resize(mask, (img_size, img_size))\n                masks.append(mask)                                        \n            img = cv2.resize(img, (img_size, img_size))            \n            x_batch += [img]\n            y_batch += [masks] \n\n            img_names = img_names[img_names != fn]   \n        \n        x_batch = np.array(x_batch)\n        y_batch = np.transpose(np.array(y_batch), (0, 2, 3, 1))        \n\n        yield x_batch, y_batch\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import ResNet50\nbackbone = 'resnet50'\npreprocess_input = sm.get_preprocessing(backbone)\n\nmodel = sm.Unet(\n           backbone_name ='resnet50',\n           classes=4,\n           activation='sigmoid',\n           input_shape=(img_size, img_size, 3))\n\nmodel.compile(optimizer=optimizers.Adam(lr=9e-3), loss=bce_dice_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EpochBegin(keras.callbacks.Callback):\n    def on_epoch_begin (self, epoch, logs={}):\n        global new_ep\n        new_ep = True\nEpoch_Begin_Clb = EpochBegin()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nbatch_size = 16\nmodel.fit_generator(keras_generator(batch_size),\n              steps_per_epoch=200,                    \n              epochs=20,                    \n              verbose=1,\n              callbacks=[Epoch_Begin_Clb]\n              )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntest_img = []\ntestfiles=os.listdir(path + 'test_images/')\nfor fn in tqdm_notebook(testfiles):     \n        img = cv2.imread( path + 'test_images/'+fn )\n        img = cv2.resize(img,(img_size,img_size))       \n        test_img.append(img)\nlen(test_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npredict = model.predict(np.asarray(test_img))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npred_rle = []\nfor img in predict:      \n    img = cv2.resize(img, (525, 350))\n    tmp = np.copy(img)\n    tmp[tmp<np.mean(img)] = 0\n    tmp[tmp>0] = 1\n    for i in range(tmp.shape[-1]):\n        pred_rle.append(mask2rle(tmp[:,:,i]))\nlen(pred_rle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(5, figsize=(20, 20))\naxs[0].imshow(cv2.resize(plt.imread(path + 'test_images/' + testfiles[0]),(525, 350)))\nfor i in range(4):\n    axs[i+1].imshow(rle2mask(pred_rle[i], img.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv( path + 'sample_submission.csv', converters={'EncodedPixels': lambda e: ' '} )\nsub['EncodedPixels'] = pred_rle\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}