{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom skimage.io import imread\nfrom skimage.morphology import label\nfrom matplotlib import pyplot as plt\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.optimizers import *\nimport random\n\nDATA_PATH   = '../input/airbus-ship-detection/'\nTRAIN_PATH  = DATA_PATH+'train_v2/'\nTEST_PATH   = DATA_PATH+'test_v2/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(DATA_PATH+'train_ship_segmentations_v2.csv')\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(['Ships', 'No Ships'], \n    [len(df[~df.EncodedPixels.isna()].ImageId.unique()),\n    len(df[df.EncodedPixels.isna()].ImageId.unique())]);\nplt.ylabel('Number of Images');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df['ImageId'] != '6384c3e78.jpg']\ndf.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def area_isnull(x):\n    if x == x:\n        return 0\n    else:\n        return 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['isnan'] = df['EncodedPixels'].apply(area_isnull)\ndf['isnan'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.sort_values('isnan', ascending=False)\ndf = df.iloc[100000:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_to_mask(rle_list, SHAPE):\n    tmp_flat = np.zeros(SHAPE[0]*SHAPE[1])\n    if len(rle_list) == 1:\n        mask = np.reshape(tmp_flat, SHAPE).T\n    else:\n        strt = rle_list[::2]\n        length = rle_list[1::2]\n        for i,v in zip(strt,length):\n            tmp_flat[(int(i)-1):(int(i)-1)+int(v)] = 255\n        mask = np.reshape(tmp_flat, SHAPE).T\n    return mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_area_for_rle(rle_str):\n    rle_list = [int(x) if x.isdigit() else x for x in str(rle_str).split()]\n    if len(rle_list) == 1:\n        return 0\n    else:\n        area = np.sum(rle_list[1::2])\n        return area","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['area'] = df['EncodedPixels'].apply(calc_area_for_rle)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_isship = df[df['area'] > 0]\ndf_isship.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_smallarea = df_isship['area'][df_isship['area'] < 10]\ndf_smallarea.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_smallarea.shape[0]/df_isship.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gp = df.groupby('ImageId').sum()\ngp = gp.reset_index()\ngp.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_class(area):\n    area = area / (768*768)\n    if area == 0:\n        return 0\n    elif area < 0.005:\n        return 1\n    elif area < 0.015:\n        return 2\n    elif area < 0.025:\n        return 3\n    elif area < 0.035:\n        return 4\n    elif area < 0.045:\n        return 5\n    else:\n        return 6","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gp['class'] = gp['area'].apply(calc_class)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gp['class'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, val = train_test_split(gp, test_size=0.01, stratify=gp['class'].tolist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_isship_list = train['ImageId'][train['isnan']==0].tolist()\ntrain_isship_list = random.sample(train_isship_list, len(train_isship_list))\ntrain_nanship_list = train['ImageId'][train['isnan']==1].tolist()\ntrain_nanship_list = random.sample(train_nanship_list, len(train_nanship_list))\n\nval_isship_list = val['ImageId'][val['isnan']==0].tolist()\nval_nanship_list = val['ImageId'][val['isnan']==1].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_isship_list),len(train_nanship_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ndg_args = dict(featurewise_center = False, \n                  samplewise_center = False,\n                  rotation_range = 15, \n                  width_shift_range = 0.1, \n                  height_shift_range = 0.1, \n                  shear_range = 0.01,\n                  zoom_range = [0.9, 1.25],  \n                  horizontal_flip = True, \n                  vertical_flip = True,\n                  brightness_range = [0.7, 1.3],\n                  fill_mode = 'reflect',\n                   data_format = 'channels_last')\n\nimage_gen = ImageDataGenerator(**dg_args)\nlabel_gen = ImageDataGenerator(**dg_args)\n\n\ndef mygenerator(isship_list, nanship_list, batch_size, cap_num):\n    train_img_names_nanship = nanship_list[:cap_num]\n    train_img_names_isship = isship_list[:cap_num]\n    k = 0\n    while True:\n        if k+batch_size//2 >= cap_num:\n            k = 0\n        batch_img_names_nan = train_img_names_nanship[k:k+batch_size//2]\n        batch_img_names_is = train_img_names_isship[k:k+batch_size//2]\n        batch_img = []\n        batch_mask = []\n        for name in batch_img_names_nan:\n            tmp_img = imread(TRAIN_PATH + name)\n            batch_img.append(tmp_img)\n            batch_mask.append(0)\n        for name in batch_img_names_is:\n            tmp_img = imread(TRAIN_PATH + name)\n            batch_img.append(tmp_img)\n            batch_mask.append(1)\n        img = np.stack(batch_img, axis=0)\n        mask = np.stack(batch_mask, axis=0)\n\n        g_x = image_gen.flow(img, mask,\n                             batch_size = img.shape[0], \n                             shuffle=True,\n                             seed=None)\n        \n        \n       \n        \n        imgaug, maskaug = next(g_x)\n        \n        k += batch_size//2\n        \n        yield imgaug/ 255.0, maskaug","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 4\nCAP_NUM = min(len(train_isship_list),len(train_nanship_list))\ndatagen = mygenerator(train_isship_list, train_nanship_list, batch_size=BATCH_SIZE, cap_num=CAP_NUM)\nvalgen = mygenerator(val_isship_list, val_nanship_list, batch_size=50, cap_num=CAP_NUM)\n\n\nnumvalimages = 100\nval_x, val_y = next(valgen)\nval_y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = Input(shape=(768,768,3))\n\nc1 = Conv2D(32, (3, 3), activation='relu', padding='same') (inputs)\np1 = MaxPooling2D((2, 2)) (c1)  # 384x384\n\nc2 = Conv2D(64, (3, 3), activation='relu', padding='same') (p1)\np2 = MaxPooling2D((2, 2)) (c2)  # 192x192\n\nc3 = Conv2D(128, (3, 3), activation='relu', padding='same') (p2)\np3 = MaxPooling2D((2, 2)) (c3)  # 96x96\n\nc4 = Conv2D(256, (3, 3), activation='relu', padding='same') (p3)\np4 = MaxPooling2D(pool_size=(2, 2)) (c4)  # 48x48\n\nc5 = Conv2D(512, (3, 3), activation='relu', padding='same') (p4)\np5 = MaxPooling2D(pool_size=(2, 2)) (c5) # 24x24\n\nc6 = Conv2D(512, (3, 3), activation='relu', padding='same') (p5)\np6 = MaxPooling2D(pool_size=(2, 2)) (c6) # 12x12\n\nc7 = Conv2D(512, (3, 3), activation='relu', padding='same') (p6)\np7 = MaxPooling2D(pool_size=(2, 2)) (c7) # 6x6\n\nflatp7 = Flatten() (p6)\nd1 = Dense(128, activation='relu') (flatp7)\n#d2 = Dropout(0.2)(d1)\n\nd3 = Dense(1, activation='sigmoid') (d1)\n\nmodel = Model(inputs=[inputs], outputs=[d3])\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.optimizers import Adam\nopt=Adam(1e-4, decay=0.0)\nmodel.compile(loss='binary_crossentropy',optimizer=opt,metrics=['acc'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=20\nhistory=model.fit_generator(datagen,\n                            steps_per_epoch=100,\n                            epochs=epochs,\n                            validation_data=(val_x, val_y),\n                            verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(2, 1, figsize=(15,15))\n\naxs[0].plot(history.history['loss'])\naxs[0].plot(history.history['val_loss'])\naxs[0].title.set_text('Training Loss vs Validation Loss')\naxs[0].legend(['Train', 'Validation'])\n\naxs[1].plot(history.history['acc'])\naxs[1].plot(history.history['val_acc'])\naxs[1].title.set_text('Training Accuracy vs Validation Accuracy')\naxs[1].legend(['Train', 'Validation'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('seg_model_ship_classifier.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_list = val['ImageId'].tolist()\ntrain_list = train['ImageId'].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_data(image_list):\n    batch_img = []\n    batch_mask = []\n    for name in image_list:\n        tmp_img = imread(TRAIN_PATH + name)\n        batch_img.append(tmp_img)\n        mask_list = df['EncodedPixels'][df['ImageId'] == name].tolist()\n        one_mask = np.zeros((768, 768, 1))\n        for item in mask_list:\n            rle_list = str(item).split()\n            tmp_mask = rle_to_mask(rle_list, (768, 768))\n            one_mask[:,:,0] += tmp_mask\n        if np.any(one_mask):\n            batch_mask.append(1)\n        else:\n            batch_mask.append(0)\n    img = np.stack(batch_img, axis=0)\n    mask = np.stack(batch_mask, axis=0)\n    img = img / 255.0\n    return img, mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_list = val_list[0:10]\n\nfor i in range(len(image_list)):\n    img = imread(TRAIN_PATH + image_list[i])\n    input_img, gt_mask = create_data([image_list[i]])\n    pred_mask = model.predict(input_img)\n    \n    fig = plt.figure(figsize=(10,10))\n    plt.imshow(img)\n    plt.xlabel(pred_mask)\n    plt.ylabel(gt_mask)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}