{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head ../input/train_ship_segmentations_v2.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\n# from fastai.torch_imports import *\n# from fastai.transforms import *\n# from fastai.conv_learner import *\n# from fastai.model import *\n# from fastai.dataset import *\n# from fastai.sgdr import *\n# from fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels = pd.read_csv('../input/train_ship_segmentations_v2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### No of times image is repeating in the labels csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = df_labels.ImageId\nidx.value_counts()[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels['hasShip'] = df_labels['EncodedPixels'].apply(lambda x: 0 if pd.isnull(x) else 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Add a column with counts of no of times ImageId is repeating\ndf_labels['count'] = df_labels.groupby('ImageId')['ImageId'].transform('count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make count 0 if no ship present. For count more than 1 multiple ships would be present\ndf_labels.loc[df_labels['hasShip'] == 0, 'count'] = 0 \n# df1.loc[df1['stream'] == 2, 'feat'] = 10\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels['count'][:15]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_c = df_labels[['ImageId','count']]\ndf_c.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_c = df_c.drop_duplicates().reset_index()\ndf_c.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.loc[:5, 'EncodedPixels']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Select 20000 images from training dataset for be used initially"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ls = df_c.sample(n=10000, replace = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ls.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_imgs = os.listdir(\"../input/test_v2\")\ntrain_imgs = os.listdir('../input/train_v2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_imgs[:5], train_imgs[-5:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/train_v2/'\ntest_path = '../input/test_v2/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nfor i in range(9):\n    image = mpimg.imread(train_path+train_imgs[i])\n    imgs.append(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_imgs), len(test_imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs_s = [train_imgs[i] for i in df_ls.index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axs = plt.subplots(3, 3, figsize=(18, 18))\naxs = axs.flatten()\nfor img, ax in zip(imgs, axs):\n    ax.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bs = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input')\npath","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_img = path/'train_v2'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fnames = get_image_files(path_img)\n# fnames[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ls.to_csv('labels.csv', columns = ['ImageId', 'count'], index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ls.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head labels.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = ImageDataBunch.from_csv(\"\", '../input/train_v2', valid_pct = 0.2, size=128, delimiter=',',\n    ds_tfms=get_transforms(flip_vert=True, max_lighting=0.1, max_warp=0.)).normalize(imagenet_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data = ImageDataBunch.from_df(path, df_ls, 'train_v2', valid_pct = 0.2, label_col='hasShip', size=128,\n#     ds_tfms=get_transforms(flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.)).normalize(imagenet_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(rows=3, figsize=(10,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.train_ds, data.valid_ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(data.classes)\nlen(data.classes),data.c","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training: resnet34"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data, models.resnet34, metrics=accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(2, max_lr=slice(1e-5,1e-3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('stage-10K')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls models/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('stage-10K')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = learn.data.valid_ds[2][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data.valid_ds.items[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = learn.data.valid_ds[0]\nimg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = {}\nfor i in range(10):\n    img = learn.data.valid_ds[i][0]\n    fname =learn.data.valid_ds.items[i]\n    imgs[fname] = img\nimgs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = {}\nfor name,img in imgs.items():\n    r = learn.predict(img)\n    result[name] = r","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name,res in result.items():\n    print(name, \":\", res)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"   ## Test some real images from test set**"},{"metadata":{"trusted":true},"cell_type":"code","source":"im = open_image(test_path+test_imgs[3])\ntest_imgs[3] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.predict(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_r = {}\nfor i in range(9):\n    image = open_image(test_path+test_imgs[i+10])\n    r = learn.predict(image)\n    test_r[test_imgs[i+10]]=r","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name,r in test_r.items():\n    print(name, \":\", r)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im=open_image('../input/test_v2/d7ad50e7b.jpg')\nim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nfor i in range(9):\n    image = mpimg.imread(test_path+test_imgs[i+10])\n    imgs.append(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axs = plt.subplots(3, 3, figsize=(18, 18))\naxs = axs.flatten()\nfor img, ax in zip(imgs, axs):\n    ax.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}