{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport zipfile\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nsns.set(style=\"darkgrid\")\n\nfrom numpy import array\nfrom numpy import argmax\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom keras.utils import to_categorical\nfrom keras import layers\nfrom keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\nfrom keras.layers import Input, Dense, Activation, BatchNormalization, Flatten, Conv2D\nfrom keras.layers import AveragePooling2D, MaxPooling2D, Dropout\nfrom keras.models import Model\n\nfrom keras import backend as K\nK.tensorflow_backend._get_available_gpus()\nfrom keras.models import Sequential\n\nprint(os.listdir(\"../input\"))\nimport warnings\nwarnings.simplefilter(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"## Taking a look at the beautiful pictures\npath = '../input/train/'\npath_test = '../input/test/'\nfig = plt.figure(figsize = (80, 30))\n\nfor num, items in enumerate(os.listdir(path)[1500:1510]):\n    y = fig.add_subplot(2, 5, num + 1)\n    img = cv2.imread(path + items)\n    new_img = cv2.resize(img, (128, 128), interpolation = cv2.INTER_AREA)\n    y.imshow(new_img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86b4a8853c597889f92a4a37c118c968a4ee585e"},"cell_type":"code","source":"print(\"Number of images in the train set: \", len([iq for iq in os.scandir(path)]))\nprint(\"Number of images in the test set: \", len([iq for iq in os.scandir(path_test)]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ec55e29a95f290f2fd3855852253de0ff6e4f88"},"cell_type":"code","source":"## Importing the data frame\ndf = pd.read_csv(\"../input/train.csv\")\nprint(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd753a0336e077976987cb0f9abdf2cfbab975ce"},"cell_type":"code","source":"df.head().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f3c4f940843dece3cc3ba804edf382f0e7fcb82"},"cell_type":"code","source":"## Since, a test dataframe is not given... Using the image names from the test set will do\ntest_df = pd.DataFrame(columns = {\"Image\", \"Id\"})\n\nfor image_id in os.listdir(path_test):\n    test_df = test_df.append({\"Image\" : image_id}, ignore_index = True)\n\nprint(test_df.shape) ## Make sure that the size of the dataframe is same as the size of the pictures in the directory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64140dc933fa788c5cbd49b0b9ae7fa0033046a8"},"cell_type":"code","source":"test_df.head().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18e1b464afef0097210eaae1c30a43f520ae3282"},"cell_type":"code","source":"df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9969de5ffdbb58bce11eaf50b97df379f10c4588"},"cell_type":"code","source":"## Shuffling up the dataset using pandas sample function\nn = 5 ## Set this number for shuffling the dataset these many number of times\nfor _ in range(n):\n    df = df.sample(frac = 1)\n\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77e9ee7d8a9062848185a844718b110d1390be8a"},"cell_type":"code","source":"## Checking out the top 10 frequently occuring whale Ids\nwhales = df.groupby('Id')['Image'].nunique()\nwhales.sort_values(ascending=False)[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f92a0cdb7aac839156ac5d30be172bc3c51ec347"},"cell_type":"code","source":"## Looking at these images with the top 10 Id\nw_id = ['new_whale', 'w_23a388d', 'w_9b5109b', 'w_9c506f6', 'w_0369a5c',\n        'w_700ebb4', 'w_3de579a', 'w_564a34b', 'w_fd3e556', 'w_88e4537']\nfig = plt.figure(figsize = (80, 30))\n\nfor num, ids in enumerate(w_id):\n    img_name = df.loc[df[\"Id\"] == ids]['Image'].tolist() ## Getting the images names from the dataframe\n    y = fig.add_subplot(2, 5, num + 1)\n    img = cv2.imread(path + img_name[3]) ## Read only the first type of each id\n    new_img = cv2.resize(img, (128, 128), interpolation = cv2.INTER_AREA)\n    y.imshow(new_img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54bce1caa4783c60b28196c712b77850ca3404a6"},"cell_type":"code","source":"## All we bascially need are the Fins of the whales, not the background of the images i.e. the water\n## Doing some edge detection to pick out the whale fins only\nfig = plt.figure(figsize = (80, 30))\n\nfor num, ids in enumerate(w_id):\n    img_name = df.loc[df[\"Id\"] == ids]['Image'].tolist() ## Getting the images names from the dataframe\n    y = fig.add_subplot(2, 5, num + 1)\n    img = cv2.imread(path + img_name[3]) ## Read only the first type of each id\n    new_img = cv2.resize(img, (128, 128), interpolation = cv2.INTER_AREA)\n    \n    # Blurring the image\n    median = cv2.medianBlur(new_img, 5)\n    # Applying canny edge detection\n    canny = cv2.Canny(median, 150, 150)\n    y.imshow(canny)\n\n## There are various other ways to do this -- but this is Baseline\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94513f719f7b252c6e41657dd189689c50de2b15"},"cell_type":"code","source":"## Converting the training images into numpy arrays\ndef convert_images(df):\n    \n    # Parameters for our images\n    number_of_images = df.shape[0]\n    IMAGE_WIDTH = 128\n    IMAGE_HEIGHT = 128\n    N_CHANNELS = 1\n    \n    # Creating an empty array of size 25361 x 128 x 128 x 3\n    dataset = np.ndarray((number_of_images, IMAGE_HEIGHT, IMAGE_WIDTH), dtype = np.float32)\n    print(\"Size of dataset: \", dataset.shape)\n    \n    i = 0\n    # Iterate through the images and convert them to arrays\n    for image_names, df_items in zip(os.listdir(path), df.itertuples()):\n        img = cv2.imread(path + df_items[1], 1)\n        new_img = cv2.resize(img, (128, 128), interpolation = cv2.INTER_AREA)\n        # Blurring the image\n        median = cv2.medianBlur(new_img, 5)\n        # Applying canny edge detection\n        canny = cv2.Canny(median, 150, 150)\n        dataset[i] = canny\n        i += 1\n        if i % 2500 == 0: # Print all along the way\n            print(\"Size of the last image was: \", dataset.shape, \"\")\n            print(\"{} number of images are converted\\n\".format(i))\n            \n    print(\"--- All images are converted ---\\nSize of the dataset is: \", dataset.shape)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed96a329a2891fa2fdc699cbe93ba7b0c33e9e8f","scrolled":true},"cell_type":"code","source":"dataset = convert_images(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac91fee7395f6e54411adfd91bf6c8679ccae312"},"cell_type":"code","source":"## Getting our labels\nY = df[\"Id\"].values\nprint(Y.shape)\n\n## Need to one-hot encode the targets -- from \"machinelearningmastery\"\ndata = array(Y)\n# integer encode\nlabel_encoder = LabelEncoder()\ninteger_encoded = label_encoder.fit_transform(data)\n# binary encode\nonehot_encoder = OneHotEncoder(sparse=False)\ninteger_encoded = integer_encoded.reshape(len(integer_encoded), 1)\nonehot_encoded = onehot_encoder.fit_transform(integer_encoded)\nprint(onehot_encoded)\n\n# invert first example\ninverted = label_encoder.inverse_transform([argmax(onehot_encoded[0, :])])\nprint(inverted)\n\nY = onehot_encoded\nprint(\"\\nTarget variable size: \", Y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26977f49767323d7ffe7b36ad10015d86f8343cc"},"cell_type":"code","source":"## Converting the dataset to a range btw 0 - 1\nfinal_dataset = dataset / 255 # Since, the values in the pixels range from 0 - 255\nfinal_dataset = final_dataset.reshape(df.shape[0], 128, 128, 1)\nprint(final_dataset.shape) ## All three channels of the first pixel of the first picture","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04a4aa0dea7f3636b5a21839b598a17d77ab4068"},"cell_type":"code","source":"# define the CNN model\ndef cnn_model():\n    # create model\n    model = Sequential()\n    \n    model.add(Conv2D(64, (5, 5), input_shape=(128, 128, 1), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(Conv2D(32, (3, 3), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(Dropout(0.2))\n    model.add(Flatten())\n    \n    model.add(Dense(500, activation='relu'))\n    model.add(Dense(100, activation='relu'))\n    model.add(Dense(Y.shape[1], activation='softmax'))\n    \n    # Compile model\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"620046c507d51adfd1bb02d0658f43a7e4a1c660"},"cell_type":"code","source":"model = cnn_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6905c8115a0b0ff3138ae29ce9cac7e13bd2e32"},"cell_type":"code","source":"# %%time\n# ## Fitting the model\n# history = model.fit(final_dataset, Y, epochs = 10, batch_size = 1024, verbose = 1)\n# ## Getting the accuracy scores\n# scores = model.evaluate(final_dataset, Y, verbose=0)\n# print('Final CNN accuracy: ', scores[1]*100, \"%\")\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73e00c674e56a486ff2a5b1bf77c3032a170df9d"},"cell_type":"code","source":"# # Plotting the history of accuracy \n# plt.plot(history.history['acc'])\n# plt.title('Model Accuracy')\n# plt.ylabel('Accuracy')\n# plt.xlabel('Epoch')\n# plt.legend(['Train'], loc = 'upper left')\n# plt.show()\n# # Summarizing the history of loss\n# plt.plot(history.history['loss'])\n# plt.title('Model Loss')\n# plt.xlabel('Epoch')\n# plt.ylabel('Loss')\n# plt.legend(['Train'], loc = 'upper left')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5a79f1f058ca8421f39a98f45fdc3215607d732"},"cell_type":"code","source":"# test_dataset = convert_images(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abdc31f0f6dd3a5dbb2bb13a3a1e932e48523eb8"},"cell_type":"code","source":"# ## Converting the dataset to a range btw 0 - 1\n# test_final_dataset = test_dataset / 255 # Since, the values in the pixels range from 0 - 255\n# test_final_dataset = test_final_dataset.reshape(df.shape[0], 128, 128, 1)\n# print(test_final_dataset.shape) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"119967fea2bb285d62e8d4f582c681cd0f87b2e0"},"cell_type":"code","source":"# predictions = model.predict(test_final_dataset, verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"230d083a703913218431c95f69b849a63a648ff2"},"cell_type":"code","source":"# for i, pred in enumerate(predictions):\n#     test_df.loc[i, 'Id'] = ' '.join(label_encoder.inverse_transform(pred.argsort()[-5:][::-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d359f4440c651e24b506e9bb536332922417c37a"},"cell_type":"code","source":"# test_df.head(10)\n# # test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2f80430f19da541401bc8e9c6b5704b5c917fad"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}