{"cells":[{"metadata":{"_uuid":"799553e03d37b0f2885c3b038e27fa2a2d757139"},"cell_type":"markdown","source":"Content:\n* [Introduction](#1):\n* [Import Data Set](#2):\n* [Preparing Images](#3):\n* [Normalize the Data](#4):\n* [Label Encoding](#5):\n* [Implementing with Keras](#6):\n* [Set the Optimizer and Annealer](#7):\n* [Compile Model](#8):\n* [Data Augmentation](#9):\n* [Epochs and Batch Size](#10):\n* [Fit the Model](#11):\n* [Evaluate the model](#12):\n* [Predict Test Data](#13):\n* [Conclusion](#14):"},{"metadata":{"_uuid":"1f812f64883f3a5642475c1b6837a3a3511c0b4d"},"cell_type":"markdown","source":"<a id=\"1\"></a> <br>\n# INTRODUCTION\n* In this kernel, we will be working on Humpback Whale Identification Dataset (Implementing with Keras)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# import warnings\nimport warnings\n# filter warnings\nwarnings.filterwarnings('ignore')\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3227f393967e2965014b3933f2bbecccb6e0fce1"},"cell_type":"markdown","source":"<a id=\"2\"></a> <br>\n## Import Dataset"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0770f23781b0dcef8257913bb0f397effb66a91c"},"cell_type":"code","source":"# Display the content of data\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e2106d6437cc74dd09ae02151ecd707d5efc9b1"},"cell_type":"code","source":"# shape gives number of rows and columns in a tuple\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"618649281910fdcf244eb586541f86de6992032b"},"cell_type":"code","source":"train.Id.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90f4532c68fbc36e95e26089bbdca7e5a56ff1c6"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eccf185c560ae28fe9930a6f0d1f822a0b1ec2b4"},"cell_type":"code","source":"train.tail(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff4d2efdc663d7600e8eadac2da440ba74238887"},"cell_type":"code","source":"# put labels into y_train variable\ny_train = train[\"Id\"]\n# Drop 'Id' column\nX_train = train.drop(labels = [\"Id\"], axis = 1)\ny_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06b4b3f9846db404938092d1f907107f4ed06530"},"cell_type":"code","source":"# Indicates sum of values in our data\ntrain.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77bd220beea4b3e258faf54573ff0d1b82565d3e"},"cell_type":"markdown","source":"<a id=\"3\"></a> <br>\n## Preparing Images"},{"metadata":{"trusted":true,"_uuid":"a442900077066c404b5071835f279a3b33b9de00"},"cell_type":"code","source":"from keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\n\ndef prepareImages(train, shape, path):\n    \n    x_train = np.zeros((shape, 100, 100, 3))\n    count = 0\n    \n    for fig in train['Image']:\n        \n        #load images into images of size 100x100x3\n        img = image.load_img(\"../input/\"+path+\"/\"+fig, target_size=(100, 100, 3))\n        x = image.img_to_array(img)\n        x = preprocess_input(x)\n\n        x_train[count] = x\n        if (count%500 == 0):\n            print(\"Processing image: \", count+1, \", \", fig)\n        count += 1\n    \n    return x_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f4dcb4353af7c7fb2aba26405d332b04f83a05e"},"cell_type":"code","source":"x_train = prepareImages(train, train.shape[0], \"train\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b7332cb21da0e745c02f6bd4e6ef079295f5b0a"},"cell_type":"markdown","source":"<a id=\"4\"></a> <br>\n### Normalize the Data"},{"metadata":{"trusted":true,"_uuid":"1ddb7426fe0a41f744e0b5a6735d9a33ca2a1723"},"cell_type":"code","source":"x_train = x_train / 255.0\nprint(\"x_train shape: \",x_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ed0b487e49ac03044aa34aa62266096729835fd"},"cell_type":"markdown","source":"#### Let's look at some samples"},{"metadata":{"trusted":true,"_uuid":"ce467f0044244cba0c01e5f81c78b5d4979af587"},"cell_type":"code","source":"# Some examples(first one)\nplt.imshow(x_train[0][:,:,0], cmap=\"gray\")\nplt.title(plt.title(train.iloc[0,0]))\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c92338fab9d194d2d5c7be467bc44753f7d7a055"},"cell_type":"code","source":"# Some examples(last one)\nplt.imshow(x_train[25360][:,:,0], cmap=\"gray\")\nplt.title(plt.title(train.iloc[25360,0]))\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c26b6aa5d6747098a1c3369975ab3dfe8db02df"},"cell_type":"code","source":"# Some examples(55th)\nplt.imshow(x_train[55][:,:,0], cmap=\"gray\")\nplt.title(plt.title(train.iloc[55,0]))\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d872cce63eff66c135a1adc50de7db4e97f11ab6"},"cell_type":"markdown","source":"<a id=\"5\"></a> <br>\n## Label Encoding"},{"metadata":{"trusted":true,"_uuid":"5aac3699095d894fac19c33f98da1110141bb9aa"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlabel_encoder = LabelEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b58de925d4f6114baef3425c71603b67448405ab"},"cell_type":"code","source":"y_train = label_encoder.fit_transform(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3c34e79bb48a484aeb647536e4f63d715b814d4"},"cell_type":"code","source":"#let's look at first 10 values\ny_train[0:10]  # => new_whale :)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b007e54b71b9f40f8faaaf8b71d4fc3ff15a4dc"},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61f567210c5ded0fe1025885a10d40fd7abeafb8"},"cell_type":"code","source":"# convert to one-hot-encoding(one hot vectors)\n# we have 5005 class look at from=> train.Id.describe()\n\nfrom keras.utils.np_utils import to_categorical\ny_train = to_categorical(y_train, num_classes = 5005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"762a2b56e410e0dd7a335bbf67d92995b2cd3608"},"cell_type":"code","source":"#converted\nprint(y_train.shape)\ny_train #let's look at vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"016f0d9eac4aa69330a765f5e4843bc21059533a"},"cell_type":"markdown","source":"<a id=\"6\"></a> <br>\n## Implementing with Keras"},{"metadata":{"trusted":true,"_uuid":"8dd65ba957c4cd043e4bf071051d963f811288b6"},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential # to create a cnn model\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 16, kernel_size = (5,5), padding = 'Same', activation = 'relu', input_shape = (100,100,3)))\nmodel.add(Conv2D(filters = 16, kernel_size = (5,5), padding = 'Same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 32, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(Conv2D(filters = 32, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n# fully connected\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = 'relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dense(y_train.shape[1], activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12e36e7c1ea90b61eb9a99998ae5942b023d61d9"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8dfb7daae609b8f09fab030fe98b7e8f51a1122"},"cell_type":"markdown","source":"<a id=\"7\"></a> <br>\n### Set the Optimizer and Annealer\n* Adam optimizer: Changes the learning rate during training"},{"metadata":{"trusted":true,"_uuid":"4fbaa0d38c495cd22b83e8b94a0a655ffecac40f"},"cell_type":"code","source":"# Define the optimizer\noptimizer = Adam(lr = 0.001, beta_1 = 0.9, beta_2 = 0.999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec0d00d397c47309f3c9c12ed49fd7c496e42746"},"cell_type":"code","source":"# # Define the optimizer\n# optimizer = RMSprop(lr = 0.001, rho=0.9, epsilon=1e-08, decay=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"feceef37bd38232686c76b10bdc227a8b39585c3"},"cell_type":"code","source":"# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6438bc89eeff87b7eb09635e7f41640bbc4d8725"},"cell_type":"markdown","source":"<a id=\"8\"></a> <br>\n### Compile Model"},{"metadata":{"trusted":true,"_uuid":"3b8429bdd886e0b2e0ea30a7c02a2dfa14579ce0"},"cell_type":"code","source":"model.compile(optimizer = optimizer, loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"014d05813744d9482e475017b583f6d145d067f1"},"cell_type":"markdown","source":"<a id=\"9\"></a>\n### Data Augmentation"},{"metadata":{"trusted":true,"_uuid":"a2fb62374ab5aa532e000e5b939dd3dc0e94c79d"},"cell_type":"code","source":"# if you want to use Data Augmentation,Activate it.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e0478d11ccbb4f3fcfd59d479e6e9543cac3f78"},"cell_type":"code","source":"# # With data augmentation to prevent overfitting\n\n# datagen = ImageDataGenerator(\n#         featurewise_center=False,  # set input mean to 0 over the dataset\n#         samplewise_center=False,  # set each sample mean to 0\n#         featurewise_std_normalization=False,  # divide inputs by std of the dataset\n#         samplewise_std_normalization=False,  # divide each input by its std\n#         zca_whitening=False,  # apply ZCA whitening\n#         rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n#         zoom_range = 0.1, # Randomly zoom image \n#         width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n#         height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n#         horizontal_flip=False,  # randomly flip images\n#         vertical_flip=False)  # randomly flip images\n\n\n# datagen.fit(x_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21e9f1aed8030797edf25d1e42a4194b505d9da6"},"cell_type":"markdown","source":"For the data augmentation, i choosed to :\n\n* ** Randomly rotate some training images by 10 degrees **\n* ** Randomly zoom by 10% some training images **\n* ** Randomly shift images horizontally by 10% of the width **\n* ** Randomly shift images vertically by 10% of the height **\n"},{"metadata":{"_uuid":"c6e2f551dcc95ac144cc2d538ca927077c2f9aaa"},"cell_type":"markdown","source":"<a id=\"10\"></a> <br>\n### Epochs and Batch Size"},{"metadata":{"trusted":true,"_uuid":"f9d0869df19ed300b65b25773b60ca54b8ade85e"},"cell_type":"code","source":"epochs = 100  # for better result increase the epochs\nbatch_size = 1000","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ada634fd21bcbf498d116e8418e6b5dc0dee1807"},"cell_type":"markdown","source":"<a id=\"11\"></a>\n### Fit the Model"},{"metadata":{"trusted":true,"_uuid":"14453f2298e27adca1ed2e81eb98203e1b667547"},"cell_type":"code","source":"#if you don't want to use data augmentation ,Use this code.\nhistory = model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=2, callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8663eda063fc209fc6e379efb7b5fa4f794117a3"},"cell_type":"markdown","source":"### We get good accuracy, but we need to play with the hyperparameters to get better results from the submission result."},{"metadata":{"_uuid":"19cb368c8f57b89071187c6b6d92c31213647eff"},"cell_type":"markdown","source":"### If you want to use data augmentation,Enable below code and disable above code."},{"metadata":{"trusted":true,"_uuid":"85eba6bdd49a4e5b5adf9a9223e8647786e5ff05"},"cell_type":"code","source":"# history = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n#                               epochs=100, verbose = 2, \n#                               steps_per_epoch=x_train.shape[0] // batch_size,\n#                               callbacks=[learning_rate_reduction]) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b69029d6ca96952a1d6aad9012b053c54451feb"},"cell_type":"markdown","source":"** * we get good accuracy, but we need to play with the hyperparameters to get better results from the submission result. **"},{"metadata":{"trusted":true,"_uuid":"2f785574a77ffe3d41a28c8aa93dc3999eb0b832"},"cell_type":"markdown","source":"<a id=\"12\"></a> <br>\n### Evaluate the model\n* Validation and Loss visualization"},{"metadata":{"trusted":true,"_uuid":"a718f81dfc0e166d7b72c4cbf6197f0f0c21c58f"},"cell_type":"code","source":"# Plot the loss curve for training\nplt.plot(history.history['loss'], color='r', label=\"Train Loss\")\nplt.title(\"Train Loss\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f679a8904e0e794ab64edca460677242a2a2ef4"},"cell_type":"code","source":"# Plot the accuracy curve for training\nplt.plot(history.history['acc'], color='g', label=\"Train Accuracy\")\nplt.title(\"Train Accuracy\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"640a32e140eef785fe43f4fe60d1f0cc125c348c"},"cell_type":"code","source":"print('Train accuracy of the model: ',history.history['acc'][-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59ec14bf8736fce5a5e5f00c5fdb933947bdafc2"},"cell_type":"code","source":"print('Train loss of the model: ',history.history['loss'][-1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"df243245816ba5e9ec7f977d6934066ecfef6558"},"cell_type":"markdown","source":"<a id=\"13\"></a> <br>\n## Predict Test Data"},{"metadata":{"trusted":true,"_uuid":"dd2d1bee2d60ea43d51c0e871538217007d199c7"},"cell_type":"code","source":"test = os.listdir(\"../input/test/\")\nprint(len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb4dde2ad5c27d0c07e23b9dc2826cac78b19c95"},"cell_type":"code","source":"col = ['Image']\ntest_data = pd.DataFrame(test, columns=col)\ntest_data['Id'] = ''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d0e517b48e53d38e2efc3882ffddd836e11b5f6"},"cell_type":"code","source":"x_test = prepareImages(test_data, test_data.shape[0], \"test\")\nx_test /= 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3766a5e2ad087421b4cb1507112fa14a121075b1"},"cell_type":"code","source":"predictions = model.predict(np.array(x_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4793a39bf94081f27769a7e90c41fc0e9f7b1e1c"},"cell_type":"code","source":"for i, pred in enumerate(predictions):\n    test_data.loc[i, 'Id'] = ' '.join(label_encoder.inverse_transform(pred.argsort()[-5:][::-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15f00fa51743207a8f2af7d7b3e06c623450edba"},"cell_type":"code","source":"test_data.head(10)\ntest_data.to_csv('submission_3.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d332d71efe42a0c82b328cbed7e0e88bc20bdd52"},"cell_type":"markdown","source":"## To be continued..."},{"metadata":{"_uuid":"74631d7f96b42d6abf0c17c7d778460535668b42"},"cell_type":"markdown","source":"<a id=\"14\"></a> <br>\n# Conclusion\n* If you like it, please upvote.\n* If you have any question, I will be appreciate to hear it."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}