{"cells":[{"metadata":{"_uuid":"4b4c6d3178aaddd67174252362c1c0802994242b"},"cell_type":"markdown","source":"## References\n\n    1.https://www.kaggle.com/youhanlee/small-data-many-class-data-augmentation \n    2.https://www.kaggle.com/satian/keras-mobilenet-starter\n    3.https://medium.com/ymedialabs-innovation/data-augmentation-techniques-in-cnn-using-tensorflow-371ae43d5be9\n\n## Overview\n\nWe are going to try to get a good percentage of our predictions using the mobilenet pretrained network, but we will focus our efforts on improving the input data.\nThe goal of this competition is identifying individual whales in images. The train dataset includes 25k images and 5k unique whale ids. In addition, ~10k of images show unique whales ('new_whale' label).\n\nTo improve the dataset we are going to carry out different data augmentation techniques:\n\n- Feature Standardization\n- ZCA Whitening\n- Random Rotations\n- Random Shifts\n- Random Flips"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mplimg\nimport seaborn as sns\nfrom matplotlib.pyplot import imshow\n\nfrom keras.backend import clear_session\n\nfrom keras import applications\nfrom keras import layers\nfrom keras.layers import Flatten, Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D, Dropout  \nfrom keras.models import Sequential, Model, load_model  \nfrom keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\nimport keras.backend as K  \nfrom keras.callbacks import ModelCheckpoint  \nfrom keras.callbacks import EarlyStopping\n\nfrom sklearn.preprocessing import LabelEncoder,OneHotEncoder\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.preprocessing.image import ImageDataGenerator\n\nclear_session()\n\n# Any results you write to the current directory are saved as output.\n\n## funtions\n   \ndef graph_acc_loss(model):\n    \n    sns.set(style = 'darkgrid')\n    plt.figure(figsize = (24, 8))\n    plt.subplot(2, 2, 1)\n    #plt.plot(range(100), model.history['acc'])\n    plt.plot(model.history['acc'],'r')  \n    plt.plot(model.history['val_acc'],'g')  \n    plt.ylabel('TRAINING ACCURACY')\n    plt.title('TRAINING ACCURACY vs EPOCHS')\n    plt.legend(['train','validation'])\n    \n    plt.subplot(2, 2, 2)\n    plt.plot(model.history['loss'],'r')  \n    plt.plot(model.history['val_loss'],'g')  \n    plt.ylabel('TRAINING LOSS')\n    plt.title('TRAINING LOSS vs EPOCHS')\n    plt.legend(['train','validation'])\n    \n    plt.subplot(2, 2, 3)\n    plt.plot(model.history['categorical_accuracy'],'b')  \n    plt.xlabel('EPOCHS')\n    plt.ylabel('TRAINING CATEGORICAL ACCURACY')\n    plt.title('TRAINING CATEGORICAL ACCURACY vs EPOCHS')\n    plt.legend(['categorical_accuracy'])\n    \n    plt.subplot(2, 2, 4)\n    plt.plot(model.history['categorical_crossentropy'],'b')  \n    plt.xlabel('EPOCHS')\n    plt.ylabel('TRAINING CATEGORICAL CROSSENTROPY')\n    plt.title('TRAINING CATEGORICAL CROSSENTROPY vs EPOCHS')\n    plt.legend(['categorical_crossentropy'])\n    \n    \ndef prepare_data(df,width,heigth, channel):\n    n_of_images = df.shape[0]\n    channel = 3\n    # preparing X numpy array with the images content\n    #X = np.zeros((15697,48,48,3))\n    X = np.zeros((n_of_images,width,heigth, channel))\n    count = 0\n    \n    for file in df['Image']:\n        img = image.load_img('../input/train/%s' % file,target_size=(width,heigth, channel))\n        x = image.img_to_array(img)\n        x = preprocess_input(x)\n        \n        X[count] = x\n        \n        if(count%4000==0):\n            print(\"ProcessingImage : \" , count+1,\", \",file)\n        count += 1\n    \n    # preparing Y numpy with de name of files , labelencoded and onehot encoded apply\n    y_encoded = df['Id'].values\n    values = np.array(y_encoded)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n    #print(integer_encoded)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded),1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n    #print(onehot_encoded)\n    y = onehot_encoded\n   \n    # split dataset in 20% validate and rest to train\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)\n    \n    return X_train, X_test, y_train, y_test, integer_encoded\n\ndef prepare_labels(y):\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n    # print(integer_encoded)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n    # print(onehot_encoded)\n\n    y = onehot_encoded\n    # print(y.shape)\n    return y, label_encoder\n\ndef prepareImages(data, m, dataset):\n    print(\"Preparing images\")\n    X_train = np.zeros((m, 96, 96, 3))\n    count = 0\n    \n    for fig in data['Image']:\n        #load images into images of size 100x100x3\n        img = image.load_img(\"../input/\"+dataset+\"/\"+fig, target_size=(96, 96, 3))\n        x = image.img_to_array(img)\n        x = preprocess_input(x)\n\n        X_train[count] = x\n        if (count%500 == 0):\n            print(\"Processing image: \", count+1, \", \", fig)\n        count += 1\n    \n    return X_train","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0eed0b3dd33bcd55ee29c273c7e6d620d12cd2a"},"cell_type":"markdown","source":"## Read the train.csv and explore "},{"metadata":{"_uuid":"f6b93ff1179d4e0cf557e7e8497a15878b45ecd9"},"cell_type":"markdown","source":"first we read the csv file, then using describe(), we can see how many unique types there are."},{"metadata":{"trusted":true,"_uuid":"fe42afdadb698a7b39cadf033ec999edb2249849","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"df_ = pd.read_csv('../input/train.csv', encoding='utf8')\nprint(df_['Id'].describe())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16700904c114306bd72c3b3d50b31103ea5411ee"},"cell_type":"markdown","source":"okay, we have a total of 25361, of which there are 5005 different types, but one of them is \"new_whale\", or what is the same, without cataloguing, so we will create a new dataframe with the unique ids removing it. We are also going to count them and save them in our variable \"number_of_clases\""},{"metadata":{"trusted":true,"_uuid":"8eeb6812cdeab92aefefa1c497c605ec3cb035b2","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"df = df_.loc[df_['Id'] != 'new_whale']\nnumber_of_clases = len(df[\"Id\"].value_counts())\nprint (\"Number of Classes: %s\" % number_of_clases)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0073d33f6d4a4eee9a346068b685149a5206e12f"},"cell_type":"markdown","source":"We created a graph to have a vision of the distribution of the labels."},{"metadata":{"trusted":true,"_uuid":"ad2a33f9e2d5c27bd9703f4a9605623556283cc0","_kg_hide-input":true},"cell_type":"code","source":"train = df\ncounted = train.groupby(\"Id\").count().rename(columns={\"Image\":\"image_count\"})\ncounted.loc[counted[\"image_count\"] > 60,'image_count'] = 60\nplt.figure(figsize=(25,4))\nsns.countplot(data=counted, x=\"image_count\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11d85f7e893ddb1f250cb5ccf3c64abe98bc94da"},"cell_type":"markdown","source":"with the function \"prepare_data\" we create arrays of numpy with the content of the images, also we prepare the and with the name of the files (we make the labelencoder and the onehot encoded to be able to pass numerical values to the neural network). Finally the same function already separates the values for test and for validation."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test, integer_encoded = prepare_data(df,96,96,3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22c19b6cbe748127ccdd9c63bc3cb4c882ec4b1a"},"cell_type":"markdown","source":"## **MobileNet**"},{"metadata":{"_uuid":"f5b00bb92fffe38b5361cf78450ffb5e15e372bd"},"cell_type":"markdown","source":"To start we will train using MobileNet (https://arxiv.org/abs/1704.04861) we will train and see what results it gives us with the data we have, then we will continue training the network but expanding the dataset with the various methods of augmentation discussed above. "},{"metadata":{"trusted":true,"_uuid":"d205e129f6e9244571128aaf0e564916ed029523"},"cell_type":"code","source":"from keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom keras.optimizers import Adam\nfrom keras.applications import MobileNet\nfrom keras.applications.mobilenet import preprocess_input\n\nmodel = MobileNet(input_shape=(96, 96, 3), alpha=1., weights=None, classes=5004)\nmodel.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',\n              metrics=['acc','mse',categorical_crossentropy, categorical_accuracy])\n#model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d845ed47c5a850ebc2b36d4fce5346e77c87c6f2","_kg_hide-output":true},"cell_type":"code","source":"mobilenet = model.fit(x=X_train/255, y=y_train, epochs=600, batch_size=100, verbose=1, validation_data=(X_test/255, y_test), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd67bc5464bcd9cadaa1e32802e18ac1a80d0c56","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#print(\"Baseline Error: %.2f%%\" % (100-scores[4]*100)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38bdbac6dd756bbb2fcd9252caa303e245cbcab2"},"cell_type":"code","source":"graph_acc_loss(mobilenet)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"23aaf016c7a7adf5dde0288b70fcc27176862034"},"cell_type":"code","source":"##Save partly trained model \nmodel.save('00_mobilenet_trained.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9166c0c2441d9a7dbe4db65a8504208e2e31d59"},"cell_type":"markdown","source":"## ** AUGMENTATION**\nhttps://machinelearningmastery.com/image-augmentation-deep-learning-keras/"},{"metadata":{"trusted":true,"_uuid":"36262b8bbc4c5c12bb602b751ff32e0f9b953a76"},"cell_type":"code","source":"# Feature Standardization\ndef augmentation_feature_standardization(X_train):\n    X_train_clone = X_train\n    datagen = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True)\n    datagen.fit(X_train_clone)\n    return X_train_clone\n\n# Random Flips\ndef augmentation_random_flips(X_train):\n    X_train_clone = X_train\n    datagen = ImageDataGenerator(horizontal_flip=True, vertical_flip=True)\n    datagen.fit(X_train_clone)\n    return X_train_clone\n\n# Random Rotations\ndef augmentation_random_rotations(X_train):\n    X_train_clone = X_train\n    datagen = ImageDataGenerator(rotation_range=90)    \n    datagen.fit(X_train_clone)\n    return X_train_clone\n\n# Random shifts\ndef augmentation_random_shifts(X_train):\n    X_train_clone = X_train\n    datagen = ImageDataGenerator(width_shift_range=shift, height_shift_range=shift)    \n    datagen.fit(X_train_clone)\n    return X_train_clone\n\n# ZCA whitening\ndef augmentation_random_zca(X_train):\n    X_train_clone = X_train\n    datagen = ImageDataGenerator(zca_whitening=True)   \n    datagen.fit(X_train_clone)\n    return X_train_clone","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0a415186ca8fc82170dea2dee6c734005cbf2b0","_kg_hide-output":true},"cell_type":"code","source":"X_train_featureStandarization = augmentation_feature_standardization(X_train)\nmobilenet1 = model.fit(x=X_train_featureStandarization/255, y=y_train, epochs=50, batch_size=100, verbose=1, validation_data=(X_test/255, y_test), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"ac2ebc94c21a6c6faa02793e9d2941e113766fa9"},"cell_type":"code","source":"##Save partly trained model \nmodel.save('01_mobilenet_featuredStandarization_trained.h5')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"956ce1c3258f18656f5cb7059054cc0575a7d3b7"},"cell_type":"code","source":"graph_acc_loss(mobilenet1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1df3203a35f3caecacfd16ac226df562f79cd533","_kg_hide-output":true},"cell_type":"code","source":"X_train_randomFlips = augmentation_random_flips(X_train)\nmobilenet2 = model.fit(x=X_train_randomFlips/255, y=y_train, epochs=50, batch_size=100, verbose=1, validation_data=(X_test/255, y_test), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true,"_uuid":"5d1e2057f71d8605970cd9823f33361e7f0d9923"},"cell_type":"code","source":"##Save partly trained model \nmodel.save('02_mobilenet_randomFlips_trained.h5')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"08415429b4eb5e733888bd8b21750cb5c198a25c"},"cell_type":"code","source":"graph_acc_loss(mobilenet2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4eee757ad1b26b0fd135cb30c308c98a1c31686f","_kg_hide-output":true},"cell_type":"code","source":"X_train_randomRotations = augmentation_random_rotations(X_train)\nmobilenet3 = model.fit(x=X_train_randomRotations/255, y=y_train, epochs=50, batch_size=100, verbose=1, validation_data=(X_test/255, y_test), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true,"_uuid":"5e07d4802e023564d5067b286df14f8df0487778"},"cell_type":"code","source":"##Save partly trained model \nmodel.save('03_mobilenet_randomRotation_trained.h5')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"7509c7bb8ba9eb3e74b541c55b2f10ace5482ed0"},"cell_type":"code","source":"graph_acc_loss(mobilenet3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eca53e908f999e57bb1f2079d403c2888c97c212","_kg_hide-output":true},"cell_type":"code","source":"X_train_randomShifts = augmentation_random_shifts(X_train)\nmobilenet4 = model.fit(x=X_train_randomShifts/255, y=y_train, epochs=50, batch_size=100, verbose=1, validation_data=(X_test/255, y_test), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"bf4e50ebc1f7aee56a7138a582f3acdbde78e4f2"},"cell_type":"code","source":"##Save partly trained model \nmodel.save('04_mobilenet_randomShifts_trained.h5')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b0292c1248e6afdcf76077627c6f8022254fa2f0"},"cell_type":"code","source":"graph_acc_loss(mobilenet4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d804c44cfea320cf9bc937ed046138b1afd53a12","_kg_hide-input":true},"cell_type":"code","source":"#X_train_randomZca = augmentation_random_zca(X_train)\n#mobilenet5 = model.fit(x=X_train_randomZca/255, y=y_train, epochs=50, batch_size=100, verbose=1, validation_data=(X_test/255, y_test), shuffle=True)\n#graph_acc_loss(mobilenet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58b8ba7b413a5b602af51dbcb76c7db5a0b0a0f7","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"for X_batch, y_batch in datagen.flow(X_train, y_train, batch_size=9):\n    # create a grid of 3x3 images\n    for i in range(0, 9):\n        plt.subplot(330 + 1 + i)\n        #plt.imshow(X_batch[i].reshape(96, 96), cmap=plt.get_cmap('gray'))\n        #plt.imshow(X_batch[i], cmap=plt.get_cmap('gray'))\n        plt.imshow(X_batch[i])\n    # show the plot\n    plt.show()\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f3e4901de1596508c769742fc92d0a927ba2282","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"##Save partly trained model \n#model.save('augmentation_mobilenet_trained.h5') \n#del model \n##Reload model \n#model = load_model('partly_trained.h5') ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a57484ea52cd76f687628e4c415d607818e7c97b"},"cell_type":"markdown","source":"## **SUBMISSION**"},{"metadata":{"_kg_hide-output":true,"trusted":true,"scrolled":false,"_uuid":"cdab729923f8d8ed0c4310ca91b87e559f6e532c"},"cell_type":"code","source":"test = os.listdir(\"../input/test/\")\nprint(len(test))\ncol = ['Image']\ntest_df = pd.DataFrame(test, columns=col)\n\ntest_df['Id'] = ''\nX = prepareImages(test_df, test_df.shape[0], \"test\")\nX /= 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ba8a4eb26ad23b3250ffad1038d4f7a782eb96f"},"cell_type":"code","source":"y_, label_encoder = prepare_labels(df['Id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d278d0550e686e27f478828efb4a7b3a1cc7e5d0"},"cell_type":"code","source":"prediction = model.predict(np.array(X), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e3052c93c5777d470b57de77ffba07b76594a8c"},"cell_type":"code","source":"for i, pred in enumerate(prediction):\n    #print (pred)\n    test_df.loc[i, 'Id'] = ' '.join(label_encoder.inverse_transform(pred.argsort()[-5:][::-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f26444ce151bf8bfe98bcf4533f1c25323bfc57"},"cell_type":"code","source":"test_df.to_csv('submission_v2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c66b51425e50e9584df86ca857edc74c4bc5cdd"},"cell_type":"code","source":"test_df.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}