{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import warnings\nwarnings.simplefilter(\"ignore\", category=DeprecationWarning)\nimport time\n#!pip install -U tf-nightly-gpu\nimport numpy as np \nimport pandas as pd \nimport os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mplimg\nfrom matplotlib.pyplot import imshow\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport tqdm\nfrom PIL import Image\n\nfrom collections import OrderedDict\nimport cv2\n\n# The tf.keras and keras, they are not compatible. Choose only one.\n#import keras\nfrom keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\nfrom keras.utils.np_utils import to_categorical\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"total_df = pd.read_csv(\"../input/humpback-whale-identification/train.csv\")\ntotal_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"There are {len(os.listdir('../input/humpback-whale-identification/train'))} images in train dataset with {total_df.Id.nunique()} unique classes.\")\nprint(f\"There are {len(os.listdir('../input/humpback-whale-identification/test'))} images in test dataset.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_df.Id.value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_df.Id.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FILTER FOR TAILS"},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport glob\nimport numpy as np\nimport os, sys, time\nimport numpy as numpy\n\n\ndef filterForTails (img):\n    draw = False\n    waitValue = 10\n\n    imgray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    blur = cv2.GaussianBlur(imgray, (7, 7), 0)\n    ret,thresh = cv2.threshold(blur,127,255,0)\n\n    blur2 = cv2.GaussianBlur(thresh, (7, 7), 0)\n    thresh2 = cv2.threshold(blur2, 250, 250, cv2.THRESH_BINARY)[1]\n    #blur3 = cv2.GaussianBlur(thresh2, (7, 7), 0)\n\n    contours2, hierarchy = cv2.findContours(thresh2, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n\n    for c in contours2:\n\n        approx = cv2.approxPolyDP(c, 0.01 * cv2.arcLength(c, True), True)\n\n        x, y, w, h = cv2.boundingRect(c)\n        roi = img[y:h+y, x:w+x]\n        imgray2 = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)\n\n        if (len(approx) != 1):\n            draw = True\n\n    if (draw):\n        img = cv2.drawContours(img, contours2, -3, (255, 255, 255), -8)\n\n        draw = False\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepareImages(train, shape, image_size, channel, path, filterOn):\n    dt_proc = np.zeros((shape, image_size, image_size, channel))\n    count = 0\n    \n    for fig in train['Image']:\n        \n        #load images into images of size 100x100x3\n        img = cv2.imread(\"../input/humpback-whale-identification/\"+path+\"/\"+fig)\n        if(filterOn):\n            img = filterForTails(img)\n        img = cv2.resize(img, (100, 100))\n        x = image.img_to_array(img)\n        x = preprocess_input(x)\n\n        dt_proc[count] = x\n        if (count%500 == 0):\n            print(\"Processing image: \", count+1, \", \", fig)\n        count += 1\n    \n    return dt_proc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = 100\nchannel = 3\nIMG_SHAPE = (image_size, image_size, channel)\n\n#change to False to not use filter on images\nfilterOn = True\n\nstart = time.time()\ndt_proc = prepareImages(total_df, total_df.shape[0], image_size, channel, \"train\", filterOn)\nend = time.time()\n\nelapsed = end - start\nprint(\"Tempo de carregamento imagens: \", elapsed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dt_proc = dt_proc / 255.0\nprint(\"dt_proc shape: \",dt_proc.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_encoder = LabelEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_total = total_df[\"Id\"]\ny_total = label_encoder.fit_transform(y_total)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# we have 5005 class look at from=> train.Id.describe()\ny_total = to_categorical(y_total, num_classes = 5005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#  Shuffle\n#x,y = shuffle(x_train,y_train, random_state=2)\n\nX_train, X_test, Y_train, Y_test = train_test_split(dt_proc, y_total, test_size=0.2, random_state=2)\ndt_proc = 0\ny_total = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # With data augmentation to prevent overfitting\n\n# datagen = ImageDataGenerator(\n#         featurewise_center=False,  # set input mean to 0 over the dataset\n#         samplewise_center=False,  # set each sample mean to 0\n#         featurewise_std_normalization=False,  # divide inputs by std of the dataset\n#         samplewise_std_normalization=False,  # divide each input by its std\n#         zca_whitening=False,  # apply ZCA whitening\n#         rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n#         zoom_range = 0.1, # Randomly zoom image \n#         width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n#         height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n#         horizontal_flip=False,  # randomly flip images\n#         vertical_flip=False)  # randomly flip images\n\n\n# datagen.fit(x_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the base model from the pre-trained model MobileNet\nbase_model = tf.keras.applications.MobileNet(input_shape=IMG_SHAPE,\n                                            include_top=False,\n                                            weights='imagenet')\n\nbase_model.trainable = False #Freeze the convolutional base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([\n  base_model,\n  tf.keras.layers.GlobalAveragePooling2D(),\n  tf.keras.layers.Dense(5005, activation='sigmoid')\n])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.0001),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 10\n'''steps_per_epoch = train_generator.n // batch_size\nvalidation_steps = validation_generator.n // batch_size\n\nhistory = model.fit_generator(train_generator,\n                              steps_per_epoch = steps_per_epoch,\n                              epochs=epochs,\n                              workers=4,\n                              validation_data=validation_generator,\n                              validation_steps=validation_steps)'''\n\nstart = time.time()\nhistory = model.fit(X_train, Y_train, epochs=epochs, batch_size=4, verbose=1)\nend = time.time()\n\nelapsed = end - start","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Tempo de treino: \", elapsed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the loss curve for training\nplt.plot(history.history['loss'], color='r', label=\"Train Loss\")\nplt.title(\"Train Loss\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the accuracy curve for training\nplt.plot(history.history['acc'], color='g', label=\"Train Accuracy\")\nplt.title(\"Train Accuracy\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluating the model\nscore = model.evaluate(X_test, Y_test, verbose=0)\nprint('Test Loss:', score[0])\nprint('Test accuracy:', score[1])\n\n\n# Plot Acurracy vs Loss values\nplt.plot(history.history['acc'])\nplt.plot(history.history['loss'])\nplt.title('Model Accuracy vs Loss')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Accuracy (' + str(round(score[1],2)) + ')', 'Loss (' + str(round(score[0],2)) + ')'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train accuracy of the model: ',history.history['acc'][-1])\nprint('Train loss of the model: ',history.history['loss'][-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(np.array(X_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluating the model\nscore = model.evaluate(X_test, Y_test, verbose=0)\nprint('Test Loss:', score[0])\nprint('Test accuracy:', score[1])\n\n\n# Plot Acurracy vs Loss values\nplt.plot(history.history['acc'])\nplt.plot(history.history['loss'])\nplt.title('Model Accuracy vs Loss')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Accuracy (' + str(round(score[1],2)) + ')', 'Loss (' + str(round(score[0],2)) + ')'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}