{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport cv2\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# We get csv files as dataframe\n# csv dosyasını pandas dataframe olarak alıyoruz\ndataFrame = pd.read_csv(\"../input/humpback-whale-identification/train.csv\")\ndataFrame.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"974d66c44f80cd080264644d8ad20e924f31293e"},"cell_type":"code","source":"# We won't use new_whale images\n# new_whale sınıfında ki resimleri kullanmıyoruz\ndataFrame = dataFrame[dataFrame.Id != \"new_whale\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e77c269c33dc6252a566659f9ea987161d040fe5"},"cell_type":"code","source":"# We turn and split train set inputs as numpy array.\n# Eğitim setinin girdilerini numpy array olarak ayırıyoruz\ntrainImageNames = dataFrame.Image.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5c5f04853e814d0b397409b759b2ebe91f43aac"},"cell_type":"code","source":"classSampleCountMinLimit = 3\n# We find nıt unique classes and create a list with them. Our minimum instance limit is classSampleCountMinLimit.\n# Burda classSampleCountMinLimit'dan fazla örneği bulunan sınıfların listesini oluşturuyoruz.\nlabelsCountList = dataFrame.Id.value_counts()\nnonUniqueLabelsList = []\nfor i in range(len(labelsCountList)):\n    if labelsCountList[i] >= classSampleCountMinLimit and labelsCountList.index[i] not in nonUniqueLabelsList:\n        nonUniqueLabelsList.append(labelsCountList.index[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee766f989ba3e94693e13becfdf93dce4733065a"},"cell_type":"code","source":"# We won't use train_test_split because we must use nonUniqueLabelsList when we split\n# train_test_split yerine validation setimizi kendimiz ayırıyoruz.\n\ncount = 0\nX_train_names = [] \nX_val_names = []\n\nfor i in range(len(trainImageNames)):\n    if dataFrame.Id.values[i] in nonUniqueLabelsList and count < 1500:\n        X_val_names.append(trainImageNames[i])\n        count = count + 1\n    else:\n        X_train_names.append(trainImageNames[i])\n        \nX_train_names = np.array(X_train_names)\nX_val_names = np.array(X_val_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"230c0c7f51a3b17ad1450c7a9d64dee47a64be58"},"cell_type":"code","source":"#from sklearn.model_selection import train_test_split\n#X_train_names, X_val_names, y_train, y_val = train_test_split(trainImageNames, labelNames, test_size = 0.1, random_state = 11)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c6f76eddff86f51ad387fdb64605856f1c10119"},"cell_type":"code","source":"print(\"Shapes ->\\nX_train = %s\\nX_val = %s\" % (X_train_names.shape, X_val_names.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"714b24a5bc331c0d159f07b6e8f9de9513985160"},"cell_type":"code","source":"# We split outputs.\ncount = 0\ny_train_names = []\ny_val_names = []\n\nfor i in range(len(dataFrame.Id.values)):\n    if dataFrame.Id.values[i] in nonUniqueLabelsList and count < 1500:\n        y_val_names.append(dataFrame.Id.values[i])\n        count = count + 1\n    else:\n        y_train_names.append(dataFrame.Id.values[i])\n\ny_train_names = np.array(y_train_names)\ny_val_names = np.array(y_val_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1509e1108a05e595c9129cd94ca8a97210b740bb"},"cell_type":"code","source":"print(\"The classes found in validation set(Top 10)\")\npd.value_counts(pd.Series(y_val_names))[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efa8b1ef0ef8772a38023a233e46d680ae949b38"},"cell_type":"code","source":"#y_train_names, y_val_names = train_test_split(dataFrame.Id.values, test_size = 0.1, random_state = 11)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"169338cd16b431b0b9a82311275c1b1ab50d8bbb"},"cell_type":"code","source":"# To use data flow from directory we need special directory structure\n# We need a directory for every class.\n\n# Keras'ın data_generatörü için gereken dizin yapısını oluşturuyoruz\n# Örnek dizin yapısında her sınıf için bir klasör gerekli\n\n#data/\n#    train/\n#        dogs/\n#            dog001.jpg\n#            dog002.jpg\n#            ...\n#        cats/\n#            cat001.jpg\n#            cat002.jpg\n#            ...\n#    validation/\n#        dogs/\n#            dog001.jpg\n#            dog002.jpg\n#            ...\n#        cats/\n#            cat001.jpg\n#            cat002.jpg\n#            ...\n\nos.mkdir(\"./data\")\n\nos.mkdir(\"./data/validation\")\nfor i in y_train_names:\n    if not os.path.exists(\"./data/validation/\"+i):\n        os.mkdir(\"./data/validation/\"+i)\n    \nos.mkdir(\"./data/train\")\nfor i in y_train_names:\n    if not os.path.exists(\"./data/train/\"+i):\n        os.mkdir(\"./data/train/\"+i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc587f9d9173543c61f6ab8e06addd858d371f44"},"cell_type":"code","source":"# Copying our images to their new places.\n# Oluşturduğumuz dizin yapısına resimlerimizi kopyalıyoruz\n\nimport shutil\nfor i in range(len(X_val_names)):\n    shutil.copy(\"../input/humpback-whale-identification/train/\"+X_val_names[i], \"./data/validation/\"+y_val_names[i]+\"/\"+X_val_names[i])\nfor i in range(len(X_train_names)):    \n    shutil.copy(\"../input/humpback-whale-identification/train/\"+X_train_names[i], \"./data/train/\"+y_train_names[i]+\"/\"+X_train_names[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9db53d7e8f87ae0e6a2eb524a7d88bb1f950d812"},"cell_type":"code","source":"# We get pretrained resnet50 and delete last layer.\n# PreTrained resnet50'yi alıp son katmanı atıyoruz\n\nfrom tensorflow.python.keras.applications import ResNet50\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Dropout\n\nnum_classes = np.unique(y_train_names).shape[0]\nresnet_weights_path = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\nmodel = Sequential()\nmodel.add(ResNet50(include_top=False, pooling='avg', weights=resnet_weights_path))\n#model.add(Dropout(0.25))\nmodel.add(Dense(num_classes, activation='softmax'))\n\n# We need to write this to say keras that don't train the first layers of resnet50\n# Resnet'in ilk katmanları zaten eğitilmiş olduğu için bunu belirtmemiz lazım\nmodel.layers[0].trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ed9b37d49c49979a3008204790dd834cfc8e3ab"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6062ad72f03e6ffd4c276f7e61f0b45702df2075"},"cell_type":"code","source":"model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41b7b757f896d83cbf4a4ad91c1878de48b89e2a"},"cell_type":"code","source":"# https://stackoverflow.com/a/46709583\n# We need some samples to fit data_image_generator. Click above for more info\n\n# Sample veriye ihtiyacımız var data_augmention generator'un std normalizasyonu için.\nX_sample = []\ntrainHeight = 224\ntrainWidth = 224\nfor i in os.listdir('../input/humpback-whale-identification/train/')[:500]:\n    img = cv2.imread('../input/humpback-whale-identification/train/'+i) # Resimleri tek tek alıyoruz\n    img = cv2.resize(img, dsize=(trainWidth, trainHeight), interpolation=cv2.INTER_LINEAR) # Resize ediyoruz INTER_LINEAR yerine başka alogritmalar ilede resim küçültme yapabiliriz\n    X_sample.append(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb493265c6d69ef63d43db457f9ce723db8f23f3"},"cell_type":"code","source":"from tensorflow.python.keras.applications.resnet50 import preprocess_input\nfrom tensorflow.python.keras.preprocessing.image import ImageDataGenerator\n\n# resNet's resolution\n# resNet 224x224 resolution resimler ile eğitilmiş\nimage_size = 224\n\n# Data augmention\n# burda data augmention yapıyoruz\ndata_generator = ImageDataGenerator(\n        preprocessing_function=preprocess_input,\n        #rotation_range=40,\n        #width_shift_range=0.2,\n        #height_shift_range=0.2,\n        #shear_range=0.2,\n        zoom_range=0.2,\n        horizontal_flip=True,\n        vertical_flip=True,\n        featurewise_center=True, \n        featurewise_std_normalization=True,\n        #rescale=1./255,\n        fill_mode='nearest')\n\ndata_generator.fit(X_sample)\n\n# train ve validation verilerimizi çekiyoruz\ntrain_generator = data_generator.flow_from_directory(\n        'data/train',\n        target_size=(image_size, image_size),\n        batch_size=64,\n        class_mode='categorical')\n\nvalidation_generator = data_generator.flow_from_directory(\n        'data/validation',\n        target_size=(image_size, image_size),\n        batch_size=64,\n        class_mode='categorical')\n\n# We fit model here\n# modeli eğitiyoruz\nmodel.fit_generator(\n        train_generator,\n        epochs = 20,\n        steps_per_epoch=X_train_names.shape[0] // 64,\n        validation_data=validation_generator,\n        validation_steps=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"386e41f15334f1bd3ea376274810a7b7a99b8320"},"cell_type":"code","source":"# We get sample submisson file. We use it as a template.\n# Sample submisson dosyasını çekiyoruz predictionları o tablo üzerine yeniden dolduracağız\nsample_df = pd.read_csv(\"../input/humpback-whale-identification/sample_submission.csv\")\ntestImagesNames = list(sample_df.Image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df6f0908d9395dc54533cfa9a1fe709e06797a41"},"cell_type":"code","source":"# We get test images with openCV\nX_test = []\nfor i in testImagesNames:\n    img = cv2.imread('../input/humpback-whale-identification/test/'+i) # Resimleri tek tek alıyoruz\n    img = cv2.resize(img, dsize=(trainWidth, trainHeight), interpolation=cv2.INTER_LINEAR) # Resize ediyoruz INTER_LINEAR yerine başka alogritmalar ilede resim küçültme yapabiliriz\n    X_test.append(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aafbfec46b54179b0dea93fd81a51aa1ced20b47"},"cell_type":"code","source":"# We normalize test images.\n\nX_test = np.array(X_test)\n\nk = X_test.shape[0]//1000\nfor i in range(k):\n    X_test[i*1000:i*1000+1000] = X_test[i*1000:i*1000+1000] / 255.0\n    if i == k-1:\n        X_test[i*1000:] = X_test[i*1000:] / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1287f22fee161cfe45f0e590e64ec908ce6a20e4"},"cell_type":"code","source":"# We predict test images\npredictions = model.predict(X_test, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97423facf57a1f8ccaff4946621d4dc5e7306250"},"cell_type":"code","source":"# If a pediction is higher than 0.4 we will get it's index num.\n# Eğer 0.4'ten daha yukarıda bir sınıfı tahmin etmişsek onun index numarasını alıyoruz.\nnonNewWhalePredIndexList = [indx for indx,i in enumerate(predictions) for j in i if j > 0.4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3b2e360d50021e7ac5357b54b2fe58637e4e55b"},"cell_type":"code","source":"print(\"Non new_whale class image count =\",np.unique(np.array(nonNewWhalePredIndexList)).shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c16d656e3c5dd8c95e143fe9b6fc1ebbc303c4a","scrolled":true},"cell_type":"code","source":"# We add 1.0 to begin of prediction array if it's in new_whale class\n# Prediction listesinin başına 1.0 ekliyoruz ki new_whale class'ı olduğu kesinleşsin.\npred2 = []\nfor indx in range(len(predictions)):\n    if indx not in nonNewWhalePredIndexList:\n        newPredItem = np.insert(predictions[indx], 0, 1.0)\n        pred2.append(newPredItem)\n    else:\n        pred2.append(predictions[indx])\npred2 = np.array(pred2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4abf3b0b5875c9c5b65c7a2587bafb06dba07fe5"},"cell_type":"code","source":"# We must delete our files to get rid of kaggle commit errors.\nshutil.rmtree(\"./data/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eec2bdf9931a6780695d9e9d21de58bc88646009"},"cell_type":"code","source":"# In here we create our submission file.\nlabels_list = list(np.unique(y_train_names))\nlabels_list = ['new_whale'] + labels_list\npred_list = [[labels_list[i] for i in p.argsort()[-5:][::-1]] for p in pred2]\npred_dic = dict((key, value) for (key, value) in zip(testImagesNames,pred_list))\npred_list_cor = [' '.join(pred_dic[id]) for id in testImagesNames]\ndf = pd.DataFrame({'Image':testImagesNames,'Id': pred_list_cor})\ndf.to_csv('submission.csv', header=True, index=False)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f04b23a7bbe496f13196ff3eb28399ff2f6b014"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}