{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os, shutil\nimport keras\nfrom keras import layers, Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom keras.layers import Conv2D,MaxPooling2D, Dropout, Dense, BatchNormalization, Flatten\n\nimport tensorflow as tf\ntf.set_random_seed(0)\n\nimport matplotlib.pyplot as plt \nfrom glob import glob\n%matplotlib inline\n\nnp.random.seed(101)\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"os.listdir('../input/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8cab0345cb85013595230db09970369e6249a8b"},"cell_type":"code","source":"train_dir = '../input/train/'\ndata = pd.DataFrame({'path': glob(os.path.join(train_dir,'*.tif'))                    \n                    })\ndata.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac1e4b460e927ddaecb4d67b50618c8f3f5b3f02"},"cell_type":"code","source":"data['id'] = data.path.apply(lambda x: str(x).split('/')[3].split('.')[0])\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"301b86b3d428e146459ea655d434c5a8b9fb3590"},"cell_type":"code","source":"df = pd.read_csv('../input/train_labels.csv')\ndata = data.merge(df, on='id')\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"996292332082e9c6233873dcaac29f229fff1be6"},"cell_type":"code","source":"training_dir= '../training_dir'\nvalidation_dir= '../validation_dir'\nos.mkdir(training_dir)\nos.mkdir(validation_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6286e9217034e97938c506d43784f8bf574ce02f"},"cell_type":"code","source":"df = data\ndf_0 = df[df.label==0]\ndf_1 = df[df.label==1]\ndf_1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f27ba1a5a55190fa5f80d01253a4eb8db7c36ee"},"cell_type":"code","source":"print(df_1.shape)\ndf_0.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8d6ac2cf4da02d949ad6a185c169507e5085e8e"},"cell_type":"code","source":"categories = [0,1]\nfor category in categories:\n    os.mkdir(os.path.join(training_dir,str(category))) #../training_dir/0 or 1\n    os.mkdir(os.path.join(validation_dir,str(category)))\n    \n # CREATING TRAINING DIRECTORY            \nfor category in categories:\n    cdir= os.path.join(training_dir,str(category)) #creates '../1 or 0'\n    for sample_count, path in enumerate(df[df.label ==category].path):\n        id = path.split('/')[3] #generate destination id_name\n        src = path\n        dst = os.path.join(cdir,id) #destination\n        shutil.copyfile(src,dst)\n        if sample_count==70000: break\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"933c7503ca8c8e2be31835b4e205d31e5a47a6d3"},"cell_type":"code","source":"#CREATING VALIDATION DIRECTORY\nfor category in categories:\n    cdir= os.path.join(validation_dir,str(category)) #creates '../1 or 0'\n    for sample_count, path in enumerate(df[df.label ==category].path):\n        if sample_count>70000:\n            id = path.split('/')[3] #generate destination id_name\n            src = path\n            dst = os.path.join(cdir,id) #destination\n            shutil.copyfile(src,dst)\n            if sample_count==89000: break\n        else: continue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6c43b54f60d4e74ac8bb2e2ebb7ca23314a08ac"},"cell_type":"code","source":"len(os.listdir(os.path.join(validation_dir,'1')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a22c9725c70bd823f7a2037a34b6d49796aa317a"},"cell_type":"code","source":"os.listdir(validation_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"258a64ad721638f53211dbd2f4ec42586344ab12"},"cell_type":"code","source":"data_gen = ImageDataGenerator(rotation_range=40,\n                          rescale=1./255, width_shift_range=0.2, \n                              height_shift_range=0.2, \n                             shear_range=0.2,\n                             zoom_range=0.2, horizontal_flip=True,\n                             fill_mode='nearest')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c19080c353557c4624c2d8e283ab26ea5030e6da"},"cell_type":"code","source":"batch_size=20\ntrain_generator = data_gen.flow_from_directory(training_dir, \n                                               target_size=(96,96),\n                                               class_mode='binary',\n                                               batch_size=batch_size)\ntest_gen = ImageDataGenerator(rescale=1./255)\n\nvalidation_generator_shuffled=test_gen.flow_from_directory(validation_dir,\n                                                 target_size=(96,96),\n                                                 class_mode='binary',\n                                                 batch_size=batch_size)\n\nvalidation_generator=test_gen.flow_from_directory(validation_dir,\n                                                 target_size=(96,96),\n                                                 class_mode='binary',\n                                                 batch_size=batch_size,\n                                                 shuffle=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba5474405876ac8ee4bca2694bb83628577c9523"},"cell_type":"code","source":"pool_size= (2,2)\n\nmodel = Sequential()\nmodel.add(layers.Conv2D(32,3,input_shape=(96,96,3), activation='relu'))\nmodel.add(layers.Conv2D(32,3,activation='relu'))\nmodel.add(layers.Conv2D(32,3,activation='relu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPool2D(pool_size))\n\nmodel.add(layers.Conv2D(64,3,activation='relu'))\nmodel.add(layers.Conv2D(64,3,activation='relu'))\nmodel.add(layers.Conv2D(64,3,activation='relu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPool2D(pool_size))\n\nmodel.add(layers.Conv2D(128,3,activation='relu'))\nmodel.add(layers.Conv2D(128,3,activation='relu'))\nmodel.add(layers.Conv2D(128,3,activation='relu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D(pool_size))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(256, activation='relu'))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31cb1014695f847a4ee47b14a8a4a0b72e0d30ca"},"cell_type":"code","source":"model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eabf15e53fecd8358476c0479e60cc2eb8e11fbd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b57063ce33bad7a4018749e78379086bafa95b15"},"cell_type":"code","source":"train_samples = 140000\ncheckpoint = ModelCheckpoint(f'../model.h5', save_best_only=True)\nstop = EarlyStopping(patience=4)\nmodel.fit_generator(train_generator, \n                    steps_per_epoch=train_samples//batch_size,\n                   validation_data=validation_generator_shuffled,\n                   epochs=10,\n                   callbacks=[checkpoint,stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"175cc619b7d862c83f85e131fb1f222ccab20276"},"cell_type":"code","source":"history = model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06c9dc2e7ad93b4ad4e3aa672d74eea6435f156b"},"cell_type":"code","source":"plt.plot(model.history.epoch, history['acc'],label='training_acc')\nplt.plot(model.history.epoch, history['val_acc'],c='green', label='Validation Accuracy')\nplt.xlabel('Epochs')\nplt.legend()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc7b69ce56cf6525906accbb562ab0999fe458c3"},"cell_type":"code","source":"plt.plot(model.history.epoch, history['loss'],label='training_loss')\nplt.plot(model.history.epoch, history['val_loss'],c='green', label='Validation Loss')\nplt.xlabel('Epochs')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"565988f20dbbfcaad256e18fa3543ea68623e013"},"cell_type":"code","source":"os.listdir('../')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29f670010f7b015c3a0976a28c4fe2c87eb41f87"},"cell_type":"code","source":"from keras.models import load_model\npredictor = load_model('../model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd050dce6347b3f18beee51fc8c438899cb20fd6"},"cell_type":"code","source":"validation_generator.filenames[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeca51b69b23e7a3588ce4dc2c8f09541ddb69da"},"cell_type":"code","source":"test_dir = '../input/test/'\ndf_test = pd.DataFrame({'path': glob(os.path.join(test_dir,'*.tif'))                    \n                    })\ndf_test['id'] = df_test.path.apply(lambda x: str(x).split('/')[3].strip())\n# df_test.drop('path', axis=1, inplace=True)\ndf_test.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da1f174ee51930ec6775a6b829aca4631ee7c733"},"cell_type":"code","source":"test_batch_size=2\ntest_generator = test_gen.flow_from_dataframe(df_test,directory='../input/test/',x_col='id',\n                                              target_size=(96,96),class_mode=None,\n                                             shuffle=False,\n                                             batch_size=test_batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"594bd9ab86d040a3ba9306556559b9ddcffd7ecd"},"cell_type":"code","source":"df_test.drop('path', inplace=True, axis=1)\ndf_test.id = test_generator.filenames\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4950f0a42dcff1d99808ff9d3df2c1eff968bc40"},"cell_type":"code","source":"for bb in test_generator:\n    plt.imshow(bb[0])\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc13226c7ee111c4f5d6653383e9efef5ddbd8d2"},"cell_type":"code","source":"no_of_samples = 57458\npredictions = predictor.predict_generator(test_generator,\n                                         steps=no_of_samples//test_batch_size, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"628d1b45475fba834518a3d32617906463df8f47"},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"271f2fec0b671969681df610ad1cd6739cc8f659"},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bb6c8675e33a8a46b1ce55ccbb0dfd85e2ae31f"},"cell_type":"code","source":"results = pd.DataFrame({'label': predictions.reshape(-1,)}, index=range(0,no_of_samples))\nresults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80a0b21741ed82ce81fb9dcfe2d693080353ceeb"},"cell_type":"code","source":"# def ro(x):\n#     if x>=0.5: x=1\n#     else: x=0\n#     return x\n\ndd = results\ndd.label = dd.label.apply(round)\ndd.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c381748170751f74a04045c0a814902144669e13"},"cell_type":"code","source":"results = pd.DataFrame({'label': predictions.reshape(-1,)}, index=range(0,no_of_samples))\nresults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7736458cd72e2b03c983cd4862d586ac92acd54d"},"cell_type":"code","source":"# df_test.drop('path', axis=1, inplace=True)\ndf_test.id = df_test.id.apply(lambda x: x.split('.')[0])\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33faa3f7bec2e32bdd8c898e9532089241abfe22"},"cell_type":"code","source":"submission = pd.concat([df_test,results],axis=1)\n# submission2 = pd.concat([df_test,dd],axis=1)\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2ae7c4a900f8aa4223034af72da058564760537"},"cell_type":"code","source":"submission.to_csv('submissions.csv', index=False)\n# submission2.to_csv('submissionswhole.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d490d42b40bac4c4579753693b075941cadaac6e"},"cell_type":"code","source":"pd.read_csv('submissions.csv').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"babd5f3ff189843d2af8f5bc34e4a2446a9c8ba7"},"cell_type":"code","source":"# def roundup(x):\n#     if x>=0.5: x = 1\n#     else: x=0\n#     return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d2e759a66ebfbd97ab1dc67a64d3054a2534c5b"},"cell_type":"code","source":"# results.label.apply(roundup).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c21d7d515efe32ea63d08be40b2d7a0ce6e3cc8d"},"cell_type":"code","source":"# val_batch_labels=np.zeros(len(validation_generator.classes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c723c1f73dbe899c1884b63f34ca805ec801791"},"cell_type":"code","source":"# i = 0\n# for b, l in (validation_generator):\n#     val_batch_labels[i*batch_size:batch_size*(i+1)] = l\n#     i+=1\n#     if i==3000:\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d30b033c9091f7032e2fecc60ec5f2391340a7dc"},"cell_type":"code","source":"# print('done')\n# pd.DataFrame(val_batch_labels)[0].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76a7cf621c8bcc9be23e9eccd70c4cde5d3121d8"},"cell_type":"code","source":"\nfrom sklearn.metrics import roc_curve,auc, confusion_matrix, classification_report\nval_pred = predictor.predict_generator(validation_generator,\n                                      steps=len(validation_generator.classes)//batch_size,\n                                      verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79d1054765ac258594233ca678cf714400af9da1"},"cell_type":"code","source":"val_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab852e45041e3d9222d3ac5e4bdf8499e5987116"},"cell_type":"code","source":"false_positive_rate,true_positive_rate,threshold = roc_curve(validation_generator.classes,\n                                                            val_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20681d9268a24d3a859600cd1987f7feaedec6c8"},"cell_type":"code","source":"val_pred_whole = np.where(val_pred>=0.5,1,0)\nAUC = auc(false_positive_rate,true_positive_rate)\nprint(AUC)\nprint(classification_report(validation_generator.classes,val_pred_whole))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5873a86aa64dc55f4df9314330798b1ae5853a2c"},"cell_type":"code","source":"dg = pd.DataFrame(val_pred_whole, columns=['label'])\ndg.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"959252a9684c64880e551ce9e2f8826994226092"},"cell_type":"code","source":"validation_generator.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cc54c6d87c1866ed4145156812305ecf3d27264"},"cell_type":"code","source":"shutil.rmtree(validation_dir)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c909a699aeb617edfd1cc45f1cb9eeac1c4497ea"},"cell_type":"code","source":"# os.mkdir('../test_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0d64b9886d70fd766107e2da5d512cf3d00c212"},"cell_type":"code","source":"# test_path= '../input/test'\n# test_dir='../test_dir'\n# # for image in os.listdir(test_path):\n# for img in os.listdir(test_path):\n#     src= os.path.join(test_path,img)\n#     dst = os.path.join(test_dir,img)\n#     shutil.copyfile(src,dst)\n# print('done')\n# len(os.listdir(test_dir))\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e900395a94eb0a6d9df1cb891a7a756ef638c43"},"cell_type":"code","source":"# os.mkdir('../test_images')\n# test_image_path = '../test_images'\n# shutil.move(test_dir, test_image_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad93e1da1d3152fadd98e86c7ef0299508f48392"},"cell_type":"code","source":"# test_generatorr = test_gen.flow_from_directory(test_image_path,target_size=(96,96),\n#                                               shuffle=False, batch_size=test_batch_size,\n#                                               class_mode='binary')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"531133c622995eca5b6dfcade26826e6118b4718"},"cell_type":"code","source":"# test_image_path= os.path.join(test_image_path,'test_dir')\n# os.listdir(test_image_path)[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27803bd48e8c1d4e9f68ece8959fce3d8aeb37d3"},"cell_type":"code","source":"# data = pd.DataFrame({'path': glob(os.path.join(test_image_path,'*.tif'))                    \n#                     })\n# data.head()\n\n# data['id'] = data.path.apply(lambda x: str(x).split('/')[3].split('.')[0])\n# data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e803c11199a6ed6275a1ead9f2c292c99ea4e4b"},"cell_type":"code","source":"# new_pred= predictor.predict_generator(test_generatorr, \n#                                       steps=len(test_generatorr.classes)//test_batch_size,\n#                                     verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c22bad39c3a417f1495e1f51d4dc8b579c6c53d"},"cell_type":"code","source":"# results = pd.DataFrame({'label': new_pred.reshape(-1,)}, index=range(0,no_of_samples))\n# results.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6df21e0b0bf3eedc5c5c1fcc9870ab0b6a53a65a"},"cell_type":"code","source":"# results.label.apply(roundup).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce9ce70efb382bb3dd728d4a35da243ac9029d6b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}