{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nPATH = \"/kaggle/input/applications-of-deep-learning-wustl-fall-2020/final-kaggle-data/\"\nPATH_TRAIN = os.path.join(PATH, \"train.csv\")\nPATH_TEST = os.path.join(PATH, \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(PATH_TRAIN)\ndf_test = pd.read_csv(PATH_TEST)\n\ndf_train = df_train[df_train.id != 1300]\n\ndf_train['filename'] = df_train[\"id\"].astype(str)+\".png\"\ndf_train['stable'] = df_train['stable'].astype(str)\n\ndf_test['filename'] = df_test[\"id\"].astype(str)+\".png\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_PCT = 0.2\nTEST_CUT = int(len(df_train) * TEST_PCT)\n\ndf_train_cut = df_train[TEST_CUT:]\ndf_validate_cut = df_train[0:TEST_CUT]\n\nprint(f\"Training size: {len(df_train_cut)}\")\nprint(f\"Validate size: {len(df_validate_cut)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport keras_preprocessing\nfrom keras_preprocessing import image\nfrom keras_preprocessing.image import ImageDataGenerator\n\nWIDTH = 640\nHEIGHT = 400\ntraining_datagen = ImageDataGenerator(\n  rescale = 1./255,\n  horizontal_flip=True,\n#   featurewise_center = True,\n#   zca_epsilon = 0.001,\n#   zca_whitening = True,\n#   zoom_range = [0.7,0.8],\n  brightness_range = [1.2,1.6],\n  #Original set it to True, I will set False\n  vertical_flip = False,\n  fill_mode='nearest')\n\ntrain_generator = training_datagen.flow_from_dataframe(\n        dataframe=df_train_cut,\n        directory=PATH,\n        x_col=\"filename\",\n        y_col=\"stable\",\n        target_size=(HEIGHT, WIDTH),\n        batch_size=8,\n        class_mode='binary')\n\nvalidation_datagen = ImageDataGenerator(rescale = 1./255,\n#                                           featurewise_center = True,\n#                                           zca_epsilon = 0.001,\n#                                           zca_whitening = True,\n#                                           zoom_range = [0.7,0.8],\n                                          brightness_range = [1.2,1.5])\n\nval_generator = validation_datagen.flow_from_dataframe(\n        dataframe=df_validate_cut,\n        directory=PATH,\n        x_col=\"filename\",\n        y_col=\"stable\",\n        target_size=(HEIGHT, WIDTH),\n        class_mode='binary')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Learning Rate Schedule\ndef lr_schedule(epoch):\n    lr = 0.0001\n    if epoch > 20:\n        lr = 0.000001\n    elif epoch > 15:\n        lr = 0.00001\n    elif epoch > 10:\n        lr = 0.00006\n    elif epoch > 5:\n        lr = 0.0001\n    print('Learning rate: ', lr)\n    return lr\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lr_schedule, verbose=True)\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lr_schedule, verbose=True)\n\n# #Learning Rate Annealer\n# from keras.callbacks import ReduceLROnPlateau\n# lrr= ReduceLROnPlateau(monitor='val_loss',   factor=.01,   patience=1,  min_lr=1e-6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.layers import Conv2D, Dense, Dropout, Flatten,BatchNormalization, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import RMSprop\nfrom keras.applications import *\nfrom keras.models import Model\nimport keras\n\n# base_model = tf.keras.applications.Xception(input_shape=(HEIGHT,WIDTH, 3), include_top=False)\n# base_model.trainable = True\n\n# base_model = ResNet50(include_top=False, input_shape=(HEIGHT,WIDTH, 3))\n# base_model.trainable = True\n\n\nmonitor = EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=8, verbose=1, mode='auto',\nrestore_best_weights=True)\nmodel = Sequential()\nbase_model = ResNet101(include_top=False, input_shape=(HEIGHT,WIDTH, 3))\nbase_model.trainable = True\nmodel.add(base_model)\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Flatten())\nmodel.add(Dense(1, activation='sigmoid'))\ncallback_list = [monitor, lr_callback]\nmodel.compile(loss = 'binary_crossentropy', optimizer='adam')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_generator,  \n  verbose = 1, \n  validation_data=val_generator, \n  steps_per_epoch=500, \n  validation_steps=200,\n  callbacks=callback_list, \n  epochs=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(\"./ResNet101_Nov9.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'ResNet101_Nov9.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build Submission\n\nNow that the neural network is trained; we need to generate a submit CSV file to send to Kaggle.  We will use nearly the same technique to build the submit file.  However, these essential points that we must address:\n\n* We do not want the data generator to create an infinite date like we did when training.  We have a fixed number of cases to score for the Kaggle submit; we only want to process them.\n* We do not want the data generator to randomize the samples' order like it did when training. Therefore we set shuffle to false.\n* We want to always start at the beginning of the data, so we reset the generator.\n\nThese ensure that the predictions align with the id's."},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_datagen = ImageDataGenerator(rescale = 1./255)\n\nsubmit_generator = submit_datagen.flow_from_dataframe(\n        dataframe=df_test,\n        directory=PATH,\n        x_col=\"filename\",\n        batch_size = 1,\n        shuffle = False,\n        target_size=(HEIGHT, WIDTH),\n        class_mode=None)\n\nsubmit_generator.reset()\npred = model.predict(submit_generator,steps=len(df_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit = pd.DataFrame({\"id\":df_test['id'],'stable':pred.flatten()})\ndf_submit.to_csv(\"./submit.csv\",index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'submit.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Further Fine Tune with Brighter Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"training_datagen = ImageDataGenerator(\n  rescale = 1./255,\n  horizontal_flip=False,\n  brightness_range = [0.9,1.6],\n  #Original set it to True, I will set False\n  vertical_flip = False,\n  fill_mode='nearest')\n\ntrain_generator = training_datagen.flow_from_dataframe(\n        dataframe=df_train_cut,\n        directory=PATH,\n        x_col=\"filename\",\n        y_col=\"stable\",\n        target_size=(HEIGHT, WIDTH),\n        batch_size=1,\n        class_mode='binary')\n\nvalidation_datagen = ImageDataGenerator(rescale = 1./255,\n#                                           featurewise_center = True,\n#                                           zca_epsilon = 0.001,\n#                                           zca_whitening = True,\n#                                           zoom_range = [0.7,0.8],\n                                          brightness_range = [1.2,1.5])\n\nval_generator = validation_datagen.flow_from_dataframe(\n        dataframe=df_validate_cut,\n        directory=PATH,\n        x_col=\"filename\",\n        y_col=\"stable\",\n        target_size=(HEIGHT, WIDTH),\n        class_mode='binary')\n\nfrom keras.models import load_model\nmodel_tune = load_model(\"ResNet101_Nov9.h5\")\n\ndef lr_schedule2(epoch):\n    lr = 3e-6\n    if epoch > 5:\n        lr = 1e-6\n    return lr\n\nlr_callback2 = tf.keras.callbacks.LearningRateScheduler(lr_schedule2, verbose=True)\nmonitor2 = EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=10, verbose=1, mode='auto',\n        restore_best_weights=True)\ncallback_list2 = [monitor2, lr_callback2]\n\n\nhistory = model_tune.fit(train_generator,  \n  verbose = 1, \n  validation_data=val_generator, \n  steps_per_epoch=500, \n  validation_steps=200,\n  epochs=10,callbacks=callback_list2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_tune.save(\"./ResNet101_Nov9_Tune.h5\")\nfrom IPython.display import FileLink\nFileLink(r'ResNet101_Nov9_Tune.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}