{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os \n\npath = \"/kaggle/input/applications-of-deep-learning-wustl-fall-2020/final-kaggle-data/\"\n\npath_train = os.path.join(path, \"train.csv\" )\npath_test = os.path.join(path, \"test.csv\")\nsubmit_path = path_test = os.path.join(path, \"submit.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What version of Python do you have?\nimport sys\nimport tensorflow.keras\nimport sklearn as sk\nimport tensorflow as tf\n\nfrom tensorflow.keras.models import Model\nfrom keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.layers import Dense,GlobalAveragePooling2D\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nprint(f\"Tensorflow Version : {tf.__version__}\")\nprint(f\"Tensor Flow Keras Version : {tensorflow.keras.__version__}\" )\nprint(\"GPU is\", \"available\" if tf.test.is_gpu_available()\\\n             else \"Not Available\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"Read the data and prepare"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(path_train)\ndf_test = pd.read_csv(path_test)\ndf_submit = pd.read_csv(submit_path)\n\ndf_train = df_train[df_train.id != 1300] # why exclude ??\n\ndf_train['filename'] = df_train['id'].astype(str)+\".png\"\ndf_train['stable'] = df_train['stable'].astype(str)\n\ndf_test['filename'] = df_test['id'].astype(str)+\".png\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Perform a basic balance plot\n\ndf_train.stable.value_counts().plot(kind='bar')\nplt.title('Label counts')\nplt.xlabel('Stable')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"create train and validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_pct = .9\n# train_cut = int(len(df_train) * train_pct)\n\n# df_train_cut = df_train[0:train_cut]\n# df_validate_cut = df_train[train_cut:]\n\ndf_train_cut, df_validate_cut = train_test_split(df_train, \n                                test_size = .2, random_state = 42)\n\nprint(f\"Training Size : {len(df_train_cut)}\")\nprint(f\"Validation Size : {len(df_validate_cut)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions \n\nGet image path\nadd / remove defective images\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_path(image_id):\n    return os.path.join(path, f\"{image_id}.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_remove_defective_images(df):\n    print(path+row['filename'])\n#     defected = []\n#     for index, row in df.iterrows():\n#         try:\n#             print(path)\n#             image = tf.io.read_file(path+row['filename'])\n#             image = tf.image.decode_png(image, channels = 3)\n#         except:\n#             defected.append(row('filename'))\n#     return df[~df['filename'].isin(defected)], defected\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for idx, row in df_validate_cut.iterrows():\n#     print(row['filename'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_cut.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Normalize the data "},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf \nimport keras_preprocessing \nfrom keras_preprocessing import image \nfrom keras_preprocessing.image import ImageDataGenerator\n\nwidth = 150\nheight = 150\n\ntraining_datagen = ImageDataGenerator(\n                rescale = 1.0/255,\n                horizontal_flip = True,\n                vertical_flip = True,\n                fill_mode = 'nearest')\n                \ntrain_generator = training_datagen.flow_from_dataframe(\n         dataframe = df_train_cut,\n         directory= path,\n         x_col = 'filename',\n         y_col = 'stable',\n         target_size =(height, width),\n         batch_size = 126,\n         class_mode = 'binary')\n         \nvalidation_datagen = ImageDataGenerator(rescale = 1./255)\n\nval_generator = validation_datagen.flow_from_dataframe(\n        dataframe = df_validate_cut,\n        directory = path,\n        x_col = 'filename',\n        y_col = 'stable',\n        target_size = (height, width),\n        batch_size = 126, \n        class_mode = 'binary')\n                                      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_generator[1][0][0][0][0]))\nprint(len(train_generator[1][0][0][0]))\nprint(len(train_generator[1][0][0]))\nprint(len(train_generator[1][0]))\nprint(len(train_generator[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_generator[0][0][0][0][0]))\nprint(len(train_generator[0][0][0][0]))\nprint(len(train_generator[0][0][0]))\nprint(len(train_generator[0][0]))\nprint(len(train_generator[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator[1][0][0][0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We now create the neural network and fit it. Some essential concepts are going on here.\n\n* Batch Size - The number of training samples that should be evaluated per training step. Smaller batch sizes, or mini-batches, are generally preferred.\n* Step - A training step is one complete run over the batch. At the end of a step, the weights are updated, and the neural network learns.\n* Epoch - An arbitrary point at which to measure results or checkpoint the model. Generally, an epoch is one complete pass over the training set. However, when generators are used, the training set size is theoretically infinite. Because of this, we set a steps_per_epoch parameter.\n* validation steps - The validation set may also be infinite; because of this, we must specify how many steps we wish to validate at the end of each Epoch."},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\n\n# model = tf.keras.models.Sequential([\n    \n#     tf.keras.layers.Conv2D(16, (3,3), activation='relu', input_shape= (height, width, 3)),\n#     tf.keras.layers.MaxPooling2D(2 ,2),\n    \n#     # The second convolution\n\n#     tf.keras.layers.Conv2D(32, (3,3), activation = 'relu'),\n#     tf.keras.layers.Dropout(0.5),\n#     tf.keras.layers.MaxPooling2D(2,2),\n    \n#     # The 3rd convolution\n#     tf.keras.layers.Conv2D(64, (3,3), activation = 'relu'),\n#     tf.keras.layers.Dropout(0.5),\n#     tf.keras.layers.MaxPooling2D(2,2),\n    \n#     # The 4th layer\n#     tf.keras.layers.Conv2D(64, (3,3), activation = 'relu'),\n#     tf.keras.layers.MaxPooling2D(2,2),\n    \n#     # The 5th layer \n#     tf.keras.layers.Conv2D(64, (3,3), activation = 'relu'),\n# #     tf.keras.layers.Dropout(0.5),\n# #     tf.keras.layers.Conv2D(64, (3,3), padding=\"same\", activation = 'relu'),\n# #     tf.keras.layers.Dropout(0.5),\n# #     tf.keras.layers.Conv2D(64, (3,3), activation = 'relu'),\n#     tf.keras.layers.MaxPooling2D(2,2),\n\n   \n    \n#     # Flatten to feed into DNN\n#     tf.keras.layers.Flatten(),\n#     tf.keras.layers.Dropout(0.5),\n    \n#     # 512 neuron hidden layer \n#     tf.keras.layers.Dense(512, activation = 'relu'),\n#     # output neuron \n#     tf.keras.layers.Dense(1, activation = 'relu')\n# ])\n\n\n# import Resnet 50\nbase_model = ResNet50(weights='imagenet', pooling=max, include_top = False) \nlen(base_model.layers)\n\n#model.summary()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x= base_model.output\nx=GlobalAveragePooling2D()(x)\nx=Dense(1024,activation='relu')(x) \nx=Dense(1024,activation='relu')(x) \npreds=Dense(1,activation='relu')(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Model(inputs = base_model.input, outputs = preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in model.layers[:166]:\n    layer.trainable=False\nfor layer in model.layers[166:]:\n    layer.trainable=True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch_steps = 30\nvaliation_steps = len(df_validate_cut)\nmodel.compile(loss = 'binary_crossentropy', optimizer = 'adam')\n\nmonitor = EarlyStopping(monitor='val_loss', min_delta = 1e-3, patience = 5,\n                       verbose = 1, mode = 'auto', restore_best_weights = True)\n\nhistory = model.fit(train_generator, epochs = 30, steps_per_epoch = 20,\n                   validation_data = val_generator,\n                   verbose = 1, validation_steps = 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Build Submission\nNow that the neural network is trained; we need to generate a submit CSV file to send to Kaggle. We will use nearly the same technique to build the submit file. However, these essential points that we must address:\n\n* We do not want the data generator to create an infinite date like we did when training. We have a fixed number of cases to score for the Kaggle submit; we only want to process them.\n* We do not want the data generator to randomize the samples' order like it did when training. Therefore we set shuffle to false.\n* We want to always start at the beginning of the data, so we reset the generator.\n\nThese ensure that the predictions align with the id's."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['loss'], label = 'train')\nplt.plot(history.history['val_loss'], label = 'Validation')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_datagen = ImageDataGenerator(rescale = 1. / 255)\n\nsubmit_generator = submit_datagen.flow_from_dataframe(\n            dataframe = df_test,\n            directory = path,\n            x_col = 'filename',\n            batch_size = 1,\n            shuffle = False,\n            target_size=(height, width),\n            class_mode = None )\n\nsubmit_generator.reset()\npred = model.predict(submit_generator, steps = len(df_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit = pd.DataFrame({\"id\": df_test['id'], 'stable': pred.flatten()})\ndf_submit.to_csv(\"/kaggle/working/submit.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}