{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Implementing a residual neural network from scratch.\n\n### references: \nhttps://www.kaggle.com/xhlulu/reducing-image-sizes-to-32x32\n\nhttps://www.coursera.org/learn/convolutional-neural-networks/home/welcome\n\nhttps://www.kaggle.com/xhlulu/cnn-baseline-iwildcam-2019"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Python ≥3.5 is required\nimport sys\nassert sys.version_info >= (3, 5)\n\n# Scikit-Learn ≥0.20 is required\nimport sklearn\nassert sklearn.__version__ >= \"0.20\"\n\n# Common imports\nimport numpy as np\nimport os\nimport gc\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score, recall_score, precision_score\n\n\nfrom keras import layers\nfrom keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D\nfrom keras.models import Model, load_model\nfrom keras.preprocessing import image\nfrom keras.utils import layer_utils, to_categorical\nfrom keras.utils.data_utils import get_file\nfrom keras.callbacks import Callback\nfrom keras.applications.imagenet_utils import preprocess_input\nimport pydot\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\nfrom keras.utils import plot_model\nfrom keras.initializers import glorot_uniform\nimport scipy.misc\nfrom matplotlib.pyplot import imshow\nimport keras.backend as K\nK.set_image_data_format('channels_last')\nK.set_learning_phase(1)\n\n%matplotlib inline \n#plotting directly without requering the plot()\n\nimport warnings\nwarnings.filterwarnings(action=\"ignore\") #ignoring most of warnings, cleaning up the notebook for better visualization\n\npd.set_option('display.max_columns', 500) #fixing the number of rows and columns to be displayed\npd.set_option('display.max_rows', 500)\n\nprint(os.listdir(\"../input\")) #showing all the files in the ../input directory\n\n# Any results you write to the current directory are saved as output. Kaggle message :D","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"X_train = np.load('../input/wildcam-reduced/X_train.npy')\nX_test = np.load('../input/wildcam-reduced/X_test.npy')\nY_train = np.load('../input/wildcam-reduced/y_train.npy')\n\nprint('Training set shape: {}'.format(X_train.shape))\nprint('Testing set shape: {}'.format(X_test.shape))\nprint('Training labels: {}'.format(Y_train.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.astype('float32')\nX_test = X_test.astype('float32')\nX_train /= 255.\nX_test /= 255.\n\nprint (\"number of training examples = \" + str(X_train.shape[0]))\nprint (\"number of test examples = \" + str(X_test.shape[0]))\nprint (\"X_train shape: \" + str(X_train.shape))\nprint (\"Y_train shape: \" + str(Y_train.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/iwildcam-2019-fgvc6/sample_submission.csv')\ntrain = pd.read_csv('../input/iwildcam-2019-fgvc6/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['category_id'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_f1s = []\n        self.val_recalls = []\n        self.val_precisions = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        X_val, y_val = self.validation_data[:2]\n        y_pred = self.model.predict(X_val)\n\n        y_pred_cat = to_categorical(\n            y_pred.argmax(axis=1),\n            num_classes=14\n        )\n\n        _val_f1 = f1_score(y_val, y_pred_cat, average='macro')\n        _val_recall = recall_score(y_val, y_pred_cat, average='macro')\n        _val_precision = precision_score(y_val, y_pred_cat, average='macro')\n\n        self.val_f1s.append(_val_f1)\n        self.val_recalls.append(_val_recall)\n        self.val_precisions.append(_val_precision)\n\n        print((f\"val_f1: {_val_f1:.4f}\"\n               f\" — val_precision: {_val_precision:.4f}\"\n               f\" — val_recall: {_val_recall:.4f}\"))\n\n        return\n\nf1_metrics = Metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#identity_block\n\ndef identity_block(X, f, filters, stage, block):\n    \"\"\"\n    Implementation of the identity block as defined in Figure 3\n    \n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    stage -- integer, used to name the layers, depending on their position in the network\n    block -- string/character, used to name the layers, depending on their position in the network\n    \n    Returns:\n    X -- output of the identity block, tensor of shape (n_H, n_W, n_C)\n    \"\"\"\n    \n    # defining name basis\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n    \n    # Retrieve Filters\n    F1, F2, F3 = filters\n    \n    # Save the input value. You'll need this later to add back to the main path. \n    X_shortcut = X\n    \n    # First component of main path\n    X = Conv2D(filters = F1, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2a', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n    \n    # Second component of main path\n    X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1,1), padding = 'same', name = conv_name_base + '2b', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path\n    X = Conv2D(filters = F3, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2c', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation\n    X = Add()([X,X_shortcut])\n    X = Activation('relu')(X)\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convolutional_block\n\ndef convolutional_block(X, f, filters, stage, block, s = 2):\n    \"\"\"\n    Implementation of the convolutional block as defined in Figure 4\n    \n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    stage -- integer, used to name the layers, depending on their position in the network\n    block -- string/character, used to name the layers, depending on their position in the network\n    s -- Integer, specifying the stride to be used\n    \n    Returns:\n    X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)\n    \"\"\"\n    \n    # defining name basis\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n    \n    # Retrieve Filters\n    F1, F2, F3 = filters\n    \n    # Save the input value\n    X_shortcut = X\n\n\n    ##### MAIN PATH #####\n    # First component of main path \n    X = Conv2D(F1, (1, 1), strides = (s,s), name = conv_name_base + '2a',padding = 'valid', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n\n    # Second component of main path\n    X = Conv2D(F2, (f, f), strides = (1,1), name = conv_name_base + '2b', padding = 'same', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path\n    X = Conv2D(F3, (1, 1), strides = (1,1), name = conv_name_base + '2c', padding = 'valid', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n\n    ##### SHORTCUT PATH ####\n    X_shortcut = Conv2D(F3, (1, 1), strides = (s,s), name = conv_name_base + '1', padding = 'valid', kernel_initializer = glorot_uniform(seed=0))(X_shortcut)\n    X_shortcut = BatchNormalization(axis = 3, name = bn_name_base + '1')(X_shortcut)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation\n    X = Add()([X,X_shortcut])\n    X = Activation('relu')(X)\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ResNet50(input_shape = (32, 32, 3), classes = 14):\n    \"\"\"\n    Implementation of the popular ResNet50 the following architecture:\n    CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3\n    -> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER\n\n    Arguments:\n    input_shape -- shape of the images of the dataset\n    classes -- integer, number of classes\n\n    Returns:\n    model -- a Model() instance in Keras\n    \"\"\"\n    \n    # Define the input as a tensor with shape input_shape\n    X_input = Input(input_shape)\n\n    \n    # Zero-Padding\n    X = ZeroPadding2D((3, 3))(X_input)\n    \n    # Stage 1\n    X = Conv2D(32, (7, 7), strides = (1, 1), name = 'conv1', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = 'bn_conv1')(X)\n    X = Activation('relu')(X)\n    X = MaxPooling2D((3, 3))(X)\n\n    # Stage 2\n    X = convolutional_block(X, f = 3, filters = [32, 32, 128], stage = 2, block='a', s = 1)\n    X = identity_block(X, 3, [32, 32, 128], stage=2, block='b')\n    X = identity_block(X, 3, [32, 32, 128], stage=2, block='c')\n\n    # Stage 3\n    X = convolutional_block(X, f = 3, filters = [64, 64, 256], stage = 3, block='a', s = 2)\n    X = identity_block(X, 3, [64, 64, 256], stage=3, block='b')\n    X = identity_block(X, 3, [64, 64, 256], stage=3, block='c')\n    X = identity_block(X, 3, [64, 64, 256], stage=3, block='d')\n\n    # Stage 4 \n    X = convolutional_block(X, f = 3, filters = [128, 128, 512], stage = 4, block='a', s = 2)\n    X = identity_block(X, 3, [128, 128, 512], stage=4, block='b')\n    X = identity_block(X, 3, [128, 128, 512], stage=4, block='c')\n    X = identity_block(X, 3, [128, 128, 512], stage=4, block='d')\n    X = identity_block(X, 3, [128, 128, 512], stage=4, block='e')\n    X = identity_block(X, 3, [128, 128, 512], stage=4, block='f')\n\n    # Stage 5 \n    X = convolutional_block(X, f = 3, filters = [256,256, 1024], stage = 5, block='a', s = 2)\n    X = identity_block(X, 3, [256,256, 1024], stage=5, block='b')\n    X = identity_block(X, 3, [256,256, 1024], stage=5, block='c')\n\n    # AVGPOOL\n    X = AveragePooling2D(pool_size=(2,2), name='avg_pool')(X)\n    \n\n    # output layer\n    X = Flatten()(X)\n    X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer = glorot_uniform(seed=0))(X)\n    \n    \n    # Create model\n    model = Model(inputs = X_input, outputs = X, name='ResNet50')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ResNet50(input_shape = (32, 32, 3), classes = 14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    x=X_train,\n    y=Y_train,\n    batch_size=64,\n    epochs=10,\n    callbacks=[f1_metrics],\n    validation_split=0.2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.subplots(figsize=(12,10))\nplt.plot(history.history['loss'], color='b', label=\"Training loss\")\nplt.plot(history.history['val_loss'], color='r', label=\"validation loss\")\nplt.legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict results\nresults = model.predict(X_test)\nsubmission_df['Predicted'] = results.argmax(axis=1)\nsubmission_df.to_csv('resnet.csv', index=False)\nsubmission_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}