{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Outline\n1. Explore Training Data\n2. Prepare Training Data\n3. Train-Valid Split\n4. Build Model\n5. Train Model\n6. Make Predictions\n"},{"metadata":{},"cell_type":"markdown","source":"## Import Dependencies:"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# visulization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nimport os\nimport gc # garbage collection\nimport glob # extract path via pattern matching\nfrom tqdm.notebook import tqdm # progressbar\nimport random\nimport math\nimport cv2 # read image\n# store to disk\nimport pickle\nimport h5py # like numpy array\n\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\n\nfrom keras.models import Sequential, Model\nfrom keras.models import load_model\nfrom keras.layers import Input, Dense, Conv2D, MaxPool2D, AveragePooling2D\nfrom keras.layers import Flatten, Dropout, BatchNormalization, Activation\nfrom keras.layers import Add\nfrom keras.optimizers import SGD, RMSprop, Adam\nfrom keras import regularizers\n\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\n# from keras.applications.vgg16 import VGG16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink('../input/pretrained-models/my_model.h5')\n# keras config file\n# !sudo $HOME/.keras/keras.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# which keras backend\nfrom keras import backend as K\nprint(\"backend:\", K.backend())\nprint(\"image_format:\", K.image_data_format())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\ntf.test.is_gpu_available() # True/False\n# Or only check for gpu's with cuda support\ntf.test.is_gpu_available(cuda_only=True) \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR = '../input/state-farm-distracted-driver-detection/imgs/train/'\nTEST_DIR = '../input/state-farm-distracted-driver-detection/imgs/test/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"driver_imgs_list = pd.read_csv('../input/state-farm-distracted-driver-detection/driver_imgs_list.csv')\nsample_submission = pd.read_csv('../input/state-farm-distracted-driver-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"driver_imgs_list.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image(path, img_height=None, img_width=None, rotate=False, color_type=0):\n    img = cv2.imread(path, color_type)\n    if img_width and img_height:\n        img = cv2.resize(img, (img_width, img_height))\n    if rotate is True:\n        rows, cols = img.shape\n        rotation_angle = random.uniform(10,-10)\n        M = cv2.getRotationMatrix2D((cols/2, rows/2), rotation_angle, 1)\n        img = cv2.warpAffine(img, M, (cols,rows))\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(image_paths):\n    image_count = len(image_paths)\n    cols = 3\n    rows = math.ceil(image_count/cols)\n    fig = plt.figure(figsize=(16, 4.5*rows))\n\n    for i in range(1,image_count+1):\n        fig.add_subplot(rows, cols, i)\n        image = get_image(image_paths[i-1])\n        plt.imshow(image, cmap=\"gray\")\n        plt.xticks([])\n        plt.yticks([])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Unique Drivers:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# display_unique_drivers\ndriver_count = driver_imgs_list.subject.nunique()\nimage_paths = [TRAIN_DIR+row.classname+'/'+row.img \n               for (index, row) in driver_imgs_list.groupby('subject').head(1).iterrows()]\nprint(driver_count)\n# plot images\nplot_images(image_paths)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Unique Classes:\nc0: normal driving   \nc1: texting - right   \nc2: talking on the phone - right   \nc3: texting - left   \nc4: talking on the phone - left   \nc5: operating the radio   \nc6: drinking   \nc7: reaching behind   \nc8: hair and makeup   \nc9: talking to passenger   "},{"metadata":{"trusted":true},"cell_type":"code","source":"# class distribution\ndriver_imgs_list.classname.value_counts().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display_unique_classes\nimage_paths = [TRAIN_DIR+row.classname+'/'+row.img \n               for (index, row) in driver_imgs_list.groupby('classname').head(1).iterrows()]\n# plot images\nplot_images(image_paths)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explore image sizes\nTaking 50 random training images, all of them have same size: 480x640.\nThat means, we can train with image sizes (aspect ratio: 3/4)."},{"metadata":{"trusted":true},"cell_type":"code","source":"random_list = np.random.permutation(len(driver_imgs_list))[:50]\ndf_copy = driver_imgs_list.iloc[random_list]\nimage_paths = [TRAIN_DIR+row.classname+'/'+row.img \n                   for (index, row) in df_copy.iterrows()]\nimage_shapes = [get_image(path).shape for path in image_paths]\nprint(set(image_shapes))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Training Data"},{"metadata":{},"cell_type":"markdown","source":"Note: Use data_generator for large datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_HEIGHT = 240\nIMG_WIDTH = 320","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_data(X, Y, train = True):\n    # normalize\n    X = X/np.float32(255)\n#     X = X.astype('float32')\n    # reshape for grayscale image\n    if len(X.shape) == 3:\n        X = np.expand_dims(X, axis=-1)\n    if train:\n        # one hot encode target\n        Y = to_categorical(Y, num_classes=10)\n        return X, Y\n    else:\n        return X","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare Small Train subset:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # load 1000 images per class from all 10 classes\n# small_df = driver_imgs_list.groupby('classname').head(1000)\n# image_paths = [TRAIN_DIR+row.classname+'/'+row.img \n#                for (index, row) in small_df.iterrows()]\n# trainx = []\n# trainy= []\n# for item in image_paths:\n#     trainx.append(get_image(item,img_height=IMG_HEIGHT,img_width=IMG_WIDTH))\n#     trainy.append(item.split('/')[-2][-1]) # as c0,c1,etc.\n# trainx = np.array(trainx)\n# trainy = np.array(trainy,'uint8')\n# #after preprocess function loaded\n# Xtrain, Ytrain = preprocess_data(trainx, trainy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_train_data(train_dir):\n    data = []\n    targets = []\n    target_classes = os.listdir(train_dir)\n\n    print(\"Image Size:\", IMG_HEIGHT, IMG_WIDTH)\n    print(target_classes)\n    \n    for c in tqdm(target_classes):\n        class_dir = os.path.join(train_dir,c)\n        items = glob.glob(os.path.join(class_dir,'*g'))\n        for item in items:\n            data.append(get_image(item,img_height=IMG_HEIGHT,img_width=IMG_WIDTH))\n            targets.append(c[1])\n    return np.array(data), np.array(targets, dtype='uint8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain, Ytrain = load_train_data(TRAIN_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del Xtrain, Ytrain\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain, Ytrain = preprocess_data(Xtrain, Ytrain)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(Xtrain.shape,Ytrain.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note: Not appropriate to load test data at once due to memory issue (skip next 3 steps) takes > 16GB"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load test data\ndef load_test_data(test_dir):\n    data = []\n    ids = []\n    items = glob.glob(os.path.join(test_dir,'*g'))\n    print(len(items))\n    for item in tqdm(items):\n        data.append(get_image(item,img_height=240,img_width=320))\n        ids.append(os.path.basename(item))\n    return np.array(data), ids\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run after memory cleanup and finished training\n# Xtest, TEST_IDS = load_test_data(TEST_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Xtest = preprocess_data(Xtest,None,train=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dir()\n# globals()\n# locals()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_pickle_file(filename, data):\n    with open(filename, \"wb\") as f:\n        pickle.dump(data, f)\n\ndef load_pickle_file(filename):\n    with open(filename, \"rb\") as f:\n        data = pickle.load(f)\n    return data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train - Validation split:"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_valid, Y_train, Y_valid = train_test_split(Xtrain, Ytrain, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# free up memory\ndel Xtrain, Ytrain\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build Model:"},{"metadata":{},"cell_type":"markdown","source":"Things to consider  in keras model building:\n- No Dropout after Conv layer\n- Use dropout after dense layer (use mostly at the end of arch to not loose data)\n- Use BatchNorm before any activation function\n\n### Regularizers:\n- Dropout\n- Weigth Decay (L2) i.e. weights should be smaller. Penalizes model complexity.\n- BatchNorm (is a most)\n- Data Augmentation: e.g. fix lightning in images\n"},{"metadata":{},"cell_type":"markdown","source":"## Basic CNN\nconv > batch_norm > relu\n"},{"metadata":{},"cell_type":"markdown","source":"### Room for Improvement:\n- Progressive Resizing\n- Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS=10\nBATCH_SIZE=32","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setup Callbacks:"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_DIR = \"saved_models\"\nif not os.path.exists(MODELS_DIR):\n    os.makedirs(MODELS_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks.callbacks import ModelCheckpoint, EarlyStopping\n\nfilepath = MODELS_DIR+'/epoch{epoch:02d}-loss{loss:.2f}-val_loss{val_loss:.2f}.hdf5'\n# checkpoint\nmodel_checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, \n                                   save_best_only=True, save_weights_only=False, mode='min', period=1)\n\n# early stopping: patience = epocs\nearly_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1,\n                               mode='min', baseline=None, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No need for adaptive learning alogrithms like: RmsProp, Adam\nfrom keras.callbacks.callbacks import LearningRateScheduler, ReduceLROnPlateau\n# This function keeps the learning rate at 0.01 for the first five epochs\n# and decreases it exponentially after that.\ndef learning_rate_scheduler(epoch):\n  if epoch < 5:\n    return 0.01\n  else:\n    return 0.01 * math.exp(0.1 * (5 - epoch))\n\nlr_scheduler = LearningRateScheduler(learning_rate_scheduler, verbose=1)\n# OR\n# lr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, \n#                                             verbose=1, mode='min', min_delta=0.0001, min_lr=0.0001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_model_loss(history):\n    plt.plot(history['loss'])\n    plt.plot(history['val_loss'])\n    plt.title('Model Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['train', 'valid'], loc='upper left')\n    plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Rmsprop"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use stride 2 in the middle to reduce size and increase no. of filters\n# use avgpool at the end (not maxpool)\ndef seq_conv_block(model, filters=32):\n    model.add(Conv2D(filters=filters, kernel_size=(3,3), strides=2, padding=\"same\"))\n    model.add(BatchNormalization(axis=-1))\n    model.add(Activation(\"relu\"))\n    return model\n\n# all conv layers  with strides=1\nmodel = Sequential(name=\"seq_conv_rmsprop\")\n\nmodel.add(Conv2D(input_shape=(IMG_HEIGHT,IMG_WIDTH,1), filters=16, kernel_size=(3,3), padding=\"same\"))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(BatchNormalization(axis=-1))\nmodel.add(Activation(\"relu\"))\n\nmodel = seq_conv_block(model, filters=32)\nmodel = seq_conv_block(model, filters=64)\nmodel = seq_conv_block(model, filters=128)\n\nmodel.add(Flatten())\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(500))\nmodel.add(Dropout(0.5))\nmodel.add(BatchNormalization(axis=-1))\nmodel.add(Activation(\"relu\"))\n\nmodel.add(Dense(10, activation=\"softmax\"))\n\n# sgd = optimizers.SGD(lr=0.01, clipvalue=0.5)\n# optimizer = RMSprop(learning_rate=0.001)\n# adad = Adam(learning_rate=0.001)\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# history1 = model.fit(x=X_train,y=Y_train, validation_data=(X_valid, Y_valid),\n#                           epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=1,\n#                           callbacks=[model_checkpoint, early_stopping])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot Model History\n# plot_model_loss(history_v1.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del X_train, X_valid, Y_train, Y_valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saving/loading whole models (architecture + weights + optimizer state)\n# creates a HDF5 file 'my_model.h5'\n# model.save('saved_models/my_model_4conv_block.h5')\n# del model  # deletes the existing model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls saved_models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# returns a compiled model\nmodel = load_model('../input/pretrained-models/my_model_4conv_block.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # save only architecture\n# json_string = model.to_json()\n# from keras.models import model_from_json\n# model = model_from_json(json_string)\n# #only weights\n# model.save_weights('my_model_weights.h5')\n# model.load_weights('my_model_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Predictions:"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss, accuracy = model.evaluate(X_valid, Y_valid)\nprint(\"Loss:\", loss)\nprint(\"Accuracy:\", accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_submission_df(predictions, ids):\n    result_df = pd.DataFrame(predictions, columns=[\"c0\",\"c1\",\"c2\",\"c3\",\"c4\",\"c5\",\"c6\",\"c7\",\"c8\",\"c9\"])\n    result_df['img'] = ids\n    return result_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare Kaggle Submission:"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_items = glob.glob(os.path.join(TEST_DIR,'*g'))\nprint(len(test_items))\n\n# pedict block-wise due to memory issue\nblock_size = 1000\ntotal_test_size = len(test_items)\nresult_df = []\ndata = []\nids = []\ncount = 0\nfor item in tqdm(test_items):\n    data.append(get_image(item, img_height=IMG_HEIGHT, img_width=IMG_WIDTH))\n    ids.append(os.path.basename(item))\n    if len(ids) == block_size or count == total_test_size-1:\n        data = np.array(data)\n        data = preprocess_data(data, None, train=False)\n        df = prepare_submission_df(model.predict(data), ids)\n        result_df.append(df)\n        data = []\n        ids = []\n    \n    count += 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_df = None\nfor df in result_df:\n    if final_df is None:\n        final_df = df\n    else:\n        final_df = final_df.append(df)\n\nprint(len(final_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Reordering DF columns\n# cols = df.columns.tolist()\n# cols = cols[-1] + cols[:-1]\n# df = df[cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install kaggle\n# !kaggle competitions submit -c state-farm-distracted-driver-detection -f submission.csv -m \"First Submission.\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Resnet Like CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef conv_layer(inputs, filters=16, num_strides=1):\n    return Conv2D(filters=filters, kernel_size=(3,3), strides=num_strides, padding='same')(inputs)\n\ndef conv_block(inputs, filters=16, num_strides=1):\n    '''\n    conv>batch_norm>relu\n    '''\n    x = conv_layer(inputs, filters, num_strides)\n    x = BatchNormalization(axis=-1)(x)\n    x = Activation('relu')(x)\n    return x\n    \ndef resnet_block(inputs, filters=16):\n    x_shortcut = inputs\n    x = conv_block(inputs, filters)\n    x = BatchNormalization(axis=-1)(x)\n    x = Add()([x,x_shortcut]) # skip connection\n    x = Activation('relu')(x)\n    return x\n    \n\ninputs = Input(shape=(IMG_HEIGHT,IMG_WIDTH,1))\n\noutput_0 = conv_block(inputs=inputs, filters=16)\noutput_0 = MaxPool2D(pool_size=(2,2), strides=(2,2))(output_0)\n\noutput_1 = conv_block(output_0, filters=32)\noutput_1 = resnet_block(output_1, filters=32)\noutput_1 = MaxPool2D(pool_size=(2,2), strides=(2,2))(output_1)\n\noutput_2 = conv_block(output_1, filters=64)\noutput_2 = resnet_block(output_2, filters=64)\noutput_2 = AveragePooling2D(pool_size=(2,2), strides=(2,2))(output_2)\n\noutput_3 = Flatten()(output_2)\noutput_3 = Dropout(0.5)(output_3)\n\noutput_4 = Dense(500)(output_3)\noutput_4 = Dropout(0.5)(output_4)\noutput_4 = BatchNormalization(axis=-1)(output_4)\noutput_4 = Activation('relu')(output_4)\n\noutput_5 = Dense(10, activation='softmax')(output_4)\n\nres_model = Model(inputs=inputs, outputs=output_5, name=\"res_model\")\n\nres_model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\nres_model.summary()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}