{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **CS452 Project (State Farm Distracted Driver Detection)**\nBy Radheem Razi (K163645), Abdul Mannan (K163620), Murtaza Multanwala (K163618), Shahyar (K163750) & Saira (K163665)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## **Introduction**\nThis notebook is part of our Deep Learning for Visual Perception (CS452) project for Spring '20\n\n\nWe are trying to solve the problem of detecting distracted drivers by participating in a featured Kaggle competition called \"State Farm Distracted Driver Detection\".\nThis dataset is available on Kaggle, under the State Farm competition: https://www.kaggle.com/c/state-farm-distracted-driver-detection\nThis notebook mostly revolves around investigating and visualizing the given dataset.\n\nThere are 24 versions and each version has saved the results of each portion of the notebook.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Import the Libraries\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport random\nimport gc\nimport pickle\nimport time\nimport tensorflow\nimport datetime\nimport math\nfrom skimage import color\n\nos.environ['KERAS_BACKEND'] = 'tensorflow'\n#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # 3 = INFO, WARNING, and ERROR messages are not printed\n\n\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\nfrom IPython.display import FileLink\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nimport seaborn as sns \n%matplotlib inline\n\nfrom IPython.display import display, Image\nimport matplotlib.image as mpimg\nimport cv2\nimport PIL\nfrom PIL import Image\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import load_files       \nfrom keras.utils import np_utils\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import log_loss\n\nfrom keras.models import Sequential, Model\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization, GlobalAveragePooling2D,MaxPool2D\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing import image\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.vgg19 import VGG19\n\n\nfrom keras.utils import to_categorical\nfrom keras.models import Model\nfrom keras import optimizers\n\nfrom keras.layers import Dense,GlobalAveragePooling2D\nfrom keras.applications import MobileNet\nfrom keras.applications.mobilenet import preprocess_input\n\nfrom sklearn.metrics import log_loss\nfrom sklearn.metrics import accuracy_score\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom sklearn.metrics import classification_report\nfrom keras.callbacks import LearningRateScheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import the Datasets\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"activity_map = {'c0': 'Safe driving', \n                'c1': 'Texting - right', \n                'c2': 'Talking on the phone - right', \n                'c3': 'Texting - left', \n                'c4': 'Talking on the phone - left', \n                'c5': 'Operating the radio', \n                'c6': 'Drinking', \n                'c7': 'Reaching behind', \n                'c8': 'Hair and makeup', \n                'c9': 'Talking to passenger'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = pd.read_csv('../input/state-farm-distracted-driver-detection/driver_imgs_list.csv')\ndataset.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n#### Import Driver Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"by_drivers = dataset.groupby('subject')\nunique_drivers = by_drivers.groups.keys()\nprint(unique_drivers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_rows = 224\nimg_cols = 224\ncolor_type = 3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\ndriver_details = pd.read_csv('../input/state-farm-distracted-driver-detection/driver_imgs_list.csv',na_values='na')\n\ntrain_image = []\nimage_label = []\n\n\nfor i in range(10):\n    print('now we are in the folder C',i)\n    imgs = os.listdir(\"../input/state-farm-distracted-driver-detection/imgs/train/c\"+str(i))\n    print(len(imgs))\n    for j in range(1300):               ########comment range(len(imgs)):#1900 (DUE TO MEMORY LIMITATIONS UNDERSAMPLED DATASET HERE)\n                                                                         #  (In colab notebook all training images are considered when training)                     \n        \n        \n        img_name = \"../input/state-farm-distracted-driver-detection/imgs/train/c\"+str(i)+\"/\"+imgs[j]\n        img = cv2.imread(img_name)\n        \n        #img = Image.open(img_name)\n        #wpercent = (img_rows/float(img.size[0]))\n        #hsize = int((float(img.size[1])*float(wpercent)))\n        #img = img.resize((img_rows,hsize), PIL.Image.ANTIALIAS)\n        #img=np.array(img)\n        #img=img/255\n        \n        #img = color.rgb2gray(img)\n        #img = img[50:,120:-50]           ########uncomment to crop\n        \n        img = cv2.resize(img,(img_rows,img_cols))\n        label = i\n        driver = driver_details[driver_details['img'] == imgs[j]]['subject'].values[0]\n        train_image.append([img,label,driver])\n        image_label.append(i)\nif not os.path.exists('cache'):        \n        os.mkdir('cache')\n        \nfile = open('cache/train224.dat', 'wb')    ##saving cache to save time \npickle.dump(train_image, file)\nfile.close()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Class imbalance problem exists hence prevented (in this notebook) by under sampling technique","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image = []\nfile = open('cache/train224.dat', 'rb')\ntrain_image=pickle.load(file)\nfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 0\nfig, ax = plt.subplots(1, 2, figsize = (img_rows,img_cols ))\nfor i in range(2):\n    #print ('Image number:',i)\n    img = train_image[i][0]\n    #img = color.rgb2gray(img)\n    #\\img = img[50:,120:-50]\n    #img = cv2.resize(img,(img_rows,img_cols))\n    \n    ax[i].imshow(img)\n    plt.show\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train and validation split based on driver ids","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Eliminating Data leakage**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"> For random drivers:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ## getting list of driver names\n\n# D = []\n# for features,labels,drivers in train_image:\n#     D.append(drivers)\n\n# ## Deduplicating drivers\n\n# deduped = []\n\n# for i in D:\n#     if i not in deduped:\n#         deduped.append(i)\n    \n\n# ## selecting random drivers for the validation set\n# driv_selected = []\n# import random\n# driv_nums = random.sample(range(len(deduped)), 4)\n# for i in driv_nums:\n#     driv_selected.append(deduped[i])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Explicitly mentioned drivers (Based on random CV)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"driv_selected =  ['p061', 'p035', 'p056', 'p052','p042'] # ['p061', 'p035', 'p056', 'p052'] ['p050', 'p015', 'p022', 'p056','p042']\n### NEW RANDOM SEARCH ['p014', 'p012', 'p064', 'p039'] 2.27 start\nX_train= []\ny_train = []\nX_test = []\ny_test = []\nD_train = []\nD_test = []\ntrue_test = []\n\nfor features,labels,drivers in train_image:\n    if drivers in driv_selected:\n        X_test.append(features)\n        y_test.append(labels)\n        D_test.append(drivers)\n        true_test.append(labels)\n    \n    else:\n        X_train.append(features)\n        y_train.append(labels)\n        D_train.append(drivers)\n    \nprint (len(X_train),len(X_test))\nprint (len(y_train),len(y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train = np.array(X_train).reshape(-1,img_rows,img_cols,color_type)\n#X_test = np.array(X_test).reshape(-1,img_rows,img_cols,color_type)\nX_train = np.array(X_train)\nX_test = np.array(X_test)\n\ny_train = to_categorical(y_train)\ny_test = to_categorical(y_test)\nimg_rows=X_train.shape[1]\nimg_cols=X_train.shape[2]\nprint(img_rows,img_cols)\n\nprint (X_train.shape)\nprint(y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_image,img\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Statistics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Statistics\nfrom glob import glob\n\n# Load the list of names\nnames = [item[17:19] for item in sorted(glob(\"../input/imgs/train/*/\"))]\ntest_files_size = len(np.array(glob(os.path.join('..', 'input/imgs', 'test', '*.jpg'))))\nx_train_size = len(X_train)\ncategories_size = len(names)\nx_test_size = len(X_test)\nprint('There are %s total images.\\n' % (test_files_size + x_train_size + x_test_size))\nprint('There are %d training images.' % x_train_size)\nprint('There are %d total training categories.' % categories_size)\nprint('There are %d validation images.' % x_test_size)\nprint('There are %d test images.'% test_files_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Dataset Visualization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot figure size\nplt.figure(figsize = (10,10))\n# Count the number of images per category\nsns.countplot(x = 'classname', data = dataset)\n# Change the Axis names\nplt.ylabel('Count')\nplt.title('Categories Distribution')\n# Show plot\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the frequency of images per driver\ndrivers_id = pd.DataFrame((dataset['subject'].value_counts()).reset_index())\ndrivers_id.columns = ['driver_id', 'Counts']\ndrivers_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plotting class distribution\ndataset['class_type'] = dataset['classname'].str.extract('(\\d)',expand=False).astype(np.float)\nplt.figure(figsize = (20,20))\ndataset.hist('class_type', alpha=0.5, layout=(1,1), bins=10)\nplt.title('Class distribution')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Images belonging to different categories**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"activity_map = {'c0': 'Safe driving', \n                'c1': 'Texting - right', \n                'c2': 'Talking on the phone - right', \n                'c3': 'Texting - left', \n                'c4': 'Talking on the phone - left', \n                'c5': 'Operating the radio', \n                'c6': 'Drinking', \n                'c7': 'Reaching behind', \n                'c8': 'Hair and makeup', \n                'c9': 'Talking to passenger'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (12, 20))\nimage_count = 1\nBASE_URL = '../input/imgs/train/'\nfor directory in os.listdir(BASE_URL):\n    if directory[0] != '.':\n        for i, file in enumerate(os.listdir(BASE_URL + directory)):\n            if i == 1:\n                break\n            else:\n                fig = plt.subplot(5, 2, image_count)\n                image_count += 1\n                image = mpimg.imread(BASE_URL + directory + '/' + file)\n                plt.imshow(image)\n                plt.title(activity_map[directory])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 4 #648\nnb_epoch = 100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **CNN model**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Training results saved in earlier version of this notebook","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nnb_epoch = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_dir = \"saved_models\"\nif not os.path.exists(models_dir):\n    os.makedirs(models_dir)\n    \ncheckpointer = ModelCheckpoint(filepath='saved_models/weights_best_vanilla.hdf5', \n                               monitor='val_loss', mode='min',\n                               verbose=1, save_best_only=True)\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\ncallbacks = [checkpointer, es]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model_v2():\n    # Optimised Vanilla CNN model\n    model = Sequential()\n\n    ## CNN 1\n    model.add(Conv2D(32,(3,3),activation='relu',input_shape=(img_rows, img_cols, color_type)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(32,(3,3),activation='relu',padding='same'))\n    model.add(BatchNormalization(axis = 3))\n    model.add(MaxPooling2D(pool_size=(2,2),padding='same'))\n    model.add(Dropout(0.3))\n\n    ## CNN 2\n    model.add(Conv2D(64,(3,3),activation='relu',padding='same'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64,(3,3),activation='relu',padding='same'))\n    model.add(BatchNormalization(axis = 3))\n    model.add(MaxPooling2D(pool_size=(2,2),padding='same'))\n    model.add(Dropout(0.3))\n\n    ## CNN 3\n    model.add(Conv2D(128,(3,3),activation='relu',padding='same'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(128,(3,3),activation='relu',padding='same'))\n    model.add(BatchNormalization(axis = 3))\n    model.add(MaxPooling2D(pool_size=(2,2),padding='same'))\n    model.add(Dropout(0.5))\n\n    ## Output\n    model.add(Flatten())\n    model.add(Dense(512,activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.5))\n    model.add(Dense(128,activation='relu'))\n    model.add(Dropout(0.25))\n    model.add(Dense(10,activation='softmax'))\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_v2 = create_model_v2()\n\nmodel_v2.summary()\n\n# Compiling the model\nmodel_v2.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nnb_train_samples = X_train.shape[0]\nnb_validation_samples = X_test.shape[0]\nprint(nb_train_samples)\nprint(nb_validation_samples)\n\ndatagen = ImageDataGenerator(\n    height_shift_range=0.5,\n    width_shift_range = 0.5,\n    zoom_range = 0.5,\n    rotation_range=30\n        )\n\ntraining_generator = datagen.flow(X_train, y_train, batch_size=batch_size)\nvalidation_generator = datagen.flow(X_test, y_test, batch_size=batch_size)\n\nhistory_v3 = model_v2.fit_generator(training_generator,\n                         steps_per_epoch = nb_train_samples // batch_size,\n                         epochs = nb_epoch, \n                         callbacks=callbacks,\n                         verbose = 1,\n                         validation_data = validation_generator,\n                         validation_steps = nb_validation_samples // batch_size)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_v2.load_weights('saved_models/weights_best_vanilla.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model_v2.evaluate(X_test, y_test, verbose=1)\nprint('Score: ', score)\n\ny_pred = model_v2.predict(X_test, batch_size=batch_size, verbose=1)\nscore = log_loss(y_test, y_pred)\nprint('Score log loss:', score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model_v2.evaluate_generator(validation_generator, nb_validation_samples // batch_size)\nprint(\"Test Score:\", score[0])\nprint(\"Test Accuracy:\", score[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_test_class(model, test_files, image_number, color_type=1):\n    img_brute = test_files[image_number]\n    img_brute = cv2.resize(img_brute,(img_rows,img_cols))\n    plt.imshow(img_brute, cmap='gray')\n\n    new_img = img_brute.reshape(-1,img_rows,img_cols,color_type)\n\n    y_prediction = model.predict(new_img, batch_size=batch_size, verbose=1)\n    print('Y prediction: {}'.format(y_prediction))\n    print('Predicted: {}'.format(activity_map.get('c{}'.format(np.argmax(y_prediction)))))\n    \n    plt.show()\n    \nplot_test_class(model_v2, test_files, 143,color_type) # The model really performs badly","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  **Transfer Learning (VGG-19)**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def vgg_std19_model(img_rows, img_cols, color_type=3):\n    nb_classes = 10\n    # Remove fully connected layer and replace\n    # with softmax for classifying 10 classes\n    vgg19_model = VGG19(weights=\"imagenet\", include_top=False)\n\n    # Freeze all layers of the pre-trained model\n    #for layer in vgg16_model.layers:\n     #   layer.trainable = False\n        \n    x = vgg19_model.output\n    x = GlobalAveragePooling2D()(x)\n    #x = Dense(1024, activation='relu')(x)\n    predictions = Dense(nb_classes, activation = 'softmax')(x)\n\n    model = Model(input = vgg19_model.input, output = predictions)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the VGG19 network\nprint(\"Loading network...\")\nmodel_vgg19 = vgg_std19_model(img_rows, img_cols)\n\nmodel_vgg19.summary()\nsgd = optimizers.SGD(lr=0.0001,momentum=0.9) # try 0.01 - didn't converge and 0.005 , 0.001 best acc of 11%\n#r=1e-4, momentum=0.9 old- lr = 0.005\n#rms=optimizers.RMSprop(lr=1e-5)\ncc=CategoricalCrossentropy(label_smoothing=5e-5) \n#model_vgg16.compile(loss='categorical_crossentropy', optimizer='rmsprop',   metrics=['accuracy'])\n#sgd = optimizers.SGD(lr = 0.005) # try 0.01 - didn't converge and 0.005 , 0.001 best acc of 11%\n\nmodel_vgg19.compile(optimizer=sgd,loss=cc,metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndatagen = ImageDataGenerator(\n    height_shift_range=0.5,\n    width_shift_range = 0.5,\n    zoom_range = 0.5,\n    rotation_range=30\n        )\n#datagen.fit(X_train)\ndata_generator = datagen.flow(X_train, y_train, batch_size = batch_size)\n#mobilenet_model=model.fit(X_train, Y_train, batch_size=32, nb_epoch=25, verbose=1, validation_data=(X_valid, Y_valid))\n\n# Fits the model on batches with real-time data augmentation:\n#mobilenet_model = model.fit_generator(data_generator,steps_per_epoch = len(X_train) / 32, callbacks=[checkpointer, earlystopper],\n #                                                         epochs = nb_epoch, verbose = 1, validation_data = (X_valid, Y_valid))\n\n\nnb_train_samples = X_train.shape[0]\nnb_validation_samples = X_test.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training the vgg Model\nmodels_dir = \"saved_models\"\nif not os.path.exists(models_dir):\n    os.makedirs(models_dir)\n    \ncheckpoint = ModelCheckpoint('saved_models/weights_best_vgg16_model.hdf5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n\nhistory_v4 = model_vgg16.fit_generator(data_generator,\n                         steps_per_epoch = nb_train_samples // batch_size,\n                         epochs = nb_epoch, \n                         callbacks=[es, checkpoint],\n                         verbose = 1,\n                         validation_data = (X_test, y_test),\n                         validation_steps = nb_validation_samples // batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_vgg19.load_weights('saved_models/weights_best_vgg19.hdf5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Transfer Learning (Xception)**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.enable()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelXception_V1():\n    import keras\n    from keras.layers import Input\n    xception_input = Input(shape = (img_rows,img_cols, color_type), name = 'Image_input')\n\n    ## The RESNET model\n\n    from keras.applications.xception import preprocess_input, decode_predictions\n    from keras.applications.xception import Xception\n\n\n    #Get the RESNET weights and layers\n\n    model_xception_conv = Xception(weights= 'imagenet', include_top=False, input_shape= (img_rows,img_cols, color_type))\n    \n    \n    output_xception_conv = model_xception_conv(xception_input)\n\n    #Add the fully-connected layers \n\n    x=GlobalAveragePooling2D()(output_xception_conv)\n    x=Dense(1024,activation='relu')(x) #we add dense layers so that the model can learn more complex functions and classify for better results.\n    x = Dropout(0.1)(x) # **reduce dropout \n    x=Dense(1024,activation='relu')(x) #dense layer 2\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(512,activation='relu')(x) #dense layer 3\n    x = Dense(10, activation='softmax', name='predictions')(x)\n\n\n    xception_pretrained = Model(input = xception_input, output = x)\n    # for layer in resnet50_pretrained.layers[:2]:\n    #     layer.trainable=False\n    # for layer in resnet50_pretrained.layers[2:]:\n    #     layer.trainable=True\n    for layer in xception_pretrained.layers:\n        print(\"{}: {}\".format(layer, layer.trainable))\n\n    xception_pretrained.summary()\n\n\n    # Compile CNN model\n    adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n    sgd = optimizers.SGD(lr = 0.005)\n    xception_pretrained.compile(loss='categorical_crossentropy',optimizer = sgd,metrics=['accuracy'])\n    \n    return xception_pretrained\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelXception()_V2:\n    import keras\n    from keras.layers import Input\n    xception_input = Input(shape = (img_rows,img_cols, color_type), name = 'Image_input')\n\n    ## The RESNET model\n\n    from keras.applications.xception import preprocess_input, decode_predictions\n    from keras.applications.xception import Xception\n\n\n    #Get the RESNET weights and layers\n\n    model_xception_conv = Xception(weights= 'imagenet', include_top=False, input_shape= (img_rows,img_cols, color_type))\n    \n    \n    output_xception_conv = model_xception_conv(xception_input)\n\n    #Add the fully-connected layers \n\n    x=GlobalAveragePooling2D()(output_xception_conv)\n    #x=Dense(1024,activation='relu')(x) #we add dense layers so that the model can learn more complex functions and classify for better results.\n    #x = Dropout(0.1)(x) # **reduce dropout \n    #x=Dense(1024,activation='relu')(x) #dense layer 2\n    #x = BatchNormalization()(x)\n    #x = Dropout(0.5)(x)\n    #x = Dense(512,activation='relu')(x) #dense layer 3\n    x = Dense(10, activation='softmax', name='predictions')(x)\n\n\n    xception_pretrained = Model(input = xception_input, output = x)\n    # for layer in resnet50_pretrained.layers[:2]:\n    #     layer.trainable=False\n    # for layer in resnet50_pretrained.layers[2:]:\n    #     layer.trainable=True\n    for layer in xception_pretrained.layers:\n        print(\"{}: {}\".format(layer, layer.trainable))\n\n    xception_pretrained.summary()\n\n\n    # Compile CNN model\n    #adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n    sgd = optimizers.SGD(lr=5e-4, momentum=0.9)#lr = 0.005)\n    cc=CategoricalCrossentropy(label_smoothing=5e-5) \n\n    xception_pretrained.compile(loss=cc,optimizer = sgd,metrics=['accuracy'])\n    \n    return xception_pretrained\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelXception=modelXception_V2()\nmodelXception.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\n\ncheckpointer = ModelCheckpoint('xception_weights_aug_extralayer_alltrained_sgd2_V2.hdf5', verbose=1, save_best_only=True)\nearlystopper = EarlyStopping(monitor='val_loss', patience=15, verbose=1)\n\n\ndatagen = ImageDataGenerator(\n    height_shift_range=0.5,\n    width_shift_range = 0.5,\n    zoom_range = 0.5,\n    rotation_range=30,\n    shear_range=0.2\n        )\n\n#datagen.fit(X_train)\ndata_generator = datagen.flow(X_train, y_train, batch_size = batch_size)\n\n# Fits the model on batches with real-time data augmentation:\nxception_model = model.fit_generator(data_generator,steps_per_epoch = len(X_train) / batch_size, callbacks=[checkpointer, earlystopper],\n                                                            epochs = nb_epoch, verbose = 1, validation_data = (X_test, y_test))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Transfer Learning (ResNet)**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Input\nresnet50_input = Input(shape = (img_rows, img_cols, 3), name = 'Image_input')\n\n## The RESNET model\n\nfrom keras.applications.resnet50 import preprocess_input, decode_predictions\nfrom keras.applications.resnet50 import ResNet50\n\n\n#Get the RESNET weights and layers\n\nmodel_resnet50_conv = ResNet50(weights= 'imagenet', include_top=False, input_shape= (img_rows,img_cols,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\n\n\nx=model_resnet50_conv.output\nx=GlobalAveragePooling2D()(x)\n\nx = Dropout(0.5)(x)\n\nx=Dense(512,activation='relu')(x)\n\npreds=Dense(10,activation='softmax')(x) #final layer with softmax activation\n\nresnet50_pretrained = Model(inputs=model_resnet50_conv.input, outputs=preds)\n    \n#output_resnet50_conv = model_resnet50_conv(resnet50_input)\n#Add the fully-connected layers \n#x=GlobalAveragePooling2D()(output_resnet50_conv)\n#x = Flatten(name='flatten')(output_resnet50_conv)\n#x = Dense(500, activation='relu', name='fc1')(x)\n#x = Dense(500, activation='relu', name='fc2')(x)\n#x = Dense(10, activation='softmax', name='predictions')(x)\n\n\n#resnet50_pretrained = Model(input = resnet50_input, output = x)\n\nresnet50_pretrained.summary()\n\n# Compile CNN model\n#adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n\ndef custom_loss(y_true, y_pred):\n    return tf.keras.losses.CategoricalCrossentropy(y_true, y_pred, label_smoothing=0.1)\n\ndef step_decay(epoch):\n    initial_lrate = 0.001\n    drop = 0.5\n    epochs_drop = 10.0\n    lrate = initial_lrate * math.pow(drop,  \n        math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nlrate = LearningRateScheduler(step_decay)\n\nsgd = optimizers.SGD( lr=5e-4, momentum=0.9)#0.001)\ncc=CategoricalCrossentropy(label_smoothing=5e-5)\n\nresnet50_pretrained.compile(loss=cc,optimizer = sgd,metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\n\n\noutput_resnet50_conv = model_resnet50_conv(resnet50_input)\n\n#Add the fully-connected layers \n\nx = Flatten(name='flatten')(output_resnet50_conv)\nx = Dense(500, activation='relu', name='fc1')(x)\nx = Dense(500, activation='relu', name='fc2')(x)\nx = Dense(10, activation='softmax', name='predictions')(x)\n\n\nresnet50_pretrained = Model(input = resnet50_input, output = x)\n\nresnet50_pretrained.summary()\n\n# Compile CNN model\n#adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n\ndef custom_loss(y_true, y_pred):\n    return tf.keras.losses.CategoricalCrossentropy(y_true, y_pred, label_smoothing=0.1)\n\ndef step_decay(epoch):\n    initial_lrate = 0.001\n    drop = 0.5\n    epochs_drop = 10.0\n    lrate = initial_lrate * math.pow(drop,  \n        math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nlrate = LearningRateScheduler(step_decay)\n\nsgd = optimizers.SGD( lr=5e-4, momentum=0.9)#0.001)\ncc=CategoricalCrossentropy(label_smoothing=5e-5)\n\nresnet50_pretrained.compile(loss=cc,optimizer = sgd,metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\n\n\nx=model_resnet50_conv.output\nx=GlobalAveragePooling2D()(x)\n\n#x = Dropout(0.001)(x)\n\n#x=Dense(512,activation='relu')(x)\n\npreds=Dense(10,activation='softmax')(x) #final layer with softmax activation\n\nresnet50_pretrained = Model(inputs=model_resnet50_conv.input, outputs=preds)\n    \n#output_resnet50_conv = model_resnet50_conv(resnet50_input)\n#Add the fully-connected layers \n#x=GlobalAveragePooling2D()(output_resnet50_conv)\n#x = Flatten(name='flatten')(output_resnet50_conv)\n#x = Dense(500, activation='relu', name='fc1')(x)\n#x = Dense(500, activation='relu', name='fc2')(x)\n#x = Dense(10, activation='softmax', name='predictions')(x)\n\n\n#resnet50_pretrained = Model(input = resnet50_input, output = x)\n\nresnet50_pretrained.summary()\n\n# Compile CNN model\n#adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n\ndef custom_loss(y_true, y_pred):\n    return tf.keras.losses.CategoricalCrossentropy(y_true, y_pred, label_smoothing=0.1)\n\ndef step_decay(epoch):\n    initial_lrate = 0.001\n    drop = 0.5\n    epochs_drop = 10.0\n    lrate = initial_lrate * math.pow(drop,  \n        math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nlrate = LearningRateScheduler(step_decay)\n\nsgd = optimizers.SGD( lr=5e-4, momentum=0.9)#0.001)\ncc=CategoricalCrossentropy(label_smoothing=5e-5)\n\nresnet50_pretrained.compile(loss=cc,optimizer = sgd,metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\n\n\nx=model_resnet50_conv.output\nx=GlobalAveragePooling2D()(x)\n\nx = Dropout(0.001)(x)\n\nx=Dense(512,activation='relu')(x)\n\npreds=Dense(10,activation='softmax')(x) #final layer with softmax activation\n\nresnet50_pretrained = Model(inputs=model_resnet50_conv.input, outputs=preds)\n    \n#output_resnet50_conv = model_resnet50_conv(resnet50_input)\n#Add the fully-connected layers \n#x=GlobalAveragePooling2D()(output_resnet50_conv)\n#x = Flatten(name='flatten')(output_resnet50_conv)\n#x = Dense(500, activation='relu', name='fc1')(x)\n#x = Dense(500, activation='relu', name='fc2')(x)\n#x = Dense(10, activation='softmax', name='predictions')(x)\n\n\n#resnet50_pretrained = Model(input = resnet50_input, output = x)\n\nresnet50_pretrained.summary()\n\n# Compile CNN model\n#adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n\ndef custom_loss(y_true, y_pred):\n    return tf.keras.losses.CategoricalCrossentropy(y_true, y_pred, label_smoothing=0.1)\n\ndef step_decay(epoch):\n    initial_lrate = 0.001\n    drop = 0.5\n    epochs_drop = 10.0\n    lrate = initial_lrate * math.pow(drop,  \n        math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nlrate = LearningRateScheduler(step_decay)\n\nsgd = optimizers.SGD( lr=5e-4, momentum=0.9)#0.001)\ncc=CategoricalCrossentropy(label_smoothing=5e-5)\n\nresnet50_pretrained.compile(loss=cc,optimizer = sgd,metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint,EarlyStopping\n\ncheckpointer = ModelCheckpoint('resnet_weights_aug_alltrained_sgd2_setval.hdf5', verbose=1, save_best_only=True)\nearlystopper = EarlyStopping(monitor='accuracy', patience=7, verbose=1)\n\n\ndatagen = ImageDataGenerator(\n    height_shift_range=0.5,\n    width_shift_range = 0.5,\n    zoom_range = 0.5,\n    rotation_range=30\n        )\n#datagen.fit(X_train)\ndata_generator = datagen.flow(X_train, y_train, batch_size = batch_size)\n\n# Fits the model on batches with real-time data augmentation:\nresnet50_model = resnet50_pretrained.fit_generator(data_generator,steps_per_epoch = len(X_train) / batch_size, callbacks=[checkpointer, earlystopper,lrate],\n                                                            epochs = 40, verbose = 1, validation_data = (X_test, y_test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Transfer Learning (Mobile Net)**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.enable()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelMobileNet():\n    base_model=MobileNet(weights='imagenet',include_top=False) #imports the mobilenet model and discards the last 1000 neuron layer.\n\n    x=base_model.output\n    x=GlobalAveragePooling2D()(x)\n\n    x=Dense(1024,activation='relu')(x) #we add dense layers so that the model can learn more complex functions and classify for better results.\n\n    x = Dropout(0.1)(x) # ****reduce dropout \n    x=Dense(1024,activation='relu')(x) #dense layer 2\n\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n\n    x=Dense(512,activation='relu')(x) #dense layer 3\n\n    preds=Dense(10,activation='softmax')(x) #final layer with softmax activation\n\n    model = Model(inputs=base_model.input, outputs=preds)\n    \n    from keras import optimizers  \n\n    #adam = optimizers.Adam(lr=0.001) #tried 0.0005 - too slow and didn't converge\n    sgd = optimizers.SGD(lr = 0.005) # try 0.01 - didn't converge and 0.005 , 0.001 best acc of 11%\n\n    model.compile(optimizer=sgd,loss='categorical_crossentropy',metrics=['accuracy']) # create object\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fine tuned mobilenet model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelMobileNet_v1():\n    base_model=MobileNet(weights='imagenet',include_top=False) #imports the mobilenet model and discards the last 1000 neuron layer.\n\n    x=base_model.output\n    x=GlobalAveragePooling2D()(x)\n\n    x=Dense(1024,activation='relu')(x) #we add dense layers so that the model can learn more complex functions and classify for better results.\n\n    x = Dropout(0.00001)(x) # ****reduce dropout \n    x=Dense(1024,activation='relu')(x) #dense layer 2\n\n    x = BatchNormalization()(x)\n    x = Dropout(0.01)(x)\n\n    x=Dense(512,activation='relu')(x) #dense layer 3\n\n    preds=Dense(10,activation='softmax')(x) #final layer with softmax activation\n\n    model = Model(inputs=base_model.input, outputs=preds)\n    \n    #for layer in model.layers[:-2]:\n    #         layer.trainable=False\n    \n    for layer in model.layers:\n        print(\"{}: {}\".format(layer, layer.trainable))\n    from keras import optimizers  \n\n    #adam = optimizers.Adam(lr=0.001) #tried 0.0005 - too slow and didn't converge\n    sgd = optimizers.SGD(lr=5e-4, momentum=0.9) # try 0.01 - didn't converge and 0.005 , 0.001 best acc of 11%\n    #r=1e-4, momentum=0.9 old- lr = 0.005\n    #rms=optimizers.RMSprop(lr=1e-5)\n    cc=CategoricalCrossentropy(label_smoothing=5e-5) #label_smoothing=0.1) #'categorical_crossentropy'\n    \n    model.compile(optimizer=sgd,loss=cc,metrics=['accuracy']) # create object\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelMobileNet_Without_extra():\n    import keras\n\n    base_model=MobileNet(weights='imagenet',include_top=False) #imports the mobilenet model and discards the last 1000 neuron layer.\n\n    x=base_model.output\n    x=GlobalAveragePooling2D()(x)\n\n    #x = Dropout(0.001)(x)\n\n    #x=Dense(512,activation='relu')(x)\n    preds=Dense(10,activation='softmax')(x) #final layer with softmax activation\n\n    model = Model(inputs=base_model.input, outputs=preds)\n    \n    from keras import optimizers  \n    def step_decay(epoch):\n        initial_lrate = 5e-4\n        drop = 0.0001\n        epochs_drop = 10.0\n        lrate = initial_lrate * math.pow(drop,  \n            math.floor((1+epoch)/epochs_drop))\n        return lrate\n\n    lrate = LearningRateScheduler(step_decay)\n\n    #adam = optimizers.Adam(lr=0.001) #tried 0.0005 - too slow and didn't converge\n    #adam = keras.optimizers.Adam(lr=5e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0)\n    sgd = optimizers.SGD(lr=5e-4, momentum=0.9) # try 0.01 - didn't converge and 0.005 , 0.001 best acc of 11%\n    cc=CategoricalCrossentropy(label_smoothing=5e-5) #'categorical_crossentropy'\n    #RMSprop= keras.optimizers.RMSprop(lr=5e-4)\n    \n    \n    model.compile(optimizer=sgd,loss=cc,metrics=['accuracy']) # create object\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model=modelMobileNet_v1()\nmodel=modelMobileNet_Without_extra()\nmodel.summary()\ndef step_decay(epoch):\n    initial_lrate = 5e-4\n    drop = 0.0001\n    epochs_drop = 10.0\n    lrate = initial_lrate * math.pow(drop,  \n        math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nlrate = LearningRateScheduler(step_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\n\n#model=modelMobileNet()\n\ncheckpointer = ModelCheckpoint('mobilenet_sgd_extra_layersft_cv.hdf5', verbose=1, save_best_only=True)\nearlystopper = EarlyStopping(monitor='val_loss', patience=20, verbose=1)\n\ndatagen = ImageDataGenerator(\n    \n    height_shift_range=0.5,\n    width_shift_range = 0.5,\n    zoom_range = 0.5,\n    rotation_range=10\n    #shear_range=0.2\n        )\n#datagen.fit(X_train)\ndata_generator = datagen.flow(X_train, y_train, batch_size = batch_size)\n\n# Fits the model on batches with real-time data augmentation:\nmobilenet_model = model.fit_generator(data_generator,steps_per_epoch = len(X_train) / batch_size, callbacks=[checkpointer, earlystopper],\n                                                            epochs = nb_epoch, verbose = 1, validation_data = (X_test, y_test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://i.imgur.com/hUHTQJe.png)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndel model,checkpointer,earlystopper,data_generator,mobilenet_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions on validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#model= modelMobileNet_v1()\nmodel=modelMobileNet_Without_extra()\nmodel.load_weights('../input/imbalance/mobilenet_sgd_extra_layers1.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel4=resnet50_pretrained.load_weights('../input/imbalance/resnet_weights_aug_alltrained_sgd2_setval2.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_valid_res = resnet50_pretrained.predict(X_test, batch_size=batch_size, verbose=1)\nscore = log_loss(y_test, predictions_valid_res)\nprint('Score log_loss: ', score)\nval_pred=np.argmax(predictions_valid_res,axis=1)\nval=np.argmax(y_test,axis=1)\nprint(classification_report(val, val_pred,target_names=activity_map))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions on test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image = []\ni = 0\nfig, ax = plt.subplots(1, 20, figsize = (img_rows,img_cols))\n\nfiles = os.listdir('../input/state-farm-distracted-driver-detection/imgs/test/')\nnums = np.random.randint(low=1, high=len(files), size=20)\nfor i in range(20):\n    print ('Image number:',i)\n    img = cv2.imread('../input/state-farm-distracted-driver-detection/imgs/test/'+files[nums[i]])\n    #img = color.rgb2gray(img)\n    #img = img[50:,120:-50]\n    img = cv2.resize(img,(img_rows,img_cols))\n    test_image.append(img)\n    ax[i].imshow(img,cmap = 'gray')\n    plt.show","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model=modelMobileNet()\n#model= vgg_std16_model(img_rows, img_cols)\n#model=model_v2\n#model.load_weights('saved_models/weights_best_vanilla.hdf5')\ntest = []\n\nfor img in test_image:\n    test.append(img)\n    \n#model.load_weights('mobilenet_sgd_extra_layers.hdf5') #mobilenet\n#model.load_weights('saved_models/weights_best_vgg16.hdf5') #vgg16\n\ntest = np.array(test).reshape(-1,img_rows,img_cols,color_type)\nprediction = model.predict(test)\nprediction[0:1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags = { \"C0\": \"safe driving\",\n\"C1\": \"texting - right\",\n\"C2\": \"talking on the phone - right\",\n\"C3\": \"texting - left\",\n\"C4\": \"talking on the phone - left\",\n\"C5\": \"operating the radio\",\n\"C6\": \"drinking\",\n\"C7\": \"reaching behind\",\n\"C8\": \"hair and makeup\",\n\"C9\": \"talking to passenger\" }\n\ni = 0\nfig, ax = plt.subplots(20, 1, figsize = (img_rows,img_cols))\n\nfor i in range(20):\n    ax[i].imshow(test[i].squeeze())\n    #predicted_class = 'C'+str(np.where(prediction[i] == np.amax(prediction[i]))[0][0])\n    #ax[i].set_title(tags[predicted_class])\n    plt.show","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating submission csv","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_rows=112\nimg_cols = 112\ncolor_type = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntest_image = []\ntest_id=[]\ni = 0\n#fig, ax = plt.subplots(1, 20, figsize = (50,50 ))\n\nfiles = sorted(os.listdir('../input/state-farm-distracted-driver-detection/imgs/test/'))\nprint(len(files))\nj=len(files)/4   #####As memory keeps getting exhausted we handled the predictions on test set iteratively\nprint(j)\na=int(j)+int(j)\nb=a+int(j)\nprint(b)\n\nfor i in range(int(j),a):\n    print ('Image number:',str(i)+' File: '+files[i])\n    \n    img = cv2.imread('../input/state-farm-distracted-driver-detection/imgs/test/'+files[i])\n    #img = Image.open('../input/state-farm-distracted-driver-detection/imgs/test/'+files[i])\n    #wpercent = (img_rows/float(img.size[0]))\n    \n    #hsize = int((float(img.size[1])*float(wpercent)))\n    #img = img.resize((img_rows,hsize), PIL.Image.ANTIALIAS)\n    #img=np.array(img)\n    \n    #img = color.rgb2gray(img)\n    #img = img[50:,120:-50]\n    img = cv2.resize(img,(img_rows,img_cols))\n    test_image.append(img)\n    test_id.append(files[i])\n    #ax[i].imshow(img,cmap = 'gray')\n    #plt.show","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 0\nfig, ax = plt.subplots(1, 3, figsize = (img_rows,img_cols))\nfor i in range(3):\n    #print ('Image number:',i)\n    img = test[i]\n    #img = color.rgb2gray(img)\n    #\\img = img[50:,120:-50]\n    #img = cv2.resize(img,(img_rows,img_cols))\n    \n    ax[i].imshow(img)\n    plt.show\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntest = []\n\nfor img in test_image:\n    test.append(img)\n    \n\n#model.load_weights('mobilenet_sgd_extra_layers.hdf5')\n\ntest = np.array(test).reshape(-1,img_rows,img_cols,color_type)\ndel test_image\ngc.collect()\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ensemble for test ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel1=model.load_weights('../input/imbalance/mobilenet_sgd_extra_layers1.hdf5')\nmodel2=modelXception.load_weights('../input/imbalance/xception_weights_aug_extralayer_alltrained_sgd2_V2.hdf5')\n#model3=model_vgg16.load_weights('../input/models/weights_best_vgg16.hdf5')\n#model_vgg19.load_weights('../input/vgg19model/weights_best_vgg19.hdf5')\nmodel3=resnet50_pretrained.load_weights('../input/imbalance/resnet_weights_aug_alltrained_sgd2_setval2.hdf5')\n\n\npred1=model.predict(test)\npred2=modelXception.predict(test)\n#pred3=model_vgg19.predict(test)\npred3=resnet50_pretrained.predict(test)\n\nensemble_prediction=(pred1+pred2*0.3+pred3)/3\nlen(ensemble_prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submission(predictions, test_id, info):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    now = datetime.datetime.now()\n    if not os.path.isdir('subms'):\n        os.mkdir('subms')\n    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\n    sub_file = os.path.join('subms', 'submission_' + suffix + '.csv')\n    result1.to_csv(sub_file, index=False)\n    \n\ninfo_string='ens-half4'\ncreate_submission(ensemble_prediction, test_id, info_string)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**KNN FOR TEST**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"## Defining the input\n\nfrom keras.layers import Input\nvgg16_input = Input(shape = (img_rows,img_cols,3), name = 'Image_input')\n\n\n## The VGG model\n\nfrom keras.applications.vgg16 import VGG16, preprocess_input\n\n#Get back the convolutional part of a VGG network trained on ImageNet\nmodel_vgg16_conv = VGG16(weights='imagenet', include_top=False, input_tensor = vgg16_input)\n#model_vgg16_conv.summary()\nprint('Model Loaded')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" %%time\n#7min\nvgg16_features_output=model_vgg16_conv.predict(test)\nvgg16_features_output.shape\nknn_input=np.reshape(vgg16_features_output,(vgg16_features_output.shape[0],-1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg16_features_output.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file = open('test100_knninput.dat', 'wb') #saving output\npickle.dump( knn_input, file)\nfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.neighbors import NearestNeighbors #KNN ALGORITHM TEST (FINAL K-D TREES IN COLAB NOTEBOOK)\nnbrs = NearestNeighbors(n_neighbors=10).fit(knn_input)\nprint(\"Done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndistances, indices = nbrs.kneighbors(knn_input)\nlen(indices)#56 min\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indices[0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file = open('test32_indices.dat', 'wb')\npickle.dump( [distances, indices], file)\nfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"distancess,indices=test_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indices[0]  #[    0, 10535, 56026, 55935, 28105, 44194, 57421, 68745, 46932,50244])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(10, 1, figsize = (56,56))\n\nimg = 0\n\nfor i in indices[0]:  \n    ax[img].imshow(test[i])\n    plt.show\n    img = img+1\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nknn_predictions = []\npredictions = []\n\nprint(indices.shape[0])\nj=indices.shape[0]/5\nprint(j)\na=int(j)+int(j)\nb=a+int(j)\nc=b+int(j)\nprint(b)\nprint(c)\n\n\nfor i in range(0,20):\n    print(str(0),str(int(j)))\n    mean_prediction = []\n    for j in range(len(ensem[0])-1):\n        print(str(len(ensem[0])-1))\n        k=0\n        for img in indices[i]:\n            print(\"Image Number: \"+str(img))\n            print('')\n               \n            ensemble_prediction=ensem[img][:10]\n            print( ensem[img][10],img)\n                    \n            print(ensemble_prediction[j])\n            predictions.append(ensemble_prediction[j])\n            k=k+1\n            if k>=9:\n                break\n        print(\"herrer\")\n        trimmed_value = (sum(predictions) - max(predictions) - min(predictions))/(len(predictions) -2 )\n        mean_value = np.mean(predictions)\n        predictions = []\n        \n    \n        mean_prediction.append(trimmed_value)\n    \n    \n    mean_prediction = np.asarray(mean_prediction)/sum(mean_prediction)\n    print(\"mean calculated\")\n    knn_predictions.append(mean_prediction)\n    print(\"knn calculated\")\n    \nknn_predictions = np.asarray(knn_predictions)           \n        \n\n        \nprint('The code is done')\nknn_class = []\n\nfor i in range(len(knn_predictions)):\n    knn_class.append(np.where(knn_predictions[i] == np.amax(knn_predictions[i]))[0][0])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#prediction=knn_predictions\nprediction=knn_predictions\n\ntags = { \"C0\": \"safe driving\",\n\"C1\": \"texting - right\",\n\"C2\": \"talking on the phone - right\",\n\"C3\": \"texting - left\",\n\"C4\": \"talking on the phone - left\",\n\"C5\": \"operating the radio\",\n\"C6\": \"drinking\",\n\"C7\": \"reaching behind\",\n\"C8\": \"hair and makeup\",\n\"C9\": \"talking to passenger\" }\n\ni = 0\nfig, ax = plt.subplots(20, 1, figsize = (img_rows,img_cols))\n\nfor i in range(20):\n    ax[i].imshow(test[i].squeeze())\n    predicted_class = 'C'+str(np.where(prediction[i] == np.amax(prediction[i]))[0][0])\n    ax[i].set_title(tags[predicted_class])\n    plt.show","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = []\n\nfor img in test_image:\n    test.append(img)\n    \n\n#model.load_weights('mobilenet_sgd_extra_layers.hdf5')\n\ntest = np.array(test).reshape(-1,img_rows,img_cols,color_type)\nprediction = model.predict(test)\nprediction[0:1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('../input/others/mobilenet_sgd_extra_layersft-0.197.hdf5')\npred0=model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\nmodel1=model.load_weights('../input/imbalance/mobilenet_sgd_extra_layers1.hdf5')\nmodel2=modelXception.load_weights('../input/imbalance/xception_weights_aug_extralayer_alltrained_sgd2_V2.hdf5')\n#model3=model_vgg16.load_weights('../input/models/weights_best_vgg16.hdf5')\nmodel_vgg19.load_weights('../input/vgg19model/weights_best_vgg19.hdf5')\nmodel3=resnet50_pretrained.load_weights('../input/imbalance/resnet_weights_aug_alltrained_sgd2_setval_1.hdf5')\n\n\npred5=model.predict(X_test)\npred6=modelXception.predict(X_test)\npred7=model_vgg19.predict(X_test)\npred8=resnet50_pretrained.predict(X_test)\n\n#finalpred=(pred1*0.6+pred4*0.9+pred2*0.3+pred3*0.3)/4\n#finalpred=(pred1*0.7+pred4*0.9)/2\nfinalpred=(pred5+pred6*0.3+pred8)/3\n#finalpred=(pred1+pred2+pred3)/3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1=model.load_weights('../input/mobile/mobilenet_sgd_extra_layers (1).hdf5')\nmodel2=modelXception.load_weights('../input/modelss/xception_weights_aug_extralayer_alltrained_sgd2_V2.hdf5')\n#model3=model_vgg16.load_weights('../input/models/weights_best_vgg16.hdf5')\nmodel_vgg19.load_weights('../input/vgg19model/weights_best_vgg19.hdf5')\nmodel4=resnet50_pretrained.load_weights('../input/resnet/resnet_weights_aug_alltrained_sgd2_setval.hdf5')\n\n\npred1=model.predict(X_test)\npred2=modelXception.predict(X_test)\npred3=model_vgg19.predict(X_test)\npred4=resnet50_pretrained.predict(X_test)\n\nprediction=(pred1*0.6+pred4*0.9+pred2*0.3+pred3*0.3)/4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#predictions=(pred0*0.9+pred1*0.4+pred4*0.9+pred2*0.05+pred3*0.05)/5\n\npred=ensemble_predictions\n#pred=(pred5+pred6*0.05+pred8*0.7)/3\n\n#pred=(pred0*0.9+pred1*0.4+pred4*0.9+pred2*0.05+pred3*0.05+pred5+pred6*0.05+pred8*0.7)/8\n\nscore = log_loss(y_test, pred)\nprint('Score log_loss: ', score)\nval_pred=np.argmax(pred,axis=1)\n\nval=np.argmax(y_test,axis=1)\nprint(classification_report(val, val_pred,target_names=activity_map))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Network ensemble predictions on test set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from statistics import mean,median\n\npredictions = []\nensemble_predictions = []\n\nfor i in range(len(X_test)):\n#for i in range(1):\n    mean_prediction = []\n    \n    for j in range(10):\n        predictions.append(pred0[i][j]*0.9)\n        predictions.append(pred1[i][j]*0.4)\n        predictions.append(pred2[i][j]*0.05)\n        predictions.append(pred3[i][j]*0.05)\n        predictions.append(pred4[i][j]*0.9)\n        \n        predictions.append(pred5[i][j])\n        predictions.append(pred6[i][j]*0.05)\n        predictions.append(pred8[i][j]*0.7)\n       \n        \n        trimmed_value = (sum(predictions) - max(predictions) - min(predictions))/(len(predictions) -2 )\n        \n        mean_value =np.mean(predictions)\n\n        predictions = []\n        mean_prediction.append(trimmed_value)\n        #mean_prediction.append(mean_value)\n    \n    mean_prediction = mean_prediction/ sum(mean_prediction)\n    ensemble_predictions.append(mean_prediction)\n        \n    \n#ensemble_predictions = np.asarray(ensemble_predictions)        \nprint('Got predictions from ensemble')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* For ensemble results","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model1=model.load_weights('../input/mobile/mobilenet_sgd_extra_layers (1).hdf5')\nmodel2=modelXception.load_weights('../input/modelss/xception_weights_aug_extralayer_alltrained_sgd2_V2.hdf5')\n#model3=model_vgg16.load_weights('../input/models/weights_best_vgg16.hdf5')\nmodel_vgg16.load_weights('../input/vgg19model/weights_best_vgg19.hdf5')\nmodel4=resnet50_pretrained.load_weights('../input/resnet/resnet_weights_aug_alltrained_sgd2_setval.hdf5')\n\n\n\ntest = []\n\nfor img in test_image:\n    test.append(img)\n    \n\n#model.load_weights('mobilenet_sgd_extra_layers.hdf5')\n\ntest = np.array(test).reshape(-1,img_rows,img_cols,color_type)\npred1=model.predict(test)\npred2=modelXception.predict(test)\npred3=model_vgg19.predict(test)\npred4=resnet50_pretrained.predict(test)\n\nprediction=(pred1*0.6+pred4*0.9+pred2*0.3+pred3*0.3)/4\nprediction[0:1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submission(predictions, test_id, info):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    now = datetime.datetime.now()\n    if not os.path.isdir('subms'):\n        os.mkdir('subms')\n    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\n    sub_file = os.path.join('subms', 'submission_' + suffix + '.csv')\n    result1.to_csv(sub_file, index=False)\n    \n\ninfo_string='ens-half4'\ncreate_submission(prediction, test_id, info_string)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del prediction,test,img,test_image,test_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.enable()\ngc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}