{"cells":[{"metadata":{"_uuid":"1253ba418616021aae1cf19531df65b87645b465"},"cell_type":"markdown","source":"# Xplore the files"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"from os import listdir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7eae6f6e2fc1a1d430bcd71be1015ead8194f2ff"},"cell_type":"code","source":"train_path = '../input/train/'\nlistdir(train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a24311197bbf8d3382a281da5de241c89fb06520"},"cell_type":"code","source":"test_path = '../input/test/'\nlistdir(test_path)[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6034eaff653ed3cc94d52a800c75d5178d4899c9"},"cell_type":"code","source":"len(listdir(test_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4f27675e1e4bfb664d65f06d56eb085f7073a94","collapsed":true},"cell_type":"code","source":"test_path_array = listdir(test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0bdb07340286c1767855cf740ff42bcf64285407"},"cell_type":"code","source":"matching = [s for s in test_path_array if \"img_1.jpg\" in s]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bce9b2eb570a602f8d7675155132f028a1eb30eb"},"cell_type":"code","source":"matching","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2c4b19fce15faa565560d554f902df3cd55bf4e","collapsed":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0148809562c58ccd590f0df192ca9564989307a2","collapsed":true},"cell_type":"code","source":"driver_imgs = pd.read_csv('../input/driver_imgs_list.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"256a25322ac06de26a6374dd68c8bef51f3fe31f"},"cell_type":"code","source":"driver_imgs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16226556052fe8820681e78dedae5fc052ef2f11","collapsed":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0420171214dcbb42557adc296d1e53983b29483f"},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d8a16e5860f61e163d7c81b019841115724b3fe"},"cell_type":"code","source":"listdir(train_path + 'c0/')[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bce4caa5bcde58d3fafb8f26b9cbdac7fca676f3"},"cell_type":"markdown","source":"# Label preparation"},{"metadata":{"trusted":true,"_uuid":"aa1226dca22ec7a202a4460db3417b7331d18fb2","collapsed":true},"cell_type":"code","source":"from tqdm import tqdm\ndef loadBatchImages(path):\n    catList = listdir(path)\n    loadedImages = []\n    loadedLabels = []\n    for cat in catList:\n        if not cat.startswith('.'):\n            deepPath = path+cat+\"/\"\n            imageList = listdir(deepPath)\n            for images in tqdm(imageList):\n                img = deepPath + images\n                loadedLabels.append(int(cat[1:]))\n                loadedImages.append(img)\n            \n    return loadedImages, loadedLabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27e1c3f4ef60d8c51ce30b4e44f6a8172bfb83fb"},"cell_type":"code","source":"train_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4df58c06526f9fa823e9ead2d1417c710f7e77e8"},"cell_type":"code","source":"loadedImages, loadedLabels = loadBatchImages(train_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b4157c0c504851fc0d09b71ab5a9b48d7f67413"},"cell_type":"markdown","source":"# Convert to One Hot Encoding Labels"},{"metadata":{"trusted":true,"_uuid":"a79fa473a125f5441f7bb355f1b97611cac1d185","collapsed":true},"cell_type":"code","source":"num_classes = len(np.unique(loadedLabels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"911b97577929b25a55c14ee18c2c5aa884db9456"},"cell_type":"code","source":"num_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ac17106dda687af2d4a0749bfb14063674e7f6a"},"cell_type":"code","source":"# Encode labels to hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\nfrom keras.utils.np_utils import to_categorical\nlabels_Hot = to_categorical(loadedLabels, num_classes = num_classes)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"adc3f4b8c02d84b3951ea774e71d41d2d7312861"},"cell_type":"markdown","source":"# Create the Dataframe for Datagenerators"},{"metadata":{"trusted":true,"_uuid":"e3cc14dedf691a2fb2732b73c7d5f2a7d81f2879","collapsed":true},"cell_type":"code","source":"df= pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4051dd5972dab690106fce2c6b34c3f3d7e58ffc","collapsed":true},"cell_type":"code","source":"df['path']=loadedImages\ndf['labels'] = list(labels_Hot)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f14ace0503fbd96b5ac4f6d8914a2ff5068e62d"},"cell_type":"markdown","source":"# Create Data Generators"},{"metadata":{"trusted":true,"_uuid":"5dff97bfd88839e982b049dcb90952dfbbaa89ba","collapsed":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nIMG_SIZE = (128, 128)\ncore_idg = ImageDataGenerator(samplewise_center=True, \n                              samplewise_std_normalization=True, \n                              horizontal_flip = True, \n                              vertical_flip = False, \n                              height_shift_range= 0.05, \n                              width_shift_range=0.1, \n                              rotation_range=5, \n                              shear_range = 0.1,\n                              fill_mode = 'reflect',\n                              zoom_range=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aba5abed58178870c715191aef4557f2513a702d","collapsed":true},"cell_type":"code","source":"def flow_from_dataframe(img_data_gen, in_df, path_col, y_col, **dflow_args):\n    base_dir = os.path.dirname(in_df[path_col].values[0])\n    print('## Ignore next message from keras, values are replaced anyways')\n    df_gen = img_data_gen.flow_from_directory(base_dir, \n                                     class_mode = 'sparse',\n                                    **dflow_args)\n    df_gen.filenames = in_df[path_col].values\n    df_gen.classes = np.stack(in_df[y_col].values)\n    df_gen.samples = in_df.shape[0]\n    df_gen.n = in_df.shape[0]\n    df_gen._set_index_array()\n    df_gen.directory = '' # since we have the full path\n    print('Reinserting dataframe: {} images'.format(in_df.shape[0]))\n    return df_gen","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"408f683a1f36142ea66f6df2a7146eb4ffc28912","collapsed":true},"cell_type":"code","source":"train_df = df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2e5dace24bf6c88af67529aeca03c58795f14db"},"cell_type":"code","source":"train_gen = flow_from_dataframe(core_idg, train_df, \n                             path_col = 'path',\n                            y_col = 'labels', \n                            target_size = IMG_SIZE,\n                            batch_size = 32)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"975387fb1bfeb370c6d741e2b772d263cd6d0acd","collapsed":true},"cell_type":"code","source":"t_x, t_y = next(train_gen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9c920a1db3b064a2c61a9d6fe930afe516289e3"},"cell_type":"code","source":"t_x.shape[1:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"386c2a62f099bc0cfb1cd1423a090c55f4552661"},"cell_type":"markdown","source":"# Vgg16"},{"metadata":{"trusted":true,"_uuid":"0bcaf0faf8e68c6efa154815d69f7fc53f77f984","collapsed":true},"cell_type":"code","source":"from keras.applications import VGG16\nfrom keras.applications.vgg16 import preprocess_input\n\nimport keras\nfrom keras import backend as K\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Model,Sequential, model_from_json\nfrom keras.optimizers import SGD, RMSprop, Adam, Adagrad, Adadelta\nfrom keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Conv2D, MaxPool2D, MaxPooling2D\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33f4cf1b611ac3809de5828e0635faed1189811f"},"cell_type":"code","source":"pretrained_model_1 = VGG16(include_top=False, input_shape=t_x.shape[1:])\nbase_model = pretrained_model_1 # Topless\noptimizer1 = keras.optimizers.Adam()\n# Add top layer\nx = base_model.output\nx = Conv2D(100, kernel_size = (3,3), padding = 'valid')(x)\nx = Flatten()(x)\nx = Dropout(0.75)(x)\npredictions = Dense(num_classes, activation='softmax')(x)\nmodel = Model(inputs=base_model.input, outputs=predictions)\n# Train top layer\nfor layer in base_model.layers:\n    layer.trainable = False\nmodel.compile(loss='categorical_crossentropy', \n              optimizer=optimizer1, \n              metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"942b7cf601e4330de1d93f95eefe28f3d56af001"},"cell_type":"code","source":"model.fit_generator(train_gen,steps_per_epoch=100,epochs = 10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1402e4f6034bd93b16d0c8c43161fadce6f803a"},"cell_type":"markdown","source":"# Test Data preparation"},{"metadata":{"trusted":true,"_uuid":"a885d88d3d7a7fdf23496d5bf1f9fca9075457dd","collapsed":true},"cell_type":"code","source":"import glob\nfrom glob import glob\ntest_image_paths = glob('../input/test/*.jpg', recursive=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"77f11db9786ed21b6026e5e88baeee9b83206eba"},"cell_type":"code","source":"X_test = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"990194985fa42611384639e7010885a02765b95a"},"cell_type":"code","source":"X_test['path'] = test_image_paths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3b1dbc6213e205f317b5573da2db579966242c86"},"cell_type":"code","source":"X_test['labels'] = X_test['path'].map(lambda x: os.path.splitext(os.path.basename(x))[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e7c26e5bd28d8b716d121761671f87e34c4c79b"},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1aec9cef30101cf881b33508fa6653594fea0116"},"cell_type":"code","source":"X_test['labels'] = X_test['labels'] + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"331fa3b4a6400b76b937917e7336a7bef8bc9736"},"cell_type":"code","source":"X_test['labels'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f6783c4370ae62a91d7686742c5e4f7d9718441"},"cell_type":"code","source":"test_gen = flow_from_dataframe(core_idg, X_test, \n                             path_col = 'path',\n                            y_col = 'labels', \n                            target_size = IMG_SIZE,\n                            batch_size = 256) # we can use much larger batches for evaluation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e717eed22a032ea432fde18088a1889758ef474"},"cell_type":"code","source":"len(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d56a39198f272f069366dc8965e17dc6a33e4132"},"cell_type":"code","source":"pred_Y =  model.predict_generator(test_gen,verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"799b5d1e3ef56ac6a5d20f91e75913d06f418d48"},"cell_type":"code","source":"pred_Y[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f4061accf77693f77bbad37f7f077527bf3d7a5a"},"cell_type":"code","source":"def do_clip(arr, mx): return np.clip(arr, (1-mx)/9, mx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8fd1d9cd0461b7b140d6b4ebcef0f31eae2427f3"},"cell_type":"code","source":"pred_Y = do_clip(pred_Y,0.93)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0df5833e15adae8e494a870d693ac47ded9d9c4"},"cell_type":"code","source":"len(pred_Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d5dee47e7c1575fd9646226f23619e81dd5f59d"},"cell_type":"code","source":"pred_Y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c2a1c9dcb3d19bfa29ee9f08271f872ee309b89"},"cell_type":"code","source":"len(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16dcf189b3e91d48ad17a61579e60023674c9fb8"},"cell_type":"code","source":"X_test['labels'][:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"36b23dc76659dd62f9b0947123500f59651241fd"},"cell_type":"code","source":"submission = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ae92086b5de08a090478499f1b39fb5643b21d50"},"cell_type":"code","source":"submission['img'] = X_test['labels']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8533b0e6384c6d071500b21b6fe7610191211dec"},"cell_type":"code","source":"submission['c0'] = pred_Y[:,0]\nsubmission['c1'] = pred_Y[:,1]\nsubmission['c2'] = pred_Y[:,2]\nsubmission['c3'] = pred_Y[:,3]\nsubmission['c4'] = pred_Y[:,4]\nsubmission['c5'] = pred_Y[:,5]\nsubmission['c6'] = pred_Y[:,6]\nsubmission['c7'] = pred_Y[:,7]\nsubmission['c8'] = pred_Y[:,8]\nsubmission['c9'] = pred_Y[:,9]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48cf663eac4b4ee33619e2eb31b07fd2eaeace9a","collapsed":true},"cell_type":"code","source":"submission.to_csv(\"predictions.csv\",index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"416e57fdcecf86d876a04fc66074a2fdb46c33eb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}