{"cells":[{"metadata":{},"cell_type":"markdown","source":"* > **WARNING**\n* > ****Please Change the Loss Function and Metrics and Use your own Custom Model****\n* > ****This is just a Blueprint and will not work if you commit or submit****\n* > ****The time took for the current test model to be created is around 60s and fitting the model I genuinely have no idea, I tried training it for 10 records it took around an hour on GPU. I've changed the code and have no idea if it works.****"},{"metadata":{},"cell_type":"markdown","source":"# Importing Required Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas\nimport os\nimport numpy\nfrom tqdm import tqdm_notebook\nimport gc\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Making Symbolic Links"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_images images\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_maps maps\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_lidar lidar","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hyperparameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"shape = (100, 100, 3)\nMAX_VALUE = 140","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Understanding Directories"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# Directories :-\nPATH = '/kaggle/input/3d-object-detection-for-autonomous-vehicles/'\nTRAIN_PATHS = [i for i in os.listdir(PATH) if 'train' in i]\nTEST_PATHS = [i for i in os.listdir(PATH) if 'test' in i]\n\nTRAIN_PATHS, TEST_PATHS","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Installing and Making the LyftDataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install lyft-dataset-sdk\nfrom pyquaternion import Quaternion\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset\nlyft_data = LyftDataset(data_path = '.', json_path = PATH + 'train_data', verbose = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Getting the training data and necessary features"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pandas.read_csv(PATH + 'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categories = [i['name'] for i in lyft_data.category]\ncategories","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['confidence' ,'center_x', 'center_y', \"center_z\", 'width', 'length', 'height', 'rotate_w', 'rotate_x', 'rotate_y', 'rotate_z', 'class']\nsensors = lyft_data.sensor\nsensors = [i['channel'] for i in sensors]\nsensors = [i for i in sensors if 'LIDAR' not in i]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utility Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def getImageFileNames(token : str):\n    \n    list_of_filenames = []\n    \n    for sensor in sensors:\n        filename = lyft_data.get('sample_data', lyft_data.get('sample', token)['data'][sensor])['filename']\n        list_of_filenames.append(filename)\n        \n    return list_of_filenames        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getData(token):\n    \n    list_of_values = []\n    list_of_anns = lyft_data.get('sample', token)['anns']\n    \n    for annotation_token in list_of_anns:\n        values = [1.0]\n        sample_data = lyft_data.get('sample_annotation', annotation_token)\n        values = values + sample_data['translation'] + sample_data['size'] + sample_data['rotation'] + one_hot_encoding(sample_data['category_name'])\n        list_of_values.append(values)\n        \n    for _ in range(MAX_VALUE - len(list_of_anns)):\n        list_of_values.append([0]*11 + one_hot_encoding('other_vehicle'))\n    \n    return numpy.array(list_of_values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating a Model with Custom Metrics and Loss Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Dropout, concatenate, GlobalAveragePooling2D, add\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nimport tensorflow as tf\n\ndef mse(y_true, y_pred):\n     return K.mean(K.square(y_pred - y_true), axis=-1)\n\ndef LossFunction(y_true, y_pred):\n     return mse(y_true[:,0], y_pred[:,0]) + mse(y_true[:,1:4], y_pred[:,:4]) + mse(y_true[:,4:7], y_pred[:,4:7]) + mse(y_true[:,7:11], y_pred[:,7:11]) + K.categorical_crossentropy(y_true[:,11:], y_pred[:,11:])\n\ndef IOU_Metric(true, pred): #any shape can go - can't be a loss function\n    \n    def iou_loss_core(true,pred):  #this can be used as a loss if you make it negative\n        intersection = true * pred\n        notTrue = 1 - true\n        union = true + (notTrue * pred)\n\n        return (K.sum(intersection, axis=-1) + K.epsilon()) / (K.sum(union, axis=-1) + K.epsilon())\n    \n    def castF(x):\n        return K.cast(x, K.floatx())\n\n    def castB(x):\n        return K.cast(x, bool)\n    \n    tresholds = [0.5 + (i*.05)  for i in range(10)]\n\n    #flattened images (batch, pixels)\n    true = K.batch_flatten(true)\n    pred = K.batch_flatten(pred)\n    pred = castF(K.greater(pred, 0.5))\n\n    #total white pixels - (batch,)\n    trueSum = K.sum(true, axis=-1)\n    predSum = K.sum(pred, axis=-1)\n\n    #has mask or not per image - (batch,)\n    true1 = castF(K.greater(trueSum, 1))    \n    pred1 = castF(K.greater(predSum, 1))\n\n    #to get images that have mask in both true and pred\n    truePositiveMask = castB(true1 * pred1)\n\n    #separating only the possible true positives to check iou\n    testTrue = tf.boolean_mask(true, truePositiveMask)\n    testPred = tf.boolean_mask(pred, truePositiveMask)\n\n    #getting iou and threshold comparisons\n    iou = iou_loss_core(testTrue,testPred) \n    truePositives = [castF(K.greater(iou, tres)) for tres in tresholds]\n\n    #mean of thressholds for true positives and total sum\n    truePositives = K.mean(K.stack(truePositives, axis=-1), axis=-1)\n    truePositives = K.sum(truePositives)\n\n    #to get images that don't have mask in both true and pred\n    trueNegatives = (1-true1) * (1 - pred1) # = 1 -true1 - pred1 + true1*pred1\n    trueNegatives = K.sum(trueNegatives) \n\n    return (truePositives + trueNegatives) / castF(K.shape(true)[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create your own Model\n# This is a test model\ndef custom_model():\n    \n    inputs = []\n    x = []\n    for i in range(len(sensors)):\n        inputs.append(Input(shape, batch_size = 1, name = 'CAM_INPUT_' + str(i)))\n\n    for i in range(len(sensors)):\n        y = Conv2D(64, (3, 3))(inputs[i])\n        y = Conv2D(64, (3, 3))(y)\n        y = Dropout(0.05)(y)\n\n        y = Conv2D(128, (3, 3))(y)\n        y = MaxPool2D()(y)\n\n        x.append(y)\n        \n    x = add(x)\n    x = GlobalAveragePooling2D()(x)\n\n    outputs = []\n    for i in range(MAX_VALUE):\n\n        confidence = Dense(1, activation = 'sigmoid', name = 'CONFIDENCE_' + str(i+1))(x)\n        center = Dense(3, activation = 'linear', name = 'CENTER_' + str(i+1))(x)\n        size = Dense(3, activation = 'linear', name = 'SIZE_' + str(i+1))(x)\n        rotation = Dense(4, activation = 'linear', name = 'ROTATION_' + str(i+1))(x)\n        category = Dense(10, activation = 'softmax', name = 'CATEGORY_' + str(i+1))(x)\n\n        output_layer = concatenate([confidence, center, size, rotation, category], name = 'OUTPUT_LAYER_' + str(i))\n        \n        outputs.append(output_layer)\n        \n    model = Model(inputs, outputs)\n    model.compile(loss = LossFunction, optimizer = 'adam', metrics = ['accuracy', IOU_Metric])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def one_hot_encoding(value):\n    global categories\n    \n    x = categories.index(value)\n    \n    return [0] * (x) + [1] + [0] * (len(categories) - x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_fit(model):\n    \n    for token in train_data['Id']:\n        values = getData(token)\n        values = values.reshape((values.shape[0],) + (1, ) + (values.shape[1], ))\n        filenames = getImageFileNames(token)\n        \n        images = [numpy.asarray(Image.open(i).resize(shape[:-1])).reshape((1, ) + shape) for i in filenames]\n        \n        print(images[0].shape, values.shape)\n        print(\"Training the model . . . . . \")\n        model.fit(images, values.tolist())\n        print(\"Model Trained for token : {}\".format(token))\n        \n        del images, values\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel = custom_model()\n# print(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fit(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TODO : Prediction and Testing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !rm images\n# !rm maps\n# !rm lidar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_images images\n# !ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_maps maps\n# !ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_lidar lidar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_lyft_data = LyftDataset(data_path = '.', json_path = PATH + 'test_data', verbose = True)\n# test_data = pandas.read_csv(PATH + 'sample_submission.csv')\n\n# x = test_lyft_data.get('sample_data', test_lyft_data.get('sample', test_data['Id'][0])['data']['CAM_BACK'])\n# y = test_lyft_data.get('calibrated_sensor', x['calibrated_sensor_token'])\n# z = test_lyft_data.get('ego_pose', x['ego_pose_token'])\n\n# y, z, test_lyft_data.get_sample_data(test_lyft_data.get('sample_data', test_lyft_data.get('sample', test_data['Id'][0])['data']['CAM_BACK'])['token'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}