{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import load_img\nimport numpy as np \nimport pandas as pd \n\nimport numpy as np\nimport cv2\nimport random\nimport math\nimport time, os\nfrom PIL import Image, ImageEnhance\nfrom matplotlib import pyplot as plt\nfrom glob import glob\nfrom scipy.misc import imread\nfrom math import sin, cos\nPATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)\ntrain = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'sample_submission.csv')\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T04:20:38.376939Z","iopub.execute_input":"2021-05-27T04:20:38.377686Z","iopub.status.idle":"2021-05-27T04:20:38.451366Z","shell.execute_reply.started":"2021-05-27T04:20:38.37761Z","shell.execute_reply":"2021-05-27T04:20:38.450424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The fuction below is for transform  PredictionString to list of dicts**","metadata":{}},{"cell_type":"code","source":"def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords","metadata":{"execution":{"iopub.status.busy":"2021-05-27T03:00:21.030765Z","iopub.execute_input":"2021-05-27T03:00:21.031148Z","iopub.status.idle":"2021-05-27T03:00:21.0398Z","shell.execute_reply.started":"2021-05-27T03:00:21.031084Z","shell.execute_reply":"2021-05-27T03:00:21.038516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  **Image Augmentation:**\n\nsorce:https://www.kaggle.com/niuddd/image-augmentations-for-pku-self-driving-car\n\nFor image augmentation we do (1)Contrast  (2)Brightness  (3)Add noise\n\nThe fuction will ramdom select images from train\n\nthen create some enhanced images for augmentation\n\nfinally return (1)dataframe of these images (2)list of these PIL Images \n\nnote:   Add noise will take a little long time","metadata":{}},{"cell_type":"code","source":"def Contrast_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    Contrast_data=[]\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'), img_number)\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        img = Image.open(fname)\n        ##add contrast here\n        enh = ImageEnhance.Contrast(img)\n        img_enh = enh.enhance(np.random.uniform(1.5, 2))#PIL.Image\n        ##save image \n        img.save('Contrast_'+str(i)+\".jpg\")\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        Contrast_data.append([img_enh,str2coords(pstring)])\n      \n    train.reset_index(inplace=True)    \n    return Contrast_data\n        \n    \n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Brightness_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    Brightness_data=[]\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'), img_number)\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        img = Image.open(fname)\n        ##add Brightness here\n        enh = ImageEnhance.Brightness(img)\n        img_enh = enh.enhance(np.random.uniform(0.5, 1.0))#PIL.Image\n        ##save image\n        img.save('Brightness_'+str(i)+\".jpg\")\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        Brightness_data.append([img_enh,str2coords(pstring)])\n    train.reset_index(inplace=True)    \n    return Brightness_data\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_noise(image):\n    \"\"\"gauss noise\"\"\"\n    row,col,ch= image.shape\n    mean = 0\n    var = np.random.random()*0.01 #0.001~0.01\n    sigma = var**0.5\n    gauss = np.random.normal(mean,sigma,(row,col,ch))\n    gauss = gauss.reshape(row,col,ch)\n    noisy = image + gauss\n    noisy = np.clip(noisy, 0, 1)\n    return noisy\ndef Noise_enhance(img_number):\n    seed = np.random.randint(1, 2019)\n    np.random.seed(seed)\n    Noise_data=[]\n    fname_list = np.random.choice(glob('../input/pku-autonomous-driving/train_images/*'),img_number)\n    train.set_index(\"ImageId\" , inplace=True)\n    for i,ax in enumerate(fname_list):\n        fname = fname_list[i]\n        img = plt.imread(fname)\n        img = (img/255).astype('float32')\n        ##add noise here\n        img_enh = add_noise(img)#gauss\n        ##create enhanced image's PredictionString from oringinal image\n        pstring=train.loc[fname.split('/')[-1].split('.jpg')[0]][0]\n        Noise_data.append([img_enh,str2coords(pstring)])\n    train.reset_index(inplace=True)    \n    return Noise_data\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Image preprocessing:**","metadata":{}},{"cell_type":"code","source":"IMG_WIDTH = 1024\nIMG_HEIGHT = IMG_WIDTH // 16 * 5\nMODEL_SCALE = 8\n\ndef rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\n\ndef _regr_preprocess(regr_dict, flip=False):\n    if flip:\n        for k in ['x', 'pitch', 'roll']:\n            regr_dict[k] = -regr_dict[k]\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef preprocess_image(img, flip=False):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 6]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    if flip:\n        img = img[:,::-1]\n    return (img / 255).astype('float32')\n\ndef get_mask_and_regr(img, labels, flip=False):\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    coords = str2coords(labels)\n    xs, ys = get_img_coords(labels)\n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        #print(x,img.shape[0] // 2,y, img.shape[1] // 6)\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        #x=(x*1/2)*(IMG_HEIGHT / MODEL_SCALE)/(img.shape[0] // 2)\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // 6) * IMG_WIDTH / (img.shape[1] * 4/3) / MODEL_SCALE\n        #y=(y* 4/3)*(IMG_WIDTH / MODEL_SCALE)/((img.shape[1] * 3/4) )\n\n        y = np.round(y).astype('int')\n        #print(x,y)\n\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict, flip)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n    if flip:\n        mask = np.array(mask[:,::-1])\n        regr = np.array(regr[:,::-1])\n    return mask, regr\n\ndef get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    #img_p[:, 0] /= zs\n    #img_p[:, 1] /= zs\n    \n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\n       ","metadata":{"execution":{"iopub.status.busy":"2021-05-27T03:00:26.121915Z","iopub.execute_input":"2021-05-27T03:00:26.122277Z","iopub.status.idle":"2021-05-27T03:00:26.249958Z","shell.execute_reply.started":"2021-05-27T03:00:26.122218Z","shell.execute_reply":"2021-05-27T03:00:26.248214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Generate mask images as target images**\n\nI just save mask images without regr","metadata":{}},{"cell_type":"code","source":"train.set_index(\"ImageId\" , inplace=True)\nfor filename in os.listdir('../input/pku-autonomous-driving/train_images'):\n    img0 = imread('../input/pku-autonomous-driving/train_images/'+filename)\n    img = preprocess_image(img0)\n    pstring=train.loc[filename.split('.jpg')[0]]\n    mask, regr = get_mask_and_regr(img0, train.loc[filename.split('.jpg')]['PredictionString'][0])\n    #print(train.loc[filename.split('.jpg')]['PredictionString'][0])\n    #cv2.imwrite('/kaggle/working/'+filename,mask) \n    np.save(filename,  mask)\ntrain.reset_index(inplace=True)  ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"check the ouput of mask images:","metadata":{}},{"cell_type":"code","source":"from IPython.display import Image, display\nfrom tensorflow.keras.preprocessing.image import load_img\nimport PIL\nfrom PIL import ImageOps\n\n# Display input image #7\ndisplay(Image(filename=input_img_paths[9]))\n\n# Display auto-contrast version of corresponding target (per-pixel categories)\nmask =np.load(target_img_paths[9])\nplt.imshow(mask)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-05-27T05:15:09.245802Z","iopub.execute_input":"2021-05-27T05:15:09.246239Z","iopub.status.idle":"2021-05-27T05:15:09.413222Z","shell.execute_reply.started":"2021-05-27T05:15:09.246179Z","shell.execute_reply":"2021-05-27T05:15:09.411784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_dir = \"../input/pku-autonomous-driving/train_images/\"\ntarget_dir = \"/kaggle/working/\"\nimg_size = (320,1024)\ntarget_size=(40, 128)\nnum_classes = 3\nbatch_size = 32\ninput_img_paths = sorted(\n    [\n        os.path.join(input_dir, fname)\n        for fname in os.listdir(input_dir)\n        if fname.endswith(\".jpg\")\n    ]\n)\ntarget_img_paths = sorted(\n    [\n        os.path.join(target_dir, fname)\n        for fname in os.listdir(target_dir)\n        if fname.endswith(\".npy\") and not fname.startswith(\".\")\n    ]\n)\n\nprint(\"Number of samples:\", len(input_img_paths),len(target_img_paths))\n\nfor input_path, target_path in zip(input_img_paths[:10], target_img_paths[:10]):\n    print(input_path, \"|\", target_path)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T04:11:19.864457Z","iopub.execute_input":"2021-05-27T04:11:19.864889Z","iopub.status.idle":"2021-05-27T04:11:19.926779Z","shell.execute_reply.started":"2021-05-27T04:11:19.864826Z","shell.execute_reply":"2021-05-27T04:11:19.922983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Create cardataset**","metadata":{}},{"cell_type":"code","source":"class CarDataset(keras.utils.Sequence):\n    \"\"\"Helper to iterate over the data (as Numpy arrays).\"\"\"\n\n    def __init__(self, batch_size, img_size,target_size, input_img_paths, target_img_paths):\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.target_size = target_size\n        self.input_img_paths = input_img_paths\n        self.target_img_paths = target_img_paths\n\n    def __len__(self):\n        return len(self.target_img_paths) // self.batch_size\n\n    def __getitem__(self, idx):\n        \"\"\"Returns tuple (input, target) correspond to batch #idx.\"\"\"\n        i = idx * self.batch_size\n        batch_input_img_paths = self.input_img_paths[i : i + self.batch_size]\n        batch_target_img_paths = self.target_img_paths[i : i + self.batch_size]\n        x = np.zeros((self.batch_size,) + self.img_size + (3,), dtype=\"float32\")\n        for j, path in enumerate(batch_input_img_paths):\n            img = load_img(path, target_size=self.img_size)\n            x[j] = img\n        y = np.zeros((self.batch_size,) + self.target_size + (1,), dtype=\"uint8\")\n        for j, path in enumerate(batch_target_img_paths):\n            #img = load_img(path, target_size=self.target_size, color_mode=\"grayscale\")\n            img =np.load(path)\n            y[j] = np.expand_dims(img, 2)\n            # Ground truth labels are 1, 2, 3. Subtract one to make them 0, 1, 2:\n            y[j] -= 1\n        return x, y","metadata":{"execution":{"iopub.status.busy":"2021-05-27T05:44:01.917537Z","iopub.execute_input":"2021-05-27T05:44:01.917939Z","iopub.status.idle":"2021-05-27T05:44:01.932781Z","shell.execute_reply.started":"2021-05-27T05:44:01.917882Z","shell.execute_reply":"2021-05-27T05:44:01.931105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**generate object of  train and  val:**","metadata":{}},{"cell_type":"code","source":"val_samples = 1000\nrandom.Random(1337).shuffle(input_img_paths)\nrandom.Random(1337).shuffle(target_img_paths)\ntrain_input_img_paths = input_img_paths[:-val_samples]\ntrain_target_img_paths = target_img_paths[:-val_samples]\nval_input_img_paths = input_img_paths[-val_samples:]\nval_target_img_paths = target_img_paths[-val_samples:]\n\n# Instantiate data Sequences for each split\ntrain_gen = CarDataset(\n    batch_size, img_size,target_size, train_input_img_paths, train_target_img_paths\n)\nval_gen = CarDataset(batch_size, img_size,target_size, val_input_img_paths, val_target_img_paths)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T05:44:04.364897Z","iopub.execute_input":"2021-05-27T05:44:04.365426Z","iopub.status.idle":"2021-05-27T05:44:04.387103Z","shell.execute_reply.started":"2021-05-27T05:44:04.365381Z","shell.execute_reply":"2021-05-27T05:44:04.385226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**model:**","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers\n\n\ndef get_model(img_size, num_classes):\n    inputs = keras.Input(shape=img_size + (3,))\n\n    ### [First half of the network: downsampling inputs] ###\n\n    # Entry block\n    x = layers.Conv2D(32, 3, strides=2, padding=\"same\")(inputs)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation(\"relu\")(x)\n\n    previous_block_activation = x  # Set aside residual\n\n    # Blocks 1, 2, 3 are identical apart from the feature depth.\n    for filters in [64, 128, 256]:\n        x = layers.Activation(\"relu\")(x)\n        x = layers.SeparableConv2D(filters, 3, padding=\"same\")(x)\n        x = layers.BatchNormalization()(x)\n\n        x = layers.Activation(\"relu\")(x)\n        x = layers.SeparableConv2D(filters, 3, padding=\"same\")(x)\n        x = layers.BatchNormalization()(x)\n\n        x = layers.MaxPooling2D(3, strides=2, padding=\"same\")(x)\n\n        # Project residual\n        residual = layers.Conv2D(filters, 1, strides=2, padding=\"same\")(\n            previous_block_activation\n        )\n        x = layers.add([x, residual])  # Add back residual\n        previous_block_activation = x  # Set aside next residual\n\n    ### [Second half of the network: upsampling inputs] ###\n\n    for filters in [256, 128, 64, 32]:\n        x = layers.Activation(\"relu\")(x)\n        x = layers.Conv2DTranspose(filters, 3, padding=\"same\")(x)\n        x = layers.BatchNormalization()(x)\n\n        x = layers.Activation(\"relu\")(x)\n        x = layers.Conv2DTranspose(filters, 3, padding=\"same\")(x)\n        x = layers.BatchNormalization()(x)\n\n        x = layers.UpSampling2D(2)(x)\n\n        # Project residual\n        residual = layers.UpSampling2D(2)(previous_block_activation)\n        residual = layers.Conv2D(filters, 1, padding=\"same\")(residual)\n        x = layers.add([x, residual])  # Add back residual\n        previous_block_activation = x  # Set aside next residual\n\n    # Add a per-pixel classification layer\n    outputs = layers.Conv2D(num_classes, 3, activation=\"softmax\", padding=\"same\")(x)\n\n    # Define the model\n    model = keras.Model(inputs, outputs)\n    return model\n\n\n# Free up RAM in case the model definition cells were run multiple times\nkeras.backend.clear_session()\n\n# Build model\nmodel = get_model(img_size, num_classes)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T05:28:12.542888Z","iopub.execute_input":"2021-05-27T05:28:12.543504Z","iopub.status.idle":"2021-05-27T05:28:14.386204Z","shell.execute_reply.started":"2021-05-27T05:28:12.543433Z","shell.execute_reply":"2021-05-27T05:28:14.384948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **TRAIN MODEL**","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer=\"rmsprop\", loss=\"sparse_categorical_crossentropy\")\n\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(\"oxford_segmentation.h5\", save_best_only=True)\n]\n\n# Train the model, doing validation at the end of each epoch.\nepochs = 15\nmodel.fit(train_gen, epochs=epochs, validation_data=val_gen, callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T05:44:11.269298Z","iopub.execute_input":"2021-05-27T05:44:11.269653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for test the shape of dataset:","metadata":{}},{"cell_type":"code","source":"#mask=np.expand_dims(mask, 2)\nx = np.zeros((batch_size,) + img_size + (3,), dtype=\"float32\")\ny = np.zeros((batch_size,) + target_size + (1,), dtype=\"uint8\")\nimg = load_img(input_img_paths[9], target_size=img_size)\nx[0]=img\nmask=np.load(target_img_paths[9])\ny[0]=np.expand_dims(mask, 2)\n#print('img.shape', img.shape)\n\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\n\nprint('x.shape',x.shape, 'std:', np.std(mask))\n\nprint('y.shape',y.shape, 'std:', np.std(mask))","metadata":{},"execution_count":null,"outputs":[]}]}