{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Image Segmentation (modified Vladimir Zotov)","metadata":{}},{"cell_type":"markdown","source":"## Соревнование Carvana Image Masking Challenge.\n#### За основу был взят ноутбук Image segmentation U-Net. \n### Метрика на приватном лидерборде составляла 0.98163\n\n---\n\n#### Пробовал добавить количества примеров за счет аугментации (повороты, контрастность), но прироста не давало.\n#### Дополнительно добавил 3 слоя двойных сверток с пулингами, а затем 3 деконволюции для возврата к исходной размерности.\n### Итоговая метрика на приватном лидерборде составила 0.99129\n\n---","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nfrom zipfile import ZipFile\nimport keras.backend as K\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport os\nimport cv2\nimport imageio","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:11.815517Z","iopub.execute_input":"2022-01-24T11:11:11.815991Z","iopub.status.idle":"2022-01-24T11:11:11.824246Z","shell.execute_reply.started":"2022-01-24T11:11:11.815945Z","shell.execute_reply":"2022-01-24T11:11:11.822872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_zip = \"/kaggle/input/carvana-image-masking-challenge/train.zip\"\nwith ZipFile(train_zip, 'r') as zip_: \n    zip_.extractall('/kaggle/working')","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:12.862953Z","iopub.execute_input":"2022-01-24T11:11:12.863392Z","iopub.status.idle":"2022-01-24T11:11:24.013737Z","shell.execute_reply.started":"2022-01-24T11:11:12.86336Z","shell.execute_reply":"2022-01-24T11:11:24.012584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_mask_zip = \"/kaggle/input/carvana-image-masking-challenge/train_masks.zip\"\nwith ZipFile(train_mask_zip, 'r') as zip_: \n    zip_.extractall('/kaggle/working')","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:24.015693Z","iopub.execute_input":"2022-01-24T11:11:24.016127Z","iopub.status.idle":"2022-01-24T11:11:25.977806Z","shell.execute_reply.started":"2022-01-24T11:11:24.016082Z","shell.execute_reply":"2022-01-24T11:11:25.976198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train set:  \", len(os.listdir(\"/kaggle/working/train\")))\nprint(\"Train masks:\", len(os.listdir(\"/kaggle/working/train_masks\")))","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:28.835823Z","iopub.execute_input":"2022-01-24T11:11:28.836212Z","iopub.status.idle":"2022-01-24T11:11:28.852811Z","shell.execute_reply.started":"2022-01-24T11:11:28.836179Z","shell.execute_reply":"2022-01-24T11:11:28.851724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"car_ids = []\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/working/train'):\n    for filename in filenames:\n        path = os.path.join(dirname, filename)    \n        paths.append(path)\n        \n        car_id = filename.split(\".\")[0]\n        car_ids.append(car_id)\n\nd = {\"id\": car_ids, \"car_path\": paths}\ndf = pd.DataFrame(data = d)\ndf = df.set_index('id')\ndf","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:30.176014Z","iopub.execute_input":"2022-01-24T11:11:30.176554Z","iopub.status.idle":"2022-01-24T11:11:30.278584Z","shell.execute_reply.started":"2022-01-24T11:11:30.176505Z","shell.execute_reply":"2022-01-24T11:11:30.276639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"car_ids = []\nmask_path = []\nfor dirname, _, filenames in os.walk('/kaggle/working/train_masks'):\n    for filename in filenames:\n        path = os.path.join(dirname, filename)\n        mask_path.append(path)\n        \n        car_id = filename.split(\".\")[0]\n        car_id = car_id.split(\"_mask\")[0]\n        car_ids.append(car_id)\n\n        \nd = {\"id\": car_ids,\"mask_path\": mask_path}\nmask_df = pd.DataFrame(data = d)\nmask_df = mask_df.set_index('id')\nmask_df","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:41.40854Z","iopub.execute_input":"2022-01-24T11:11:41.408911Z","iopub.status.idle":"2022-01-24T11:11:41.453452Z","shell.execute_reply.started":"2022-01-24T11:11:41.40888Z","shell.execute_reply":"2022-01-24T11:11:41.452234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"mask_path\"] = mask_df[\"mask_path\"]\ndf","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:42.728416Z","iopub.execute_input":"2022-01-24T11:11:42.728769Z","iopub.status.idle":"2022-01-24T11:11:42.7514Z","shell.execute_reply.started":"2022-01-24T11:11:42.728738Z","shell.execute_reply":"2022-01-24T11:11:42.75009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = [256,256]\n\ndef data_augmentation(car_img, mask_img):\n\n    if tf.random.uniform(()) > 0.5:\n        car_img = tf.image.flip_left_right(car_img)\n        mask_img = tf.image.flip_left_right(mask_img)\n\n    return car_img, mask_img\n\n# ADD_VZ\n# def data_augmentation2(car_img, mask_img):\n\n#     if tf.random.uniform(()) > 0.5:\n#         car_img = tf.image.adjust_contrast(car_img, 2.)\n#         # mask_img = tf.image.adjust_contrast(mask_img)\n\n#     return car_img, mask_img\n\n# def data_augmentation3(car_img, mask_img):\n\n#     if tf.random.uniform(()) > 0.5:\n#         car_img = tf.image.rot90(car_img)\n#         mask_img = tf.image.rot90(mask_img)\n\n#     return car_img, mask_img\n#_ADD_VZ\n\ndef preprocessing(car_path, mask_path):\n    car_img = tf.io.read_file(car_path) \n    car_img = tf.image.decode_jpeg(car_img, channels=3)\n    car_img = tf.image.resize(car_img, img_size)\n    car_img = tf.cast(car_img, tf.float32) / 255.0\n    \n    mask_img = tf.io.read_file(mask_path)\n    mask_img = tf.image.decode_jpeg(mask_img, channels=3)\n    mask_img = tf.image.resize(mask_img, img_size)\n    mask_img = mask_img[:,:,:1]    \n    mask_img = tf.math.sign(mask_img)\n    \n    \n    return car_img, mask_img\n\ndef create_dataset(df, train = False):\n    if not train:\n        ds = tf.data.Dataset.from_tensor_slices((df[\"car_path\"].values, df[\"mask_path\"].values))\n        ds = ds.map(preprocessing, tf.data.AUTOTUNE)\n    else:\n        ds = tf.data.Dataset.from_tensor_slices((df[\"car_path\"].values, df[\"mask_path\"].values))\n        ds = ds.map(preprocessing, tf.data.AUTOTUNE)\n        ds = ds.map(data_augmentation, tf.data.AUTOTUNE)\n        # ds = ds.map(data_augmentation2, tf.data.AUTOTUNE) # vz\n        # ds = ds.map(data_augmentation3, tf.data.AUTOTUNE) # vz\n\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:44.476817Z","iopub.execute_input":"2022-01-24T11:11:44.477241Z","iopub.status.idle":"2022-01-24T11:11:44.491541Z","shell.execute_reply.started":"2022-01-24T11:11:44.477168Z","shell.execute_reply":"2022-01-24T11:11:44.490237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, valid_df = train_test_split(df, random_state=42, test_size=.25)\ntrain = create_dataset(train_df, train = True)\nvalid = create_dataset(valid_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:45.864775Z","iopub.execute_input":"2022-01-24T11:11:45.865198Z","iopub.status.idle":"2022-01-24T11:11:48.452944Z","shell.execute_reply.started":"2022-01-24T11:11:45.865161Z","shell.execute_reply":"2022-01-24T11:11:48.451842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_LENGTH = len(train_df)\nBATCH_SIZE = 16\nBUFFER_SIZE = 1000","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:48.454912Z","iopub.execute_input":"2022-01-24T11:11:48.455365Z","iopub.status.idle":"2022-01-24T11:11:48.461557Z","shell.execute_reply.started":"2022-01-24T11:11:48.455321Z","shell.execute_reply":"2022-01-24T11:11:48.459928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = train.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()\ntrain_dataset = train_dataset.prefetch(buffer_size=tf.data.AUTOTUNE)\nvalid_dataset = valid.batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:50.477557Z","iopub.execute_input":"2022-01-24T11:11:50.477932Z","iopub.status.idle":"2022-01-24T11:11:50.493834Z","shell.execute_reply.started":"2022-01-24T11:11:50.4779Z","shell.execute_reply":"2022-01-24T11:11:50.492656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display(display_list):\n    plt.figure(figsize=(15, 15))\n\n    title = ['Input Image', 'True Mask', 'Predicted Mask']\n\n    for i in range(len(display_list)):\n        plt.subplot(1, len(display_list), i+1)\n        plt.title(title[i])\n        plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))\n        plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:52.002004Z","iopub.execute_input":"2022-01-24T11:11:52.002493Z","iopub.status.idle":"2022-01-24T11:11:52.009827Z","shell.execute_reply.started":"2022-01-24T11:11:52.002459Z","shell.execute_reply":"2022-01-24T11:11:52.008102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n   for image, mask in train.take(i):\n        sample_image, sample_mask = image, mask\n        display([sample_image, sample_mask])","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:11:52.95498Z","iopub.execute_input":"2022-01-24T11:11:52.955411Z","iopub.status.idle":"2022-01-24T11:11:56.512239Z","shell.execute_reply.started":"2022-01-24T11:11:52.955378Z","shell.execute_reply":"2022-01-24T11:11:56.511095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Модель","metadata":{}},{"cell_type":"markdown","source":"Мы будем использовать модель U-Net. U-Net состоит из кодера (downsampler) и декодера (upsampler). Для того чтобы выучить надежные характеристики и уменьшить количество обучаемых параметров, в качестве кодера можно использовать предварительно обученную модель. В качестве кодера будет использоваться предварительно обученная модель MobileNetV2, подготовленная и готовая к использованию в tf.keras.applications.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model = tf.keras.applications.MobileNetV2(input_shape=[256, 256, 3], include_top=False)\n\n# Use the activations of these layers\nlayer_names = [\n    'block_1_expand_relu',   # 64x64\n    'block_3_expand_relu',   # 32x32\n    'block_6_expand_relu',   # 16x16\n    'block_13_expand_relu',  # 8x8\n    'block_16_project',      # 4x4\n]\nbase_model_outputs = [base_model.get_layer(name).output for name in layer_names]\n\n# Create the feature extraction model\ndown_stack = tf.keras.Model(inputs=base_model.input, outputs=base_model_outputs)\ndown_stack.trainable = False","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:12:01.832291Z","iopub.execute_input":"2022-01-24T11:12:01.83268Z","iopub.status.idle":"2022-01-24T11:12:03.821671Z","shell.execute_reply.started":"2022-01-24T11:12:01.832647Z","shell.execute_reply":"2022-01-24T11:12:03.820494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upsample(filters, size, norm_type='batchnorm', apply_dropout=False):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    \n    result = tf.keras.Sequential()\n    result.add(\n      tf.keras.layers.Conv2DTranspose(filters, size, strides=2,\n                                      padding='same',\n                                      kernel_initializer=initializer,\n                                      use_bias=False))\n\n    if norm_type.lower() == 'batchnorm':\n        result.add(tf.keras.layers.BatchNormalization())\n    elif norm_type.lower() == 'instancenorm':\n        result.add(InstanceNormalization())\n\n    if apply_dropout:\n        result.add(tf.keras.layers.Dropout(0.5))\n\n        result.add(tf.keras.layers.ReLU())\n\n    return result\n\nup_stack = [\n    upsample(512, 3),  # 4x4 -> 8x8\n    upsample(256, 3),  # 8x8 -> 16x16\n    upsample(128, 3),  # 16x16 -> 32x32\n    upsample(64, 3),   # 32x32 -> 64x64\n]","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:12:03.823514Z","iopub.execute_input":"2022-01-24T11:12:03.823957Z","iopub.status.idle":"2022-01-24T11:12:03.865171Z","shell.execute_reply.started":"2022-01-24T11:12:03.823911Z","shell.execute_reply":"2022-01-24T11:12:03.863916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unet_model(output_channels):\n    inputs = tf.keras.layers.Input(shape=[256, 256, 3])\n\n    # Downsampling through the model\n    skips = down_stack(inputs)\n    x = skips[-1]\n    skips = reversed(skips[:-1])\n\n  # Upsampling and establishing the skip connections\n    for up, skip in zip(up_stack, skips):\n        x = up(x)\n        concat = tf.keras.layers.Concatenate()\n        x = concat([x, skip])\n\n        \n    # ADD_VZ\n    x = tf.keras.layers.Conv2D(128, (3, 3), padding='same', name='add_vz_1')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.Conv2D(128, (3, 3), padding='same', name='add_vz_2')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    \n    x = tf.keras.layers.Conv2D(64, (3, 3), padding='same', name='add_vz_3')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.Conv2D(64, (3, 3), padding='same', name='add_vz_4')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    \n    x = tf.keras.layers.Conv2D(32, (3, 3), padding='same', name='add_vz_5')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.Conv2D(32, (3, 3), padding='same', name='add_vz_6')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    \n    x = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same', name='add_vz_7')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    \n    x = tf.keras.layers.Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same', name='add_vz_8')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    \n    x = tf.keras.layers.Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same', name='add_vz_9')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    #_ADD_VZ\n    \n    \n  # This is the last layer of the model\n    last = tf.keras.layers.Conv2DTranspose(\n      output_channels, 3, strides=2, activation='sigmoid',\n      padding='same')  #64x64 -> 128x128\n\n    x = last(x)\n\n    return tf.keras.Model(inputs=inputs, outputs=x)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:12:04.70639Z","iopub.execute_input":"2022-01-24T11:12:04.706838Z","iopub.status.idle":"2022-01-24T11:12:04.729673Z","shell.execute_reply.started":"2022-01-24T11:12:04.706797Z","shell.execute_reply":"2022-01-24T11:12:04.728257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dice_coef(y_true, y_pred, smooth=1):\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3])\n    return K.mean( (2. * intersection + smooth) / (union + smooth), axis=0)\n\ndef dice_loss(in_gt, in_pred):\n    return 1-dice_coef(in_gt, in_pred)\n\nmodel = unet_model(1)\n\nmodel.compile(optimizer='adam',\n              loss = dice_loss,\n              metrics=[dice_coef,'binary_accuracy'])\n\ntf.keras.utils.plot_model(model, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:12:06.194436Z","iopub.execute_input":"2022-01-24T11:12:06.194924Z","iopub.status.idle":"2022-01-24T11:12:08.146621Z","shell.execute_reply.started":"2022-01-24T11:12:06.194875Z","shell.execute_reply":"2022-01-24T11:12:08.145501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for images, masks in train_dataset.take(1):\n    for img, mask in zip(images, masks):\n        sample_image = img\n        sample_mask = mask\n        break\ndef visualize(display_list):\n    plt.figure(figsize=(15, 15))\n    title = ['Input Image', 'True Mask', 'Predicted Mask']\n    for i in range(len(display_list)):\n        plt.subplot(1, len(display_list), i+1)\n        plt.title(title[i])\n        plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))\n        plt.axis('off')\n    plt.show()\n\ndef show_predictions(sample_image, sample_mask):\n    pred_mask = model.predict(sample_image[tf.newaxis, ...])\n    pred_mask = pred_mask.reshape(img_size[0],img_size[1],1)\n    visualize([sample_image, sample_mask, pred_mask])\n    \nshow_predictions(sample_image, sample_mask)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:12:11.997338Z","iopub.execute_input":"2022-01-24T11:12:11.997724Z","iopub.status.idle":"2022-01-24T11:12:42.72466Z","shell.execute_reply.started":"2022-01-24T11:12:11.997689Z","shell.execute_reply":"2022-01-24T11:12:42.72047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:12:48.988816Z","iopub.execute_input":"2022-01-24T11:12:48.989208Z","iopub.status.idle":"2022-01-24T11:12:49.044596Z","shell.execute_reply.started":"2022-01-24T11:12:48.989171Z","shell.execute_reply":"2022-01-24T11:12:49.043487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stop = tf.keras.callbacks.EarlyStopping(patience=5,restore_best_weights=True)\n\nclass DisplayCallback(tf.keras.callbacks.Callback):\n    def on_epoch_begin(self, epoch, logs=None):\n        if (epoch + 1) % 3 == 0:\n            show_predictions(sample_image, sample_mask)\nEPOCHS = 70\nSTEPS_PER_EPOCH = TRAIN_LENGTH // BATCH_SIZE\n\nmodel_history = model.fit(train_dataset, epochs=EPOCHS,\n                          steps_per_epoch=STEPS_PER_EPOCH,\n                          validation_data=valid_dataset,\n                          callbacks=[DisplayCallback(), early_stop])","metadata":{"execution":{"iopub.status.busy":"2022-01-24T11:13:02.541694Z","iopub.execute_input":"2022-01-24T11:13:02.544778Z","iopub.status.idle":"2022-01-24T12:00:43.3205Z","shell.execute_reply.started":"2022-01-24T11:13:02.544728Z","shell.execute_reply":"2022-01-24T12:00:43.319335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('/kaggle/working/model_v2')","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:02:17.345803Z","iopub.execute_input":"2022-01-24T12:02:17.346197Z","iopub.status.idle":"2022-01-24T12:02:58.910027Z","shell.execute_reply.started":"2022-01-24T12:02:17.34616Z","shell.execute_reply":"2022-01-24T12:02:58.908915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_zip = \"/kaggle/input/carvana-image-masking-challenge/test.zip\"\nwith ZipFile(test_zip, 'r') as zip_test: \n    zip_test.extractall('/kaggle/working')","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:06:06.102277Z","iopub.execute_input":"2022-01-24T12:06:06.102714Z","iopub.status.idle":"2022-01-24T12:09:35.598293Z","shell.execute_reply.started":"2022-01-24T12:06:06.102673Z","shell.execute_reply":"2022-01-24T12:09:35.59602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"test set:  \", len(os.listdir(\"/kaggle/working/test\")))","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:09:35.601363Z","iopub.execute_input":"2022-01-24T12:09:35.601819Z","iopub.status.idle":"2022-01-24T12:09:35.680464Z","shell.execute_reply.started":"2022-01-24T12:09:35.601776Z","shell.execute_reply":"2022-01-24T12:09:35.678358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"test set:  \", os.listdir(\"/kaggle/working/test\")[:5])","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:09:35.6828Z","iopub.execute_input":"2022-01-24T12:09:35.683249Z","iopub.status.idle":"2022-01-24T12:09:35.758572Z","shell.execute_reply.started":"2022-01-24T12:09:35.683174Z","shell.execute_reply":"2022-01-24T12:09:35.757235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dirname2, _2, filenames2 in os.walk('/kaggle/working/test')\n# test_paths = [os.path.join(dirname2, filename2) for filename2 in filenames2]\n# test_paths[:5]\n\ntest_paths = []\nfor filename2 in os.listdir(\"/kaggle/working/test\"):\n    test_paths.append(os.path.join(\"/kaggle/working/test\", filename2))\ntest_paths[:5]","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:16:02.077849Z","iopub.execute_input":"2022-01-24T12:16:02.078264Z","iopub.status.idle":"2022-01-24T12:16:02.378951Z","shell.execute_reply.started":"2022-01-24T12:16:02.078217Z","shell.execute_reply":"2022-01-24T12:16:02.37749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMG_ROWS = img_size[0]\n# IMG_COLS = img_size[1]\n\ndef preprocessing_test(car_path):\n    car_img = tf.io.read_file(car_path) \n    car_img = tf.image.decode_jpeg(car_img, channels=3)\n    car_img = tf.image.resize(car_img, img_size)\n    car_img = tf.cast(car_img, tf.float32) / 255.0\n    \n    return car_img\n\n\ndef test_img_generator(test_paths): \n    for path in test_paths: \n        yield np.array([preprocessing_test(path)])\n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:16:08.712242Z","iopub.execute_input":"2022-01-24T12:16:08.712606Z","iopub.status.idle":"2022-01-24T12:16:08.71953Z","shell.execute_reply.started":"2022-01-24T12:16:08.712575Z","shell.execute_reply":"2022-01-24T12:16:08.71822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(test_img_generator(test_paths[:10]), len(test_paths[:10]))","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:16:10.268353Z","iopub.execute_input":"2022-01-24T12:16:10.268804Z","iopub.status.idle":"2022-01-24T12:16:13.0242Z","shell.execute_reply.started":"2022-01-24T12:16:10.268771Z","shell.execute_reply":"2022-01-24T12:16:13.022978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_IMG_ROWS = 1918\nTEST_IMG_COLS = 1280\n\nfig = plt.figure(0, figsize=(20, 10)) \nk = 5\nfig.add_subplot(2, 2, 1) \nplt.imshow(imageio.imread(test_paths[k])) \nfig.add_subplot(2, 2, 2) \nplt.imshow(np.squeeze(cv2.resize(pred[k], (TEST_IMG_ROWS, TEST_IMG_COLS))), cmap='gray') \nfig.add_subplot(2, 2, 3) \nplt.imshow(imageio.imread(test_paths[k+1])) \nfig.add_subplot(2, 2, 4) \nplt.imshow(np.squeeze(cv2.resize(pred[k+1], (TEST_IMG_ROWS, TEST_IMG_COLS))), cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:16:15.193365Z","iopub.execute_input":"2022-01-24T12:16:15.193822Z","iopub.status.idle":"2022-01-24T12:16:17.144769Z","shell.execute_reply.started":"2022-01-24T12:16:15.193782Z","shell.execute_reply":"2022-01-24T12:16:17.143229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Подготавливаем данные для отправки","metadata":{}},{"cell_type":"code","source":"def rle_encode(mask):\n    pixels = mask.flatten()\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] = runs[1::2] - runs[:-1:2]\n    return runs","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:16:35.208658Z","iopub.execute_input":"2022-01-24T12:16:35.209072Z","iopub.status.idle":"2022-01-24T12:16:35.215748Z","shell.execute_reply.started":"2022-01-24T12:16:35.209039Z","shell.execute_reply":"2022-01-24T12:16:35.21432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('submit.txt', 'w') as dst:\n    dst.write('img,rle_mask\\n')\n    for path in test_paths:\n        img = np.array([preprocessing_test(path)])\n        pred_mask = model.predict(img)[0]\n        bin_mask = 255. * cv2.resize(pred_mask, (TEST_IMG_ROWS, TEST_IMG_COLS))\n        bin_mask[bin_mask<=127] = 0\n        bin_mask[bin_mask>127] = 1\n        rle = rle_encode(bin_mask.astype(np.uint8))\n        rle = ' '.join(str(x) for x in rle)\n        dst.write('%s,%s\\n' % (path.split('/')[-1], rle))","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:16:41.88899Z","iopub.execute_input":"2022-01-24T12:16:41.889446Z","iopub.status.idle":"2022-01-24T14:34:36.961312Z","shell.execute_reply.started":"2022-01-24T12:16:41.889408Z","shell.execute_reply":"2022-01-24T14:34:36.959153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from subprocess import check_output\nprint(check_output([\"ls\", \"../working\"]).decode(\"utf8\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-24T14:44:11.522584Z","iopub.execute_input":"2022-01-24T14:44:11.523001Z","iopub.status.idle":"2022-01-24T14:44:11.686219Z","shell.execute_reply.started":"2022-01-24T14:44:11.522952Z","shell.execute_reply":"2022-01-24T14:44:11.684781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv\n\nwith open('../working/submit.txt', 'r') as in_file:\n    stripped = (line.strip() for line in in_file)\n    lines = (line.split(\",\") for line in stripped if line)\n    with open('submission.csv', 'w') as out_file:\n        writer = csv.writer(out_file)\n        writer.writerows(lines)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T14:34:37.240113Z","iopub.execute_input":"2022-01-24T14:34:37.24054Z","iopub.status.idle":"2022-01-24T14:35:11.320907Z","shell.execute_reply.started":"2022-01-24T14:34:37.240502Z","shell.execute_reply":"2022-01-24T14:35:11.319556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Private Score: 0.99129","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}}]}