{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Keras starter kit: single-pixel classification with a residual convnet (trained on sample 1)","metadata":{"_uuid":"748d3554-7759-4443-b1b8-916e19ca50ed","_cell_guid":"a18c7d0f-a17a-4603-956d-2b531016c536","trusted":true}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport numpy as np\n\nimport time\nimport glob\nimport PIL.Image as Image\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom tqdm import tqdm\n\n# Data config\nDATA_DIR = '/kaggle/input/vesuvius-challenge-ink-detection/'\nBUFFER = 32  # Half-size of papyrus patches we'll use as model inputs\nZ_DIM = 16   # Number of slices in the z direction. Max value is 64 - Z_START\nZ_START = 25  # Offset of slices in the z direction\n\n# Model config\nBATCH_SIZE = 64\n\nplt.imshow(Image.open(DATA_DIR + \"/train/1/ir.png\"), cmap=\"gray\")","metadata":{"_uuid":"91db1348-a896-4607-8686-f6c6df6419ed","_cell_guid":"ff3c9fb9-0c86-4acf-9162-c741c46e53a4","collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-03-19T02:36:55.319754Z","iopub.execute_input":"2023-03-19T02:36:55.320890Z","iopub.status.idle":"2023-03-19T02:37:08.014875Z","shell.execute_reply.started":"2023-03-19T02:36:55.320830Z","shell.execute_reply":"2023-03-19T02:37:08.013622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load up the data","metadata":{}},{"cell_type":"code","source":"def load_mask(split, index):\n    img = Image.open(f\"{DATA_DIR}/{split}/{index}/mask.png\").convert('1')\n    return tf.convert_to_tensor(img, dtype=\"bool\")\n\ndef load_labels(split, index):\n    img = Image.open(f\"{DATA_DIR}/{split}/{index}/inklabels.png\")\n    return tf.convert_to_tensor(img, dtype=\"bool\")\n\nmask = load_mask(split=\"train\", index=1)\nlabels = load_labels(split=\"train\", index=1)\n\nfig, (ax1, ax2) = plt.subplots(1, 2)\nax1.set_title(\"mask.png\")\nax1.imshow(mask, cmap='gray')\nax2.set_title(\"inklabels.png\")\nax2.imshow(labels, cmap='gray')\nplt.show()","metadata":{"_uuid":"351fefa9-30dd-4e8d-bf3b-1aaa0fb33905","_cell_guid":"43a9acbe-f2c5-4976-b9ee-027e62c27a83","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-19T02:37:08.017284Z","iopub.execute_input":"2023-03-19T02:37:08.018019Z","iopub.status.idle":"2023-03-19T02:37:14.734576Z","shell.execute_reply.started":"2023-03-19T02:37:08.017975Z","shell.execute_reply":"2023-03-19T02:37:14.733508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nEach 3D input is represented as a .tif image stack of 16-bit grayscale images. Each image represents a \"slice\" in the z-direction, going from below the papyrus, to above the papyrus.\n","metadata":{"_uuid":"74a6ae91-2aa2-4eb3-9f3c-956dc54cf017","_cell_guid":"09e95f98-439b-49c3-aae2-550fadd533df","trusted":true}},{"cell_type":"code","source":"def load_volume(split, index):\n    # Load the 3d x-ray scan, one slice at a time\n    z_slices_fnames = sorted(glob.glob(f\"{DATA_DIR}/{split}/{index}/surface_volume/*.tif\"))[Z_START:Z_START + Z_DIM]\n    z_slices = []\n    for z, filename in  tqdm(enumerate(z_slices_fnames)):\n        img = Image.open(filename)\n        z_slice = np.array(img, dtype=np.int16)\n        z_slices.append(z_slice)\n\n    return tf.stack(z_slices, axis=-1)\n\nvolume = load_volume(split=\"train\", index=1)\n\nprint(f\"volume {volume.shape} - {volume.dtype}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:14.736324Z","iopub.execute_input":"2023-03-19T02:37:14.737160Z","iopub.status.idle":"2023-03-19T02:37:35.452434Z","shell.execute_reply.started":"2023-03-19T02:37:14.737114Z","shell.execute_reply":"2023-03-19T02:37:35.451229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize some slices of the papyrus","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 5, figsize=(15, 3))\nfor z, ax in enumerate(axes):\n    ax.imshow(volume[:, :, z], cmap='gray')\n    ax.set_xticks([]); ax.set_yticks([])\nfig.tight_layout()\nplt.show()","metadata":{"_uuid":"929b9e7a-d30c-4462-a7b9-0765a76cdb4e","_cell_guid":"f75858f9-06ad-43cc-be5f-ab09738a58c1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-19T02:37:35.455322Z","iopub.execute_input":"2023-03-19T02:37:35.456030Z","iopub.status.idle":"2023-03-19T02:37:42.276406Z","shell.execute_reply.started":"2023-03-19T02:37:35.455968Z","shell.execute_reply":"2023-03-19T02:37:42.272045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Select a validation area\n\nNext, let's hold out a validation area. We use a small rectangle around the letter \"rho\" for our evaluation, and we'll exclude those pixels from the training set.","metadata":{}},{"cell_type":"code","source":"val_location = (3500, 1100)\nval_zone_size = (800, 800)\n\nfig, ax = plt.subplots()\nax.imshow(labels)\npatch = patches.Rectangle([val_location[1], val_location[0]], val_zone_size[0], val_zone_size[1], linewidth=2, edgecolor='g', facecolor='none')\nax.add_patch(patch)\nplt.show()","metadata":{"_uuid":"39a855d9-38f5-4f2d-ac7f-1f65529d1a3e","_cell_guid":"62d60bbe-85f1-4df4-b1e6-f4337268b11c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-19T02:37:42.278173Z","iopub.execute_input":"2023-03-19T02:37:42.278861Z","iopub.status.idle":"2023-03-19T02:37:43.778623Z","shell.execute_reply.started":"2023-03-19T02:37:42.278821Z","shell.execute_reply":"2023-03-19T02:37:43.777503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a dataset that randomly samples coordinates (locations)\n\nWe want the training windows to be within the mask and outside the validation area.","metadata":{}},{"cell_type":"code","source":"def sample_random_location(shape):\n    random_train_x = tf.random.uniform(shape=(), minval=BUFFER, maxval=shape[0] - BUFFER - 1, dtype=\"int32\")\n    random_train_y = tf.random.uniform(shape=(), minval=BUFFER, maxval=shape[1] - BUFFER - 1, dtype=\"int32\")\n    random_train_location = tf.stack([random_train_x, random_train_y])\n    return random_train_location\n\ndef is_in_masked_zone(location, mask):\n    return mask[location[0], location[1]]\n\nsample_random_location_train = lambda x: sample_random_location(mask.shape)\nis_in_mask_train = lambda x: is_in_masked_zone(x, mask)\n\ndef is_in_val_zone(location, val_location, val_zone_size):\n    x = location[0]\n    y = location[1]\n    x_match = val_location[0] - BUFFER <= x <= val_location[0] + val_zone_size[0] + BUFFER\n    y_match = val_location[1] - BUFFER <= y <= val_location[1] + val_zone_size[1] + BUFFER\n    return x_match and y_match\n\ndef is_proper_train_location(location):\n    return not is_in_val_zone(location, val_location, val_zone_size) and is_in_mask_train(location)\n\ntrain_locations_ds = tf.data.Dataset.from_tensor_slices([0]).repeat().map(sample_random_location_train, num_parallel_calls=tf.data.AUTOTUNE)\ntrain_locations_ds = train_locations_ds.filter(is_proper_train_location)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:43.780322Z","iopub.execute_input":"2023-03-19T02:37:43.781469Z","iopub.status.idle":"2023-03-19T02:37:44.080130Z","shell.execute_reply.started":"2023-03-19T02:37:43.781425Z","shell.execute_reply":"2023-03-19T02:37:44.078942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize training patch locations","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.imshow(labels)\n\nfor x, y in train_locations_ds.take(200):\n    patch = patches.Rectangle([y - BUFFER // 2, x - BUFFER // 2], 2 * BUFFER, 2 * BUFFER, linewidth=2, edgecolor='r', facecolor='none')\n    ax.add_patch(patch)\n\nval_patch = patches.Rectangle([val_location[1], val_location[0]], val_zone_size[0], val_zone_size[1], linewidth=2, edgecolor='g', facecolor='none')\nax.add_patch(val_patch)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:44.081979Z","iopub.execute_input":"2023-03-19T02:37:44.082369Z","iopub.status.idle":"2023-03-19T02:37:47.315057Z","shell.execute_reply.started":"2023-03-19T02:37:44.082328Z","shell.execute_reply":"2023-03-19T02:37:47.313773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a training dataset that yields volume patches + their labels","metadata":{}},{"cell_type":"code","source":"def extract_subvolume(location, volume):\n    x = location[0]\n    y = location[1]\n    subvolume = volume[x-BUFFER:x+BUFFER, y-BUFFER:y+BUFFER, :]\n    subvolume = tf.cast(subvolume, dtype=\"float32\") / 65535.\n    return subvolume\n\ndef extract_labels(location, labels):\n    x = location[0]\n    y = location[1]\n    label = labels[x, y]\n    label = tf.cast(label, dtype=\"float32\")\n    return label\n\ndef extract_subvolume_and_label(location):\n    subvolume = extract_subvolume(location, volume)\n    label = extract_labels(location, labels)\n    return subvolume, label\n\nshuffle_buffer_size = BATCH_SIZE * 4\n\ntrain_ds = train_locations_ds.map(extract_subvolume_and_label, num_parallel_calls=tf.data.AUTOTUNE)\ntrain_ds = train_ds.prefetch(tf.data.AUTOTUNE).batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:47.320005Z","iopub.execute_input":"2023-03-19T02:37:47.322767Z","iopub.status.idle":"2023-03-19T02:37:49.034919Z","shell.execute_reply.started":"2023-03-19T02:37:47.322720Z","shell.execute_reply":"2023-03-19T02:37:49.033735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for subvolume_batch, label_batch in train_ds.take(1):\n    print(f\"subvolume shape: {subvolume_batch.shape[1:]}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:49.038575Z","iopub.execute_input":"2023-03-19T02:37:49.039431Z","iopub.status.idle":"2023-03-19T02:37:50.492526Z","shell.execute_reply.started":"2023-03-19T02:37:49.039397Z","shell.execute_reply":"2023-03-19T02:37:50.489756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a validation dataset that yields patches from the validation area and their labels","metadata":{}},{"cell_type":"code","source":"val_locations_stride = 3\nval_locations = []\nfor x in range(val_location[0], val_location[0] + val_zone_size[0], val_locations_stride):\n    for y in range(val_location[1], val_location[1] + val_zone_size[1], val_locations_stride):\n        val_locations.append((x, y))\n\nval_ds = tf.data.Dataset.from_tensor_slices(val_locations).map(extract_subvolume_and_label, num_parallel_calls=tf.data.AUTOTUNE)\nval_ds = val_ds.prefetch(tf.data.AUTOTUNE).batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:50.500269Z","iopub.execute_input":"2023-03-19T02:37:50.500728Z","iopub.status.idle":"2023-03-19T02:37:50.882631Z","shell.execute_reply.started":"2023-03-19T02:37:50.500680Z","shell.execute_reply":"2023-03-19T02:37:50.881481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check dataset throughput","metadata":{}},{"cell_type":"code","source":"t0 = time.time()\nn = 200\nfor _ in train_ds.take(n):\n    pass\nprint(f\"Time per batch: {(time.time() - t0) / n:.4f}s\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:50.885695Z","iopub.execute_input":"2023-03-19T02:37:50.886560Z","iopub.status.idle":"2023-03-19T02:37:57.452611Z","shell.execute_reply.started":"2023-03-19T02:37:50.886497Z","shell.execute_reply":"2023-03-19T02:37:57.451232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute a trivial baseline: best validation accuracy achievable without lookng at the papyrus","metadata":{}},{"cell_type":"code","source":"def trivial_baseline(dataset):\n    total = 0\n    matches = 0.\n    for _, batch_label in tqdm(dataset):\n        matches += tf.reduce_sum(tf.cast(batch_label, \"float32\"))\n        total += tf.reduce_prod(tf.shape(batch_label))\n    return 1. - matches / tf.cast(total, \"float32\")\n\nscore = trivial_baseline(val_ds).numpy()\nprint(f\"Best validation score achievable trivially: {score * 100:.2f}% accuracy\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:37:57.454088Z","iopub.execute_input":"2023-03-19T02:37:57.454786Z","iopub.status.idle":"2023-03-19T02:38:20.147099Z","shell.execute_reply.started":"2023-03-19T02:37:57.454752Z","shell.execute_reply":"2023-03-19T02:38:20.145826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augment the data\n","metadata":{"_uuid":"cfbc9413-b6ff-4a69-ae60-22dddef18f11","_cell_guid":"e7b84e1e-7903-4290-9e09-3bf2cb62b222","trusted":true}},{"cell_type":"code","source":"augmenter = keras.Sequential([\n    layers.RandomFlip(\"horizontal\"),\n])\n\ndef augment_train_data(data, label):\n    data = augmenter(data)\n    return data, label\n\naugmented_train_ds = train_ds.map(augment_train_data, num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:38:20.148857Z","iopub.execute_input":"2023-03-19T02:38:20.149267Z","iopub.status.idle":"2023-03-19T02:38:20.310310Z","shell.execute_reply.started":"2023-03-19T02:38:20.149226Z","shell.execute_reply":"2023-03-19T02:38:20.309286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train a Keras model\n\nNow we'll train the model. We're training on 2 GPUs with a `MirroredStrategy`, just for the hell of it. The speed up from working on 2 T4s rather than 1 P100 is small, but you can use larger batch sizes at least.","metadata":{}},{"cell_type":"code","source":"def get_model(input_shape):\n    inputs = keras.Input(input_shape)\n    x = layers.Conv2D(kernel_size=3, filters=128, activation=\"relu\")(inputs)\n\n    for size in (128, 256, 512):\n        residual = x\n        x = layers.BatchNormalization()(x)\n        x = layers.SeparableConv2D(kernel_size=3, filters=size, activation=\"relu\", padding=\"same\")(x)\n        x = layers.SeparableConv2D(kernel_size=3, filters=size, activation=\"relu\", padding=\"same\")(x)\n        x = layers.MaxPooling2D(2, padding=\"same\")(x)\n        residual = layers.Conv2D(kernel_size=1, filters=size, strides=2, padding=\"same\")(residual)\n        x = layers.Add()([residual, x])\n\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.3)(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n    model = keras.Model(inputs, outputs)\n    return model\n\nstrategy = tf.distribute.MirroredStrategy()\nwith strategy.scope():\n    model = get_model(input_shape=(BUFFER * 2, BUFFER * 2, Z_DIM))\n    model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n\nmodel.summary()\n\n# Because train_ds is infinitely streaming, you need to specify steps_per_epoch\n# i.e. the dataset has no epoch boundary on its own\nmodel.fit(augmented_train_ds,\n          validation_data=val_ds,\n          epochs=5,\n          steps_per_epoch=2000,\n#           class_weight={0: 1., 1: 4.},  # We over-weight the rarer class (1)\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:38:20.311984Z","iopub.execute_input":"2023-03-19T02:38:20.312353Z","iopub.status.idle":"2023-03-19T02:57:32.861859Z","shell.execute_reply.started":"2023-03-19T02:38:20.312294Z","shell.execute_reply":"2023-03-19T02:57:32.860582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"model.keras\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:57:32.863593Z","iopub.execute_input":"2023-03-19T02:57:32.864015Z","iopub.status.idle":"2023-03-19T02:57:33.031248Z","shell.execute_reply.started":"2023-03-19T02:57:32.863973Z","shell.execute_reply":"2023-03-19T02:57:33.029600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Free up some memory","metadata":{}},{"cell_type":"code","source":"del volume\ndel mask\ndel labels\ndel train_ds\ndel val_ds\n\n# Manually trigger garbage collection\nkeras.backend.clear_session()\nimport gc\ngc.collect()\n\nmodel = keras.models.load_model(\"model.keras\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:57:33.033486Z","iopub.execute_input":"2023-03-19T02:57:33.034300Z","iopub.status.idle":"2023-03-19T02:57:33.800651Z","shell.execute_reply.started":"2023-03-19T02:57:33.034245Z","shell.execute_reply":"2023-03-19T02:57:33.799573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute predictions on test samples","metadata":{}},{"cell_type":"code","source":"INFER_BATCH_SIZE = 128\n\ndef compute_predictions_map(split, index):\n    print(f\"Load data for {split}/{index}\")\n\n    test_volume = load_volume(split=split, index=index)\n    test_mask = load_mask(split=split, index=index)\n\n    test_locations = []\n    stride = BUFFER // 2\n    for x in range(BUFFER, test_volume.shape[0] - BUFFER, stride):\n        for y in range(BUFFER, test_volume.shape[1] - BUFFER, stride):\n            test_locations.append((x, y))\n\n    print(f\"{len(test_locations)} test locations (before filtering by mask)\")\n\n    sample_random_location_test = lambda x: sample_random_location(test_mask.shape)\n    is_in_mask_test = lambda x: is_in_masked_zone(x, test_mask)\n    extract_subvolume_test = lambda x: extract_subvolume(x, test_volume)\n\n    test_locations_ds = tf.data.Dataset.from_tensor_slices(test_locations).filter(is_in_mask_test)\n    test_ds = test_locations_ds.map(extract_subvolume_test, num_parallel_calls=tf.data.AUTOTUNE)\n\n    predictions_map = np.zeros(test_volume.shape[:2], dtype=\"float16\")\n\n    print(f\"Compute predictions\")\n\n    for loc_batch, patch_batch in tqdm(zip(test_locations_ds.batch(INFER_BATCH_SIZE), test_ds.batch(INFER_BATCH_SIZE))):\n        predictions = model.predict_on_batch(patch_batch)\n        for (x, y), pred in zip(loc_batch, predictions):\n            predictions_map[x, y] = pred\n            \n    del test_volume\n    del test_mask\n    return predictions_map","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:57:33.802382Z","iopub.execute_input":"2023-03-19T02:57:33.802730Z","iopub.status.idle":"2023-03-19T02:57:33.813162Z","shell.execute_reply.started":"2023-03-19T02:57:33.802700Z","shell.execute_reply":"2023-03-19T02:57:33.811892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_map_1 = compute_predictions_map(split=\"train\", index=1)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T02:57:33.814733Z","iopub.execute_input":"2023-03-19T02:57:33.815915Z","iopub.status.idle":"2023-03-19T03:01:51.709025Z","shell.execute_reply.started":"2023-03-19T02:57:33.815870Z","shell.execute_reply":"2023-03-19T03:01:51.707728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_map_a = compute_predictions_map(split=\"test\", index=\"a\")\npredictions_map_b = compute_predictions_map(split=\"test\", index=\"b\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:01:51.711157Z","iopub.execute_input":"2023-03-19T03:01:51.712800Z","iopub.status.idle":"2023-03-19T03:08:02.955424Z","shell.execute_reply.started":"2023-03-19T03:01:51.712756Z","shell.execute_reply":"2023-03-19T03:08:02.954172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inspect predictions\n\nActually visualizing the prediction maps doesn't really work out, so let's compute some stats and compare top to the prediction map we got from the training papyrus. This is just a sanity check.","metadata":{}},{"cell_type":"code","source":"def stats(predictions_map):\n    size = predictions_map.shape[0] * predictions_map.shape[1]\n    for thr in (0.1, 0.25, 0.5, 0.75):\n        num_nonzero = np.count_nonzero(predictions_map_1 > thr)\n        percent_nonzero = 100. * num_nonzero / size\n        print(f\"Percent above {thr}: {percent_nonzero:.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:08:02.957643Z","iopub.execute_input":"2023-03-19T03:08:02.958056Z","iopub.status.idle":"2023-03-19T03:08:02.964175Z","shell.execute_reply.started":"2023-03-19T03:08:02.958015Z","shell.execute_reply":"2023-03-19T03:08:02.962902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(predictions_map_1)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:08:02.965880Z","iopub.execute_input":"2023-03-19T03:08:02.966567Z","iopub.status.idle":"2023-03-19T03:08:04.700148Z","shell.execute_reply.started":"2023-03-19T03:08:02.966507Z","shell.execute_reply":"2023-03-19T03:08:04.698918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(predictions_map_a)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:08:04.701971Z","iopub.execute_input":"2023-03-19T03:08:04.702394Z","iopub.status.idle":"2023-03-19T03:08:06.407113Z","shell.execute_reply.started":"2023-03-19T03:08:04.702350Z","shell.execute_reply":"2023-03-19T03:08:06.405778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(predictions_map_b)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:08:06.411089Z","iopub.execute_input":"2023-03-19T03:08:06.411720Z","iopub.status.idle":"2023-03-19T03:08:08.238497Z","shell.execute_reply.started":"2023-03-19T03:08:06.411674Z","shell.execute_reply":"2023-03-19T03:08:08.237098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate submission file","metadata":{}},{"cell_type":"code","source":"def rle(predictions_map, threshold):\n    flat_img = predictions_map.flatten()\n    flat_img = np.where(flat_img > threshold, 1, 0).astype(np.uint8)\n\n    starts = np.array((flat_img[:-1] == 0) & (flat_img[1:] == 1))\n    ends = np.array((flat_img[:-1] == 1) & (flat_img[1:] == 0))\n    starts_ix = np.where(starts)[0] + 2\n    ends_ix = np.where(ends)[0] + 2\n    lengths = ends_ix - starts_ix\n    return \" \".join(map(str, sum(zip(starts_ix, lengths), ())))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:08:08.239983Z","iopub.execute_input":"2023-03-19T03:08:08.240692Z","iopub.status.idle":"2023-03-19T03:08:08.249720Z","shell.execute_reply.started":"2023-03-19T03:08:08.240648Z","shell.execute_reply":"2023-03-19T03:08:08.248535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.5\n\nrle_a = rle(predictions_map_a, threshold=threshold)\nrle_b = rle(predictions_map_b, threshold=threshold)\nprint(\"Id,Predicted\\na,\" + rle_a + \"\\nb,\" + rle_b, file=open('submission.csv', 'w'))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T03:08:08.252731Z","iopub.execute_input":"2023-03-19T03:08:08.253684Z","iopub.status.idle":"2023-03-19T03:08:10.333852Z","shell.execute_reply.started":"2023-03-19T03:08:08.253638Z","shell.execute_reply":"2023-03-19T03:08:10.332789Z"},"trusted":true},"execution_count":null,"outputs":[]}]}