{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! nvidia-smi","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-14T06:17:33.428896Z","iopub.execute_input":"2023-05-14T06:17:33.429609Z","iopub.status.idle":"2023-05-14T06:17:34.618229Z","shell.execute_reply.started":"2023-05-14T06:17:33.429573Z","shell.execute_reply":"2023-05-14T06:17:34.616972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nfrom glob import glob\nfrom matplotlib import pyplot as plt\nimport matplotlib.patches as patches\nimport matplotlib\nimport seaborn as sns\nfrom dataclasses import dataclass, field, asdict\nfrom pprint import pprint as pp\nfrom typing import List\nimport altair as alt\nimport cv2 as cv\nfrom skimage import exposure\nfrom PIL import Image\nimport imageio\nimport os\nfrom pathlib import Path\nfrom typing import Tuple, Optional\nimport numba\nfrom skimage import color, transform, exposure","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-14T06:17:34.620914Z","iopub.execute_input":"2023-05-14T06:17:34.621658Z","iopub.status.idle":"2023-05-14T06:17:36.541711Z","shell.execute_reply.started":"2023-05-14T06:17:34.621615Z","shell.execute_reply":"2023-05-14T06:17:36.540721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastcore.all import *\nfrom fastprogress import progress_bar","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-14T06:17:36.543390Z","iopub.execute_input":"2023-05-14T06:17:36.544097Z","iopub.status.idle":"2023-05-14T06:17:42.844987Z","shell.execute_reply.started":"2023-05-14T06:17:36.544060Z","shell.execute_reply":"2023-05-14T06:17:42.844007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = \"/kaggle/input/understanding_cloud_organization\"\npath = Path(x)\nprint(path.ls())\nos.listdir(x)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-14T06:17:42.847921Z","iopub.execute_input":"2023-05-14T06:17:42.848653Z","iopub.status.idle":"2023-05-14T06:17:42.864404Z","shell.execute_reply.started":"2023-05-14T06:17:42.848616Z","shell.execute_reply":"2023-05-14T06:17:42.863462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(path/'train.csv')\ntrain_fns = sorted(glob.glob(str(path) + '/train_images/*.jpg'))\ntest_fns = sorted(glob.glob(str(path) + '/test_images/*.jpg'))\nprint('{} images train set.'.format(len(train_df)))\nprint('{}  images  test set.'.format(len(test_fns)))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-14T06:17:42.867585Z","iopub.execute_input":"2023-05-14T06:17:42.870815Z","iopub.status.idle":"2023-05-14T06:17:47.657723Z","shell.execute_reply.started":"2023-05-14T06:17:42.870778Z","shell.execute_reply":"2023-05-14T06:17:47.656768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:47.662013Z","iopub.execute_input":"2023-05-14T06:17:47.664279Z","iopub.status.idle":"2023-05-14T06:17:47.689065Z","shell.execute_reply.started":"2023-05-14T06:17:47.664241Z","shell.execute_reply":"2023-05-14T06:17:47.687814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_img_label(img_lbl):\n    \"\"\"Return image and label from file name like '0011165.jpg_Flower'\"\"\"\n    s = img_lbl.split(\"_\")\n    assert len(s) == 2\n    return s[0], s[1]","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:47.694274Z","iopub.execute_input":"2023-05-14T06:17:47.696867Z","iopub.status.idle":"2023-05-14T06:17:47.704546Z","shell.execute_reply.started":"2023-05-14T06:17:47.696832Z","shell.execute_reply":"2023-05-14T06:17:47.703520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split Image_Label\ntrain_df[\"Image\"] = train_df[\"Image_Label\"].apply(lambda img_lbl: split_img_label(img_lbl)[0])\ntrain_df[\"Label\"] = train_df[\"Image_Label\"].apply(lambda img_lbl: split_img_label(img_lbl)[1])\ndel train_df[\"Image_Label\"]\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:47.706033Z","iopub.execute_input":"2023-05-14T06:17:47.707994Z","iopub.status.idle":"2023-05-14T06:17:47.799368Z","shell.execute_reply.started":"2023-05-14T06:17:47.707956Z","shell.execute_reply":"2023-05-14T06:17:47.798342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_mask = train_df.dropna(subset=[\"EncodedPixels\"])\nax = train_with_mask[\"Label\"].value_counts().plot(kind=\"pie\", autopct='%1.1f%%', title=\"Shares of each classes\", figsize=(10, 6))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:47.804007Z","iopub.execute_input":"2023-05-14T06:17:47.806536Z","iopub.status.idle":"2023-05-14T06:17:48.123898Z","shell.execute_reply.started":"2023-05-14T06:17:47.806498Z","shell.execute_reply":"2023-05-14T06:17:48.122924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# utility functions\n\ndef rle_decode(encoded: str, shape: Tuple[int, int], value: int = 1) -> np.ndarray:\n    \"\"\"Decodes an RLE-encoded string.\n    \n    Parameters\n    ----------\n    encoded\n        RLE mask.\n    shape\n        Mask shape in (height, width) format.\n    value\n        Value to fill in the mask.\n        \n    Returns\n    -------\n    mask\n        The decoded mask as 2D image of shape (height, width).\n    \"\"\"\n#     rle_string = encoded\n      \n#     rows, cols = 1400,2100\n    \n#     if rle_string == -1:\n#         return np.zeros((1400, 2100))\n#     else:\n#         rle_numbers = [int(num_string) for num_string in rle_string.split(' ')]\n#         rle_pairs = np.array(rle_numbers).reshape(-1,2)\n#         img = np.zeros(rows*cols, dtype=np.uint8)\n#         for index, length in rle_pairs:\n#             index -= 1\n#             img[index:index+length] = value\n#         img = img.reshape(rows,cols)\n#         img = img.T\n#     return img\n    mask_rle = encoded\n    \n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = value\n    return img.reshape(shape)\n    \n    \n#     numbers = list(map(int, encoded.split()))\n#     starts, runs = [np.asarray(xs) for xs in (numbers[::2], numbers[1::2])]\n\n#     # pixels enumerations starts from 1 but arrays are\n#     # indexed staring from 0 so need to make an adjustment\n#     starts -= 1\n\n#     mask = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n#     for start, run in zip(starts, runs):\n#         mask[start:start + run] = value\n\n#     # In NumPy arrays, first goes height and then goes width; also,\n#     # the pixels in the mask are enumerated from top to bottom and\n#     # from left to right, but the mask was filled in a different\n#     # order so need to transpose\n#     return mask.reshape(shape[1], shape[0]).T\n\n\ndef rle_encode(mask: np.ndarray, threshold: Optional[float] = None) -> str:\n    \"\"\"Encoders a binary mask into RLE-string.\n    \n    References\n    ----------\n    [1] https://www.kaggle.com/hackerpoet/even-faster-run-length-encoder\n    \"\"\"\n\n    pixels = mask.T.flatten()\n\n    if threshold is not None:\n        pixels = np.where(pixels > threshold, 1, 0)\n\n    pixels = pixels.astype(np.uint8)\n    pixels = np.concatenate([[0], pixels, [0]])\n\n    mask_start = (pixels[:-1] == 0) & (pixels[1:] == 1)\n    mask_end = (pixels[:-1] == 1) & (pixels[1:] == 0)\n    [start_idx] = np.where(mask_start)\n    [end_idx] = np.where(mask_end)\n    lengths = end_idx - start_idx\n\n    encoded = np.zeros(start_idx.shape[0] + lengths.shape[0])\n    encoded[::2] = start_idx + 2  # adjust for counting from 1\n    encoded[1::2] = lengths\n\n    return \" \".join([str(x) for x in encoded.astype(int)])\n\n\n@numba.njit()\ndef rle_numba(pixels):\n    size = len(pixels)\n    points = []\n    if pixels[0] == 1:\n        points.append(0+1)\n    flag = True\n    for i in range(1, size):\n        if pixels[i] != pixels[i-1]:\n            if flag:\n                points.append(i+1)\n                flag = False\n            else:\n                points.append(i+1 - points[-1])\n                flag = True\n    if pixels[-1] == 1:\n        points.append(size-points[-1]+1)\n    return points\n\n\ndef rle_numba_encode(image):\n    pixels = image.flatten(order=\"F\")\n    points = rle_numba(pixels)\n    return \" \".join(str(x) for x in points)\n\n\ndef on_kaggle() -> bool:\n    return os.environ.get(\"KAGGLE_URL_BASE\", False)\n\n\ndef get_dataset_size(debug: bool = False):\n    n_total = len(get_image_files(DATA_ROOT))\n    return n_total // 10 if debug else n_total\n\n\ndef get_combined_code(*class_names):\n    assert 1 <= len(class_names) <= 4\n    combined_code = 0\n    for class_name in class_names:\n        combined_code |= MASK_CODES[class_name]\n    label = \"+\".join(class_names)\n    return label, combined_code\n\n\ndef equalize(data: np.array, adaptive: bool) -> np.ndarray:\n    \"\"\"Histogram equalization to normalize images before previewing.\"\"\"\n    \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    method = (\n        exposure.equalize_adapthist\n        if adaptive\n        else exposure.equalize_hist\n    )\n    return method(data)\n\n\ndef predict(learn, test_ids, metadata, n_sample=None):    \n    if n_sample is not None:\n        test_ids = np.random.choice(test_ids, size=n_sample, replace=False)\n    \n    with learn.no_bar():\n        predicted = []\n    \n        for test_id in progress_bar(test_ids):\n            mask, *_ = learn.predict(metadata[test_id].full_path)\n            predicted.append(mask.numpy())\n    \n    return predicted, test_ids\n\n\ndef view_results(test_ids, predictions, metadata, figsize_mult=4, n_rows=5, n_cols=4, overlay_resize=(128, 128)):\n    f, axes = plt.subplots(\n        n_rows, n_cols,\n        figsize=(n_rows * figsize_mult, n_cols * figsize_mult)\n    )\n    for test_id, mask, ax in zip(test_ids, predictions, axes.flat):\n        fn = metadata[test_id].full_path\n        img = np.asarray(PIL.Image.open(fn))\n        img = equalize(img, True)\n        img = transform.resize(img, mask.shape)\n        img = np.amax(img) - img\n        overlay = color.label2rgb(mask, img, kind=\"overlay\", alpha=0.5)\n        overlay = transform.resize(overlay, overlay_resize)\n        ax.imshow(overlay)\n        ax.set_axis_off()\n    return f\n\n\nif on_kaggle():\n    DATA_ROOT = Path(\"/kaggle/input/understanding_cloud_organization\")\n    OUTPUT_DIR = Path(\"/kaggle/working\")\n    \nelse:\n    # local setup\n    DATA_ROOT = Path(\"/mnt/fast/data/uco\")\n    OUTPUT_DIR = Path(\"/mnt/fast/data/uco_prepared\")\n    \ndef rle_to_mask(rle_string, width, height):\n    '''\n    convert RLE(run length encoding) string to numpy array\n\n    Parameters: \n    rle_string (str): string of rle encoded mask\n    height (int): height of the mask\n    width (int): width of the mask\n\n    Returns: \n    numpy.array: numpy array of the mask\n    '''\n    \n    rows, cols = height, width\n    \n    if rle_string == -1:\n        return np.zeros((height, width))\n    else:\n        rle_numbers = [int(num_string) for num_string in rle_string.split(' ')]\n        rle_pairs = np.array(rle_numbers).reshape(-1,2)\n        img = np.zeros(rows*cols, dtype=np.uint8)\n        for index, length in rle_pairs:\n            index -= 1\n            img[index:index+length] = 255\n        img = img.reshape(cols,rows)\n        img = img.T\n        return img\n    \ndef get_mask(line_id, shape = (2100,1400)):\n    '''\n    Function to visualize the image and the mask.\n    INPUT:\n        line_id - id of the line to visualize the masks\n        shape - image shape\n    RETURNS:\n        np_mask - numpy segmentation map\n    '''\n    # replace null values with '-1'\n    im_df = train_df.fillna('-1')\n    \n    # convert rle to mask\n    rle = im_df.loc[line_id]['EncodedPixels']\n    if rle != '-1':\n        np_mask = rle_to_mask(rle, shape[0], shape[1])\n        np_mask = np.clip(np_mask, 0, 1)\n    else:\n        # empty mask\n        np_mask = np.zeros((shape[0],shape[1]), dtype=np.uint8)\n        \n    return np_mask\n\n\n\n# helper function to get segmentation mask for an image by filename\ndef get_mask_by_image_id(image_id, label):\n    '''\n    Function to visualize several segmentation maps.\n    INPUT:\n        image_id - filename of the image\n    RETURNS:\n        np_mask - numpy segmentation map\n    '''\n    im_df = train_df[train_df['Image'] == image_id.split('/')[-1]].fillna('-1')\n\n    image = np.asarray(Image.open(image_id))\n\n    rle = im_df[im_df['Label'] == label]['EncodedPixels'].values[0]\n    if rle != '-1':\n        np_mask = rle_to_mask(rle, np.asarray(image).shape[1], np.asarray(image).shape[0])\n        np_mask = np.clip(np_mask, 0, 1)\n    else:\n        # empty mask\n        np_mask = np.zeros((np.asarray(image).shape[0], np.asarray(image).shape[1]), dtype=np.uint8)\n        \n    return np_mask\n    \nTRAIN_CSV = DATA_ROOT/\"train.csv\"\n\nUNIQUE_CLASSES = ['Fish', 'Flower', 'Gravel', 'Sugar']\n\nMASK_CODES = dict(zip(UNIQUE_CLASSES, (0b001, 0b010, 0b100, 0b1000)))\n\nCOMBINED_MASK_CODES = [\n    (\"none\", 0),\n    *list(MASK_CODES.items()),\n    get_combined_code(\"Fish\", \"Flower\"),\n    get_combined_code(\"Fish\", \"Gravel\"),\n    get_combined_code(\"Fish\", \"Sugar\"),\n    get_combined_code(\"Flower\", \"Gravel\"),\n    get_combined_code(\"Flower\", \"Sugar\"),\n    get_combined_code(\"Gravel\", \"Sugar\"),\n    get_combined_code(\"Flower\",\"Gravel\", \"Sugar\",),\n    get_combined_code(\"Fish\",\"Gravel\", \"Sugar\"),\n    get_combined_code(\"Fish\", \"Flower\", \"Sugar\"),\n    get_combined_code(\"Fish\", \"Flower\",\"Gravel\"),\n    get_combined_code(\"Fish\", \"Flower\",\"Gravel\", \"Sugar\"),\n]","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:48.134785Z","iopub.execute_input":"2023-05-14T06:17:48.138150Z","iopub.status.idle":"2023-05-14T06:17:48.466881Z","shell.execute_reply.started":"2023-05-14T06:17:48.138111Z","shell.execute_reply":"2023-05-14T06:17:48.465590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_masks_rle(img):\n    \"\"\"Get RLE-encoded masks for this image\"\"\"\n    img = str(img).split(\"/\")[-1]  # get filename only\n    return train_df.loc[img, UNIQUE_CLASSES].to_list()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:48.475230Z","iopub.execute_input":"2023-05-14T06:17:48.477570Z","iopub.status.idle":"2023-05-14T06:17:48.484864Z","shell.execute_reply.started":"2023-05-14T06:17:48.477533Z","shell.execute_reply":"2023-05-14T06:17:48.483852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MASK_CODES,COMBINED_MASK_CODES","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:48.486227Z","iopub.execute_input":"2023-05-14T06:17:48.486840Z","iopub.status.idle":"2023-05-14T06:17:48.502982Z","shell.execute_reply.started":"2023-05-14T06:17:48.486807Z","shell.execute_reply":"2023-05-14T06:17:48.501850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_seed(1, reproducible=True)\nDEBUG = True\nDATASET_SIZE = 3000 if DEBUG else 20000  # read only a small subset of data to train some basic model\nDATA_ROOT = path","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:48.507682Z","iopub.execute_input":"2023-05-14T06:17:48.508639Z","iopub.status.idle":"2023-05-14T06:17:48.523570Z","shell.execute_reply.started":"2023-05-14T06:17:48.508603Z","shell.execute_reply":"2023-05-14T06:17:48.522414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fnames= get_image_files(path)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:48.527457Z","iopub.execute_input":"2023-05-14T06:17:48.528048Z","iopub.status.idle":"2023-05-14T06:17:54.184201Z","shell.execute_reply.started":"2023-05-14T06:17:48.527999Z","shell.execute_reply":"2023-05-14T06:17:54.183116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fnames[0].parts","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.188827Z","iopub.execute_input":"2023-05-14T06:17:54.191335Z","iopub.status.idle":"2023-05-14T06:17:54.201794Z","shell.execute_reply.started":"2023-05-14T06:17:54.191297Z","shell.execute_reply":"2023-05-14T06:17:54.200951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_slice_id(fname):\n    return fname.parts[5]","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.206575Z","iopub.execute_input":"2023-05-14T06:17:54.215248Z","iopub.status.idle":"2023-05-14T06:17:54.221281Z","shell.execute_reply.started":"2023-05-14T06:17:54.215214Z","shell.execute_reply":"2023-05-14T06:17:54.220200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_slice_id(fnames[0])","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.225228Z","iopub.execute_input":"2023-05-14T06:17:54.225896Z","iopub.status.idle":"2023-05-14T06:17:54.236830Z","shell.execute_reply.started":"2023-05-14T06:17:54.225864Z","shell.execute_reply":"2023-05-14T06:17:54.235837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.240093Z","iopub.execute_input":"2023-05-14T06:17:54.247255Z","iopub.status.idle":"2023-05-14T06:17:54.277237Z","shell.execute_reply.started":"2023-05-14T06:17:54.247210Z","shell.execute_reply":"2023-05-14T06:17:54.276404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MASK_CODES = dict(zip(UNIQUE_CLASSES, (0b001, 0b010, 0b100, 0b1000)))\nMASK_CODES ","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.281121Z","iopub.execute_input":"2023-05-14T06:17:54.282250Z","iopub.status.idle":"2023-05-14T06:17:54.297937Z","shell.execute_reply.started":"2023-05-14T06:17:54.282216Z","shell.execute_reply":"2023-05-14T06:17:54.297079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_items():\n    return fnames\ndef get_y(fname):\n    return get_mask_image(fname.parts[5])","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.301231Z","iopub.execute_input":"2023-05-14T06:17:54.301932Z","iopub.status.idle":"2023-05-14T06:17:54.315741Z","shell.execute_reply.started":"2023-05-14T06:17:54.301900Z","shell.execute_reply":"2023-05-14T06:17:54.314961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nempty_masks = set()\n\nmasks_dir = OUTPUT_DIR/\"masks\"\n\nif not masks_dir.exists():\n    masks_dir.mkdir()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:17:54.317302Z","iopub.execute_input":"2023-05-14T06:17:54.317934Z","iopub.status.idle":"2023-05-14T06:17:54.327077Z","shell.execute_reply.started":"2023-05-14T06:17:54.317901Z","shell.execute_reply":"2023-05-14T06:17:54.326146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor sample_id, df_group in progress_bar(train_df.groupby(\"Image\")):\n    shape = ( 2100,1400)\n    decoded_mask = np.zeros(shape, dtype=np.uint8)\n\n    for _, row in df_group.iterrows():\n        rle_mask = row.EncodedPixels\n        if isinstance(rle_mask, str):\n            decoded_mask |= rle_decode(rle_mask, shape, value=MASK_CODES[row[\"Label\"]])\n    \n    if not decoded_mask.any():\n        empty_masks.add(sample_id)\n    \n    mask_image = Image.fromarray(decoded_mask.T)\n    mask_path = masks_dir/f\"{row['Image']}.png\"\n    mask_image.save(mask_path)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:22:08.268439Z","iopub.execute_input":"2023-05-14T06:22:08.268904Z","iopub.status.idle":"2023-05-14T06:31:22.336527Z","shell.execute_reply.started":"2023-05-14T06:22:08.268868Z","shell.execute_reply":"2023-05-14T06:31:22.335455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nimages_dir = OUTPUT_DIR/\"images\"\n\nif not images_dir.exists():\n    images_dir.mkdir()\n\n\n\nsource_folder =  x+'/train_images/*'\ndestination_folder = images_dir\n\n! cp -r {source_folder} {destination_folder}","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:31:22.341341Z","iopub.execute_input":"2023-05-14T06:31:22.344257Z","iopub.status.idle":"2023-05-14T06:32:28.751512Z","shell.execute_reply.started":"2023-05-14T06:31:22.344219Z","shell.execute_reply":"2023-05-14T06:32:28.750102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_items(source_dir: Path):\n    return get_image_files(source_dir.joinpath(\"images\"))\n\ndef get_y(fn: Path):\n    return fn.parent.parent.joinpath(\"masks\").joinpath(f\"{fn.stem}.jpg.png\")\n\n# This was the code available in fastai\n@ToTensor\ndef encodes(self, o:PILMask): return o._tensor_cls(image2tensor(o)[0])\n\n# And this is how we customize it to suit our needs\n@ToTensor\ndef encodes(self, o:PILMask): return o._tensor_cls(image2tensor(o))","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:32:28.755622Z","iopub.execute_input":"2023-05-14T06:32:28.758866Z","iopub.status.idle":"2023-05-14T06:32:28.769983Z","shell.execute_reply.started":"2023-05-14T06:32:28.758825Z","shell.execute_reply":"2023-05-14T06:32:28.769074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seg = DataBlock(blocks=(ImageBlock, MaskBlock(COMBINED_MASK_CODES)),\n                get_items=get_items,\n                get_y=get_y,\n                splitter=RandomSplitter(),\n               )\n\ndls = seg.dataloaders(OUTPUT_DIR, bs=32)\n\nlearn = unet_learner(dls, resnet18, metrics=DiceMulti)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:32:28.775106Z","iopub.execute_input":"2023-05-14T06:32:28.777677Z","iopub.status.idle":"2023-05-14T06:33:16.803809Z","shell.execute_reply.started":"2023-05-14T06:32:28.777641Z","shell.execute_reply":"2023-05-14T06:33:16.802784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T06:33:30.224323Z","iopub.execute_input":"2023-05-14T06:33:30.225341Z","iopub.status.idle":"2023-05-14T06:33:44.821461Z","shell.execute_reply.started":"2023-05-14T06:33:30.225278Z","shell.execute_reply":"2023-05-14T06:33:44.817737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}