{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Visualization for HuBMAP + HPA - Hacking the Human Body**\n##### **Segment multi-organ functional tissue units**","metadata":{}},{"cell_type":"markdown","source":"### **Importing necessary modules & packages**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\n\nimport imageio, tifffile\nimport cv2, math, sys\nimport glob, os, joblib","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-27T06:07:12.746170Z","iopub.execute_input":"2022-09-27T06:07:12.746652Z","iopub.status.idle":"2022-09-27T06:07:19.541664Z","shell.execute_reply.started":"2022-09-27T06:07:12.746552Z","shell.execute_reply":"2022-09-27T06:07:19.540203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Loading Training DataFrame**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/hubmap-organ-segmentation/train.csv')\ndisplay(train.head())\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:19.545939Z","iopub.execute_input":"2022-09-27T06:07:19.546823Z","iopub.status.idle":"2022-09-27T06:07:20.053264Z","shell.execute_reply.started":"2022-09-27T06:07:19.546767Z","shell.execute_reply":"2022-09-27T06:07:20.051954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_WIDTH = train['img_width'].max()\nMAX_HEIGHT = train['img_height'].max()\nN_CHANNELS = 3\nIMG_SIZE = 640\nPATCH_SIZE = 640\nN_PATCHES_PER_IMAGE = (IMG_SIZE // PATCH_SIZE) ** 2\nN_SAMPLES = len(train)\nN_PATCHES = N_SAMPLES * N_PATCHES_PER_IMAGE\n\nprint(f'N_SAMPLES: {N_SAMPLES}, N_PATCHES: {N_PATCHES}, MAX_WIDTH: {MAX_WIDTH}, MAX_HEIGHT: {MAX_HEIGHT}')\nprint(f'IMG_SIZE: {IMG_SIZE}, PATCH_SIZE: {PATCH_SIZE}, N_PATCHES_PER_IMAGE: {N_PATCHES_PER_IMAGE}')","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.054908Z","iopub.execute_input":"2022-09-27T06:07:20.055787Z","iopub.status.idle":"2022-09-27T06:07:20.063164Z","shell.execute_reply.started":"2022-09-27T06:07:20.055741Z","shell.execute_reply":"2022-09-27T06:07:20.062285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Explorative Data Analysis**","metadata":{}},{"cell_type":"code","source":"train[['img_height', 'img_width']].value_counts().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.065714Z","iopub.execute_input":"2022-09-27T06:07:20.066308Z","iopub.status.idle":"2022-09-27T06:07:20.081164Z","shell.execute_reply.started":"2022-09-27T06:07:20.066276Z","shell.execute_reply":"2022-09-27T06:07:20.080065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train['data_source'].value_counts().to_frame())  # distribution of data source\ndisplay(train['pixel_size'].value_counts().to_frame())  # distribution of pixel size\ndisplay(train['tissue_thickness'].value_counts().to_frame())  # tissue thickness","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.084756Z","iopub.execute_input":"2022-09-27T06:07:20.085121Z","iopub.status.idle":"2022-09-27T06:07:20.108924Z","shell.execute_reply.started":"2022-09-27T06:07:20.085090Z","shell.execute_reply":"2022-09-27T06:07:20.108160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# age distribution bar graph\nplt.figure(figsize=(8,5))\ntrain['age'].plot(kind='hist')\nplt.title('Age Distribution', size=24)\nplt.xlim(0, 100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.109988Z","iopub.execute_input":"2022-09-27T06:07:20.111153Z","iopub.status.idle":"2022-09-27T06:07:20.389705Z","shell.execute_reply.started":"2022-09-27T06:07:20.111119Z","shell.execute_reply":"2022-09-27T06:07:20.388482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# organ distribution pi-chart\nax = plt.figure(figsize=(8, 8), facecolor='white')\ntrain['organ'].value_counts().plot(kind='pie', autopct='%1.1f%%', title='Organ Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.391247Z","iopub.execute_input":"2022-09-27T06:07:20.391678Z","iopub.status.idle":"2022-09-27T06:07:20.569675Z","shell.execute_reply.started":"2022-09-27T06:07:20.391648Z","shell.execute_reply":"2022-09-27T06:07:20.568462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pi-chart distribution based on gender\nax = plt.figure(figsize=(8, 8), facecolor='white')\ntrain['sex'].value_counts().plot(kind='pie', autopct='%1.1f%%', title='Sex Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.571333Z","iopub.execute_input":"2022-09-27T06:07:20.573483Z","iopub.status.idle":"2022-09-27T06:07:20.881556Z","shell.execute_reply.started":"2022-09-27T06:07:20.573445Z","shell.execute_reply":"2022-09-27T06:07:20.879853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Plotting Samples**","metadata":{}},{"cell_type":"code","source":"def resize_tensor(tensor):\n    return cv2.resize(tensor, [IMG_SIZE, IMG_SIZE], interpolation=cv2.INTER_CUBIC).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.883648Z","iopub.execute_input":"2022-09-27T06:07:20.885092Z","iopub.status.idle":"2022-09-27T06:07:20.893803Z","shell.execute_reply.started":"2022-09-27T06:07:20.885010Z","shell.execute_reply":"2022-09-27T06:07:20.892057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mask(image_id):\n    row = train.loc[train['id'] == image_id].squeeze()\n    h, w = row[['img_height', 'img_width']]\n    mask = np.zeros(shape=[h * w], dtype=np.uint8)\n    s = row['rle'].split()\n    starts, lengths = [ np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2]) ]\n    starts -= 1\n    ends = starts + lengths\n    for lo, hi in zip(starts, ends):\n        mask[lo : hi] = 1\n        \n    mask = mask.reshape([h, w]).T\n    mask = resize_tensor(mask)\n    mask = np.expand_dims(mask, axis=2)\n    return mask","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.902012Z","iopub.execute_input":"2022-09-27T06:07:20.904120Z","iopub.status.idle":"2022-09-27T06:07:20.922111Z","shell.execute_reply.started":"2022-09-27T06:07:20.904045Z","shell.execute_reply":"2022-09-27T06:07:20.919894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image(image_id, negative=False):\n    image = tifffile.imread(f'/kaggle/input/hubmap-organ-segmentation/train_images/{image_id}.tiff')\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n        \n    # Reverse pixels to make tissue colored and background black\n    if negative:\n        image = image - image.min()\n        image = image / (image.max() - image.min())\n        image = image * 255\n        image = 255 - image.astype(np.uint8)\n        \n    image = resize_tensor(image)\n    return image\n\nimage = get_image(train.loc[0, 'id'], True)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:20.924624Z","iopub.execute_input":"2022-09-27T06:07:20.925946Z","iopub.status.idle":"2022-09-27T06:07:21.665605Z","shell.execute_reply.started":"2022-09-27T06:07:20.925868Z","shell.execute_reply":"2022-09-27T06:07:21.664670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image_and_masks(rows=4, cols=4):\n    # Figure\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*8, rows*6))\n    # Unique Image Ids\n    image_ids = train['id'].unique()\n    \n    for r in range(rows):\n        image_id = image_ids[r]\n        df_row = train.loc[train['id'] == image_id].head(1).squeeze()\n        # Rad Image\n        image = get_image(image_id)\n        image_negative = get_image(image_id, negative=True)\n        \n        # Image Original\n        axes[r, 0].imshow(image)\n        axes[r, 0].set_title(f'Image {image_id} Raw', size=16)\n        axes[r, 0].axis(False)\n        \n        # Image Negative Original\n        axes[r, 1].imshow(image_negative)\n        axes[r, 1].set_title(f'Image {image_id} Negative min: {image_negative.min()} max: {image_negative.max()}', size=16)\n        axes[r, 1].axis(False)\n        \n        # Mask\n        mask = get_mask(image_id)\n        axes[r, 2].imshow(mask)\n        axes[r, 2].set_title('Mask', size=16)\n        axes[r, 2].axis(False)\n        \n        # Image with Mask\n        axes[r, 3].imshow(image)\n        axes[r, 3].imshow((mask * np.array([255, 0, 0])), alpha=0.50)\n        axes[r, 3].set_title('Image and Mask', size=16)\n        axes[r, 3].axis(False)\n            \n    # Adjust Vertical Space Between Subplots\n    fig.subplots_adjust(wspace=0.10)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:21.666885Z","iopub.execute_input":"2022-09-27T06:07:21.667398Z","iopub.status.idle":"2022-09-27T06:07:21.678636Z","shell.execute_reply.started":"2022-09-27T06:07:21.667357Z","shell.execute_reply":"2022-09-27T06:07:21.677831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_image_and_masks(rows=8)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:21.679956Z","iopub.execute_input":"2022-09-27T06:07:21.680538Z","iopub.status.idle":"2022-09-27T06:07:32.890800Z","shell.execute_reply.started":"2022-09-27T06:07:21.680506Z","shell.execute_reply":"2022-09-27T06:07:32.889489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Creating Train Arrays**","metadata":{}},{"cell_type":"code","source":"def extract_patches(image):\n    _, _, c = image.shape\n    image = tf.expand_dims(image, 0)\n    image_patches = tf.image.extract_patches(image, [1,PATCH_SIZE,PATCH_SIZE,1], [1, PATCH_SIZE, PATCH_SIZE, 1], [1, 1, 1, 1], padding='SAME')\n    image_patches = tf.reshape(image_patches, [N_PATCHES_PER_IMAGE, PATCH_SIZE, PATCH_SIZE, c])\n    image_patches = image_patches.numpy()\n\n    return image_patches","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:32.892365Z","iopub.execute_input":"2022-09-27T06:07:32.893755Z","iopub.status.idle":"2022-09-27T06:07:32.905474Z","shell.execute_reply.started":"2022-09-27T06:07:32.893699Z","shell.execute_reply":"2022-09-27T06:07:32.904334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_mask(image_id):    \n    image = get_image(image_id, True)\n    image_patches = extract_patches(image)\n    \n    mask = get_mask(image_id)\n    mask_patches = extract_patches(mask)\n    \n    organ = str.encode(train.loc[train['id'] == image_id, 'organ'].squeeze())\n    return image_patches, mask_patches, organ","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:32.907167Z","iopub.execute_input":"2022-09-27T06:07:32.908123Z","iopub.status.idle":"2022-09-27T06:07:32.916891Z","shell.execute_reply.started":"2022-09-27T06:07:32.908079Z","shell.execute_reply":"2022-09-27T06:07:32.915675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_CHUNKS = len(train)\nCHUNKS = np.array_split(train['id'].values, N_CHUNKS)\n\nprint(f'N_CHUNKS: {N_CHUNKS}, CHUNK_SIZE: {len(CHUNKS[0])}')","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:32.918853Z","iopub.execute_input":"2022-09-27T06:07:32.919630Z","iopub.status.idle":"2022-09-27T06:07:32.939041Z","shell.execute_reply.started":"2022-09-27T06:07:32.919590Z","shell.execute_reply":"2022-09-27T06:07:32.937189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_tf_records(chunks):\n    N_SAMPLES_PER_TFRECORD = []\n    ORGAN_PER_TFRECORD = []\n    for chunk_idx, image_id in enumerate(tqdm(chunks)):\n        # Get Image and Mask Patches\n        image_patches, mask_patches, organ = get_image_mask(image_id.squeeze())\n        tfrecord_name = f'batch_{chunk_idx}.tfrecords'\n        \n        # Create the actual TFRecords\n        with tf.io.TFRecordWriter(tfrecord_name) as file_writer:\n            sample_count = 0\n            for image, mask in zip(image_patches, mask_patches):\n                sample_count += 1\n\n                image_serialized = tf.io.serialize_tensor(image).numpy()\n                mask_serialized = tf.io.serialize_tensor(mask).numpy()\n\n                record_bytes = tf.train.Example(features=tf.train.Features(feature={\n                    # Image\n                    'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_serialized])),\n                    # Mask\n                    'mask': tf.train.Feature(bytes_list=tf.train.BytesList(value=[mask_serialized])),\n                    \n                    # Organ\n                    'organ': tf.train.Feature(bytes_list=tf.train.BytesList(value=[organ])),\n                })).SerializeToString()\n                file_writer.write(record_bytes)\n                    \n            # Add Sample Count\n            N_SAMPLES_PER_TFRECORD.append(sample_count)\n            # Add organ\n            ORGAN_PER_TFRECORD.append(organ)\n            \n    # Save Number of Samples per TFRecord to determine step count during training\n    np.save('N_SAMPLES_PER_TFRECORD.npy', np.array(N_SAMPLES_PER_TFRECORD, dtype=np.int16))\n    # Save organ per TFRecord for stratifying kfolds\n    np.save('ORGAN_PER_TFRECORD.npy', np.array(ORGAN_PER_TFRECORD, dtype=str))\n\n# Create TFRecords\nto_tf_records(CHUNKS)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:07:32.941558Z","iopub.execute_input":"2022-09-27T06:07:32.942558Z","iopub.status.idle":"2022-09-27T06:11:02.961722Z","shell.execute_reply.started":"2022-09-27T06:07:32.942475Z","shell.execute_reply":"2022-09-27T06:11:02.960689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Checking TFRecord**","metadata":{}},{"cell_type":"code","source":"def decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string),\n        'organ': tf.io.FixedLenFeature([], tf.string),\n    })\n    \n    image = tf.io.parse_tensor(features['image'], out_type=tf.uint8)\n    image = tf.reshape(image, [PATCH_SIZE, PATCH_SIZE, N_CHANNELS])\n    \n    mask = tf.io.parse_tensor(features['mask'], out_type=tf.uint8)\n    mask = tf.reshape(mask, [PATCH_SIZE, PATCH_SIZE, 1])\n\n    organ = features['organ']\n    \n    # Explicit reshape needed for TPU, tell cimpiler dimensions of image\n    image = tf.reshape(image, [PATCH_SIZE, PATCH_SIZE, N_CHANNELS])\n    # Explicit reshape needed for TPU, tell cimpiler dimensions of image\n    mask = tf.reshape(mask, [PATCH_SIZE, PATCH_SIZE, 1])\n    \n    return image, mask, organ","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:02.963244Z","iopub.execute_input":"2022-09-27T06:11:02.963863Z","iopub.status.idle":"2022-09-27T06:11:02.974120Z","shell.execute_reply.started":"2022-09-27T06:11:02.963819Z","shell.execute_reply":"2022-09-27T06:11:02.972989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(bs):\n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob('./*.tfrecords')\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=1)\n    train_dataset = train_dataset.map(decode_tfrecord, num_parallel_calls=cpu_count())\n    train_dataset = train_dataset.batch(bs)\n    \n    return train_dataset","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:02.975532Z","iopub.execute_input":"2022-09-27T06:11:02.975917Z","iopub.status.idle":"2022-09-27T06:11:02.994305Z","shell.execute_reply.started":"2022-09-27T06:11:02.975882Z","shell.execute_reply":"2022-09-27T06:11:02.993025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Plotting Patches**","metadata":{}},{"cell_type":"code","source":"def show_batch(dataset, rows=10, cols=2):\n    images, masks, organs = next(iter(dataset))\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*6, rows*6))\n    for r in range(rows):\n        axes[r, 0].imshow(images[r])\n        organ = organs[r].numpy().decode(\"UTF-8\")\n        axes[r, 0].set_title(f'Organ: {organ}', size=16)\n        axes[r, 1].imshow(masks[r])\n        axes[r, 1].set_title('Mask', size=16)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:02.996227Z","iopub.execute_input":"2022-09-27T06:11:02.997117Z","iopub.status.idle":"2022-09-27T06:11:03.008849Z","shell.execute_reply.started":"2022-09-27T06:11:02.997065Z","shell.execute_reply":"2022-09-27T06:11:03.007460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_train_dataset(10)\nshow_batch(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:03.011770Z","iopub.execute_input":"2022-09-27T06:11:03.012590Z","iopub.status.idle":"2022-09-27T06:11:08.138775Z","shell.execute_reply.started":"2022-09-27T06:11:03.012549Z","shell.execute_reply":"2022-09-27T06:11:08.137517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Reconstruct Image/Mask from Patches**","metadata":{}},{"cell_type":"code","source":"def show_image_mask_from_patches(dataset, n_samples=10):\n    s = int(N_PATCHES_PER_IMAGE ** 0.50)\n    counter = 0\n    dataset_iter = iter(dataset)\n    \n    for _ in tqdm(range(n_samples)):\n        # Main plot\n        fig = plt.figure(figsize=(30, 10))\n        subfigs = fig.subfigures(1, 3)\n        plt.suptitle('Patched Image/Mask/Combined', fontsize=48, y=1.1)\n        images, masks, organs = next(dataset_iter)\n        o = organs[0].numpy().decode()\n        \n        # Make subplots\n        ax_images = subfigs[0].subplots(s, s)\n        subfigs[0].suptitle(f'Image ({o})', size=32)\n        ax_masks = subfigs[1].subplots(s, s)\n        subfigs[1].suptitle('Mask', size=32)\n        ax_combined = subfigs[2].subplots(s, s)\n        subfigs[2].suptitle('Combined', size=32)\n        \n        count = 0\n        for r in range(s):\n            for c in range(s):\n                idx = r * s + c\n                \n                if s > 1:\n                    # Image\n                    ax_images[r,c].imshow(images[idx].numpy())\n                    ax_images[r,c].set_title(f'σ{images[idx].numpy().std():.2f}', c='red')\n                    ax_images[r,c].axis(False)\n                    # Mask\n                    ax_masks[r,c].imshow(masks[idx].numpy() * [255,255,0])\n                    ax_masks[r,c].axis(False)\n                    # Combined\n                    ax_combined[r,c].imshow(images[idx].numpy())\n                    ax_combined[r,c].imshow((masks[idx] * np.array([255, 0, 0])), alpha=0.50)\n                    ax_combined[r,c].axis(False)\n                else:\n                    # Image\n                    ax_images.imshow(images[idx].numpy())\n                    ax_images.set_title(f'σ{images[idx].numpy().std():.2f}', c='red')\n                    ax_images.axis(False)\n                    # Mask\n                    ax_masks.imshow(masks[idx].numpy() * [255,255,0])\n                    ax_masks.axis(False)\n                    # Combined\n                    ax_combined.imshow(images[idx].numpy())\n                    ax_combined.imshow((masks[idx] * np.array([255, 0, 0])), alpha=0.50)\n                    ax_combined.axis(False)\n                \n                count += 1\n                \n        # Show plot\n        fig.subplots_adjust(wspace=0.05)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:08.140765Z","iopub.execute_input":"2022-09-27T06:11:08.141150Z","iopub.status.idle":"2022-09-27T06:11:08.158372Z","shell.execute_reply.started":"2022-09-27T06:11:08.141115Z","shell.execute_reply":"2022-09-27T06:11:08.157422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_train_dataset(N_PATCHES_PER_IMAGE)\nshow_image_mask_from_patches(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:08.159872Z","iopub.execute_input":"2022-09-27T06:11:08.160912Z","iopub.status.idle":"2022-09-27T06:11:19.282345Z","shell.execute_reply.started":"2022-09-27T06:11:08.160868Z","shell.execute_reply":"2022-09-27T06:11:19.281135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Mean/Standard Deviation Computation**","metadata":{}},{"cell_type":"code","source":"MEAN = np.zeros(3, dtype=np.float32)\nSTD = np.zeros(3, dtype=np.float32)\n\nfor image, _, _ in tqdm(get_train_dataset(1), total=N_PATCHES):\n    # Update Mean and STD\n    image_np = image.numpy().squeeze()\n    MEAN += (image_np / 255).mean(axis=(0,1)) / N_PATCHES\n    STD += (image_np /  255).std(axis=(0,1)) / N_PATCHES\n\ndisplay(pd.Series(MEAN).to_frame('Mean'))\ndisplay(pd.Series(STD).to_frame('Standard Deviation'))","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:19.284156Z","iopub.execute_input":"2022-09-27T06:11:19.284556Z","iopub.status.idle":"2022-09-27T06:11:33.319992Z","shell.execute_reply.started":"2022-09-27T06:11:19.284521Z","shell.execute_reply":"2022-09-27T06:11:33.319210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('MEAN.npy', MEAN)\nnp.save('STD.npy', STD)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T06:11:33.321457Z","iopub.execute_input":"2022-09-27T06:11:33.322538Z","iopub.status.idle":"2022-09-27T06:11:33.328211Z","shell.execute_reply.started":"2022-09-27T06:11:33.322490Z","shell.execute_reply":"2022-09-27T06:11:33.327224Z"},"trusted":true},"execution_count":null,"outputs":[]}]}