{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport numpy as np\nimport random\n\n# Dicom\nimport pydicom\n\n# Enum\nfrom enum import Enum\n\n# CV\nimport cv2\n\n# Tensorflow\nimport tensorflow as tf\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.models import (\n    Model,\n    load_model\n)\nfrom tensorflow.keras.callbacks import (\n    Callback, \n    ModelCheckpoint, \n    EarlyStopping\n)\nfrom tensorflow.keras.layers import (\n    Input,\n    Conv3D,\n    BatchNormalization,\n    MaxPooling3D,\n    MaxPool3D,\n    Flatten,\n    Dense,\n    Dropout,\n    Resizing,\n    Rescaling,\n    RandomFlip,\n    RandomRotation,\n    concatenate,\n    GlobalAveragePooling3D,\n    Reshape,\n    LeakyReLU,\n    ReLU\n)\n\n# Keras\nimport keras\nfrom keras.utils.vis_utils import plot_model\n\n#submission = pd.read_csv(\"/kaggle/input/cnn-cv-brain-prediction/submission.csv\")\n#submission.head(20)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-23T07:48:25.019171Z","iopub.execute_input":"2023-09-23T07:48:25.019506Z","iopub.status.idle":"2023-09-23T07:48:33.594502Z","shell.execute_reply.started":"2023-09-23T07:48:25.019473Z","shell.execute_reply":"2023-09-23T07:48:33.593375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission[\"BraTS21ID\"] = submission[\"BraTS21ID\"].apply(lambda x: f'{x:05d}')\n\n#submission = submission.groupby('BraTS21ID', sort=False)['MGMT_value'].mean().reset_index()\n#submission.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.596273Z","iopub.execute_input":"2023-09-23T07:48:33.596936Z","iopub.status.idle":"2023-09-23T07:48:33.602238Z","shell.execute_reply.started":"2023-09-23T07:48:33.596907Z","shell.execute_reply":"2023-09-23T07:48:33.601170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission['BraTS21ID'] = submission['BraTS21ID'].apply(lambda x: f'{x:05d}')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.603640Z","iopub.execute_input":"2023-09-23T07:48:33.604221Z","iopub.status.idle":"2023-09-23T07:48:33.615152Z","shell.execute_reply.started":"2023-09-23T07:48:33.604189Z","shell.execute_reply":"2023-09-23T07:48:33.614120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.617847Z","iopub.execute_input":"2023-09-23T07:48:33.618181Z","iopub.status.idle":"2023-09-23T07:48:33.625254Z","shell.execute_reply.started":"2023-09-23T07:48:33.618150Z","shell.execute_reply":"2023-09-23T07:48:33.624368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv',dtype=\"string\")\n#df['MGMT_value'] = submission['MGMT_value']","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.628467Z","iopub.execute_input":"2023-09-23T07:48:33.628739Z","iopub.status.idle":"2023-09-23T07:48:33.634579Z","shell.execute_reply.started":"2023-09-23T07:48:33.628716Z","shell.execute_reply":"2023-09-23T07:48:33.633364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.head(500)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.635927Z","iopub.execute_input":"2023-09-23T07:48:33.636171Z","iopub.status.idle":"2023-09-23T07:48:33.645550Z","shell.execute_reply.started":"2023-09-23T07:48:33.636150Z","shell.execute_reply":"2023-09-23T07:48:33.644622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission = df.copy()\n#submission.to_csv('submission.csv',index=False)\n\n#submission","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.647034Z","iopub.execute_input":"2023-09-23T07:48:33.647427Z","iopub.status.idle":"2023-09-23T07:48:33.655354Z","shell.execute_reply.started":"2023-09-23T07:48:33.647400Z","shell.execute_reply":"2023-09-23T07:48:33.654290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_PYDICOM_IMPORTED = True\nif IS_PYDICOM_IMPORTED:\n    \"\"\"\n    This block of code checks if the code is being run with PYDICOM Lib.\n    \"\"\"\n    try:\n        import pydicom\n        import os\n        import glob\n        import cv2\n        from enum import Enum\n        import numpy as np\n        import pandas as pd\n        from concurrent.futures import ThreadPoolExecutor, as_completed\n        from tqdm import tqdm\n\n    except ImportError:\n        print(f\"Missing some imports: {ImportError}\")\n\n    class ImageFormat(Enum):\n        \"\"\"\n        Enum to represent image formats.\n\n        - 'W-H-D-C' = Width-Height-Depth-Channel\n        - 'D-W-H-C' = Depth-Width-Height-Channel\n        \"\"\"\n        WHDC = \"W-H-D-C\"  # Format (128, 128, 64, 1)\n        DWHC = \"D-W-H-C\"  # Format (64, 128, 128, 1)\n\n        def swap_dimensions(image, image_format):\n            \"\"\"\n            Swap dimensions of an image NumPy array based on the specified permutation type.\n\n            Parameters:\n            ----------\n            image : numpy.ndarray\n                Input image with dimensions to be swapped.\n            permutation_type : str\n                The category of the scan to load.\n                - 'WxHxDxC' = Width-Height-Depth-Channel'\n                - 'DxWxHxC' = (Depth-Width-Height-Channel)\n\n            Returns:\n            numpy.ndarray: Image with swapped dimensions.\n            \"\"\"\n            if image_format == ImageFormat.DWHC:\n                # Permutation: (128, 128, 64, 1) vers (64, 128, 128, 1)\n                image = np.transpose(image, (2, 1, 0, 3))\n\n            elif image_format == ImageFormat.WHDC:\n                # Permutation: (64, 128, 128, 1) vers (128, 128, 64, 1)\n                image = np.transpose(image, (2, 1, 0, 3))\n\n            return image\n\n    class DICOMLoader():\n        def __init__(self,\n            df,\n            input_path,\n            scan_categories,\n            num_imgs                        = None,\n            size                            = (224, 224),\n            scale                           = 1.0,\n            rotate_angle                    = 0,\n            enable_center_focus             = False,\n            id_column_name                  = \"ID\",\n            label_column_name               = \"Label\",\n            image_format                    = ImageFormat.WHDC,\n            max_threads                     = 8,\n            image_file_sorter               = lambda x: int(x[:-4].split(\"-\")[-1]),\n            debug_mode                      = False\n        ):\n            \"\"\"\n            This class is designed for loading DICOM images from a given directory and\n            creating a dataset for medical image analysis.\n\n            Parameters:\n            -----------\n            df : DataFrame\n                The DataFrame containing metadata and labels for DICOM images.\n            input_path : str\n                The path to the directory containing DICOM image files.\n            scan_categories : list\n                A list of scan categories to include in the dataset.\n            num_imgs : int, optional\n                The number of images to load per scan. If specified, must be divisible\n                by 2 when 'enable_center_focus' is True. Default is None.\n            size : tuple, optional\n                The size to which images should be resized. Default is (224, 224).\n            scale : float, optional\n                The scaling factor applied to the images. Default is 1.0.\n            rotate_angle : int, optional\n                The angle in degrees by which images should be rotated. Default is 0.\n            enable_center_focus : bool, optional\n                If True, focus on the central images when 'num_imgs' is specified.\n                Default is False.\n            id_column_name : str, optional\n                The name of the column containing unique IDs in 'df'. Default is \"ID\".\n            label_column_name : str, optional\n                The name of the column containing labels in 'df'. Default is \"Label\".\n            image_format : ImageFormat, optional\n                The format of the DICOM images (e.g., ImageFormat.WHDC). Default is\n                ImageFormat.WHDC.\n            max_threads : int, optional\n                The maximum number of threads to use for image loading. Default is 8.\n            image_file_sorter : function, optional\n                A function used to sort image files. Default sorts by numeric value\n                at the end of the filename.\n            debug_mode: bool\n                Debug mode\n\n            Raises:\n            -------\n            ValueError\n                - If 'num_imgs' is not divisible by 2 when 'enable_center_focus' is True.\n                - If 'rotate_angle' is not in the range [0, 360].\n                - If 'id_column_name' or 'label_column_name' is not in 'df.columns'.\n            \"\"\"\n            if num_imgs is not None and num_imgs % 2 != 0 and enable_center_focus:\n                raise ValueError(\"num_imgs must be divisible by 2 for central image\")\n\n            if not (0 <= rotate_angle <= 360):\n                raise ValueError(\"Rotation value must be between 0 and 360\")\n\n            for col in [id_column_name, label_column_name]:\n                if col not in df.columns:\n                    raise ValueError(f\"Columns {col} must be in dataset\")\n\n            self.__df                  = df.copy()\n            self.__num_imgs            = num_imgs\n            self.__id_column_name      = id_column_name\n            self.__label_column_name   = label_column_name\n            self.__input_path          = input_path\n            self.__scan_categories     = scan_categories\n            self.__max_threads         = max_threads\n            self.__size                = size\n            self.__scale               = scale\n            self.__rotate_angle        = rotate_angle\n            self.__image_format        = image_format\n            self.__image_file_sorter   = image_file_sorter\n            self.__enable_center_focus = enable_center_focus\n            self.__debug               = InternalDebug(debug_mode=debug_mode)\n\n        # ---------------- #\n        # Public\n        # ---------------- #\n\n        # -------- #\n        # Property\n        # -------- #\n\n        @property\n        def scan_categories(self):\n            return self.__scan_categories\n\n        @property\n        def num_imgs(self):\n            return self.__num_imgs\n\n        @property\n        def image_format(self):\n            return self.__image_format\n\n        @property\n        def df(self):\n            return self.__df.copy()\n\n        @property\n        def len(self):\n            \"\"\"\n            Returns the length of the DataFrame.\n\n            Returns\n            -------\n            int\n                The number of rows in the DataFrame.\n            \"\"\"\n            return len(self.__df)\n\n        # -------- #\n        # Get\n        # -------- #\n\n        def get_id(self, row):\n            \"\"\"\n            Retrieves the ID for a given row in the DataFrame.\n\n            Parameters\n            ----------\n            row : int\n                The row index in the DataFrame.\n\n            Returns\n            -------\n            str\n                The ID corresponding to the row.\n            \"\"\"\n            return self.__df.loc[row, self.__id_column_name]\n\n        def gel_label(self, row):\n            \"\"\"\n            Retrieves the label for a given row in the DataFrame.\n\n            Parameters\n            ----------\n            row : int\n                The row index in the DataFrame.\n\n            Returns\n            -------\n            str\n                The label corresponding to the row.\n            \"\"\"\n            return self.__df.loc[row, self.__label_column_name]\n\n        # -------- #\n        # Format\n        # -------- #\n\n        def format(self, images, type):\n            if type == \"normalize\" and self.image_format == ImageFormat.WHDC:\n                return ImageFormat.swap_dimensions(images, ImageFormat.DWHC)\n\n            elif type == \"default\" and self.image_format == ImageFormat.WHDC:\n                return ImageFormat.swap_dimensions(images, ImageFormat.WHDC)\n\n            else:\n                return images\n\n        # -------- #\n        # Load\n        # -------- #\n\n        def load_scan(self, row, scan_category, show_progress=True):\n            \"\"\"\n            Loads a scan for a given row and scan category.\n\n            Parameters\n            ----------\n            row : int\n                The row index in the DataFrame.\n            scan_category : str\n                The category of the scan to load.\n            show_progress : bool, optional\n                Whether to show a progress bar.\n\n            Returns\n            -------\n            list\n                A list of loaded images.\n            \"\"\"\n            self.__debug.log(\"== load_scan ==\")\n            # Get the patient ID and construct the scan path\n            # ----\n            patient_id = str(self.__df.loc[row, self.__id_column_name]).zfill(5)\n            scans_path = os.path.join(self.__input_path, patient_id, scan_category)\n\n            # Check if the scan path exists\n            # ----\n            if not os.path.exists(scans_path):\n                raise FileNotFoundError(f\"The folder {scans_path} doesn't exist.\")\n\n            # Get the list of image files\n            # ----\n            image_files = sorted(\n                glob.glob(os.path.join(scans_path, \"*\")),\n                key = self.__image_file_sorter\n            )\n\n            # Check if any image files were found\n            # ----\n            if not image_files:\n                raise ValueError(f\"No image files found in {scans_path}.\")\n\n            # Select images based on enable_center_focus and num_imgs\n            # ----\n            image_files = self._select_subset_image_files(image_files)\n\n            # Initialize the list to store loaded images\n            # ----\n            loaded_images = []\n\n            # Load images using ThreadPoolExecutor\n            # ----\n            with ThreadPoolExecutor(self.__max_threads) as executor:\n                image_data_iterable = executor.map(self._load_dicom_image, image_files)\n\n                # Show progress if enabled\n                # ----\n                if show_progress:\n                    image_data_iterable = tqdm(image_data_iterable, total=len(image_files), desc=\"Loading images\")\n\n                # Append loaded images to the list\n                # ----\n                for image_data in image_data_iterable:\n                    if image_data is not None:\n                        loaded_images.append(image_data)\n\n            # Check if any images were loaded\n            # ----\n            if not loaded_images:\n                raise ValueError(\"No images were loaded, and num_imgs is set. Cannot proceed.\")\n\n            # Fill up the list to num_imgs if necessary\n            # ----\n            if self.__num_imgs is not None:\n                while len(loaded_images) < self.__num_imgs:\n                    zero_image = np.zeros_like(loaded_images[0])\n                    loaded_images.append(zero_image)\n\n            loaded_images = np.array(loaded_images)\n\n            return self.format(loaded_images, \"default\")\n\n        def load_all_scans(\n            self,\n            row,\n            show_progress=True\n        ):\n            \"\"\"\n            Loads all scans for a given row in the DataFrame.\n\n            Parameters\n            ----------\n            row : int\n                The row index in the DataFrame.\n            show_progress : bool, optional\n                Whether to show a progress bar.\n\n            Returns\n            -------\n            dict\n                A dictionary containing all loaded images, categorized by scan type.\n            \"\"\"\n            self.__debug.log(\"== load_all_scans ==\")\n            # Initialize an empty dictionary to store images for each MRI type\n            # ----\n            all_images = {}\n\n            # Use a ThreadPoolExecutor to load images for each MRI type concurrently\n            # ----\n            with ThreadPoolExecutor(self.__max_threads) as executor:\n                future_to_scan_category = {\n                    executor.submit(\n                        self.load_scan,\n                        row,\n                        scan_category,\n                        False\n                    ): scan_category for scan_category in self.__scan_categories\n                }\n\n                # Initialize a tqdm progress bar\n                # ----\n                if show_progress:\n                    progress_bar = tqdm(total=len(self.__scan_categories), desc=\"Loading scan types\")\n\n                # As each task completes, store the loaded images in the all_images dictionary and update the progress bar\n                # ----\n                for future in as_completed(future_to_scan_category):\n                    scan_category = future_to_scan_category[future]\n                    image_data = future.result()\n\n                    if image_data is not None:\n                        all_images[scan_category] = image_data\n                        if show_progress:\n                            progress_bar.update(1)\n\n                # Close the progress bar if it exists\n                # ----\n                if show_progress:\n                    progress_bar.close()\n\n            # Ordering by categories\n            # ----\n            return {key: all_images.get(key, []) for key in self.__scan_categories}\n\n        # -------- #\n        # Show\n        # -------- #\n\n        def _show_scan(self, scan_category, images, color_map):\n            images = self.format(images, \"normalize\")\n            show_text(\"h4\", scan_category, False)\n            show_images(images, color_map=color_map)\n\n        def show(self, row, scan_category, color_map='gray'):\n            \"\"\"\n            Displays the images for a given row and scan category.\n\n            Parameters\n            ----------\n            row : int\n                The row index in the DataFrame.\n            scan_category : str\n                The category of the scan to display.\n            color_map : str, optional\n                The color map to use for displaying the images.\n\n            \"\"\"\n            images = self.load_scan(row, scan_category)\n            self._show_scan(scan_category, images, color_map)\n\n\n        def show_all(self, row, color_map='gray'):\n            \"\"\"\n            Displays all images for a given row in the DataFrame.\n\n            Parameters\n            ----------\n            row : int\n                The row index in the DataFrame.\n            color_map : str, optional\n                The color map to use for displaying the images.\n\n            \"\"\"\n            loaders_images = self.load_all_scans(row)\n            for scan_category, images in loaders_images.items():\n                self._show_scan(scan_category, images, color_map)\n\n        # Overriding the summary method\n        def summary(self, train_dataset=None):\n            super().summary()\n            print(\"Additional summary details specific:\")\n            # Get the value of self.__scan_categories[0]\n            # ----\n            scan_category = self.__scan_categories[0]\n\n            if scan_category is not None:\n                images = self.load_scan(0, scan_category)\n                print(\"\\n\")\n\n                if images is not None:\n                    print(\"Size:\", self.len)\n                    print(\"Images Shape:\", images.shape)\n\n                else:\n                    print(\"Error: Loading images are empty.\")\n            else:\n                print(\"Error: scan_category is empty.\")\n            print(\"=\" * 50)\n\n        # ---------------- #\n        # Private methods\n        # ---------------- #\n\n        def _load_dicom_image(self, dicom_path):\n            \"\"\"\n            Loads a DICOM image from a given path and applies various transformations\n            such as VOI LUT, rotation, normalization, cropping, and resizing.\n\n            Parameters\n            ----------\n            dicom_path : str\n                The path to the DICOM file.\n\n            Returns\n            -------\n            2D array\n                The transformed DICOM image.\n\n            Raises\n            ------\n            FileNotFoundError\n                If the specified DICOM file does not exist.\n            IOError\n                If an error occurs while reading the DICOM file.\n\n            Notes\n            -----\n            The method performs the following transformations in order:\n            1. Applies Value of Interest Lookup Table (VOI LUT) for better visibility.\n            2. Rotates the image based on the specified angle.\n            3. Normalizes the pixel values in the image.\n            4. Crops the image to focus on the region of interest.\n            5. Resizes the image to the specified dimensions.\n            \"\"\"\n            self.__debug.log(\"== _load_dicom_image ==\")\n            # Check if the DICOM file exists\n            # ----\n            if not os.path.exists(dicom_path):\n                raise FileNotFoundError(f\"File {dicom_path} does not exist.\")\n\n            # Load the DICOM file\n            # ----\n            try:\n                dicom_file = pydicom.dcmread(dicom_path)\n            except Exception as e:\n                raise IOError(f\"An error occurred while reading the DICOM file: {e}\")\n\n            # Get image array\n            # ----\n            image = dicom_file.pixel_array\n            self.__debug.log(\"Pixel array shape:\", image.shape)\n\n            # Rotate\n            # ----\n            image = self._rotate_img(image)\n\n            # Normalization\n            # ----\n            image = self._normalization_img(image)\n\n            # Crop\n            # ----\n            image = self._crop_img(image)\n\n            # Resize\n            # ----\n            image = self._resize_img(image)\n\n            # Chanel\n            # ----\n            image = np.expand_dims(image, axis=-1)\n\n            self.__debug.log(\"Is normalized: \", self._is_normalized(image))\n            self.__debug.log(\"Image shape: \", image.shape)\n\n            return image\n\n        def _resize_img(self, image):\n            w, h = self.__size\n\n            # Resize\n            # ----\n            return cv2.resize(image, (w, h), interpolation = cv2.INTER_AREA)\n\n        def _crop_img(self, image):\n            \"\"\"\n            Crops and resizes a given image.\n\n            Parameters\n            ----------\n            image : 2D array\n                The image to be cropped and resized.\n\n            Returns\n            -------\n            2D array\n                The cropped and resized image.\n            \"\"\"\n            # Skip if no crop\n            # ----\n            if self.__scale <= 0:\n                return image\n\n            # Calculate the center of the image\n            # ----\n            center_x, center_y = image.shape[1] / 2, image.shape[0] / 2\n\n            # Calculate the dimensions of the scaled image\n            # ----\n            width_scaled, height_scaled = image.shape[1] * self.__scale, image.shape[0] * self.__scale\n\n            # Calculate the coordinates for cropping the image\n            # ----\n            left_x, right_x = center_x - width_scaled / 2, center_x + width_scaled / 2\n            top_y, bottom_y = center_y - height_scaled / 2, center_y + height_scaled / 2\n\n            # Crop the image using the calculated coordinates\n            # ----\n            return image[int(top_y):int(bottom_y), int(left_x):int(right_x)]\n\n        def _rotate_img(self, image):\n          \"\"\"\n          Rotates the image array by the specified angle.\n\n          Parameters\n          ----------\n          image : ndarray\n              The original image array.\n\n          Returns\n          -------\n          ndarray\n              The rotated image array.\n          \"\"\"\n          if self.__rotate_angle <= 0:\n              return image\n\n          #    Math of the rotation matrix\n          # ----\n          height, width = image.shape[:2]\n          center = (width / 2, height / 2)\n          rotation_matrix = cv2.getRotationMatrix2D(center, self.__rotate_angle, 1.0)\n\n          # Apply rotation\n          # ----\n          return cv2.warpAffine(image, rotation_matrix, (width, height))\n\n        def _normalization_img(self, image):\n            \"\"\"\n            Normalizes the image array to a range of 0 to 255.\n\n            Parameters\n            ----------\n            image : ndarray\n                The original image array.\n\n            Returns\n            -------\n            ndarray\n                The normalized image array.\n            \"\"\"\n            min_val = np.min(image)\n            max_val = np.max(image)\n\n            if max_val == 0:\n                # If max value is zero, return an array of zeros\n                # ----\n                return np.zeros_like(image).astype(np.uint8)\n\n            # Otherwise, proceed with normalization\n            # ----\n            image = image - min_val\n            image = image / max_val\n\n            return (image * 255).astype(np.uint8)\n\n        def _select_subset_image_files(self, image_files):\n            \"\"\"\n            Selects a subset of image files based on the object's attributes.\n\n            Parameters\n            ----------\n            image_files : list\n                List of image files to select from.\n\n            Returns\n            -------\n            list\n                A subset of the original list of image files.\n            \"\"\"\n            if self.__enable_center_focus and self.__num_imgs is not None:\n                middle = len(image_files) // 2\n                num_imgs2 = self.__num_imgs // 2\n                p1 = max(0, middle - num_imgs2)\n                p2 = min(len(image_files), middle + num_imgs2)\n                return image_files[p1:p2]\n\n            elif self.__num_imgs is not None:\n                return image_files[:self.__num_imgs]\n\n            else:\n                return image_files\n\n\n        def _is_normalized(self, image):\n            min_value = np.min(image)\n            max_value = np.max(image)\n\n            return min_value >= 0.0 and max_value <= 255\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.656938Z","iopub.execute_input":"2023-09-23T07:48:33.657181Z","iopub.status.idle":"2023-09-23T07:48:33.717999Z","shell.execute_reply.started":"2023-09-23T07:48:33.657159Z","shell.execute_reply":"2023-09-23T07:48:33.717140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InternalDebug:\n    def __init__(self, debug_mode=False, debug_prefix=None):\n        self.__debug_mode = debug_mode\n        self.__debug_prefix = debug_prefix\n\n    @property\n    def __prefix(self):\n        if self.__debug_prefix is not None:\n            return self.__debug_prefix\n        else:\n            return \"\"\n\n    def log(self, *args):\n        if self.__debug_mode:\n            if self.__debug_prefix is not None:\n                print(self.__debug_prefix, *args)\n            else:\n                print(*args)\n\n    def separator(self, character=\"=\", length=50):\n        separator = character * length\n        self.log(separator)\n\n    def info(self, *args):\n        if self.__debug_mode:\n            prefix = self.__prefix + \"[INFO]\"\n            self.log(prefix, *args)\n\n    def warning(self, *args):\n        if self.__debug_mode:\n            prefix = self.__prefix + \"[WARNING]\"\n            self.log(prefix, *args)\n\n    def error(self, *args):\n        if self.__debug_mode:\n            prefix = self.__prefix + \"[ERROR]\"\n            self.log(prefix, *args)\n\n    def set_debug_mode(self, debug_mode):\n        self.__debug_mode = debug_mode\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.719677Z","iopub.execute_input":"2023-09-23T07:48:33.720277Z","iopub.status.idle":"2023-09-23T07:48:33.732765Z","shell.execute_reply.started":"2023-09-23T07:48:33.720222Z","shell.execute_reply":"2023-09-23T07:48:33.731823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ScanDataset(Sequence):\n    def __init__(\n        self,\n        dicom_loader,\n        batch_size,\n        subset     = \"train\",\n        shuffle    = True,\n        debug_mode = False\n    ):\n        \"\"\"\n        Initializes the ScanDataset object.\n\n        Parameters\n        ----------\n        dicom_loader : object\n            The DICOMLoader object to load DICOM images.\n        batch_size : int\n            The size of each batch.\n        subset: Subset of the data to return.\n          One of \"training\", \"validation\" or \"other\".\n          training and validation give the y_bath\n        shuffle : bool, optional\n            Whether to shuffle the dataset.\n        debug_mode : bool, optional\n            Whether to print debug information.\n        \"\"\"\n        self.__dicom_loader = dicom_loader\n        self.__batch_size   = batch_size\n        self.__is_trainable = subset.lower() in [\"validation\", \"train\"]\n        self.__shuffle      = shuffle\n        self.__debug        = InternalDebug(debug_mode=debug_mode)\n        self.__indices      = np.arange(self.__dicom_loader.len)\n        \n        if self.__shuffle:\n            np.random.shuffle(self.__indices)\n        \n    # ---------------- #\n    # Public methods\n    # ---------------- #\n\n    def show_batch(\n        self,\n        row,\n        columns=None,\n        figure_size=(5, 5),\n        color_map='hot'\n    ):\n        # Get the batch of images and labels (if in training mode)\n        # ----\n        if self.__is_trainable:\n            x_batch, y_batch = self[row]\n        else:\n            x_batch = self[row]\n            y_batch = None\n\n        # Determine the number of columns to show\n        # ----\n        if columns == None:\n            columns = self.__dicom_loader.num_imgs\n            \n        # Determine the number of input tensors\n        # ----\n        num_input_tensors = len(self.__dicom_loader.scan_categories)\n\n        # Loop through each sub-batch in the main batch\n        # ----\n        for i in range(len(x_batch[0])):\n            # Display the batch number and label (if available)\n            # ----\n            label_info = f\" Label: {y_batch[i]}\" if y_batch is not None else \"\"\n\n            # Loop through each input tensor\n            # ----\n            for j in range(num_input_tensors):\n                images = x_batch[j][i]\n                scan_type = self.__dicom_loader.scan_categories[j]\n\n                if self.__dicom_loader.image_format == ImageFormat.WHDC:\n                    images = ImageFormat.swap_dimensions(images, ImageFormat.DWHC)\n\n                # Generate labels for each image in the set\n                # ----\n                labels = [\n                            f\"Batch: {i + 1} \\nImg: {k + 1} \\nType: {scan_type} \\nLabel:{label_info}\"\n                            for k in range(len(images))\n                         ]\n\n                # Show\n                # ----\n                show_images(\n                    images,\n                    y=labels,\n                    columns=columns,\n                    figure_size=figure_size,\n                    color_map=color_map\n                )\n\n    # Overriding the summary method\n    def summary(self, train_dataset=None):\n        super().summary()\n\n        print(\"Additional summary details specific:\")\n\n        # 0 is the index of the first batch\n        # ----\n        if self.__is_trainable:\n            batch_x, batch_y = self[0]\n        else:\n            batch_x = self[0]\n\n        # Checking the batch format\n        # ----\n        print(\"Batch_x format:\")\n        for i, x in enumerate(batch_x):\n          print(f\"- Scan type {i+1}: {x.shape}\")\n\n        if self.__is_trainable:\n            print(f\"Batch_y format: {batch_y.shape}\")\n            \n        print(\"=\" * 40)\n\n    def on_epoch_end(self):\n        \"\"\"\n        Shuffles the dataset at the end of each epoch if shuffle is True.\n        \"\"\"\n        if self.__shuffle:\n            np.random.shuffle(self.__indices)\n\n    # ---------------- #\n    # Private methods\n    # ---------------- #\n\n    def __getitem__(self, ids):\n        self.__debug.log(\"== __getitem__ ==\")\n        \"\"\"\n        Retrieves a batch of data by batch index.\n\n        Parameters\n        ----------\n        ids : int\n            The batch index.\n\n        Returns\n        -------\n        tuple\n            A tuple containing the batch of images and labels.\n        \"\"\"\n        # Calculate the start and end indices for the batch\n        # ----\n        from_id = ids * self.__batch_size\n        to_id = (ids + 1) * self.__batch_size\n\n        self.__debug.log(f\"Batch ID: {ids}\")\n         \n        # Get the indices and labels for the current batch\n        # ----\n        batch_indices = self.__indices[from_id: to_id]\n   \n        self.__debug.log(\"batch_indices:\", batch_indices.tolist())\n\n        batches_y = []\n\n        # Initialize a list to hold batches for each input tensor\n        # ----\n        batches_x = [[] for _ in range(len(self.__dicom_loader.scan_categories))]\n\n        # Loop through each index in the batch\n        # ----\n        for i in batch_indices:\n            self.__debug.log(\"Processing batch index:\", i)\n\n            # Store label\n            # ----\n            label = self.__dicom_loader.gel_label(i)\n            batches_y.append(label)\n            self.__debug.log(\"Label:\", label)\n\n            # Load all scans for the current index\n            # ----\n            batch_x_image_paths = self.__dicom_loader.load_all_scans(i, show_progress=False)\n\n            # Loop through each scan type and its corresponding images\n            # ----\n            for j, (scan_type, images) in enumerate(batch_x_image_paths.items()):\n                self.__debug.log(\"Processing Scan Type:\", scan_type, \"Number of Images Loaded:\", len(images))\n                batches_x[j].append(images)\n\n        # Convert to batch x y\n        # ----\n        batch_x = [np.array(b) for b in batches_x]\n        batch_y = np.array(batches_y)\n\n        self.__debug.log(f\"Final batch shapes - batch_x: {[x.shape for x in batch_x]}, batch_y: {batch_y}\")\n\n        # Return the image batches and labels if in training mode, otherwise just the image batches\n        # ----\n        if self.__is_trainable:\n            return batch_x, batch_y\n        else:\n            return batch_x\n\n    def __len__(self):\n        \"\"\"\n        Calculates the number of batches in the dataset.\n\n        Returns\n        -------\n        int\n            The number of batches.\n        \"\"\"\n        return int(np.ceil(self.__dicom_loader.len / self.__batch_size))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.737065Z","iopub.execute_input":"2023-09-23T07:48:33.737349Z","iopub.status.idle":"2023-09-23T07:48:33.761147Z","shell.execute_reply.started":"2023-09-23T07:48:33.737292Z","shell.execute_reply":"2023-09-23T07:48:33.760254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MRIType(Enum):\n    FLAIR = \"FLAIR\"\n    T1w = \"T1w\"\n    T1wCE = \"T1wCE\"\n    T2w = \"T2w\"\n    \nclass DatasetType(Enum):\n    TRAIN = \"train\"\n    VALIDATION = \"validation\"\n    TEST = \"test\"","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.762810Z","iopub.execute_input":"2023-09-23T07:48:33.763438Z","iopub.status.idle":"2023-09-23T07:48:33.773848Z","shell.execute_reply.started":"2023-09-23T07:48:33.763407Z","shell.execute_reply":"2023-09-23T07:48:33.772938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global\n# ----\nVERSION         = \"V1\"\nVERBOSITY       = 2\nSEED            = 123\nSCAN_CATEGORIES = [mri_type.value for mri_type in MRIType]\nEXCLUDED_IDS    = [109, 123, 709]\n\n# Paths\n# ----\nRUN_DIR = './run'\nINPUT_PATH = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"\n\n# Train\nTRAIN_DATASET_PATH = INPUT_PATH + \"/train\"\nTRAIN_DATASET_DF_DIR = INPUT_PATH + \"/train_labels.csv\"\n\n# Test\nTEST_DATASET_PATH = INPUT_PATH + \"/test\"\nTEST_DATASET_DF_DIR = INPUT_PATH + \"/sample_submission.csv\"\n\n# Submission\nSUBMISSION_DATASET_DF_DIR = '/kaggle/working/submission.csv'\n\n# TF Callback Paths\n# ----\nLOGS_PATH = f'{RUN_DIR}/logs'\nBEST_MODEL_PATH = f'{RUN_DIR}/models'\nBEST_MODEL_H5_DIR = f'{BEST_MODEL_PATH}/model_{VERSION}.h5'\n\n# Fold\n# ----\nNUM_SPLIT_FOLDS = 5\nSELECTED_VALIDATION_FOLD = 1\n\n# Dicom Loader\n# ----\nMAX_THREADS_DICOM_LOADER = 8\n\n# Image\n# ----\nIMG_WIDTH_SIZE, IMG_HEIGHT_SIZE, IMG_CHAN = (128, 128, 1)\nIMG_SIZE = (IMG_WIDTH_SIZE, IMG_HEIGHT_SIZE)\n\nIMG_SEQ                  = 32\nIMG_SCALE                = 1\nIMG_ROTATE               = 0 \nIMG_ENABLE_CENTRAL_FOCUS = True\n\nSHUFFLE    = True\n\n# Augmentation\n# ----\nAUGMENTATION_FRACTION               = 5\nAUGMENTATION_CROP_LIMITS            = (0.85, 0.95)\nAUGMENTATION_ROTATION_LIMITS        = (4, 12)\nAUGMENTATION_TRANSLATION_X_Y_LIMITS = ((2, 6), (0, 2))\nAUGMENTATION_BLUR                   = (0, 0.15)\nAUGMENTATION_CONSTRAST_BRIGHT       = ((0.8, 1.2),(-2, 2))\n\n# Model\n# ----\nINPUT_SHAPE = (IMG_WIDTH_SIZE, IMG_HEIGHT_SIZE, IMG_SEQ, IMG_CHAN) # Format sample: (128, 128, 64, 1)\n\nMODEL_NAME = \"Mult3DCNN4Input\"\nBATCH_SIZE = 8\nEPOCHS     = 26\n\nCOMPILE_OPTIMIZER = SGD(learning_rate =0.001)\nCOMPILE_LOSS = 'binary_crossentropy'\nCOMPILE_METRICS = [AUC(name='auc')]\n\n# TF Callback\n# ----\nTF_CALL_BACK_BEST_MODEL_MONITOR  = \"val_auc\"\nTF_CALL_BACK_EARLY_STOP_MONITOR  = \"auc\"\nTF_CALL_BACK_EARLY_STOP_PATIENTE = 6","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:33.775491Z","iopub.execute_input":"2023-09-23T07:48:33.775819Z","iopub.status.idle":"2023-09-23T07:48:37.330531Z","shell.execute_reply.started":"2023-09-23T07:48:33.775779Z","shell.execute_reply":"2023-09-23T07:48:37.329538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\ntrain_df = pd.read_csv(TRAIN_DATASET_DF_DIR)\ntrain_df.rename(columns = { \"BraTS21ID\": \"ID\", \"MGMT_value\": \"Label\"}, inplace=True)\n\nindex_to_remove = train_df[train_df['ID'].isin(EXCLUDED_IDS)].index\ntrain_df.drop(index_to_remove, inplace=True)\n\ntrain_df.reset_index(drop=True, inplace=True)\n\n\n# Test\ntest_df = pd.read_csv(TEST_DATASET_DF_DIR)\ntest_df.rename(columns = { \"BraTS21ID\": \"ID\", \"MGMT_value\": \"Label\"}, inplace=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:37.335382Z","iopub.execute_input":"2023-09-23T07:48:37.337648Z","iopub.status.idle":"2023-09-23T07:48:37.381935Z","shell.execute_reply.started":"2023-09-23T07:48:37.337612Z","shell.execute_reply":"2023-09-23T07:48:37.380931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DeepScanModel(Model):\n    def __init__(self, input_shape, model_name=\"My3DCNNModel\"):\n        # Define input layers\n        # ----\n        self.input_layers = [Input(shape=input_shape) for _ in range(4)]\n\n        # Build CNN models for each input\n        # ----\n        self.cnn_models = [self.build_cnn_branch(input_layer) for input_layer in self.input_layers]\n\n        # Concatenate outputs of CNN models\n        # ----\n        concatenated = concatenate(self.cnn_models)\n\n        # Add Global Average Pooling and Dense layers\n        # ----\n        x = self.build_head(concatenated)\n\n        # Define the final model\n        # ----\n        super(DeepScanModel, self).__init__(inputs=self.input_layers, outputs=x, name=model_name)\n\n    def build_cnn_branch(self, input_layer):        \n        x = Conv3D(64, 3)(input_layer)\n        x = ReLU()(x)\n        x = MaxPool3D(2)(x)\n        x = BatchNormalization()(x)\n        \n        x = Conv3D(128, 3)(x)\n        x = ReLU()(x)\n        x = MaxPool3D(2)(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.1)(x)\n\n        x = Conv3D(256, 3)(x)\n        x = ReLU()(x)\n        x = MaxPool3D(2)(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.2)(x)\n        \n        return x\n\n    def build_head(self, x):\n        x = GlobalAveragePooling3D()(x)\n\n        x = Dense(1024)(x)\n        x = ReLU()(x)\n        x = Dropout(0.3)(x)\n\n        x = Dense(1, activation=\"sigmoid\")(x)\n        \n        return x\n\n    def show_graph(self):\n        display(plot_model(self, show_shapes=True, show_layer_names=True))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:37.383641Z","iopub.execute_input":"2023-09-23T07:48:37.384292Z","iopub.status.idle":"2023-09-23T07:48:37.398594Z","shell.execute_reply.started":"2023-09-23T07:48:37.384258Z","shell.execute_reply":"2023-09-23T07:48:37.397647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dicom loader\n# ----\ntest_dicom_loader = DICOMLoader(\n    test_df,\n    input_path          = TEST_DATASET_PATH,\n    scan_categories     = SCAN_CATEGORIES,\n    num_imgs            = IMG_SEQ,\n    size                = IMG_SIZE,\n    scale               = IMG_SCALE,\n    rotate_angle        = IMG_ROTATE,\n    max_threads         = MAX_THREADS_DICOM_LOADER,\n    enable_center_focus = IMG_ENABLE_CENTRAL_FOCUS,\n    debug_mode          = False\n)\n\n# Dataset\n# ----\ntest_dataset = ScanDataset(\n    dicom_loader = test_dicom_loader,\n    batch_size   = BATCH_SIZE,\n    subset       = DatasetType.TEST.value,\n    shuffle      = False,\n    debug_mode   = False\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:37.400178Z","iopub.execute_input":"2023-09-23T07:48:37.400990Z","iopub.status.idle":"2023-09-23T07:48:37.409963Z","shell.execute_reply.started":"2023-09-23T07:48:37.400954Z","shell.execute_reply":"2023-09-23T07:48:37.409047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_model(\"/kaggle/input/model-v10/model_V10.h5\", custom_objects={'DeepScanModel': DeepScanModel})","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:37.411604Z","iopub.execute_input":"2023-09-23T07:48:37.412378Z","iopub.status.idle":"2023-09-23T07:48:38.987247Z","shell.execute_reply.started":"2023-09-23T07:48:37.412344Z","shell.execute_reply":"2023-09-23T07:48:38.986211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_predictions(model, test_dataset, test_df):\n    predictions = []\n\n    for batch_idx in range(len(test_dataset)):\n        scan_type_1, scan_type_2, scan_type_3, scan_type_4 = test_dataset[batch_idx]\n        \n        batch_predictions = model.predict([scan_type_1, scan_type_2, scan_type_3, scan_type_4])\n\n        predictions.append(batch_predictions)\n\n    # Flatten the predictions list\n    # ----\n    submission = test_df.copy()\n    submission[\"Label\"] = [item[0] for sublist in predictions for item in sublist]\n    submission.rename(columns={\"ID\": \"BraTS21ID\", \"Label\": \"MGMT_value\"}, inplace=True)\n\n    return submission\n\nsubmission = generate_predictions(model, test_dataset, test_df)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:48:38.995191Z","iopub.execute_input":"2023-09-23T07:48:38.998139Z","iopub.status.idle":"2023-09-23T07:49:42.177285Z","shell.execute_reply.started":"2023-09-23T07:48:38.998103Z","shell.execute_reply":"2023-09-23T07:49:42.176273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:49:42.178942Z","iopub.execute_input":"2023-09-23T07:49:42.179272Z","iopub.status.idle":"2023-09-23T07:49:42.198700Z","shell.execute_reply.started":"2023-09-23T07:49:42.179239Z","shell.execute_reply":"2023-09-23T07:49:42.197520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:49:42.201170Z","iopub.execute_input":"2023-09-23T07:49:42.202314Z","iopub.status.idle":"2023-09-23T07:49:42.211176Z","shell.execute_reply.started":"2023-09-23T07:49:42.202280Z","shell.execute_reply":"2023-09-23T07:49:42.210064Z"},"trusted":true},"execution_count":null,"outputs":[]}]}