{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\ni = 0\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    \n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        i += 1\n        if i > 5:\n            break\n    if i > 5:\n        break\n        \n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-09T11:24:59.970668Z","iopub.execute_input":"2021-09-09T11:24:59.971077Z","iopub.status.idle":"2021-09-09T11:25:00.007006Z","shell.execute_reply.started":"2021-09-09T11:24:59.970968Z","shell.execute_reply":"2021-09-09T11:25:00.006240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\n\npath = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/'\nonlydirs = [f for f in listdir(path) if isfile(join(path, f))]\nlen(onlydirs)","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:04.789873Z","iopub.execute_input":"2021-09-09T11:25:04.790223Z","iopub.status.idle":"2021-09-09T11:25:05.014182Z","shell.execute_reply.started":"2021-09-09T11:25:04.790189Z","shell.execute_reply":"2021-09-09T11:25:05.013227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport pydicom\n \nexamples = ['00000','00178']\nfor ex in examples: \n \n    p1, p2, p3, p4 = 'FLAIR', 'T2w', 'T1w', 'T1wCE'\n    base_path = f'/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{ex}'\n \n    print(f'Образец №{ex}:')\n    for p in [p1, p2, p3, p4]:\n        onlyfiles = [f for f in listdir(f'{base_path}/{p}/') if isfile(join(f'{base_path}/{p}/', f))]\n        img = pydicom.read_file(f'{base_path}/{p}/{onlyfiles[0]}').pixel_array\n        print(f'Количество файлов в типе {p}: {len(onlyfiles)}, примеры файлов: {onlyfiles[:2]}', \n             f'разрешение: {img.shape}')","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:07.258267Z","iopub.execute_input":"2021-09-09T11:25:07.258621Z","iopub.status.idle":"2021-09-09T11:25:08.447547Z","shell.execute_reply.started":"2021-09-09T11:25:07.258592Z","shell.execute_reply":"2021-09-09T11:25:08.446249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n#for file management\nimport json\n#for standardized data storage\nimport glob\n#for precise file selection(low verbosity)\nimport random\n#random amount generator\nimport collections\n#has premade datastructure objects that can be implemented\nimport numpy as np\n#linear algebra\nimport pandas as pd\n#succinct array and data handling\nimport pydicom\n#c based dicom modulation tool\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n#pixel handler that uses voi_lut to grab pixel data from within the frame of a window\nimport cv2\n#image data handler\nimport matplotlib.pyplot as plt\n#data visualization library\nimport seaborn as sns\n#data visualization library\n\n\n# /kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00162/T2w/Image-4.dcm","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:12.795506Z","iopub.execute_input":"2021-09-09T11:25:12.795864Z","iopub.status.idle":"2021-09-09T11:25:13.358561Z","shell.execute_reply.started":"2021-09-09T11:25:12.795832Z","shell.execute_reply":"2021-09-09T11:25:13.357705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:21.243617Z","iopub.execute_input":"2021-09-09T11:25:21.243955Z","iopub.status.idle":"2021-09-09T11:25:21.264401Z","shell.execute_reply.started":"2021-09-09T11:25:21.243920Z","shell.execute_reply":"2021-09-09T11:25:21.263638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FLAIR_path'] = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/' + train_df['BraTS21ID'].astype(str).str.zfill(5) + '/FLAIR/'\ntrain_df['T1w_path'] = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/' + train_df['BraTS21ID'].astype(str).str.zfill(5) + '/T1w/' \ntrain_df['T2w_path'] = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/' + train_df['BraTS21ID'].astype(str).str.zfill(5) + '/T2w/' \ntrain_df['T1wCE_path'] = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/train/' + train_df['BraTS21ID'].astype(str).str.zfill(5) + '/T1wCE/' \n","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:24.531453Z","iopub.execute_input":"2021-09-09T11:25:24.531792Z","iopub.status.idle":"2021-09-09T11:25:24.560996Z","shell.execute_reply.started":"2021-09-09T11:25:24.531762Z","shell.execute_reply":"2021-09-09T11:25:24.560192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rescale(arr):\n    arr_min = arr.min()\n    arr_max = arr.max()\n    if (arr_max - arr_min) == 0:\n        return arr\n    return (arr - arr_min) / (arr_max - arr_min)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport imageio\nfrom IPython.display import Image\n\ndef visualize_data_gif(images):\n    imageio.mimsave(\"/kaggle/working/1.gif\", images, duration=5.0 / images.shape[0])\n    return Image(filename=\"/kaggle/working/1.gif\", format='png')","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:30.495709Z","iopub.execute_input":"2021-09-09T11:25:30.496070Z","iopub.status.idle":"2021-09-09T11:25:30.536689Z","shell.execute_reply.started":"2021-09-09T11:25:30.496017Z","shell.execute_reply":"2021-09-09T11:25:30.535923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport re","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:36.778411Z","iopub.execute_input":"2021-09-09T11:25:36.778746Z","iopub.status.idle":"2021-09-09T11:25:39.991909Z","shell.execute_reply.started":"2021-09-09T11:25:36.778715Z","shell.execute_reply":"2021-09-09T11:25:39.991086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.config.list_physical_devices('GPU')","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:27:04.343162Z","iopub.execute_input":"2021-09-09T11:27:04.343537Z","iopub.status.idle":"2021-09-09T11:27:04.416293Z","shell.execute_reply.started":"2021-09-09T11:27:04.343504Z","shell.execute_reply":"2021-09-09T11:27:04.415280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MRIDataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, df, X_col, y_col, batch_size,\n                 input_size= (256, 256), depth_size=64,\n                 shuffle=True):\n    \n        self.df = df.copy()\n        self.X_col = X_col\n        self.y_col = y_col\n        self.depth_size = depth_size\n        self.batch_size = batch_size\n        self.input_size = input_size\n        self.shuffle = shuffle\n        self.n = len(self.df)\n    \n    def on_epoch_end(self):\n        if self.shuffle:\n            self.df = self.df.sample(frac=1).reset_index(drop=True)\n    \n    def __get_input(self, path, target_size):\n    \n        scan3d = None\n        onlyfiles = [f for f in listdir(path) if isfile(join(path, f))]\n        filepatt = 'Image-{}.dcm'\n        digits = [int(re.search('\\d+',i).group()) for i in listdir(path) if re.match(filepatt.format('\\d+\\\\'),i)]\n        digits.sort()\n        onlyfiles = [filepatt.format(dig) for dig in digits]\n        \n        center = len(onlyfiles) // 2\n        left = max(0, center - (self.depth_size // 2))\n        right = min(len(onlyfiles), center + (self.depth_size // 2))\n        onlyfiles = onlyfiles[left: right]\n        if len(onlyfiles) < self.depth_size:\n            img_shape = pydicom.read_file(f'{path}{onlyfiles[0]}').pixel_array.shape\n            add_z = self.depth_size - len(onlyfiles)\n            scan3d = np.zeros((add_z, target_size[0], target_size[1],1))\n        \n            \n        scans = []\n        for f in onlyfiles:\n            img = pydicom.read_file(f'{path}{f}')\n            img = img.pixel_array\n            img = self._rescale(img)\n            img = np.expand_dims(img, axis=-1)\n            img = tf.image.resize(img,(target_size[0], target_size[1])).numpy()\n            \n            img = self._normalize(img)\n            scans.append(img)\n        \n        if scan3d is not None:\n            return np.concatenate([np.array(scans), scan3d]) \n        else:\n            return np.array(scans)\n    \n    def _rescale(self, arr):\n        arr_min = arr.min()\n        arr_max = arr.max()\n        if (arr_max - arr_min) == 0:\n            return arr\n        return (arr - arr_min) / (arr_max - arr_min)\n    \n    def _normalize(self, arr):\n        img = arr - arr.mean()\n        # divide by the standard deviation (only if it is different from zero)\n        if np.std(img) != 0:\n            img = img / np.std(img)\n        return img\n    \n    def __get_data(self, batches):\n        if self.X_col is None:\n            PATHS = ['FLAIR_path', 'T1w_path', 'T2w_path', 'T1wCE_path']\n            X_batch = []\n            for p in PATHS:\n                batch_part_path = batches[p]\n                X_batch.append(np.asarray([self.__get_input(x,  self.input_size) for x in batch_part_path]))\n            y_batch = batches[self.y_col].values\n            X_batch = np.concatenate(X_batch, axis=4)\n            \n        else:\n            path_batch = batches[self.X_col]\n            X_batch = np.asarray([self.__get_input(x,  self.input_size) for x in path_batch])\n            y_batch = batches[self.y_col].values\n        return X_batch, y_batch\n    \n    def __getitem__(self, index):\n#         print('Star_loading')\n        batches = self.df[index * self.batch_size:(index + 1) * self.batch_size]\n        X, y = self.__get_data(batches)\n#         print(X.shape)\n        return X, y\n    \n    def __len__(self):\n        return self.n // self.batch_size     ","metadata":{"execution":{"iopub.status.busy":"2021-09-09T11:25:41.305749Z","iopub.execute_input":"2021-09-09T11:25:41.306142Z","iopub.status.idle":"2021-09-09T11:25:41.327285Z","shell.execute_reply.started":"2021-09-09T11:25:41.306107Z","shell.execute_reply":"2021-09-09T11:25:41.326198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MRIDataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, df, X_col, y_col, batch_size,\n                 input_size= (256, 256), depth_size=64,\n                 shuffle=True):\n    \n        self.df = df.copy()\n        self.X_col = X_col\n        self.y_col = y_col\n        self.depth_size = depth_size\n        self.batch_size = batch_size\n        self.input_size = input_size\n        self.shuffle = shuffle\n        self.n = len(self.df)\n    \n    def on_epoch_end(self):\n        if self.shuffle:\n            self.df = self.df.sample(frac=1).reset_index(drop=True)\n    \n    def __get_input(self, path, target_size):\n        pass\n    \n    def _rescale(self, arr):\n        pass\n    \n    def _normalize(self, arr):\n        pass\n    \n    def __get_data(self, batches):\n        pass\n\n    \n    def __getitem__(self, index):\n        pass\n   \n    \n    def __len__(self):\n        return self.n // self.batch_size   ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet-3D\n!pip install keras_applications \n# import efficientnet_3D.keras as efn \nimport efficientnet_3D.tfkeras as efn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import imageio\nfrom IPython.display import Image\n\ndef visualize_data_gif(images):\n    imageio.mimsave(\"/kaggle/working/1.gif\", images, duration=5.0 / images.shape[0])\n    return Image(filename=\"/kaggle/working/1.gif\", format='png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen = MRIDataGenerator(train_df, э, 'MGMT_value', 10, (256, 256), 64, True)\niterator = iter(gen)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = next(iterator)[0] * 255\nimages = []\nfor i in range(4):\n    a = data[:,:,:,:, i]\n    images.append(np.stack([a,a,a], axis=4).reshape((*a.shape[:4], 3)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val = np.concatenate(images, axis=3)\nvisualize_data_gif(val[5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_data_gif(val[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = images[:,:,:,:, i]\nz = np.stack([a,a,a], axis=4).reshape((*a.shape[:4], 3))\nvisualize_data_gif(z[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"next(iterator)[0].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet_3D.tfkeras as efn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"depth = 64\nresolution = (192, 192)\nbatches = 8\ngen = MRIDataGenerator(train_df, 'FLAIR_path', 'MGMT_value', batches, resolution, depth, True)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-09T12:13:37.330822Z","iopub.execute_input":"2021-09-09T12:13:37.331176Z","iopub.status.idle":"2021-09-09T12:13:37.336086Z","shell.execute_reply.started":"2021-09-09T12:13:37.331146Z","shell.execute_reply":"2021-09-09T12:13:37.334844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check\n# from tensorflow.keras import Sequential\n# from tensorflow.keras.layers import Conv3D, MaxPooling3D, Flatten, Dense\n# model = Sequential()\n# model.add(Conv3D(16, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform', input_shape=(32, 150, 150, 1)))\n# model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n# model.add(Conv3D(32, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n# model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n# model.add(Conv3D(64, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n# model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n# model.add(Flatten())\n# model.add(Dense(32, activation='relu', kernel_initializer='he_uniform'))\n# model.add(Dense(1, activation='sigmoid'))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    \n    from tensorflow.keras import Sequential\n    from tensorflow.keras.layers import Conv3D, MaxPooling3D, Flatten, Dense\n    model = Sequential()\n    model.add(Conv3D(32, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform', input_shape=(64,192, 192, 1)))\n    model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n    model.add(Conv3D(64, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n    model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n    model.add(Conv3D(128, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n    model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n    model.add(Conv3D(256, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n    model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n    model.add(Flatten())\n    model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(64, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(1, activation='sigmoid'))\n    \n    model.compile(\n        optimizer='adam', \n        loss='binary_crossentropy',\n        metrics=[tf.keras.metrics.BinaryAccuracy()]\n        )\n    \n    history = model.fit(gen, steps_per_epoch = batches, \n                              verbose=1,\n                              epochs = 10\n                       )","metadata":{"execution":{"iopub.status.busy":"2021-09-09T12:26:09.556682Z","iopub.execute_input":"2021-09-09T12:26:09.557161Z","iopub.status.idle":"2021-09-09T12:31:55.920162Z","shell.execute_reply.started":"2021-09-09T12:26:09.557115Z","shell.execute_reply":"2021-09-09T12:31:55.918346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.InputLayer(input_shape=(64,64,32,1)),\n        tf.keras.layers.Conv3D(3, kernel_size=1, padding='same'),\n        efn.EfficientNetB0(include_top=False, input_shape=(64,64,32,3), pooling='avg'),\n        tf.keras.layers.Dense(64, activation='relu'),\n        tf.keras.layers.Dense(1, activation='sigmoid')\n    ])\n    \n    model.compile(\n        optimizer='adam', \n        loss='binary_crossentropy',\n        metrics=[tf.keras.metrics.AUC()]\n        )\n    \n    history = model.fit(gen, steps_per_epoch = batches, \n                              verbose=1,\n                              epochs = 3\n                       )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_layer = tf.keras.layers.InputLayer(input_shape=(depth, resolution[0], resolution[1], 1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = efn.EfficientNetB0(input_shape=(depth, resolution[0], resolution[1], 1), weights=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = tf.keras.layers.Flatten()(model.output)\nout = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_model = tf.keras.Model(inputs=model.input, outputs=out)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_model.compile(optimizer='adam', loss='binary_crossentropy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = new_model.fit(gen, steps_per_epoch = batches, \n                              verbose=1,\n                              epochs = 3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.ylabel(\"loss\")\nplt.xlabel(\"epoch\")\nplt.title(\"Training Loss Curve\")\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}