{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras_preprocessing.image.dataframe_iterator import DataFrameIterator\n\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-16T02:43:05.365169Z","iopub.execute_input":"2021-07-16T02:43:05.365809Z","iopub.status.idle":"2021-07-16T02:43:12.327506Z","shell.execute_reply.started":"2021-07-16T02:43:05.365666Z","shell.execute_reply":"2021-07-16T02:43:12.326527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Quick Notebook on making a dataframe iterator for training in tensorflow","metadata":{}},{"cell_type":"code","source":"#Built from https://gist.github.com/ReemRashwan/8c92086d3104d01978a16e05ca93a165\nclass DCMFlowFromDataFrame(DataFrameIterator):\n    def __init__(self, *arg, **kwargs):\n        self.white_list_formats = ('dcm')\n        super(DCMFlowFromDataFrame, self).__init__(*arg, **kwargs)\n        self.dataframe = kwargs['dataframe']\n        self.x = self.dataframe[kwargs['x_col']]\n        self.y = self.dataframe[kwargs['y_col']]\n        self.target_size = kwargs['target_size']\n\n    def _get_batches_of_transformed_samples(self, indices_array):\n        batch_x = np.array([self.read_dcm_as_array(dcm_path, self.target_size)\n                            for dcm_path in self.x.iloc[indices_array]])\n\n        batch_y = np.array(self.y.iloc[indices_array].astype(np.uint8))\n\n        if self.image_data_generator is not None:\n            for i, (x, y) in enumerate(zip(batch_x, batch_y)):\n                transform_params = self.image_data_generator.get_random_transform(x.shape)\n                batch_x[i] = self.image_data_generator.apply_transform(x, transform_params)\n\n        return batch_x, batch_y\n\n    @staticmethod\n    def read_dcm_as_array(dcm_path, target_size=(256, 256)):\n        image_array = pydicom.dcmread(dcm_path).pixel_array\n        image_array = cv2.resize(image_array, target_size, interpolation=cv2.INTER_NEAREST)\n        image_array = np.expand_dims(image_array, -1)\n        \n        return image_array","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:13.794174Z","iopub.execute_input":"2021-07-16T02:43:13.794537Z","iopub.status.idle":"2021-07-16T02:43:13.804701Z","shell.execute_reply.started":"2021-07-16T02:43:13.794507Z","shell.execute_reply":"2021-07-16T02:43:13.803967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating a full df of paths to valid images (i.e. images that are not empty), with the labels of each study","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/rsnamiccai-brain-tumor-radiogenomic-valid-images/valid_trains.csv\", dtype=str)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:14.954019Z","iopub.execute_input":"2021-07-16T02:43:14.954533Z","iopub.status.idle":"2021-07-16T02:43:15.403465Z","shell.execute_reply.started":"2021-07-16T02:43:14.954493Z","shell.execute_reply":"2021-07-16T02:43:15.402745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:15.988837Z","iopub.execute_input":"2021-07-16T02:43:15.989328Z","iopub.status.idle":"2021-07-16T02:43:16.015303Z","shell.execute_reply.started":"2021-07-16T02:43:15.989287Z","shell.execute_reply":"2021-07-16T02:43:16.014384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv',dtype=str)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:16.570216Z","iopub.execute_input":"2021-07-16T02:43:16.570593Z","iopub.status.idle":"2021-07-16T02:43:16.580319Z","shell.execute_reply.started":"2021-07-16T02:43:16.570562Z","shell.execute_reply":"2021-07-16T02:43:16.579265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(train_labels,left_on='sub_dir',right_on='BraTS21ID').drop('BraTS21ID',axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:17.061183Z","iopub.execute_input":"2021-07-16T02:43:17.061544Z","iopub.status.idle":"2021-07-16T02:43:17.182277Z","shell.execute_reply.started":"2021-07-16T02:43:17.061513Z","shell.execute_reply":"2021-07-16T02:43:17.181261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['path'] = np.vectorize(os.path.join)(train['main_dir'], train['sub_dir'], train['mpMRI_scan'], train['file'])","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:17.333682Z","iopub.execute_input":"2021-07-16T02:43:17.334032Z","iopub.status.idle":"2021-07-16T02:43:18.355665Z","shell.execute_reply.started":"2021-07-16T02:43:17.334001Z","shell.execute_reply":"2021-07-16T02:43:18.354714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(['main_dir','sub_dir','mpMRI_scan','file'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:18.357073Z","iopub.execute_input":"2021-07-16T02:43:18.357359Z","iopub.status.idle":"2021-07-16T02:43:18.425940Z","shell.execute_reply.started":"2021-07-16T02:43:18.357330Z","shell.execute_reply":"2021-07-16T02:43:18.425005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sample train for some rows to test on","metadata":{}},{"cell_type":"code","source":"sample = train.sample(n=320, random_state=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:19.037468Z","iopub.execute_input":"2021-07-16T02:43:19.037985Z","iopub.status.idle":"2021-07-16T02:43:19.050670Z","shell.execute_reply.started":"2021-07-16T02:43:19.037951Z","shell.execute_reply":"2021-07-16T02:43:19.049533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:46:15.309568Z","iopub.execute_input":"2021-07-16T02:46:15.309980Z","iopub.status.idle":"2021-07-16T02:46:15.320091Z","shell.execute_reply.started":"2021-07-16T02:46:15.309949Z","shell.execute_reply":"2021-07-16T02:46:15.319076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model without much thought just to test on","metadata":{}},{"cell_type":"code","source":"def make_model():\n    model = tf.keras.Sequential()\n    model.add(tf.keras.layers.Conv2D(32,(3,3),activation='relu',input_shape=(256,256,1)))\n    model.add(tf.keras.layers.MaxPooling2D((2,2)))\n    model.add(tf.keras.layers.Conv2D(32,(3,3),activation='relu'))\n    model.add(tf.keras.layers.MaxPooling2D((2,2)))\n    model.add(tf.keras.layers.Conv2D(32,(3,3),activation='relu'))\n    model.add(tf.keras.layers.MaxPooling2D((2,2)))\n    model.add(tf.keras.layers.Flatten())\n    model.add(tf.keras.layers.Dense(128,activation='relu'))\n    model.add(tf.keras.layers.Dense(32,activation='relu'))\n    model.add(tf.keras.layers.Dense(1,activation='sigmoid'))\n    \n    model.compile(loss='binary_crossentropy',optimizer=tf.keras.optimizers.Adam(),metrics=[tf.keras.metrics.AUC()])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:20.680826Z","iopub.execute_input":"2021-07-16T02:43:20.681181Z","iopub.status.idle":"2021-07-16T02:43:20.690934Z","shell.execute_reply.started":"2021-07-16T02:43:20.681152Z","shell.execute_reply":"2021-07-16T02:43:20.690022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = make_model()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:21.253553Z","iopub.execute_input":"2021-07-16T02:43:21.254199Z","iopub.status.idle":"2021-07-16T02:43:21.688556Z","shell.execute_reply.started":"2021-07-16T02:43:21.254142Z","shell.execute_reply":"2021-07-16T02:43:21.687558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:22.301500Z","iopub.execute_input":"2021-07-16T02:43:22.301884Z","iopub.status.idle":"2021-07-16T02:43:22.306577Z","shell.execute_reply.started":"2021-07-16T02:43:22.301855Z","shell.execute_reply":"2021-07-16T02:43:22.305808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = DCMFlowFromDataFrame(dataframe=sample,\n                                       x_col='path',\n                                       y_col='MGMT_value',\n                                       image_data_generator=train_datagen,\n                                       target_size=(256,256),\n                                       batch_size=32,\n                                       class_mode='binary')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:23.454275Z","iopub.execute_input":"2021-07-16T02:43:23.454779Z","iopub.status.idle":"2021-07-16T02:43:24.497584Z","shell.execute_reply.started":"2021-07-16T02:43:23.454746Z","shell.execute_reply":"2021-07-16T02:43:24.496410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_generator,\n    epochs=1\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T02:43:52.615531Z","iopub.execute_input":"2021-07-16T02:43:52.615923Z","iopub.status.idle":"2021-07-16T02:44:05.759530Z","shell.execute_reply.started":"2021-07-16T02:43:52.615887Z","shell.execute_reply":"2021-07-16T02:44:05.758601Z"},"trusted":true},"execution_count":null,"outputs":[]}]}