{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nfrom tensorflow import keras\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.preprocessing import image\nimport cv2\nimport time\nimport glob\nimport os\nimport pandas\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:12:33.499607Z","iopub.execute_input":"2021-08-26T16:12:33.499962Z","iopub.status.idle":"2021-08-26T16:12:33.504524Z","shell.execute_reply.started":"2021-08-26T16:12:33.499931Z","shell.execute_reply":"2021-08-26T16:12:33.50363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    def __init__(self,csv_path='../input/rsnasubmissionresult/result.csv',width=256,height=256,batch_size=16,shuffle=True):\n        self.df = pd.read_csv(csv_path,dtype='str')\n        self.batch_size = batch_size\n        self.base_dir = '../input/classify-tumor-best/DATATUMORONLY_TRAIN/train'\n        self.width = width\n        self.crop_length = 224\n        self.height = height\n        self.tolerance = 5\n        self.shuffle = shuffle\n        self.all_lengths = self.get_all_slices()\n        self.new_df = pandas.DataFrame(self.all_lengths)\n        self.new_df = self.new_df.rename(columns={0:\"flair\"})\n        self.result = pandas.concat([self.new_df, self.df], axis=1)\n        #self.sorted_df = self.result.sort_values('flair')\n        self.slices_list = np.array(list(self.result['flair']))\n        self.folders_list = np.array(list(self.result['folder_id']))\n        self.label_list = np.array(list(self.result['MGMT_value']))\n        self.on_epoch_end()\n    \n    def get_all_slices(self): \n        all_paths = []\n        for i in list(self.df['folder_id']):\n            i = os.path.join(self.base_dir,i)\n            all_paths.append(len(glob.glob(i+'/flair/*')))\n        return all_paths\n        \n    def on_epoch_end(self):\n        self.random_indexes = np.arange(self.slices_list.max())\n        if self.shuffle:\n            np.random.shuffle(self.random_indexes)\n            self.df = self.df.sample(frac=1) \n    \n    def __len__(self):\n        return self.slices_list.max()\n    \n    def __getitem__(self,user_index):\n        start =time.time()\n        index = self.random_indexes[user_index]\n        labels = []\n        indexes = np.where((self.slices_list >= index-self.tolerance) &(self.slices_list <= index+self.tolerance))\n        tol_slice= np.take(self.slices_list, indexes)[0]\n        tol_folder= np.take(self.folders_list, indexes)[0]\n        non_zero_slice = tol_slice[tol_slice!=0]\n        non_zero_folder = tol_folder[tol_slice!=0]\n        random_indexes = np.random.choice(len(non_zero_folder), size=self.batch_size)\n        random_folder = np.take(non_zero_folder,random_indexes)\n        random_slices = np.take(non_zero_slice,random_indexes)\n        self.max_depth = random_slices.max()\n        batch_x = self.__data_gen_batch(random_folder)\n        for i in random_folder:\n            labels.append(int(self.result.loc[self.result['folder_id']==i]['MGMT_value']))\n        #print(labels)\n        return batch_x,self.one_hot_encoder(labels)\n    \n    def one_hot_encoder(self,y):\n        b = np.zeros((len(y), 2))\n        b[np.arange(len(y)),y] = 1\n        return b\n    \n    def get_max_len(self,batch,min_depth=50):\n        max_len = 0\n        for patient_id in batch['folder_id']:\n            #print(os.path.join(self.base_dir,patient_id,'flair/*'))\n            length = len(glob.glob(os.path.join(self.base_dir,patient_id,'flair/*')))\n            if length > max_len:\n                max_len = length\n        if max_len < min_depth:\n            max_len = min_depth\n        return max_len\n\n    def __data_gen_image(self,folder_name):\n        flair_path = glob.glob(os.path.join(self.base_dir,folder_name,'flair/*'))\n        all_images = []\n        all_images = np.zeros(shape=(self.max_depth,self.height,self.height,1),dtype=np.float64)\n        for i,img_path in enumerate(flair_path):\n            img = image.load_img(img_path,target_size=(self.height,self.width),color_mode='grayscale')\n            img = image.img_to_array(img)\n            all_images[i,] = img\n        return np.transpose(all_images,(1,2,0,3))\n\n    def __data_gen_batch(self,folder_names):\n        batch_data = np.empty(shape=(self.batch_size,self.height,self.width,self.max_depth,1))\n        for i,patient_id in enumerate(folder_names):\n            batch_data[i,] = self.__data_gen_image(patient_id)\n        return batch_data\n    \n    def crop(self,image,crop_length=224):\n        img_height ,img_width = image.shape[:2]\n        start_y = (img_height - self.crop_length) // 2\n        start_x = (img_width - self.crop_length) // 2\n        cropped_image=image[start_y:(img_height - start_y), start_x:(img_width - start_x), :]\n        return cropped_image","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:12:36.87977Z","iopub.execute_input":"2021-08-26T16:12:36.880092Z","iopub.status.idle":"2021-08-26T16:12:36.903161Z","shell.execute_reply.started":"2021-08-26T16:12:36.880063Z","shell.execute_reply":"2021-08-26T16:12:36.902166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = DataGenerator(batch_size=4,height=256,width=256)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:12:40.935964Z","iopub.execute_input":"2021-08-26T16:12:40.936285Z","iopub.status.idle":"2021-08-26T16:12:44.170523Z","shell.execute_reply.started":"2021-08-26T16:12:40.936255Z","shell.execute_reply":"2021-08-26T16:12:44.16968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = datagen[0]","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:11:36.751621Z","iopub.status.idle":"2021-08-26T16:11:36.752212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(width=256, height=256, depth=None):\n    \"\"\"Build a 3D convolutional neural network model.\"\"\"\n\n    inputs = keras.Input((width, height, depth, 1))\n    x = layers.Conv3D(filters=32, kernel_size=3, activation=\"relu\")(inputs)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n\n    x = layers.Conv3D(filters=64, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n\n    x = layers.Conv3D(filters=128, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n\n    x = layers.Conv3D(filters=256, kernel_size=3, activation=\"relu\")(x)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n\n    x = layers.GlobalAveragePooling3D()(x)\n    x = layers.Dense(units=128, activation=\"relu\")(x)\n    #x = layers.Dropout(0.3)(x)\n\n    outputs = layers.Dense(units=2, activation=\"softmax\")(x)\n\n    # Define the model.\n    model = keras.Model(inputs, outputs, name=\"3dcnn\")\n    return model\n\nmodel = get_model(width=128, height=128, depth=None)\nmodel.summary()\nModel: \"3dcnn\"","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:12:44.171882Z","iopub.execute_input":"2021-08-26T16:12:44.172194Z","iopub.status.idle":"2021-08-26T16:12:46.236144Z","shell.execute_reply.started":"2021-08-26T16:12:44.172161Z","shell.execute_reply":"2021-08-26T16:12:46.234407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    loss=\"categorical_crossentropy\",\n    optimizer=tf.keras.optimizers.Adam(),\n    metrics=[\"accuracy\"]\n)\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n    \"3d_image_classification.h5\", save_best_only=True,metrics=\"accuracy\",mode=max\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:12:46.237837Z","iopub.execute_input":"2021-08-26T16:12:46.238188Z","iopub.status.idle":"2021-08-26T16:12:46.257485Z","shell.execute_reply.started":"2021-08-26T16:12:46.238151Z","shell.execute_reply":"2021-08-26T16:12:46.256558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit_generator(\n    datagen,\n    steps_per_epoch=len(datagen),\n    epochs=100,\n    #callbacks=[checkpoint_cb],\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:24:17.765607Z","iopub.execute_input":"2021-08-26T16:24:17.765982Z","iopub.status.idle":"2021-08-26T16:24:36.178274Z","shell.execute_reply.started":"2021-08-26T16:24:17.765949Z","shell.execute_reply":"2021-08-26T16:24:36.1757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d","metadata":{},"execution_count":null,"outputs":[]}]}