{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nfrom tensorflow import keras\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.preprocessing import image\nimport cv2\nimport time\nimport glob\nimport os\nimport pandas\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications.resnet50 import preprocess_input\nfrom tqdm import tqdm\nimport random","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:20:16.268781Z","iopub.execute_input":"2021-08-31T14:20:16.269147Z","iopub.status.idle":"2021-08-31T14:20:20.66091Z","shell.execute_reply.started":"2021-08-31T14:20:16.269113Z","shell.execute_reply":"2021-08-31T14:20:20.660116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataLoader(keras.utils.Sequence):\n    def __init__(self,csv='/kaggle/input/rsnasubmissionresult/result.csv',height=256,width=256,inchannels=1,batch_size=16,shuffle=True,\\\n                modeltype='imagenet',maxlen_slices=50,return_feat=True):\n        self.batch_size = batch_size\n        self.csv = csv\n        self.base_dir = '/kaggle/input/test-train-08thresh-mgmt-tumorimages/DATATUMORONLY/train/'\n        self.width = width\n        self.height = height\n        self.return_feat = return_feat\n        self.shuffle = shuffle\n        self.inchannels = inchannels\n        self.model_type = modeltype #imagenet or tumor\n        self.modelarch = 'resnet50'\n        self.maxlen_slices = maxlen_slices\n        \n        if self.inchannels == 1:\n            self.imgreadmode = 'grayscale'\n        elif self.inchannels == 1:\n            self.imgreadmode = 'rgb'\n        if self.return_feat:\n            self.load_feature_extractor()\n        self.load_data()\n        self.on_epoch_end()\n        \n    def on_epoch_end(self):\n        if self.shuffle:\n            random.shuffle(self.all_pat_ids)\n            \n    def load_data(self):#memory fill ho jaegi is se\n        df = pd.read_csv(self.csv)\n        dset = tqdm(df.iterrows(),total=len(df))\n        self.use_cols = ['FLAIR'] #,'T2w']\n        self.ignore_ids = [109, 123, 709]\n        self.min_frames = 5\n        self.data = {}\n        self.class_map = {0:'Tumor',1:'MGMT'}\n\n        for i,row in dset:\n            #folder_id, FLAIR, T1w, T1wCE, T2w, BraTS21ID, MGMT_value = row\n            pat_id = int(row['folder_id'])\n            if pat_id in self.ignore_ids:\n                print('Skipping PATID:',pat_id)\n                continue\n            target = row['MGMT_value']\n            images = []\n            pat_id = str(pat_id).zfill(5)\n            for col in self.use_cols:\n                images.extend(glob.glob(os.path.join(self.base_dir,pat_id,col.lower(),'*.png')))\n            if len(images)>self.min_frames:\n                self.data[pat_id] = {}\n                self.data[pat_id]['images'] = images[:self.maxlen_slices]#issue here\n                self.data[pat_id]['label'] = int(target)\n        \n        self.all_pat_ids = list(self.data.keys())\n        df = None\n        dset = None\n        print('DATA:>> ',len(self.data),' | IDS',len(self.all_pat_ids))\n        \n    def load_feature_extractor(self):\n        self.feat_batchsize = 16\n        if self.model_type == 'imagenet':\n            base_model = ResNet50(weights='imagenet')#input_shape\n            x = layers.Reshape((-1,1))(base_model.layers[-2].output)\n            x = layers.AveragePooling1D(pool_size=4)(x)\n            out = layers.Reshape((-1,))(x)\n            self.featext = Model(inputs=base_model.input, outputs=[out,base_model.layers[-1].output])\n            self.outdim = 512\n            base_model = None\n            self.featext_preprocessor = preprocess_input\n            self.featext_loadmode = 'rgb'\n        else:\n            path = '/kaggle/input/classifier4mods-finetune/FINAL_MODELALLMods_ep5_acc98_finetuned.h5'\n            base_model = load_model(path)\n            self.featext = Model(inputs=base_model.input, outputs=[base_model.layers[-2].output,base_model.layers[-1].output])\n            self.outdim = base_model.layers[-2].output_shape[-1]\n            base_model = None\n            self.featext_preprocessor = lambda x:x/255.\n            self.featext_loadmode = 'grayscale'\n            \n            \n    def load_image_array(self,img_path,mode='rgb'):\n        img = image.load_img(img_path,target_size=(self.height,self.width),color_mode=mode)\n        img = image.img_to_array(img)\n        return img\n    \n    def one_hot_encoder(self,y):\n        b = np.zeros((len(y), 2))\n        b[np.arange(len(y)),y] = 1\n        return b\n    \n    def __len__(self):\n        return int(len(self.data)/self.batch_size)\n    \n    def pad_depth_feat(self,featslist,shp):\n        padded_feat = np.zeros(shp)\n        mask = np.zeros((len(featslist), shp[1]))\n        for i,feat in enumerate(featslist):\n            padded_feat[i,:feat.shape[0]] = feat\n            mask[i,:feat.shape[0]] = 1\n        \n        return padded_feat,mask\n            \n    \n    def create_2d_mask(self,mask):\n        b,t = mask.shape\n        mask2d = np.zeros((b,t,t))\n        for i in range(b):\n            for j in range(t):\n                mask2d[i][j] = mask[i]\n        return mask2d\n    \n    def __getitem__(self,index):\n        full_batch_feat = []\n        full_batch_target = []\n        depthlist = []\n        for patid in self.all_pat_ids[index*self.batch_size:(index+1)*self.batch_size]:\n            full_batch_target.append(self.data[patid]['label'])\n            imagespath = self.data[patid]['images']\n            \n            if self.return_feat:\n                \n                allfeats = np.zeros((len(imagespath),self.outdim))\n                for i in range(0,len(imagespath),self.feat_batchsize):\n                    batchimgspath = imagespath[i:i+self.feat_batchsize]\n\n                    batchimgs = [self.load_image_array(path,self.featext_loadmode) for path in batchimgspath]\n                    batchimgs = [self.featext_preprocessor(img) for img in batchimgs]\n                    batchimgs = np.array(batchimgs)\n\n                    if len(batchimgs.shape)<4:\n                        batchimgs = batchimgs[:,:,:,None]#ye kyu chahiye\n\n\n                    batchfeat,_ = self.featext(batchimgs)\n                    #print('patid',patid,'batchfeat',batchfeat.shape)\n                    allfeats[i:i+self.feat_batchsize] = batchfeat\n\n                    batchfeat = None\n                    batchimgs = None\n                    _ = None\n\n                #print('patid',patid,'allfeats',allfeats.shape)\n            else:\n                allfeats = np.zeros((len(imagespath),self.height,self.width,self.inchannels))\n                for i in range(0,len(imagespath)):\n                    path = imagespath[i]\n                    img = self.load_image_array(path,self.imgreadmode)\n                    img /= 255.\n                    if len(img.shape) == 2:\n                        img = img[:,:,None]\n                        \n                    allfeats[i,:,:,:] = img\n                    \n            full_batch_feat.append(allfeats)\n            depthlist.append(allfeats.shape[0])\n        \n       \n        #print('full_batch_feat',len(full_batch_feat))\n        max_len = max(depthlist)\n        if self.return_feat:\n            shp = (len(full_batch_feat), max_len, self.outdim)\n        else:\n            shp = (len(full_batch_feat), max_len, self.height,self.width,self.inchannels)\n        full_batch_feat, mask = self.pad_depth_feat(full_batch_feat,shp)\n        mask = self.create_2d_mask(mask)\n        full_batch_target = self.one_hot_encoder(full_batch_target)\n        #print(f'OutBatch: {full_batch_feat.shape} || Label: {full_batch_target.shape} || Mask: {mask.shape}')\n\n        return [full_batch_feat,mask],full_batch_target,\n        ","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:26:52.035422Z","iopub.execute_input":"2021-08-31T14:26:52.035759Z","iopub.status.idle":"2021-08-31T14:26:52.069466Z","shell.execute_reply.started":"2021-08-31T14:26:52.035727Z","shell.execute_reply":"2021-08-31T14:26:52.068446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, mask, training):\n        attn_output = self.att(inputs, inputs,attention_mask=mask)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:26:52.356637Z","iopub.execute_input":"2021-08-31T14:26:52.357025Z","iopub.status.idle":"2021-08-31T14:26:52.365883Z","shell.execute_reply.started":"2021-08-31T14:26:52.35696Z","shell.execute_reply":"2021-08-31T14:26:52.364947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PositionEmbedding(layers.Layer):\n    def __init__(self, maxlen, embed_dim):\n        super(PositionEmbedding, self).__init__()\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        return x + positions","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:26:52.739865Z","iopub.execute_input":"2021-08-31T14:26:52.740236Z","iopub.status.idle":"2021-08-31T14:26:52.746803Z","shell.execute_reply.started":"2021-08-31T14:26:52.740203Z","shell.execute_reply":"2021-08-31T14:26:52.745615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IH,IW = 224,224\nmodeltype = 'imagenet'\nmaxlen_slices = 100\nbs = 16\ngen = DataLoader(modeltype=modeltype,height=IH,width=IW,batch_size=bs,maxlen_slices=maxlen_slices,inchannels=1,return_feat=True)\nif 1:\n    for i,sample in enumerate(gen):\n        [full_batch_feat,mask],full_batch_target = sample\n        print(full_batch_feat.shape,full_batch_target.shape,mask.shape)\n        break","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:26:53.468509Z","iopub.execute_input":"2021-08-31T14:26:53.468874Z","iopub.status.idle":"2021-08-31T14:26:59.892525Z","shell.execute_reply.started":"2021-08-31T14:26:53.468834Z","shell.execute_reply":"2021-08-31T14:26:59.891115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layers.Reshape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_dim = 512  # Embedding size for each token\nnum_heads = 8  # Number of attention heads\nff_dim = embed_dim*4  # Hidden layer size in feed forward network inside transformer\n\n\ninputs = layers.Input(shape=(None,embed_dim))\ninput_mask = layers.Input(shape=(None,None))\n#pos_layer = PositionEmbedding(maxlen_slices, embed_dim)\n#x = pos_layer(inputs)\ntransformer_block_1 = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block_1(inputs,input_mask)\ntransformer_block_2 = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block_2(x,input_mask)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(128, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(2, activation=\"softmax\")(x)\nmodel = keras.Model(inputs=[inputs,input_mask], outputs=outputs)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:27:30.340777Z","iopub.execute_input":"2021-08-31T14:27:30.34117Z","iopub.status.idle":"2021-08-31T14:27:30.594672Z","shell.execute_reply.started":"2021-08-31T14:27:30.341133Z","shell.execute_reply":"2021-08-31T14:27:30.593877Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_dim = 512  # Embedding size for each token\nnum_heads = 8  # Number of attention heads\nff_dim = embed_dim*4  # Hidden layer size in feed forward network inside transformer\n\n\ninputs = layers.Input(shape=(None,embed_dim))\ninput_mask = layers.Input(shape=(None,None))\npath = '/kaggle/input/classifier4mods-finetune/FINAL_MODELALLMods_ep5_acc98_finetuned.h5'\nbase_model = load_model(path)\nfeatext = Model(inputs=base_model.input, outputs=[base_model.layers[-2].output,base_model.layers[-1].output])\n#pos_layer = PositionEmbedding(maxlen_slices, embed_dim)\n#x = pos_layer(inputs)\ntransformer_block_1 = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block_1(inputs,input_mask)\ntransformer_block_2 = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block_2(x,input_mask)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(128, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(2, activation=\"softmax\")(x)\nmodel = keras.Model(inputs=[inputs,input_mask], outputs=outputs)\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(full_batch_feat.shape,full_batch_target.shape,mask.shape)\nout = model([full_batch_feat,mask])\nprint(out.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:27:34.757378Z","iopub.execute_input":"2021-08-31T14:27:34.757707Z","iopub.status.idle":"2021-08-31T14:27:34.802692Z","shell.execute_reply.started":"2021-08-31T14:27:34.757674Z","shell.execute_reply":"2021-08-31T14:27:34.801605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    loss=\"categorical_crossentropy\",\n    optimizer=tf.keras.optimizers.Adam(),\n    metrics=[\"accuracy\"]\n)\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n    \"3d_image_classification.h5\", save_best_only=True,metrics=\"accuracy\",mode=max\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:27:39.838433Z","iopub.execute_input":"2021-08-31T14:27:39.838751Z","iopub.status.idle":"2021-08-31T14:27:39.853564Z","shell.execute_reply.started":"2021-08-31T14:27:39.838719Z","shell.execute_reply":"2021-08-31T14:27:39.852649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit_generator(\n    gen,\n    steps_per_epoch=len(gen),\n    epochs=50,\n    verbose=1,\n    callbacks=[checkpoint_cb],\n)\nmodel.save('FULL_TRANSFORMER_MODEL.h5')","metadata":{"execution":{"iopub.status.busy":"2021-08-31T14:27:43.688693Z","iopub.execute_input":"2021-08-31T14:27:43.689037Z","iopub.status.idle":"2021-08-31T14:37:34.725064Z","shell.execute_reply.started":"2021-08-31T14:27:43.688994Z","shell.execute_reply":"2021-08-31T14:37:34.72069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}