{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport cv2\nimport random\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow.keras.layers as layers\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-01T15:30:42.739231Z","iopub.execute_input":"2021-11-01T15:30:42.739505Z","iopub.status.idle":"2021-11-01T15:30:42.746723Z","shell.execute_reply.started":"2021-11-01T15:30:42.739461Z","shell.execute_reply":"2021-11-01T15:30:42.745947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=pd.read_csv('/kaggle/input/histopathologic-cancer-detection/train_labels.csv')\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:28:50.999741Z","iopub.execute_input":"2021-11-01T14:28:51.000235Z","iopub.status.idle":"2021-11-01T14:28:51.488372Z","shell.execute_reply.started":"2021-11-01T14:28:51.000199Z","shell.execute_reply":"2021-11-01T14:28:51.487687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=data['label'].unique()\nprint(labels)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:28:54.897009Z","iopub.execute_input":"2021-11-01T14:28:54.897369Z","iopub.status.idle":"2021-11-01T14:28:54.910684Z","shell.execute_reply.started":"2021-11-01T14:28:54.897309Z","shell.execute_reply":"2021-11-01T14:28:54.909913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map={}\nfor i in tqdm(range(len(data))):\n    label_map[data.iloc[i][0]]=data.iloc[i][1]","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:28:56.107804Z","iopub.execute_input":"2021-11-01T14:28:56.108352Z","iopub.status.idle":"2021-11-01T14:29:38.789290Z","shell.execute_reply.started":"2021-11-01T14:28:56.108317Z","shell.execute_reply":"2021-11-01T14:29:38.788491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:30:02.104278Z","iopub.execute_input":"2021-11-01T14:30:02.104575Z","iopub.status.idle":"2021-11-01T14:30:02.109760Z","shell.execute_reply.started":"2021-11-01T14:30:02.104542Z","shell.execute_reply":"2021-11-01T14:30:02.108920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path='/kaggle/input/histopathologic-cancer-detection/train'\nfor file in os.listdir(path):\n    file_path=os.path.join(path,file)\n    image=cv2.imread(file_path)\n    print(image.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:30:03.026205Z","iopub.execute_input":"2021-11-01T14:30:03.026920Z","iopub.status.idle":"2021-11-01T14:30:06.474747Z","shell.execute_reply.started":"2021-11-01T14:30:03.026877Z","shell.execute_reply":"2021-11-01T14:30:06.472471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir='/kaggle/data'\ntrain_path=os.path.join(data_dir,'train')\nvalid_path=os.path.join(data_dir,'valid')\ntest_path=os.path.join(data_dir,'test')\nif not os.path.isdir(data_dir):\n    os.mkdir(data_dir)\n    os.mkdir(train_path)\n    os.mkdir(valid_path)\n    os.mkdir(test_path)\n    for label in labels:\n        os.mkdir(os.path.join(train_path,str(label)))\n        os.mkdir(os.path.join(valid_path,str(label)))\n        os.mkdir(os.path.join(test_path,str(label)))","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:30:07.916437Z","iopub.execute_input":"2021-11-01T14:30:07.917118Z","iopub.status.idle":"2021-11-01T14:30:07.928087Z","shell.execute_reply.started":"2021-11-01T14:30:07.917082Z","shell.execute_reply":"2021-11-01T14:30:07.927384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(mode,n,dest_dir):\n    file_set=random.sample(os.listdir(path),2*n)\n    n1,n0=0,0\n    if mode=='train':\n        print(f'Moving training files to {dest_dir}')\n    elif mode=='valid':\n        print(f'Moving validation files to {dest_dir}')\n    else:\n        print(f'Moving test files to {dest_dir}')\n    for file in tqdm(file_set):\n        file_path=os.path.join(path,file)\n        file_name,_=file.split('.')\n        shutil.copy(file_path,os.path.join(dest_dir,str(label_map[file_name])))\n        if label_map[file_name]==1:\n            n1+=1\n        else:\n            n0+=1\n    print(f'The number of examples for each class are {n1} and {n0}')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:30:10.469564Z","iopub.execute_input":"2021-11-01T14:30:10.470092Z","iopub.status.idle":"2021-11-01T14:30:10.479335Z","shell.execute_reply.started":"2021-11-01T14:30:10.470056Z","shell.execute_reply":"2021-11-01T14:30:10.478446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_data('train',10000,train_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:30:13.583210Z","iopub.execute_input":"2021-11-01T14:30:13.583466Z","iopub.status.idle":"2021-11-01T14:32:52.452883Z","shell.execute_reply.started":"2021-11-01T14:30:13.583435Z","shell.execute_reply":"2021-11-01T14:32:52.452174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_data('valid',7000,valid_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:32:55.119573Z","iopub.execute_input":"2021-11-01T14:32:55.120109Z","iopub.status.idle":"2021-11-01T14:34:38.898731Z","shell.execute_reply.started":"2021-11-01T14:32:55.120070Z","shell.execute_reply":"2021-11-01T14:34:38.897916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_data('test',4000,test_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:34:41.430870Z","iopub.execute_input":"2021-11-01T14:34:41.431124Z","iopub.status.idle":"2021-11-01T14:35:36.572008Z","shell.execute_reply.started":"2021-11-01T14:34:41.431094Z","shell.execute_reply":"2021-11-01T14:35:36.571179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del label_map","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:35:39.986845Z","iopub.execute_input":"2021-11-01T14:35:39.987308Z","iopub.status.idle":"2021-11-01T14:35:40.016533Z","shell.execute_reply.started":"2021-11-01T14:35:39.987273Z","shell.execute_reply":"2021-11-01T14:35:40.015734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_images_paths(path):\n    image_paths,names=[],[]\n    for folder in os.listdir(path):\n        folder_path=os.path.join(path,folder)\n        image_set=random.sample(os.listdir(folder_path),1)\n        for file in image_set:\n            file_path=os.path.join(folder_path,file)\n            image_paths.append(file_path)\n            names.append(folder)\n    return image_paths,names\nimage_paths,names=find_images_paths(train_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:35:55.997564Z","iopub.execute_input":"2021-11-01T14:35:55.998008Z","iopub.status.idle":"2021-11-01T14:35:56.020968Z","shell.execute_reply.started":"2021-11-01T14:35:55.997972Z","shell.execute_reply":"2021-11-01T14:35:56.020153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(image_paths,names,row,col):\n    fig=plt.figure(figsize=(16,16))\n    for i in range(len(names)):\n        fig.add_subplot(row,col,i+1)\n        plt.title(names[i])\n        plt.axis('off')\n        plt.imshow(cv2.imread(image_paths[i]))\n    plt.tight_layout()\n    plt.show()\nplot_images(\n    image_paths=image_paths,\n    names=names,\n    row=1,col=2\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:35:57.822308Z","iopub.execute_input":"2021-11-01T14:35:57.822785Z","iopub.status.idle":"2021-11-01T14:35:58.096884Z","shell.execute_reply.started":"2021-11-01T14:35:57.822748Z","shell.execute_reply":"2021-11-01T14:35:58.096182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_size=(96,96)\nbatch_size=32","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:01.837838Z","iopub.execute_input":"2021-11-01T14:36:01.839336Z","iopub.status.idle":"2021-11-01T14:36:01.845028Z","shell.execute_reply.started":"2021-11-01T14:36:01.839285Z","shell.execute_reply":"2021-11-01T14:36:01.842691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"string_labels=[]\nfor label in labels:\n    string_labels.append(str(label))\nprint(string_labels)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:03.764099Z","iopub.execute_input":"2021-11-01T14:36:03.764667Z","iopub.status.idle":"2021-11-01T14:36:03.770935Z","shell.execute_reply.started":"2021-11-01T14:36:03.764623Z","shell.execute_reply":"2021-11-01T14:36:03.769265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen=ImageDataGenerator(\n    preprocessing_function=tf.keras.applications.vgg16.preprocess_input,\n)\n\ntrain_data=ImageDataGenerator(\n    preprocessing_function=tf.keras.applications.vgg16.preprocess_input,\n    horizontal_flip=True\n).flow_from_directory(\n    directory=train_path,\n    target_size=target_size,\n    classes=string_labels,\n    batch_size=batch_size,\n)\n\nvalid_data=datagen.flow_from_directory(\n    directory=valid_path,\n    target_size=target_size,\n    classes=string_labels,\n    batch_size=batch_size\n)\n\ntest_data=datagen.flow_from_directory(\n    directory=test_path,\n    target_size=target_size,\n    classes=string_labels,\n    batch_size=batch_size,\n    shuffle=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:06.070469Z","iopub.execute_input":"2021-11-01T14:36:06.071106Z","iopub.status.idle":"2021-11-01T14:36:07.637990Z","shell.execute_reply.started":"2021-11-01T14:36:06.071070Z","shell.execute_reply":"2021-11-01T14:36:07.636507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MCDropout(layers.Dropout):\n    def call(self,inputs):\n        return super().call(inputs,training=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:09.478547Z","iopub.execute_input":"2021-11-01T14:36:09.479120Z","iopub.status.idle":"2021-11-01T14:36:09.487280Z","shell.execute_reply.started":"2021-11-01T14:36:09.479084Z","shell.execute_reply":"2021-11-01T14:36:09.486637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResnetLayer(layers.Layer):\n    def __init__(self,filters,n_conv=4,kernel_size=3,strides=1,**kwargs):\n        super().__init__(**kwargs)\n        self.resnet_layers=[]\n        for _ in range(n_conv):\n            self.resnet_layers.append(layers.Conv2D(\n                filters=filters,kernel_size=kernel_size,\n                strides=strides,activation='relu',padding='same'\n            ))\n            self.resnet_layers.append(layers.BatchNormalization())\n    def call(self,inputs):\n        output=inputs\n        for residual_layer in self.resnet_layers:\n            output=residual_layer(output)\n        output=layers.Concatenate()([output,inputs])\n        return tf.keras.activations.relu(output)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:10.563693Z","iopub.execute_input":"2021-11-01T14:36:10.564168Z","iopub.status.idle":"2021-11-01T14:36:10.571409Z","shell.execute_reply.started":"2021-11-01T14:36:10.564130Z","shell.execute_reply":"2021-11-01T14:36:10.570433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InceptionModule(layers.Layer):\n    def __init__(self,filter_list,**kwargs):\n        super().__init__(**kwargs)\n        self.inception_module=[]\n        for i in range(len(filter_list)):\n            if i==0:\n                self.inception_module.append(layers.Conv2D(\n                    filters=filter_list[i],kernel_size=1,\n                    strides=1,activation='relu',padding='same'\n                ))\n            elif i==len(filter_list)-1:\n                self.inception_module.append(layers.MaxPool2D(\n                    pool_size=3,strides=1,padding='same'\n                ))\n                self.inception_module.append(layers.Conv2D(\n                    filters=filter_list[i],kernel_size=1,\n                    strides=1,activation='relu',padding='same'\n                ))\n            else:\n                self.inception_module.append(layers.Conv2D(\n                    filters=filter_list[i],kernel_size=1,\n                    strides=1,activation='relu',padding='same'\n                ))\n                self.inception_module.append(layers.Conv2D(\n                    filters=filter_list[i],kernel_size=2*i-1,\n                    strides=1,activation='relu',padding='same'\n                ))\n    def call(self,inputs):\n        outputs=[]\n        for module in self.inception_module:\n            outputs.append(module(inputs))\n        final_output=layers.Concatenate()(outputs)\n        return tf.keras.activations.relu(final_output)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:14.275158Z","iopub.execute_input":"2021-11-01T14:36:14.275609Z","iopub.status.idle":"2021-11-01T14:36:14.287894Z","shell.execute_reply.started":"2021-11-01T14:36:14.275569Z","shell.execute_reply":"2021-11-01T14:36:14.286971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rate=0.45","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:16.886489Z","iopub.execute_input":"2021-11-01T14:36:16.886838Z","iopub.status.idle":"2021-11-01T14:36:16.892570Z","shell.execute_reply.started":"2021-11-01T14:36:16.886802Z","shell.execute_reply":"2021-11-01T14:36:16.891304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(target_size,rate,n_conv=4):\n    steps=int(np.log2(target_size[0]))\n    inputs=layers.Input(shape=(*target_size,3))\n    filters=8\n    cnt=1\n    model=ResnetLayer(filters=filters,n_conv=n_conv)(inputs)\n    model=layers.AvgPool2D(pool_size=2,strides=2)(model)\n    for _ in range(steps-1):\n        filters*=2\n        if cnt%2==0:\n            model=ResnetLayer(filters=filters,n_conv=n_conv)(model)\n        else:\n            model=InceptionModule(filter_list=[int(filters)/4,filters,filters*2,filters/2])(model)\n        model=layers.AvgPool2D(pool_size=2,strides=2)(model)\n        cnt+=1\n    model=MCDropout(rate)(model)\n    model=layers.Conv2D(\n        filters=4096,kernel_size=1,\n        strides=1,padding='valid',activation='relu'\n    )(model)\n    model=MCDropout(rate)(model)\n    model=layers.Conv2D(\n        filters=4096,kernel_size=1,\n        strides=1,padding='valid',activation='relu'\n    )(model)\n    model=layers.Flatten()(model)\n    model=layers.Dense(units=len(string_labels),activation='softmax')(model)\n    inception_resnet_model=tf.keras.models.Model(inputs=inputs,outputs=model)\n    inception_resnet_model.compile(\n        optimizer=Adam(learning_rate=0.0001),\n        loss='categorical_crossentropy',\n        metrics=['accuracy']\n    )\n    return inception_resnet_model","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:19.290293Z","iopub.execute_input":"2021-11-01T14:36:19.290562Z","iopub.status.idle":"2021-11-01T14:36:19.302521Z","shell.execute_reply.started":"2021-11-01T14:36:19.290530Z","shell.execute_reply":"2021-11-01T14:36:19.301430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=build_model(target_size,rate)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:21.258081Z","iopub.execute_input":"2021-11-01T14:36:21.258746Z","iopub.status.idle":"2021-11-01T14:36:24.077190Z","shell.execute_reply.started":"2021-11-01T14:36:21.258708Z","shell.execute_reply":"2021-11-01T14:36:24.076510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:28.538236Z","iopub.execute_input":"2021-11-01T14:36:28.538718Z","iopub.status.idle":"2021-11-01T14:36:28.566855Z","shell.execute_reply.started":"2021-11-01T14:36:28.538671Z","shell.execute_reply":"2021-11-01T14:36:28.565994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_path='/kaggle/check'\nif not os.path.isdir(checkpoint_path):\n    os.mkdir(checkpoint_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:35.569677Z","iopub.execute_input":"2021-11-01T14:36:35.570112Z","iopub.status.idle":"2021-11-01T14:36:35.575283Z","shell.execute_reply.started":"2021-11-01T14:36:35.570076Z","shell.execute_reply":"2021-11-01T14:36:35.574451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_checkpoint=ModelCheckpoint(\n    filepath=checkpoint_path,\n    monitor='val_accuracy',\n    save_best_only=True,\n    save_weights_only=True,\n    mode='max'\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:40.097304Z","iopub.execute_input":"2021-11-01T14:36:40.098038Z","iopub.status.idle":"2021-11-01T14:36:40.101710Z","shell.execute_reply.started":"2021-11-01T14:36:40.097995Z","shell.execute_reply":"2021-11-01T14:36:40.100971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=model.fit(\n    x=train_data,\n    batch_size=batch_size,\n    callbacks=[model_checkpoint],\n    validation_data=valid_data,\n    epochs=50\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:36:41.018633Z","iopub.execute_input":"2021-11-01T14:36:41.019154Z","iopub.status.idle":"2021-11-01T15:08:54.194132Z","shell.execute_reply.started":"2021-11-01T14:36:41.019117Z","shell.execute_reply":"2021-11-01T15:08:54.193356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights(checkpoint_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T15:09:02.209791Z","iopub.execute_input":"2021-11-01T15:09:02.210311Z","iopub.status.idle":"2021-11-01T15:09:02.702056Z","shell.execute_reply.started":"2021-11-01T15:09:02.210273Z","shell.execute_reply":"2021-11-01T15:09:02.701344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(data,steps=20):\n    y_probs=np.stack([model.predict(data) for _ in tqdm(range(steps))])\n    p=np.mean(y_probs,axis=0)\n    cm=confusion_matrix(y_true=data.classes,y_pred=np.argmax(p,axis=-1))\n    acc=cm.trace()/cm.sum()\n    return acc*100","metadata":{"execution":{"iopub.status.busy":"2021-11-01T15:11:13.405705Z","iopub.execute_input":"2021-11-01T15:11:13.406516Z","iopub.status.idle":"2021-11-01T15:11:13.412238Z","shell.execute_reply.started":"2021-11-01T15:11:13.406451Z","shell.execute_reply":"2021-11-01T15:11:13.411250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc=predict(test_data)\nprint(f'Accuracy on the test dataset is {acc}%')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T15:11:14.250719Z","iopub.execute_input":"2021-11-01T15:11:14.250967Z","iopub.status.idle":"2021-11-01T15:13:38.385251Z","shell.execute_reply.started":"2021-11-01T15:11:14.250939Z","shell.execute_reply":"2021-11-01T15:13:38.384417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_loss=history.history['val_loss']\nloss=history.history['loss']\nplt.figure()\nplt.title('Loss vs Epochs')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.plot(loss,'ro--')\nplt.plot(val_loss,'bo--')\nplt.legend(['Train','Valid'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T15:13:41.490036Z","iopub.execute_input":"2021-11-01T15:13:41.490578Z","iopub.status.idle":"2021-11-01T15:13:41.792678Z","shell.execute_reply.started":"2021-11-01T15:13:41.490542Z","shell.execute_reply":"2021-11-01T15:13:41.791969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}