{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        # print(os.path.join(dirname, filename))\n        pass\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport cv2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader,Dataset\nfrom torchvision import datasets, transforms, models\nimport torchvision.utils as utils\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom PIL import Image\nimport os\nfrom sklearn.metrics import auc,roc_auc_score\ndevice = torch.device(\"cpu\")\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport matplotlib.pyplot as plt","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-02T09:27:59.142209Z","iopub.execute_input":"2024-10-02T09:27:59.143127Z","iopub.status.idle":"2024-10-02T09:29:24.935901Z","shell.execute_reply.started":"2024-10-02T09:27:59.143068Z","shell.execute_reply":"2024-10-02T09:29:24.934859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objs as go\nimport plotly.graph_objects as go","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:24.937631Z","iopub.execute_input":"2024-10-02T09:29:24.938102Z","iopub.status.idle":"2024-10-02T09:29:24.952687Z","shell.execute_reply.started":"2024-10-02T09:29:24.938068Z","shell.execute_reply":"2024-10-02T09:29:24.951498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:24.953738Z","iopub.execute_input":"2024-10-02T09:29:24.954047Z","iopub.status.idle":"2024-10-02T09:29:24.958236Z","shell.execute_reply.started":"2024-10-02T09:29:24.954015Z","shell.execute_reply":"2024-10-02T09:29:24.957176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    # random.seed(seed)\n    # os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:24.960464Z","iopub.execute_input":"2024-10-02T09:29:24.960816Z","iopub.status.idle":"2024-10-02T09:29:24.972382Z","shell.execute_reply.started":"2024-10-02T09:29:24.960782Z","shell.execute_reply":"2024-10-02T09:29:24.971408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir='/kaggle/input/siim-isic-melanoma-classification/train.csv'\ntest_dir='/kaggle/input/siim-isic-melanoma-classification/train.csv'\ntrain=pd.read_csv(train_dir)\ntest=pd.read_csv(test_dir)\nsubmission=pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:24.973363Z","iopub.execute_input":"2024-10-02T09:29:24.973663Z","iopub.status.idle":"2024-10-02T09:29:25.146150Z","shell.execute_reply.started":"2024-10-02T09:29:24.973631Z","shell.execute_reply":"2024-10-02T09:29:25.145155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.147563Z","iopub.execute_input":"2024-10-02T09:29:25.147989Z","iopub.status.idle":"2024-10-02T09:29:25.170137Z","shell.execute_reply.started":"2024-10-02T09:29:25.147944Z","shell.execute_reply":"2024-10-02T09:29:25.169281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist=train['target'].value_counts()\ndist","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.171193Z","iopub.execute_input":"2024-10-02T09:29:25.171480Z","iopub.status.idle":"2024-10-02T09:29:25.183355Z","shell.execute_reply.started":"2024-10-02T09:29:25.171448Z","shell.execute_reply":"2024-10-02T09:29:25.182461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Benign cases are\",(32542/(32542+584))*100)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.184752Z","iopub.execute_input":"2024-10-02T09:29:25.185081Z","iopub.status.idle":"2024-10-02T09:29:25.190905Z","shell.execute_reply.started":"2024-10-02T09:29:25.185050Z","shell.execute_reply":"2024-10-02T09:29:25.189941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=train['anatom_site_general_challenge'].value_counts().index\nvalues=train['anatom_site_general_challenge'].value_counts().values\nfig = go.Figure(data=[go.Pie(labels=labels, values=values, textinfo='label+percent',\n                             insidetextorientation='radial'\n                            )])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.192096Z","iopub.execute_input":"2024-10-02T09:29:25.192685Z","iopub.status.idle":"2024-10-02T09:29:25.563470Z","shell.execute_reply.started":"2024-10-02T09:29:25.192643Z","shell.execute_reply":"2024-10-02T09:29:25.562583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=train['diagnosis'].value_counts().index[1:]\nvalues=train['diagnosis'].value_counts().values[1:]\nfig = go.Figure(data=[go.Pie(labels=labels, values=values, textinfo='label+percent',\n                             insidetextorientation='radial'\n                            )])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.567477Z","iopub.execute_input":"2024-10-02T09:29:25.567797Z","iopub.status.idle":"2024-10-02T09:29:25.588943Z","shell.execute_reply.started":"2024-10-02T09:29:25.567763Z","shell.execute_reply":"2024-10-02T09:29:25.587762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new=train.drop(labels=['image_name','patient_id','sex','age_approx','anatom_site_general_challenge','target'],axis=1)\npd.crosstab(new['diagnosis'].values,new['benign_malignant'])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.590112Z","iopub.execute_input":"2024-10-02T09:29:25.590401Z","iopub.status.idle":"2024-10-02T09:29:25.625769Z","shell.execute_reply.started":"2024-10-02T09:29:25.590368Z","shell.execute_reply":"2024-10-02T09:29:25.624942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0=train[train['target']==0].sample(2000)\ndf_1=train[train['target']==1]\ntrain=pd.concat([df_0,df_1])\ntrain=train.reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.627001Z","iopub.execute_input":"2024-10-02T09:29:25.627591Z","iopub.status.idle":"2024-10-02T09:29:25.641922Z","shell.execute_reply.started":"2024-10-02T09:29:25.627530Z","shell.execute_reply":"2024-10-02T09:29:25.641071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img_dir = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\"\ntest_img_dir = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/test/\"","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.643150Z","iopub.execute_input":"2024-10-02T09:29:25.643464Z","iopub.status.idle":"2024-10-02T09:29:25.647604Z","shell.execute_reply.started":"2024-10-02T09:29:25.643429Z","shell.execute_reply":"2024-10-02T09:29:25.646645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Benign Cases')\nbenign=[]\ndf_benign=df_0.sample(40)\ndf_benign=df_benign.reset_index()\nfor i in range(40):\n    img=cv2.imread(str(train_img_dir + df_benign['image_name'].iloc[i]+'.jpg'))\n    img = cv2.resize(img, (224,224))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = img.astype(np.float32)/255.\n    benign.append(img)\nf, ax = plt.subplots(5,8, figsize=(10,8))\nfor i, img in enumerate(benign):\n        ax[i//8, i%8].imshow(img)\n        ax[i//8, i%8].axis('off')\n        \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:25.648806Z","iopub.execute_input":"2024-10-02T09:29:25.649165Z","iopub.status.idle":"2024-10-02T09:29:31.441366Z","shell.execute_reply.started":"2024-10-02T09:29:25.649124Z","shell.execute_reply":"2024-10-02T09:29:31.440492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Malignant Cases')\nmalignant=[]\ndf_malignant=df_1.sample(40)\ndf_malignant=df_malignant.reset_index()\nfor i in range(40):\n    img=cv2.imread(str(train_img_dir + df_malignant['image_name'].iloc[i]+'.jpg'))\n    img = cv2.resize(img, (224,224))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = img.astype(np.float32)/255.\n    malignant.append(img)\nf, ax = plt.subplots(5,8, figsize=(10,8))\nfor i, img in enumerate(malignant):\n        ax[i//8, i%8].imshow(img)\n        ax[i//8, i%8].axis('off')\n        \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:31.442580Z","iopub.execute_input":"2024-10-02T09:29:31.442938Z","iopub.status.idle":"2024-10-02T09:29:34.903653Z","shell.execute_reply.started":"2024-10-02T09:29:31.442898Z","shell.execute_reply":"2024-10-02T09:29:34.902618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=[]\ndata=[]\nfor i in range(train.shape[0]):\n    data.append(train_img_dir + train['image_name'].iloc[i]+'.jpg')\n    labels.append(train['target'].iloc[i])\ndf=pd.DataFrame(data)\ndf.columns=['images']\ndf['target']=labels","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:34.904966Z","iopub.execute_input":"2024-10-02T09:29:34.905346Z","iopub.status.idle":"2024-10-02T09:29:35.004691Z","shell.execute_reply.started":"2024-10-02T09:29:34.905293Z","shell.execute_reply":"2024-10-02T09:29:35.003929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data=[]\nfor i in range(test.shape[0]):\n    test_data.append(test_img_dir + test['image_name'].iloc[i]+'.jpg')\ndf_test=pd.DataFrame(test_data)\ndf_test.columns=['images']","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.005803Z","iopub.execute_input":"2024-10-02T09:29:35.006140Z","iopub.status.idle":"2024-10-02T09:29:35.512823Z","shell.execute_reply.started":"2024-10-02T09:29:35.006107Z","shell.execute_reply":"2024-10-02T09:29:35.511801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(df['images'],df['target'], test_size=0.2, random_state=1234)\n\ntrain=pd.DataFrame(X_train)\ntrain.columns=['images']\ntrain['target']=y_train\n\nvalidation=pd.DataFrame(X_val)\nvalidation.columns=['images']\nvalidation['target']=y_val","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.513951Z","iopub.execute_input":"2024-10-02T09:29:35.514263Z","iopub.status.idle":"2024-10-02T09:29:35.525926Z","shell.execute_reply.started":"2024-10-02T09:29:35.514231Z","shell.execute_reply":"2024-10-02T09:29:35.524905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* normalizing\n* reshaping\n* augmentation(only for tarin data)","metadata":{}},{"cell_type":"code","source":"train_transform = transforms.Compose([\n        transforms.RandomRotation(10),      # rotate +/- 10 degrees\n        transforms.RandomHorizontalFlip(),  # reverse 50% of images\n        transforms.Resize(224),             # resize shortest side to 224 pixels\n        transforms.CenterCrop(224),         # crop longest side to 224 pixels at center\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ])\n\ntest_transform = transforms.Compose([\n        transforms.Resize(224),\n        transforms.CenterCrop(224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.527140Z","iopub.execute_input":"2024-10-02T09:29:35.527431Z","iopub.status.idle":"2024-10-02T09:29:35.535594Z","shell.execute_reply.started":"2024-10-02T09:29:35.527401Z","shell.execute_reply":"2024-10-02T09:29:35.534653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    \n    def __init__(self,data_paths,labels,transform=None,mode='train'):\n         self.data=data_paths\n         self.labels=labels\n         self.transform=transform\n         self.mode=mode\n    def __len__(self):\n       return len(self.data)\n    \n    def __getitem__(self,idx):\n        img_name = self.data[idx]\n        img = cv2.imread(img_name)\n        \n        if img is None:\n            print(f\"Warning: Failed to load image {img_name}\")\n            return None, None  # Or handle it differently, like returning a placeholder image\n        \n        \n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img=Image.fromarray(img)\n        if self.transform is not None:\n          img = self.transform(img)\n        img=img.cuda()\n        \n        if self.mode=='test':\n            return img,img_name\n        else:\n           \n            labels = torch.tensor(self.labels[idx]).cuda()\n\n            return img, labels","metadata":{"execution":{"iopub.status.busy":"2024-10-02T11:21:20.964946Z","iopub.execute_input":"2024-10-02T11:21:20.965800Z","iopub.status.idle":"2024-10-02T11:21:20.976428Z","shell.execute_reply.started":"2024-10-02T11:21:20.965744Z","shell.execute_reply":"2024-10-02T11:21:20.975260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset=ImageDataset(data_paths=X_train.values,labels=y_train.values,transform=train_transform)\nval_dataset=ImageDataset(data_paths=X_val.values,labels=y_val.values,transform=test_transform)\ntest_dataset=ImageDataset(data_paths=df_test['images'].values,labels=None,transform=test_transform,mode='test')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.547109Z","iopub.execute_input":"2024-10-02T09:29:35.547480Z","iopub.status.idle":"2024-10-02T09:29:35.556278Z","shell.execute_reply.started":"2024-10-02T09:29:35.547433Z","shell.execute_reply":"2024-10-02T09:29:35.555452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader=DataLoader(train_dataset,batch_size=100,shuffle=True)\nval_loader=DataLoader(val_dataset,batch_size=50,shuffle=False)\ntest_loader=DataLoader(test_dataset,batch_size=50,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.557371Z","iopub.execute_input":"2024-10-02T09:29:35.558196Z","iopub.status.idle":"2024-10-02T09:29:35.564998Z","shell.execute_reply.started":"2024-10-02T09:29:35.558160Z","shell.execute_reply":"2024-10-02T09:29:35.564168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AttentionBlock(nn.Module):\n    def __init__(self, in_features_l, in_features_g, attn_features, up_factor, normalize_attn=True):\n        super(AttentionBlock, self).__init__()\n        self.up_factor = up_factor\n        self.normalize_attn = normalize_attn\n        self.W_l = nn.Conv2d(in_channels=in_features_l, out_channels=attn_features, kernel_size=1, padding=0, bias=False)\n        self.W_g = nn.Conv2d(in_channels=in_features_g, out_channels=attn_features, kernel_size=1, padding=0, bias=False)\n        self.phi = nn.Conv2d(in_channels=attn_features, out_channels=1, kernel_size=1, padding=0, bias=True)\n    def forward(self, l, g):\n        N, C, W, H = l.size()\n        l_ = self.W_l(l)\n        g_ = self.W_g(g)\n        if self.up_factor > 1:\n            g_ = F.interpolate(g_, scale_factor=self.up_factor, mode='bilinear', align_corners=False)\n        c = self.phi(F.relu(l_ + g_)) # batch_sizex1xWxH\n        \n        # compute attn map\n        if self.normalize_attn:\n            a = F.softmax(c.view(N,1,-1), dim=2).view(N,1,W,H)\n        else:\n            a = torch.sigmoid(c)\n        # re-weight the local feature\n        f = torch.mul(a.expand_as(l), l) # batch_sizexCxWxH\n        if self.normalize_attn:\n            output = f.view(N,C,-1).sum(dim=2) # weighted sum\n        else:\n            output = F.adaptive_avg_pool2d(f, (1,1)).view(N,C) # global average pooling\n        return a, output","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.566073Z","iopub.execute_input":"2024-10-02T09:29:35.566328Z","iopub.status.idle":"2024-10-02T09:29:35.577916Z","shell.execute_reply.started":"2024-10-02T09:29:35.566299Z","shell.execute_reply":"2024-10-02T09:29:35.576854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AttnVGG(nn.Module):\n    def __init__(self, num_classes, normalize_attn=False, dropout=None):\n        super(AttnVGG, self).__init__()\n        net = models.vgg16_bn(pretrained=True)\n        self.conv_block1 = nn.Sequential(*list(net.features.children())[0:6])\n        self.conv_block2 = nn.Sequential(*list(net.features.children())[7:13])\n        self.conv_block3 = nn.Sequential(*list(net.features.children())[14:23])\n        self.conv_block4 = nn.Sequential(*list(net.features.children())[24:33])\n        self.conv_block5 = nn.Sequential(*list(net.features.children())[34:43])\n        self.pool = nn.AvgPool2d(7, stride=1)\n        self.dpt = None\n        if dropout is not None:\n            self.dpt = nn.Dropout(dropout)\n        self.cls = nn.Linear(in_features=512+512+256, out_features=num_classes, bias=True)\n        \n       # initialize the attention blocks defined above\n        self.attn1 = AttentionBlock(256, 512, 256, 4, normalize_attn=normalize_attn)\n        self.attn2 = AttentionBlock(512, 512, 256, 2, normalize_attn=normalize_attn)\n        \n       \n        self.reset_parameters(self.cls)\n        self.reset_parameters(self.attn1)\n        self.reset_parameters(self.attn2)\n    def reset_parameters(self, module):\n        for m in module.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')\n                if m.bias is not None:\n                    nn.init.constant_(m.bias, 0.)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1.)\n                nn.init.constant_(m.bias, 0.)\n            elif isinstance(m, nn.Linear):\n                nn.init.normal_(m.weight, 0., 0.01)\n                nn.init.constant_(m.bias, 0.)\n    def forward(self, x):\n        block1 = self.conv_block1(x)       # /1\n        pool1 = F.max_pool2d(block1, 2, 2) # /2\n        block2 = self.conv_block2(pool1)   # /2\n        pool2 = F.max_pool2d(block2, 2, 2) # /4\n        block3 = self.conv_block3(pool2)   # /4\n        pool3 = F.max_pool2d(block3, 2, 2) # /8\n        block4 = self.conv_block4(pool3)   # /8\n        pool4 = F.max_pool2d(block4, 2, 2) # /16\n        block5 = self.conv_block5(pool4)   # /16\n        pool5 = F.max_pool2d(block5, 2, 2) # /32\n        N, __, __, __ = pool5.size()\n        \n        g = self.pool(pool5).view(N,512)\n        a1, g1 = self.attn1(pool3, pool5)\n        a2, g2 = self.attn2(pool4, pool5)\n        g_hat = torch.cat((g,g1,g2), dim=1) # batch_size x C\n        if self.dpt is not None:\n            g_hat = self.dpt(g_hat)\n        out = self.cls(g_hat)\n\n        return [out, a1, a2]","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.579398Z","iopub.execute_input":"2024-10-02T09:29:35.579784Z","iopub.status.idle":"2024-10-02T09:29:35.598312Z","shell.execute_reply.started":"2024-10-02T09:29:35.579739Z","shell.execute_reply":"2024-10-02T09:29:35.597394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AttnVGG(num_classes=1, normalize_attn=True)\nmodel=model.cuda()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:35.599484Z","iopub.execute_input":"2024-10-02T09:29:35.599819Z","iopub.status.idle":"2024-10-02T09:29:39.979950Z","shell.execute_reply.started":"2024-10-02T09:29:35.599773Z","shell.execute_reply":"2024-10-02T09:29:39.979117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:39.981063Z","iopub.execute_input":"2024-10-02T09:29:39.981369Z","iopub.status.idle":"2024-10-02T09:29:39.988846Z","shell.execute_reply.started":"2024-10-02T09:29:39.981335Z","shell.execute_reply":"2024-10-02T09:29:39.987906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = FocalLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-4)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:39.990277Z","iopub.execute_input":"2024-10-02T09:29:39.990669Z","iopub.status.idle":"2024-10-02T09:29:39.999024Z","shell.execute_reply.started":"2024-10-02T09:29:39.990627Z","shell.execute_reply":"2024-10-02T09:29:39.998118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T09:29:40.004338Z","iopub.execute_input":"2024-10-02T09:29:40.004722Z","iopub.status.idle":"2024-10-02T09:29:40.010720Z","shell.execute_reply.started":"2024-10-02T09:29:40.004689Z","shell.execute_reply":"2024-10-02T09:29:40.009918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nstart_time = time.time()\n\nepochs = 2\n\ntrain_losses = []\ntest_losses = []\ntrain_auc=[]\nval_auc=[]\n\nfor i in range(epochs):\n\n    train_preds=[]\n    train_targets=[]\n    auc_train=[]\n    loss_epoch_train=[]\n    loss_epoch_test=[]\n    # Run the training batches\n    for b, (X_train, y_train) in tqdm(enumerate(train_loader),total=len(train_loader)):\n        \n        b+=1\n        y_pred,_,_=model(X_train)\n        loss = criterion(torch.sigmoid(y_pred.type(torch.FloatTensor)).squeeze(), y_train.type(torch.FloatTensor))   \n        loss_epoch_train.append(loss.item())\n        # For plotting purpose\n        if (i==1):\n            if (b==19):\n                I_train = utils.make_grid(X_train[0:8,:,:,:], nrow=8, normalize=True, scale_each=True)\n                __, a1, a2 = model(X_train[0:8,:,:,:])\n                \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n                 \n    try:\n        auc_train=roc_auc_score(y_train.detach().to(device).numpy(),torch.sigmoid(y_pred).detach().to(device).numpy())\n    except:\n        auc_train=0\n    train_losses.append(np.mean(loss_epoch_train))\n    train_auc.append(auc_train)\n    print(f'epoch: {i:2}   loss: {np.mean(loss_epoch_train):10.8f} AUC  : {auc_train:10.8f} ')\n    # Run the testing batches\n    \n    with torch.no_grad():\n        for b, (X_test, y_test) in enumerate(val_loader):\n            \n            y_val,_,_ = model(X_test)\n            loss = criterion(torch.sigmoid(y_val.type(torch.FloatTensor)).squeeze(), y_test.type(torch.FloatTensor))\n            loss_epoch_test.append(loss.item())\n    try:\n                                           \n        auc_val=roc_auc_score(y_test.detach().to(device).numpy(),torch.sigmoid(y_val).detach().to(device).numpy())\n    except:\n        auc_val=0\n    test_losses.append(np.mean(loss_epoch_test))\n    val_auc.append(auc_val)\n    print(f'Epoch: {i} Val Loss: {np.mean(loss_epoch_test):10.8f} AUC: {auc_val:10.8f} ')\n    \nprint(f'\\nDuration: {time.time() - start_time:.0f} seconds') # print the time elapsed","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:11:01.479034Z","iopub.execute_input":"2024-10-02T10:11:01.480101Z","iopub.status.idle":"2024-10-02T10:31:49.997220Z","shell.execute_reply.started":"2024-10-02T10:11:01.480048Z","shell.execute_reply":"2024-10-02T10:31:49.996088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_attention(I_train,a,up_factor,no_attention=False):\n    img = I_train.permute((1,2,0)).cpu().numpy()\n    # compute the heatmap\n    if up_factor > 1:\n        a = F.interpolate(a, scale_factor=up_factor, mode='bilinear', align_corners=False)\n    attn = utils.make_grid(a, nrow=8, normalize=True, scale_each=True)\n    attn = attn.permute((1,2,0)).mul(255).byte().cpu().numpy()\n    attn = cv2.applyColorMap(attn, cv2.COLORMAP_JET)\n    attn = cv2.cvtColor(attn, cv2.COLOR_BGR2RGB)\n    attn = np.float32(attn) / 255\n    # add the heatmap to the image\n    img=cv2.resize(img,(466,60))\n    if no_attention:\n        return torch.from_numpy(img)\n    else:\n        vis = 0.6 * img + 0.4 * attn\n        return torch.from_numpy(vis)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:44:47.399209Z","iopub.execute_input":"2024-10-02T10:44:47.400085Z","iopub.status.idle":"2024-10-02T10:44:47.407731Z","shell.execute_reply.started":"2024-10-02T10:44:47.400044Z","shell.execute_reply":"2024-10-02T10:44:47.406760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig=visualize_attention(I_train,a1,up_factor=2,no_attention=True)\nfirst=visualize_attention(I_train,a1,up_factor=2,no_attention=False)\nsecond=visualize_attention(I_train,a2,up_factor=4,no_attention=False)\n\nfig, (ax1, ax2,ax3) = plt.subplots(3, 1,figsize=(10, 10))\nax1.imshow(orig)\nax2.imshow(first)\nax3.imshow(second)\nax1.title.set_text('Input Images')\nax2.title.set_text('pool-3 attention')\nax3.title.set_text('pool-4 attention')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:44:54.074756Z","iopub.execute_input":"2024-10-02T10:44:54.075146Z","iopub.status.idle":"2024-10-02T10:44:54.904556Z","shell.execute_reply.started":"2024-10-02T10:44:54.075106Z","shell.execute_reply":"2024-10-02T10:44:54.903675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval();\n\nresult = {'image_name': [], 'target': []}","metadata":{"execution":{"iopub.status.busy":"2024-10-02T11:01:03.184857Z","iopub.execute_input":"2024-10-02T11:01:03.185261Z","iopub.status.idle":"2024-10-02T11:01:03.191099Z","shell.execute_reply.started":"2024-10-02T11:01:03.185221Z","shell.execute_reply":"2024-10-02T11:01:03.190043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for images, image_names in tqdm(test_loader, total=len(test_loader)):\n    if images is None:\n        print(\"F###\")\n        break\n    preds=[]\n    with torch.no_grad():\n        outputs,_,_ = model(images)\n        a=torch.sigmoid(outputs).cpu().numpy()\n        for i in a:\n            preds.append(i[0])\n\n    result['image_name'].extend(image_names)\n    result['target'].extend(preds)\n\nsubmission = pd.DataFrame(result)\nsubmission['image_name']=[x[58:70] for x in submission['image_name']]\nsubmission.to_csv('submission.csv',index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T11:01:08.365036Z","iopub.execute_input":"2024-10-02T11:01:08.365410Z","iopub.status.idle":"2024-10-02T11:01:08.481463Z","shell.execute_reply.started":"2024-10-02T11:01:08.365373Z","shell.execute_reply":"2024-10-02T11:01:08.480136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}