{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":532013,"sourceType":"datasetVersion","datasetId":253160},{"sourceId":848739,"sourceType":"datasetVersion","datasetId":251095},{"sourceId":2896320,"sourceType":"datasetVersion","datasetId":1774742},{"sourceId":2896347,"sourceType":"datasetVersion","datasetId":1774759},{"sourceId":2900791,"sourceType":"datasetVersion","datasetId":1775253}],"dockerImageVersionId":30145,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import print_function\n\nimport glob\nfrom itertools import chain\n\nimport os\nimport random\nimport zipfile\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nfrom tqdm.notebook import tqdm\n\nimport random","metadata":{"execution":{"iopub.status.busy":"2024-10-16T22:31:22.870047Z","iopub.execute_input":"2024-10-16T22:31:22.870803Z","iopub.status.idle":"2024-10-16T22:31:25.848228Z","shell.execute_reply.started":"2024-10-16T22:31:22.870681Z","shell.execute_reply":"2024-10-16T22:31:25.847024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Torch: {torch.__version__}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-16T22:31:30.524103Z","iopub.execute_input":"2024-10-16T22:31:30.524431Z","iopub.status.idle":"2024-10-16T22:31:30.530795Z","shell.execute_reply.started":"2024-10-16T22:31:30.524393Z","shell.execute_reply":"2024-10-16T22:31:30.529765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PARAMETERS SELLECTION","metadata":{}},{"cell_type":"code","source":"# Training settings\nbatch_size = 64\nepochs = 20\nlr = 5e-4\ngamma = 0.8\nseed = 42\nnum_classes = 1\ndevice = 'cuda'","metadata":{"execution":{"iopub.status.busy":"2024-10-16T22:31:33.319897Z","iopub.execute_input":"2024-10-16T22:31:33.320263Z","iopub.status.idle":"2024-10-16T22:31:33.326103Z","shell.execute_reply.started":"2024-10-16T22:31:33.320222Z","shell.execute_reply":"2024-10-16T22:31:33.324816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2024-10-16T22:31:35.940061Z","iopub.execute_input":"2024-10-16T22:31:35.941107Z","iopub.status.idle":"2024-10-16T22:31:35.952373Z","shell.execute_reply.started":"2024-10-16T22:31:35.941053Z","shell.execute_reply":"2024-10-16T22:31:35.951357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PREPROCESSING","metadata":{}},{"cell_type":"code","source":"#The Code from: https://www.kaggle.com/ratthachat/aptos-updated-albumentation-meets-grad-cam\nimport cv2\n\ndef crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n","metadata":{"execution":{"iopub.status.busy":"2024-10-16T22:31:47.85717Z","iopub.execute_input":"2024-10-16T22:31:47.857504Z","iopub.status.idle":"2024-10-16T22:31:47.87107Z","shell.execute_reply.started":"2024-10-16T22:31:47.857465Z","shell.execute_reply":"2024-10-16T22:31:47.86986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TEST IMAGE TRANSFORMATION","metadata":{}},{"cell_type":"code","source":"inPath = '../input/aptos2019-blindness-detection/test_images'\n  \n# path of the folder that will contain the modified image\ntry:\n    os.mkdir(\"test_images_transformed\")\nexcept:\n    print(\"path already exists\")\n\noutPath =\"test_images_transformed\"\n\nfor imagePath in tqdm(os.listdir(inPath)):\n    # imagePath contains name of the image \n    inputPath = os.path.join(inPath, imagePath)\n\n    image = cv2.imread(inputPath)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (224, 224))\n    image = cv2.addWeighted (image,4, cv2.GaussianBlur( image , (0,0) , 30) ,-4 ,128)\n\n    fullOutPath = os.path.join(outPath, imagePath)\n    cv2.imwrite(fullOutPath, image)\n      ","metadata":{"execution":{"iopub.status.busy":"2024-10-16T22:31:52.743634Z","iopub.execute_input":"2024-10-16T22:31:52.744031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '../input/custom-transform-blindness-2019/train_images_transformed'\ntest_dir = './test_images_transformed'\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = glob.glob(os.path.join(train_dir,'*.*'))\ntest_list = glob.glob(os.path.join(test_dir, '*.png'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LABELING","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ndf_train_old = pd.read_csv(\"../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv\")\ndf_train_old = df_train_old.rename({\"image\" : \"id_code\", \"level\" : \"diagnosis\"}, axis=1)\ndf_train = df_train.append(df_train_old).reset_index(drop=True)\n\nlabels = df_train['diagnosis'].values\nlabel_lookup = df_train.set_index('id_code')\n\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_weights = df_train['diagnosis'].value_counts()\ndfs = [df_train[df_train['diagnosis'] == i].sample(class_weights[4]) for i in range(5)]\nresampled = pd.concat(dfs, axis = 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resampled.diagnosis.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_train_list = (train_dir + '/' + resampled['id_code'].apply(lambda x: x + ('.jpg' if '_' in x else '.png'))).values\nnew_train_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ENCODING","metadata":{}},{"cell_type":"code","source":"y_train = pd.get_dummies(df_train['diagnosis']).values\n\nprint(y_train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\nprint(\"Original y_train:\", y_train.sum(axis=0))\nprint(\"Multilabel version:\", y_train_multi.sum(axis=0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_multi","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### images VISUALIZATION","metadata":{}},{"cell_type":"code","source":"get_index = lambda x : df_train[df_train.id_code == x].index[0]\ny_train_multi[get_index('0a4e1a29ffff')]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### examples!","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    name = train_list[idx].split(\"/\")[-1].split(\".\")[0]\n    ax.set_title('label = '+ str(labels[idx]) + \", file = \" + name)\n    ax.imshow(img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DATASET SPLITTING","metadata":{}},{"cell_type":"code","source":"train_list, valid_list = train_test_split(new_train_list, \n                                          test_size=0.05,\n                                          random_state=seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_list))\nprint(len(valid_list))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### AUGMENTATION","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose(\n    [\n        transforms.Resize((224, 224)),\n#         transforms.RandomResizedCrop(224),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n    ]\n)\n\nval_transforms = transforms.Compose(\n    [\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n    ]\n)\n\n\ntest_transforms = transforms.Compose(\n    [\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n    ]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Making the Dataset Class","metadata":{}},{"cell_type":"code","source":"class Blindness2019(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n\n    def __len__(self):\n        self.filelength = len(self.file_list)\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img)\n\n        label = label_lookup.loc[img_path.split(\"/\")[-1].split(\".\")[0]][0]\n#         label = torch.tensor(label).to(torch.float32)\n        image_id = img_path.split(\"/\")[-1].split(\".\")[0]\n#         label = y_train_multi[get_index(image_id)]\n#         label = y_train_multi[random.randint(0,3000)]\n        return img_transformed, label\n\nclass Blindness2019Test(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n\n    def __len__(self):\n        self.filelength = len(self.file_list)\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img)\n            \n        return img_transformed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Instantiating the dataset class","metadata":{}},{"cell_type":"code","source":"train_data = Blindness2019(train_list, transform=train_transforms)\nvalid_data = Blindness2019(valid_list, transform=test_transforms)\ntest_data = Blindness2019Test(test_list, transform=test_transforms)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset = train_data, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(dataset = valid_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset = test_data, batch_size=1, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_data), len(train_loader))\nprint(len(valid_data), len(valid_loader))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MULTIPLE MODEL DESIGNING","metadata":{}},{"cell_type":"markdown","source":"### Efficientnet","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path = [\n    '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path\nfrom efficientnet_pytorch import EfficientNet\n\nmodel = EfficientNet.from_name('efficientnet-b0')\nmodel.load_state_dict(torch.load('../input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))\nin_features = model._fc.in_features\nmodel._fc = nn.Linear(in_features, 5)\n# model.avg_pool = nn.AdaptiveAvgPool2d(output_size=(1,1))\n# model._fc = nn.Sequential(\n#                 nn.ReLU(),\n#                 nn.Linear(in_features=in_features, out_features=128, bias=True),\n#                 nn.ReLU(),\n#                 nn.Linear(in_features=128, out_features=5, bias=True, ),\n#                 nn.Sigmoid()\n#             )\nmodel = model.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### TRANSFORMER LIBRARIES.","metadata":{}},{"cell_type":"code","source":"! pip install ../input/vit-pytorch/Pillow-8.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n! pip install ../input/vit-pytorch/einops-0.3.2-py3-none-any.whl\n! pip install ../input/vit-pytorch/numpy-1.21.4-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n! pip install ../input/vit-pytorch/torch-1.10.0-cp37-cp37m-manylinux1_x86_64.whl\n! pip install ../input/vit-pytorch/torchvision-0.11.1-cp37-cp37m-manylinux1_x86_64.whl\n! pip install ../input/vit-pytorch/typing_extensions-4.0.1-py3-none-any.whl\n! pip install ../input/vit-pytorch/vit_pytorch-0.24.3-py3-none-any.whl\nfrom vit_pytorch.efficient import ViT\n\n! pip install ../input/linformer/linformer-0.2.1-py3-none-any.whl\n! pip install ../input/linformer/torch-1.10.0-cp37-cp37m-manylinux1_x86_64 (1).whl\n! pip install ../input/linformer/typing_extensions-4.0.1-py3-none-any (1).whl\nfrom linformer import Linformer","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Linformer ViT","metadata":{}},{"cell_type":"code","source":"efficient_transformer = Linformer(\n    dim=128,\n    seq_len=49+1,  # 7x7 patches + 1 cls-token\n    depth=32,\n    heads=16,\n    k=64\n)\n\nv = ViT(\n    dim=32,\n    image_size=224,\n    patch_size=16,\n    num_classes=5,\n    transformer=efficient_transformer,\n    channels=3,\n)\n\n# model = nn.Sequential(\n#                 v,\n# #                 nn.Dropout(p=.5),\n#                 nn.Linear(in_features=256, out_features=128, bias=True),\n#                 nn.Linear(in_features=128, out_features=5, bias=True, ),\n#                 nn.Sigmoid()\n#             ).to(device)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:16:06.008097Z","iopub.execute_input":"2021-12-09T15:16:06.008735Z","iopub.status.idle":"2021-12-09T15:16:06.094642Z","shell.execute_reply.started":"2021-12-09T15:16:06.008694Z","shell.execute_reply":"2021-12-09T15:16:06.093806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CaiT ViT","metadata":{}},{"cell_type":"code","source":"# import torch\n# from vit_pytorch.cait import CaiT\n\n# v = CaiT(\n#     image_size = 224,\n#     patch_size = 32,\n#     num_classes = 256,\n#     dim = 1024,\n#     depth = 12,             # depth of transformer for patch to patch attention only\n#     cls_depth = 2,          # depth of cross attention of CLS tokens to patch\n#     heads = 16,\n#     mlp_dim = 2048,\n#     dropout = 0.1,\n#     emb_dropout = 0.1,\n#     layer_dropout = 0.05    # randomly dropout 5% of the layers\n# ).to(device)\n\n# model = nn.Sequential(\n#                 v,\n# #                 nn.Dropout(p=.5),\n#                 nn.Linear(in_features=256, out_features=128, bias=True),\n#                 nn.Linear(in_features=128, out_features=5, bias=True, ),\n#                 nn.Sigmoid()\n#             ).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Facebook's LeViT","metadata":{}},{"cell_type":"code","source":"# import torch\n# from vit_pytorch.levit import LeViT\n\n# v = LeViT(\n#     image_size = 224,\n#     num_classes = 256,\n#     stages = 3,             # number of stages\n#     dim = (256, 384, 512),  # dimensions at each stage\n#     depth = 4,              # transformer of depth 4 at each stage\n#     heads = (4, 6, 8),      # heads at each stage\n#     mlp_mult = 2,\n#     dropout = 0.1\n# ).to(device)\n\n\n# model = nn.Sequential(\n#                 v,\n# #                 nn.Dropout(p=.5),\n#                 nn.Linear(in_features=256, out_features=128, bias=True),\n#                 nn.Linear(in_features=128, out_features=5, bias=True, ),\n#                 nn.Sigmoid()\n#             ).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### A CvT","metadata":{}},{"cell_type":"code","source":"# import torch\n# from vit_pytorch.cvt import CvT\n\n# v = CvT(\n#     num_classes = 256,\n#     s1_emb_dim = 64,        # stage 1 - dimension\n#     s1_emb_kernel = 7,      # stage 1 - conv kernel\n#     s1_emb_stride = 4,      # stage 1 - conv stride\n#     s1_proj_kernel = 3,     # stage 1 - attention ds-conv kernel size\n#     s1_kv_proj_stride = 2,  # stage 1 - attention key / value projection stride\n#     s1_heads = 1,           # stage 1 - heads\n#     s1_depth = 1,           # stage 1 - depth\n#     s1_mlp_mult = 4,        # stage 1 - feedforward expansion factor\n#     s2_emb_dim = 192,       # stage 2 - (same as above)\n#     s2_emb_kernel = 3,\n#     s2_emb_stride = 2,\n#     s2_proj_kernel = 3,\n#     s2_kv_proj_stride = 2,\n#     s2_heads = 3,\n#     s2_depth = 2,\n#     s2_mlp_mult = 4,\n#     s3_emb_dim = 384,       # stage 3 - (same as above)\n#     s3_emb_kernel = 3,\n#     s3_emb_stride = 2,\n#     s3_proj_kernel = 3,\n#     s3_kv_proj_stride = 2,\n#     s3_heads = 4,\n#     s3_depth = 10,\n#     s3_mlp_mult = 4,\n#     dropout = 0.\n# ).to(device)\n\n# model = nn.Sequential(\n#                 v,\n# #                 nn.Dropout(p=.5),\n#                 nn.Linear(in_features=256, out_features=128, bias=True),\n#                 nn.Linear(in_features=128, out_features=5, bias=True, ),\n#                 nn.Sigmoid()\n#             ).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deep ViT","metadata":{}},{"cell_type":"code","source":"# import torch\n# from vit_pytorch.deepvit import DeepViT\n\n# v = DeepViT(\n#     image_size = 224,\n#     patch_size = 32,\n#     num_classes = 256,\n#     dim = 1024,\n#     depth = 6,\n#     heads = 16,\n#     mlp_dim = 2048,\n#     dropout = 0.1,\n#     emb_dropout = 0.1\n# ).to(device)\n\n\n# model = nn.Sequential(\n#                 v,\n# #                 nn.Dropout(p=.5),\n#                 nn.Linear(in_features=256, out_features=128, bias=True),\n#                 nn.Linear(in_features=128, out_features=5, bias=True, ),\n#                 nn.Sigmoid()\n#             ).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### LOSS FUNCTION AND OPTIMIZER","metadata":{}},{"cell_type":"code","source":"# loss function\ncriterion = nn.CrossEntropyLoss()\n# criterion = nn.MSELoss()\n# criterion = nn.BCELoss()\n# optimizer\noptimizer = optim.Adam(model.parameters(), lr=lr)\n# scheduler\nscheduler = StepLR(optimizer, step_size=1, gamma=gamma)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:25:09.442763Z","iopub.execute_input":"2021-12-09T15:25:09.443014Z","iopub.status.idle":"2021-12-09T15:25:09.447991Z","shell.execute_reply.started":"2021-12-09T15:25:09.442985Z","shell.execute_reply":"2021-12-09T15:25:09.447162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data, label in tqdm(train_loader):\n    print(model(data.to(device)))\n    break","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:26:59.381079Z","iopub.execute_input":"2021-12-09T15:26:59.381612Z","iopub.status.idle":"2021-12-09T15:26:59.744174Z","shell.execute_reply.started":"2021-12-09T15:26:59.381577Z","shell.execute_reply":"2021-12-09T15:26:59.743031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Time to shine! ","metadata":{}},{"cell_type":"code","source":"train_loss = []\nvalidation_loss = []\nfor epoch in range(epochs):\n    epoch_loss = 0\n    epoch_accuracy = 0\n\n    for data, label in tqdm(train_loader):\n        data = data.to(device)\n        label = label.to(device)\n        output = model(data)\n        loss = criterion(output, label)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        epoch_loss += loss / len(train_loader)\n\n    with torch.no_grad():\n        epoch_val_accuracy = 0\n        epoch_val_loss = 0\n        for data, label in valid_loader:\n            data = data.to(device)\n            label = label.to(device)\n\n            val_output = model(data)\n            val_loss = criterion(val_output, label)\n\n            epoch_val_loss += val_loss / len(valid_loader)\n\n    print(\n        f\"Epoch : {epoch+1} - loss : {epoch_loss:.4f} - val_loss : {epoch_val_loss:.4f}\\n\"\n    )\n    train_loss.append(epoch_loss)\n    validation_loss.append(epoch_val_loss)\n    print(\"Learning Rate =\", scheduler.get_last_lr())\n    scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:25:10.44936Z","iopub.execute_input":"2021-12-09T15:25:10.44986Z","iopub.status.idle":"2021-12-09T15:25:10.778137Z","shell.execute_reply.started":"2021-12-09T15:25:10.449825Z","shell.execute_reply":"2021-12-09T15:25:10.777024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Generating predicitons from test set for leaderboard.","metadata":{}},{"cell_type":"code","source":"# pred = np.array([])\n# for data in tqdm(test_loader):\n#     with torch.no_grad():\n#         data = data.to(device)\n#         test_output = model(data)\n#         test_output = ((test_output > 0.5).sum(axis=1) - 1).item()\n#         pred = np.append(pred, test_output)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.array([])\nfor data in tqdm(test_loader):\n    with torch.no_grad():\n        data = data.to(device)\n        test_output = model(data)\n        test_output = np.argmax(test_output.to('cpu'), axis = 1).item()\n        pred = np.append(pred, test_output)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:12:28.237019Z","iopub.execute_input":"2021-12-09T15:12:28.237568Z","iopub.status.idle":"2021-12-09T15:13:08.868001Z","shell.execute_reply.started":"2021-12-09T15:12:28.237529Z","shell.execute_reply":"2021-12-09T15:13:08.867192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating submission.csv that the leaderboard will read from.","metadata":{}},{"cell_type":"code","source":"df_test['diagnosis'] = pred\ndf_test['diagnosis'] = df_test['diagnosis'].astype(int)\n\ndf_test.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:13:08.869724Z","iopub.execute_input":"2021-12-09T15:13:08.870065Z","iopub.status.idle":"2021-12-09T15:13:08.888427Z","shell.execute_reply.started":"2021-12-09T15:13:08.870024Z","shell.execute_reply":"2021-12-09T15:13:08.887705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Lets see if the distributions of classes in test match train","metadata":{}},{"cell_type":"code","source":"df_test.diagnosis.value_counts()/len(df_test)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:13:08.889643Z","iopub.execute_input":"2021-12-09T15:13:08.890049Z","iopub.status.idle":"2021-12-09T15:13:08.900275Z","shell.execute_reply.started":"2021-12-09T15:13:08.89001Z","shell.execute_reply":"2021-12-09T15:13:08.899171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.diagnosis.value_counts()/len(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:13:08.902897Z","iopub.execute_input":"2021-12-09T15:13:08.903317Z","iopub.status.idle":"2021-12-09T15:13:08.910901Z","shell.execute_reply.started":"2021-12-09T15:13:08.903278Z","shell.execute_reply":"2021-12-09T15:13:08.909867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sure hope they are similar!","metadata":{}},{"cell_type":"code","source":"training = pd.DataFrame()\ntraining[\"train loss\"] = train_loss\ntraining[\"val loss\"] = validation_loss\ntraining = training.applymap(lambda x : x.item())\ntraining.plot()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:13:08.912302Z","iopub.execute_input":"2021-12-09T15:13:08.913027Z","iopub.status.idle":"2021-12-09T15:13:09.144751Z","shell.execute_reply.started":"2021-12-09T15:13:08.912986Z","shell.execute_reply":"2021-12-09T15:13:09.144046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loss","metadata":{"execution":{"iopub.status.busy":"2021-12-09T15:13:09.145948Z","iopub.execute_input":"2021-12-09T15:13:09.146196Z","iopub.status.idle":"2021-12-09T15:13:09.161728Z","shell.execute_reply.started":"2021-12-09T15:13:09.146163Z","shell.execute_reply":"2021-12-09T15:13:09.161083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}