{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom typing import List\nimport os\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom PIL import Image\nimport glob\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms, models\nfrom torch.nn import functional as F\nfrom torchvision.utils import make_grid\n\n\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nfrom sklearn import metrics\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## custom dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset\nclass ClassificationDataset:\n    \"\"\"classification dataset.\"\"\"\n\n    def __init__(self, images: List[str], cats, targets: List[str], transform=None):\n        self.images = images\n        self.cats = cats\n        self.targets = targets\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        image = Image.open(self.images[idx])\n        target = self.targets[idx]\n\n        if self.transform:\n            image = self.transform(image)\n\n        cat = self.cats[idx, :]\n\n        return image, cat, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## categorical features, embedding sizes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train_df['age_approx'],kde=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.crosstab(train_df['target'],train_df['anatom_site_general_challenge']).apply(lambda r: r/r.sum(), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.crosstab(train_df['target'],train_df['sex']).apply(lambda r: r/r.sum(), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_vars = [\"sex\", \"age_approx\", \"anatom_site_general_challenge\"]\n# Convert categorical columns to category dtypes.\nfor cat in cat_vars:\n    train_df[cat] = train_df[cat].fillna(\"#na\")\n    train_df[cat] = train_df[cat].astype(\"category\")\n    test_df[cat] = test_df[cat].fillna(\"#na\")\n    test_df[cat] = test_df[cat].astype(\"category\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_szs = [len(train_df[col].cat.categories) for col in cat_vars]\nemb_szs = [(size, min(50, (size + 1) // 2)) for size in cat_szs]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"split = StratifiedShuffleSplit(n_splits=1, test_size=0.22, random_state=0)\n\nX = train_df[cat_vars].values\ny = train_df.target.values\n\nfor trn_idx, vld_idx in split.split(X, y):\n    train_index = trn_idx\n    valid_index = vld_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_df = train_df.loc[train_index]\nvalid_df = train_df.loc[valid_index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_targets = new_train_df[\"target\"].values.astype(np.float32)\nvalid_targets = valid_df[\"target\"].values.astype(np.float32)\ntest_targets = np.zeros(len(test_df)).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = [\n    f\"../input/melanoma/train_resized/train_resized/{img}.jpg\" for img in new_train_df[\"image_name\"].values\n]\nvalid_images = [\n    f\"../input/melanoma/train_resized/train_resized/{img}.jpg\" for img in valid_df[\"image_name\"].values\n]\ntest_images = [    f\"../input/melanoma/test_resized/test_resized/{img}.jpg\" for img in test_df[\"image_name\"].values]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # orig\n# train_images = [\n#     f\"../input/siim-isic-melanoma-classification/jpeg/train/{img}.jpg\" for img in new_train_df[\"image_name\"].values\n# ]\n# valid_images = [\n#     f\"../input/siim-isic-melanoma-classification/jpeg/train/{img}.jpg\" for img in valid_df[\"image_name\"].values\n# ]\n# test_images = [\n#     f\"../input/siim-isic-melanoma-classification/jpeg/test/{img}.jpg\" for img in test_df[\"image_name\"].values\n# ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cats = torch.tensor(\n    np.stack([new_train_df[col].cat.codes.values for col in cat_vars], 1), dtype=torch.int64\n)\nvalid_cats = torch.tensor(\n    np.stack([valid_df[col].cat.codes.values for col in cat_vars], 1), dtype=torch.int64\n)\ntest_cats = torch.tensor(\n    np.stack([test_df[col].cat.codes.values for col in cat_vars], 1), dtype=torch.int64\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose(\n    [\n        transforms.RandomRotation(30),      # rotate +/- 30 degrees\n        transforms.RandomHorizontalFlip(),  # flip horizontal 50%\n        transforms.RandomVerticalFlip(),  # flip vertical 50%\n        transforms.ColorJitter(brightness=0.1),\n        transforms.ToTensor(),        \n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ]\n)\n\nvalid_transform = transforms.Compose(\n    [\n        transforms.ToTensor(),        \n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ]\n)\n\n\ntrain_data = ClassificationDataset(\n    train_images, train_cats, train_targets, transform=train_transform\n)\nvalid_data = ClassificationDataset(\n    valid_images, valid_cats, valid_targets, transform=valid_transform\n)\ntest_data = ClassificationDataset(\n    test_images, test_cats, test_targets, transform=valid_transform\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_bs = 64\nvalid_bs = 911\ntest_bs = 38","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(42)\ntorch.manual_seed(42)\n\ntrain_loader = DataLoader(train_data, batch_size=train_bs, shuffle=True, pin_memory=True)\nvalid_loader = DataLoader(valid_data, batch_size=valid_bs, shuffle=False, pin_memory=True)\ntest_loader = DataLoader(test_data, batch_size=test_bs, shuffle=False, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for images, cat, labels in train_loader:\n    break\n\n# Print the labels\nprint(\"Label:\", labels.numpy())\n\nim = make_grid(images, nrow=5)  # the default nrow is 8\n\n# Inverse normalize the images\ninv_normalize = transforms.Normalize(\n    mean=[-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225],\n    std=[1 / 0.229, 1 / 0.224, 1 / 0.225],\n)\nim_inv = inv_normalize(im)\n\n# Print the images\nplt.figure(figsize=(15, 15))\nplt.imshow(np.transpose(im_inv.numpy(), (1, 2, 0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## model definition","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TabularCNNModel(nn.Module):\n    def __init__(self, emb_szs, layers, p=0.5):\n        super().__init__()\n\n        # tabular part\n        self.embeds = nn.ModuleList([nn.Embedding(ni, nf) for ni, nf in emb_szs])\n        self.emb_drop = nn.Dropout(0.04)\n        layerlist = []\n        n_emb = np.array([nf for ni, nf in emb_szs]).sum()\n        n_in = n_emb\n\n        # layers is a list of number of neurons\n        for i in layers:\n            layerlist.append(nn.Linear(n_in, i))\n            layerlist.append(nn.ReLU(inplace=True))\n            layerlist.append(nn.BatchNorm1d(i))            \n            layerlist.append(nn.Dropout(0.001))\n            n_in = i\n\n        self.tabular_layers = nn.Sequential(*layerlist)\n\n        # image part\n        # freeze model\n        self.cnn = models.resnet34(pretrained=True)\n        \n#         for param in self.cnn.parameters():\n#             param.requires_grad = False\n\n        self.cnn.fc = nn.Sequential(\n            nn.Linear(512, 200), \n            nn.ReLU(inplace=True),\n            nn.BatchNorm1d(200),\n            nn.Dropout(0.001),\n#             nn.Linear(1000, 200), \n#             nn.ReLU(inplace=True),\n#             nn.BatchNorm1d(200),\n#             nn.Dropout(0.01),            \n        )\n\n        # combined layerlist\n        self.classifier = nn.Sequential(\n            nn.Linear(400, 200), \n            nn.ReLU(inplace=True),            \n            nn.BatchNorm1d(200),\n            nn.Dropout(0.01),\n            nn.Linear(200, 1)\n        )\n\n    def forward(self, x_cat, images):\n\n        # tabular\n        embeddings = []\n        for i, e in enumerate(self.embeds):\n            embeddings.append(e(x_cat[:, i]))\n\n        x = torch.cat(embeddings, 1)\n        x = self.emb_drop(x)\n        x = self.tabular_layers(x)\n\n        # cnn\n        y = self.cnn(images)\n\n        # combined\n        xy = torch.cat([x, y], 1)\n        xy = self.classifier(xy)\n\n        return xy.squeeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = TabularCNNModel(emb_szs=emb_szs, layers=[1000, 500, 200])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpumodel = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpumodel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## loss, optimizer, scheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\n\n# optimizer = torch.optim.AdamW(gpumodel.parameters(), lr=1e-4)\n\noptimizer = torch.optim.AdamW([{'params': gpumodel.cnn.parameters(), 'lr':1e-4 },\n                              {'params': gpumodel.classifier.parameters(), 'lr':2e-2 }], \n                              lr=1e-3\n                             )\n\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, patience=3, threshold=0.001, mode=\"max\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_checkpoint(state, is_best, filename='checkpoint_34_new.pth.tar'):\n    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n    if is_best:\n        print (\"=> Saving a new best\")\n        torch.save(state, filename)  # save checkpoint\n    else:\n        print (\"=> Validation Accuracy did not improve\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 25\n\ntrain_losses = []\nval_losses = []\nval_aucs = []\n\nbest_auc = 0\n\nfor i in range(epochs):\n    start_time = time.time()\n    \n    gpumodel.train()\n\n    # Run the training batches\n    for img_train, cat_train, y_train in train_loader:\n        \n        img_cuda = img_train.cuda()\n        cat_cuda = cat_train.cuda()\n        y_cuda = y_train.cuda()\n        # predict\n        y_pred = gpumodel(cat_cuda, img_cuda)\n        train_loss = criterion(y_pred, y_cuda)\n\n        # Update parameters\n        optimizer.zero_grad()\n        train_loss.backward() \n        optimizer.step()\n\n    train_losses.append(train_loss)\n        \n    model.eval()\n    val_preds = []\n    with torch.no_grad():\n        for  img_val, cat_val, y_val in valid_loader:\n            imgv_cuda = img_val.cuda()\n            catv_cuda = cat_val.cuda()\n            yv_cuda = y_val.cuda()\n            \n            # Apply the model\n            y_pred = gpumodel(catv_cuda, imgv_cuda)\n            val_loss = criterion(y_pred, yv_cuda)\n            val_preds.append(y_pred.cpu().numpy())\n        \n        val_losses.append(val_loss)            \n        val_preds = np.vstack(val_preds).ravel()\n        val_auc = metrics.roc_auc_score(valid_targets, val_preds)\n        val_aucs.append(val_auc)\n    \n    print(f\"\"\"epoch: {i}/{epochs}: train loss: {train_loss}, \n          valid loss: {val_loss}, valid AUC {val_auc}\"\"\")\n\n    scheduler.step(val_auc)\n    \n    if val_auc > best_auc:\n        best_auc = val_auc\n        is_best= True\n    else:\n        is_best= False\n    \n    state = {'epoch': i + 1,\n            'state_dict': gpumodel.state_dict(),\n            'optim_dict' : optimizer.state_dict(),\n            'best_auc': best_auc}\n    save_checkpoint(state, is_best = is_best) # path to folder\n    \n    print(f'epoch took {(time.time()-start_time)//60} minutes')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_losses, label='train')\nplt.plot(val_losses, label='valid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"state = torch.load('checkpoint_50.pth.tar', map_location='cuda:0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"state['best_auc']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"state.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpumodel.load_state_dict(state['state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = []\nwith torch.no_grad():\n    for  img_tst, cat_tst, y_tst in test_loader:\n        img_cuda = img_tst.cuda()\n        cat_cuda = cat_tst.cuda()\n        y_cuda = y_tst.cuda()\n\n        # Apply the model\n        y_pred = gpumodel(cat_cuda, img_cuda)\n        test_preds.append(y_pred.cpu().numpy())\n    \n    test_preds = np.vstack(test_preds).ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.special import expit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['target'] = expit(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = test_df[['image_name','target']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('sub_50_6_6.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}