{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Another ditch effort, to beat the top 65% Public LB score\nMoving to pytorch, tensorflow is too magical :)\nThe older kernel is here - https://www.kaggle.com/sudhanshuraheja/siim-isic-dataset/edit/run/40536221","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport math\nimport time\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torchvision\nimport torchvision.transforms as transforms\nfrom PIL import Image\n\nfrom sklearn.metrics import *\nfrom sklearn.preprocessing import StandardScaler\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get access to cuda\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# use cpu for preprocessing and setting up the initial model to test\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The plan\nUse transfer learning to create a model to process images and get a score\n\nMust read - http://karpathy.github.io/2019/04/25/recipe/#2-set-up-the-end-to-end-trainingevaluation-skeleton--get-dumb-baselines","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED=42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data from csv and images\nclass Data():\n    def __init__(self, base, resized):\n        self.base = '/kaggle/input/' + base\n        self.resized = '/kaggle/input/' + resized\n        \n    def b(self, path):\n        return self.base + path\n    \n    def r(self, path):\n        return self.resized + path\n    \n    def read(self, file):\n        return pd.read_csv(self.b(file))\n    \ndata = Data(base='siim-isic-melanoma-classification/', resized='jpeg-melanoma-256x256/')\n\ndf_train = data.read('train.csv')\ndf_test = data.read('test.csv')\ndf_submit = data.read('sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# patients who were diagnosed\ndf_train[df_train['target']==1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_new_size(width, height, minimum, width_first=True):\n    h, w = 0, 0\n    if width >= height:\n        w = round(minimum * (width / height))\n        h = minimum\n    else:\n        w = minimum\n        h = round(minimum * (height / width))\n    if width_first:\n        return w, h\n    else:\n        return h, w\n\nprint(get_new_size(6000, 40000, 256))\nprint(get_new_size(6000, 40000, 256, width_first=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup a grid to show random images from subset\ndef show_grid(df, cols=9, rows=4):\n    if df.shape[0] == 0:\n        return\n    plt.figure(figsize=(18,9))\n    for i in range(min(df.shape[0], cols * rows)):\n        plt.subplot(rows, cols, i+1, xticks=[], yticks=[])\n        idx = np.random.randint(0, df.shape[0], 1)[0]\n        im = Image.open(data.r('train/' + df.iloc[idx]['image_name'] + '.jpg'))\n        plt.imshow(im)\n        plt.xlabel(df.iloc[idx]['benign_malignant'])\n        plt.ylabel(df.iloc[idx]['anatom_site_general_challenge'])\n    plt.show()\n\n# Check young people\n# show_grid(df_train[(df_train['age_approx'] < 40.0) & (df_train['target'] == 1)])\n# Check diagnosis\n# show_grid(df_train[(df_train['diagnosis'] == 'melanoma') & (df_train['target'] == 1)])\n# Check a single patient\npat = 'IP_0962375'\nshow_grid(df_train[(df_train['patient_id'] == pat) & (df_train['target'] == 1)])\nshow_grid(df_train[(df_train['patient_id'] == pat) & (df_train['target'] == 0)])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mark male female as 1/0\n# There are only two values, there are some missing values, which should be filled with mode\ndf_train['sex'] = df_train['sex'].replace({ 'female': 0, 'male': 1 })\ndf_test['sex'] = df_test['sex'].replace({ 'female': 0, 'male': 1 })\ndf_train['sex'].fillna(df_train['sex'].mode()[0], inplace=True)\n\n# Remove benign malignant, it's the same as target\ndf_train.drop(['benign_malignant'], inplace=True, axis=1)\n\n# Add dummies for anatom_site_general_challenge\n# Fill the nan's with a new dummy\ndef add_dummies(dataset, column, short_name):\n    dummy = pd.get_dummies(\n        dataset[column], \n        drop_first=True, \n        prefix=short_name, \n        prefix_sep='_',\n        dummy_na=True\n    )\n    merged = pd.concat([dataset, dummy], axis=1)\n    return merged.drop([column], axis=1)\n\ndf_train = add_dummies(df_train, 'anatom_site_general_challenge', 'anatom')\ndf_test = add_dummies(df_test, 'anatom_site_general_challenge', 'anatom')\n\n# Diagnosis is only in train, removing it\ndf_train.drop(['diagnosis'], inplace=True, axis=1)\n\n# Age has some missing values, fill with median\ndf_train['age_approx'].fillna(df_train['age_approx'].median(), inplace=True)\n\n# %% [code]\n# Check how many times are their images taken\ndf_train['image_count'] = df_train['patient_id'].map(df_train.groupby(['patient_id'])['image_name'].count())\ndf_test['image_count'] = df_test['patient_id'].map(df_test.groupby(['patient_id'])['image_name'].count())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# How does one scale in pytorch :(\n\nsc = StandardScaler()\n\ndef scale(df, cols_to_remove, fit=False):\n    removed = df[cols_to_remove]\n    df = df.drop(cols_to_remove, axis=1)\n    cols = df.columns\n    if fit:\n        df = sc.fit_transform(df)\n    else:\n        df = sc.transform(df)\n    df = pd.DataFrame(df, columns=cols)\n    df[cols_to_remove] = removed\n    return df\n\ndf_train = scale(df_train, fit=True, cols_to_remove=['image_name', 'patient_id', 'target'])\ndf_test = scale(df_test, cols_to_remove=['image_name', 'patient_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Experiment with pytorch transforms\n\n# ## read image\n# size = (256, 256)\n# im_path = get_path('jpeg/train/ISIC_0015719.jpg')\n# start = time.time()\n# im = Image.open(im_path)\n# print('Image.open: ', time.time() - start)\n# # print(im.format, im.size, im.mode, type(im))\n# # print(im.size[0], im.size[1])\n# # print(im.format, im.size, im.mode)\n\n# # im = im.resize(get_new_size(im.size[0], im.size[1], 256))\n# start = time.time()\n# im = transforms.Resize(get_new_size(im.size[0], im.size[1], 256, width_first=False))(im)\n# print('Resize: ', time.time() - start)\n# start = time.time()\n# im = transforms.CenterCrop(256)(im)\n# print('CenterCrop: ', time.time() - start)\n# plt.imshow(im)\n# plt.show()\n\n# # im = transforms.ColorJitter(brightness=0, contrast=0, saturation=0, hue=0)(im)\n# # im = transforms.Grayscale(1)(im)\n# # im = transforms.Pad(64, fill=(255, 255, 255), padding_mode='symmetric')(im)\n# # im2 = transforms.RandomAffine(degrees=0, translate=(0, 0), scale=None, shear=None)(im) # , scale=(1, 0.1)\n# # im2 = transforms.RandomApply([transforms.Grayscale(1)], 0.5)(im)\n# # im2 = transforms.RandomChoice([\n# #     transforms.Grayscale(1),\n# #     transforms.Pad(64, fill=(255, 255, 255), padding_mode='symmetric'),\n# # ])(im)\n# # im2 = transforms.RandomHorizontalFlip(0.99)(im)\n# # im2 = transforms.RandomOrder([\n# #     transforms.Grayscale(1),\n# #     transforms.Pad(64, fill=(255, 255, 255), padding_mode='symmetric'),\n# # ])(im)\n# # im2 = transforms.RandomPerspective(distortion_scale=0.5)(im)\n# # im2 = transforms.RandomVerticalFlip(0.99)(im)\n# # im2 = transforms.LinearTransformation(transformation_matrix=[0,0,0], mean_vector=[0,0,0])(im) # not sure\n# # im2 = transforms.ToPILImage()(\n# #     transforms.RandomErasing(p=0.99)(\n# #         transforms.ToTensor()(im)\n# #     )\n# # )\n# # im2 = transforms.functional.adjust_brightness(im, brightness_factor=2)\n# # im2 = transforms.functional.adjust_contrast(im, contrast_factor=3)\n# start = time.time()\n# im2 = transforms.functional.adjust_gamma(im, gamma=0.5)\n# print('adjust_gamma: ', time.time() - start)\n# plt.imshow(im2)\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualise data by segmenting each parameter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find and note down outliers","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Dataset for train and test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE=128\n\nclass PatientImages(Dataset):\n    \n    def __init__(self, df, is_training=True, augment=False, debug=False):\n        super(PatientImages, self).__init__()\n        self.is_training = is_training\n        self.augment = augment\n        self.debug = debug\n\n        if self.is_training:\n            self.x = df.drop(['target'], axis=1).values\n            self.y = df['target'].values\n        else:\n            self.x = df.values\n        self.columns = df.columns\n    \n    def __getitem__(self, idx):\n        x = self.x[idx]\n        y = None\n        if self.is_training:\n            y = self.y[idx]\n        \n        img = x[9]\n        train_or_test = 'train' if self.is_training == True else 'test'\n        img_path = data.r(train_or_test + '/' + img + '.jpg')\n        img = Image.open(img_path)\n\n        if self.debug:\n            plt.imshow(img)\n            plt.show()\n\n        adjusted_size = self.get_adjusted_size(img.size[0], img.size[1], SIZE)\n\n        transform = transforms.Compose([\n            # transforms.Resize(adjusted_size), # Using https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256\n            # transforms.CenterCrop(SIZE), # Using https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256\n            # transforms.ToTensor(), # Applied manually\n        ])\n\n        if self.augment:\n            transform = transforms.Compose([\n                # transforms.Resize(adjusted_size), # Using https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256\n                # transforms.CenterCrop(SIZE), # Using https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256\n                transforms.RandomHorizontalFlip(p=0.25),\n                transforms.RandomVerticalFlip(p=0.25),\n                transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n                transforms.RandomAffine(degrees=10, shear=2),\n                transforms.RandomPerspective(p=0.05, distortion_scale=0.1),\n                transforms.ToTensor(),\n#                     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n                transforms.RandomErasing(p=0.2, scale=(0.02, 0.2), value='random'),\n                transforms.ToPILImage(),\n                # transforms.ToTensor(), # Applied manually\n            ])\n            \n        img = transform(img)\n        if self.debug:\n            plt.imshow(img)\n            plt.show()\n        x_img = transforms.ToTensor()(img)\n\n        x = x[:-2]\n                    \n        if type(x).__module__ == 'numpy':\n            x = x.astype(np.float32)\n            \n        if self.is_training:\n            return (x, x_img), y\n        else:\n            return (x, x_img)\n    \n    def __len__(self):\n        return self.x.shape[0]\n    \n    def columns(self):\n        return self.columns\n    \n    def get_adjusted_size(self, width, height, minimum):\n        h, w = 0, 0\n        if width >= height:\n            w = round(minimum * (width / height))\n            h = minimum\n        else:\n            w = minimum\n            h = round(minimum * (height / width))\n            \n        # if self.debug:\n            # print(f'Original: h{height} w{width} m{minimum} - Calculated: h{h} w{w}')\n        return h, w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test Dataset\nstart = time.time()\ntrain_test = PatientImages(df_train, augment=True, debug=True)\nprint(train_test[50])\nprint(train_test.__len__())\nend = time.time()\nprint('Timed', end - start)\n\nstart = time.time()\ntest_test = PatientImages(df_test, is_training=False, debug=True)\nprint(test_test[50])\nprint(test_test.__len__())\nend = time.time()\nprint('Timed', end - start)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create DataLoader for train and test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE=64\n\nstart = time.time()\ntrain_ds = PatientImages(df_train)\ntest_ds = PatientImages(df_test, is_training=False)\nend = time.time()\nprint('Timed', end-start)\n\nstart = time.time()\ntrain_dl = DataLoader(dataset=train_ds, batch_size=BATCH_SIZE, shuffle=True)\ntest_dl = DataLoader(dataset=test_ds, batch_size=BATCH_SIZE, shuffle=True)\nend = time.time()\nprint('Timed', end-start)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test Dataloaders\nstart = time.time()\nprint(train_ds)\nprint(test_ds)\nend = time.time()\nprint('Timed _ds', end-start)\n\nstart = time.time()\nprint(train_ds[0])\nprint(test_ds[0])\nend = time.time()\nprint('Timed _ds[0]', end-start)\n\nstart = time.time()\nprint(len(train_dl))\nprint(len(test_dl))\nend = time.time()\nprint('Timed _dl', end-start)\n\nstart = time.time()\nfeatures, label = iter(train_dl).next()\nprint(features[0].shape, features[1].shape, label.shape)\nend = time.time()\nprint('Timed train_images_dl[0]', end-start)\n\nstart = time.time()\nfeatures = iter(test_dl).next()\nprint(features[0].shape, features[1].shape)\nend = time.time()\nprint('Timed test_images_dl[0]', end-start)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup model pipelines","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Start with dumb pipeline\n## Start with a dumb linear regression model\n## Define how to merge data from csv files\n## Setup initial bias properly for layers\n## Verify loss at start, should be equal to -log(1/n_classes) on softmax\n## Setup an input with all 0s to test model output\n## overfit one batch with 2 samples, check if loss is 0\n## As you add to the model, loss should go down\n## Visualise data right before it goes into the model before model(x)\n## Visualise prediction dynamics\n# Overfit\n## Start with standard models\n## Use adam, starting with lr 3e-4\n## add complexity one at a time, add signals later\n## change the learning rate decay, or keep it at 0, let me the model converge\n# Regularise\n## visualise the first layer weights\n## augment data\n## add ensembles\n## use pre-trained models\n## keep the batch size low\n## add dropout or dropout2d\n## increase weight decay penalty\n## early stopping\n## last try a larger model\n# Tune\n## random over grid search\n## hyper parameter optimisation\n# Squeeze\n## ensembles\n## leave it training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup the model\nclass NN(nn.Module):\n    \n    def __init__(self):\n        super(NN, self).__init__()\n\n        self.efn = EfficientNet.from_pretrained('efficientnet-b1')\n        self.efn._fc = nn.Linear(1280, 500, bias=True)\n        \n        self.meta = nn.Sequential(\n            nn.Linear(9, 500),\n            nn.BatchNorm1d(500),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(500, 250),\n            nn.BatchNorm1d(250),\n            nn.ReLU(),\n            nn.Dropout(p=0.2)\n        )\n        \n        self.output = nn.Linear(500 + 250, 1)\n\n        \n    def forward(self, x):\n        meta, images = x\n        cnn = self.efn(images)\n        others = self.meta(meta)\n        features = torch.cat((cnn, others), dim=1)\n        output = self.output(features)\n        return output\n\nlearning_rate = 1e-2\nmodel = NN().to(device)\ncriterion = nn.BCEWithLogitsLoss()\noptimiser = torch.optim.Adam(model.parameters(), lr=learning_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.cuda.empty_cache()\n\n# Setup the training loop\nn_epochs = 3\ntotal_steps = len(train_dl)\n\npbar = tqdm(total=len(train_dl), desc='Epoch: 0, Loss: 0.000000')\nfor epoch in range(n_epochs):\n    pbar.reset()\n    for i, (dl_data, labels) in enumerate(train_dl):\n        start = time.time()\n        meta, images = dl_data\n        \n        meta = meta.to(device)\n        images = images.to(device)\n        labels = labels.to(device)\n            \n        labels = labels.view(labels.shape[0], -1).float()\n        \n        model.train()\n        outputs = model((meta, images))\n        loss = criterion(outputs, labels)\n        \n        loss.backward()\n        optimiser.step()\n        optimiser.zero_grad()\n        \n        pbar.update(1)\n        pbar.set_description(f'Epoch: {epoch+1}, Loss: {loss.item():.6f}')\n        \npbar.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# [ (name, param.data) for name, param in model.named_parameters()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model, 'model.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup the evaluation functions\n\nwith torch.no_grad():\n    count = 0\n    op_ = None\n    lb_ = None\n    \n    pbar = tqdm(total=len(train_dl), desc='Accuracy: 00.0000%')\n\n    start = time.time()\n    for dl_data, labels in train_dl:\n        meta, images = dl_data\n        meta = meta.to(device)\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        model.eval()\n        output = model((meta, images))\n        output = torch.sigmoid(output)\n        output = output.squeeze()\n        \n        op_ = output if op_ == None else torch.cat((op_, output), 0)        \n        lb_ = labels if lb_ == None else torch.cat((lb_, labels), 0)\n\n        count += 1\n        pbar.update(1)\n    \n        op = op_.cpu()\n        op = op.round().long()\n        lb = lb_.cpu()\n        lb = lb\n        \n        pbar.set_postfix(\n            { \n                'f1': 100*f1_score(lb, op, average='weighted'), \n                'logloss': log_loss(lb, op, labels=[0, 1])\n            }\n        )\n        pbar.set_description(f'Accuracy: {accuracy_score(lb, op)*100:.4f}%')\n            \n    pbar.close()\n    print('Timed:', time.time() - start)\n            \n    print('CM', confusion_matrix(lb, op))\n    print('CR', classification_report(lb, op))\n    print(precision_recall_fscore_support(lb, op))\n    print('PRCurve', precision_recall_curve(lb, op))\n    print('Pri', precision_score(lb, op))\n    print('Rec', recall_score(lb, op))\n    print('ROC', roc_auc_score(lb, op))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare merging data","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}