{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport skimage.io\nfrom tqdm.notebook import tqdm\nimport random\nimport cv2\nimport seaborn as sns\nimport os\n# plt.rcParams['figure.figsize'] = [15,8]\nfrom __future__ import print_function, division\nimport torch\nimport os\nimport torch\nfrom skimage import io, transform\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nfrom tqdm import tqdm\n\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nplt.ion()   # interactive mode\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# !pip install tqdm\n# # ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"DATA_DIR = './'\ntrain_csv = pd.read_csv(DATA_DIR + 'train.csv')\ntrain_csv.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv['sex'].unique()\n# train_csv['sex'].nunique()\ntrain_csv.sex.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.age_approx.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.groupby(['target']).size().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.anatom_site_general_challenge.unique()\n# train_csv.anatom_site_general_challenge.value_counts()\ntrain_csv['anatom_site_general_challenge'].isnull().value_counts\n#total 32577","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.benign_malignant.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"N=33126\nrandom_index = random.randint(0,N)\n# print(train_csv.iloc[random_index])\n#train_csv.iloc[random_index]['image_name'] \nimg = DATA_DIR + 'data/train/' + train_csv.iloc[random_index]['image_name']  + '.jpg'\n# img = DATA_DIR + 'jpeg/train/' +  'ISIC_0272509' + '.jpg'\ntry:\n    img = DATA_DIR + 'data/train/' + train_csv.iloc[random_index]['image_name']  + '.jpg'\n    plt.imshow(mpimg.imread(img))\nexcept:\n    print('in except')\n    img = DATA_DIR + 'data/val/' + train_csv.iloc[random_index]['image_name']  + '.jpg'\n    plt.imshow(mpimg.imread(img))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%html\nLinks to understand if pretrained imagenet model should be used or a smaller CNN :\nhttps://ai.googleblog.com/2019/12/understanding-transfer-learning-for.html\nhttp://openaccess.thecvf.com/content_ECCVW_2018/papers/11134/Xie_Pre-training_on_Grayscale_ImageNet_Improves_Medical_Image_Classification_ECCVW_2018_paper.pdf\n\nConverting images to grayscale images for the given dataset seems like a good idea since people of different skin types are likely to be present.","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#df['Charge_Per_Line'] = df['Charge_Per_Line'].replace('-', np.nan)\ntrain_csv = train_csv.dropna(axis=0, subset=['sex'])\ntrain_csv = train_csv.dropna(axis=0, subset=['age_approx'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.groupby(['anatom_site_general_challenge']).size().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv = train_csv.dropna(axis=0, subset=['anatom_site_general_challenge'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.groupby(['diagnosis']).size().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del train_csv['diagnosis']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del train_csv['patient_id']\ndel train_csv['benign_malignant']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"encoder = {\"sex\":     {\"male\": 0, \"female\": 1},\n            \"benign_malignant\": {\"benign\": 0, \"malignant\": 1 },\n \"anatom_site_general_challenge\": {\"head/neck\":0, \"upper extremity\":1, \"lower extremity\":2, \"torso\":3, \"palms/soles\":4, \"oral/genital\":5}}\ntrain_csv.replace(encoder, inplace=True)\ntrain_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv[1:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv['anatom_site_general_challenge'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"is_infected = train_csv['target'] == 1\ndf_try = train_csv[is_infected]\nprint(len(df_try))\ntrain_csv=train_csv.append([df_try]*53,ignore_index=True)\nprint(len(train_csv))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"root_dir='./data/'\ntrain_csv['directory'] = -1\nprint(train_csv.shape[0])\ntrain_benign = 0\ntrain_malignant = 0\nval_benign = 0\nval_malignant=0\nfor idx in range(0, train_csv.shape[0]):\n    try:\n        img_name = os.path.join(root_dir, 'train',\n                                train_csv.iloc[idx, 0]) + '.jpg'\n        image = io.imread(img_name)\n        train_csv.iloc[idx, train_csv.columns.get_loc('directory')] = 0\n        if train_csv.iloc[idx, train_csv.columns.get_loc('target')] == 0 :\n            train_benign += 1 \n        else:\n            train_malignant += 1\n    except:\n        img_name = os.path.join(root_dir, 'val',\n                                train_csv.iloc[idx, 0]) + '.jpg'\n        image = io.imread(img_name)\n        train_csv.iloc[idx, train_csv.columns.get_loc('directory')] = 1\n        if train_csv.iloc[idx, train_csv.columns.get_loc('target')] == 0 :\n            val_benign += 1 \n        else:\n            val_malignant += 1\nprint(train_benign)\nprint(train_malignant)\nprint(val_benign)\nprint(val_malignant)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"total_train = train_benign + train_malignant\nbenign_weight = train_benign/total_train\nmalignant_weight = train_malignant/total_train\n# print('train: ', train)\n# print('val: ', val)\nbenign_weight = 0.98\nmalignant_weight = 0.02\nprint('benign_weight: ', benign_weight)\nprint('malignant_weight: ', malignant_weight)\ntrain_csv.directory.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"val_csv = train_csv[train_csv.directory == 1]\ntrain_csv = train_csv[train_csv.directory == 0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_csv.directory.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class MelanomaCancerDataset(Dataset):\n    \"\"\"Melanoma Cancer dataset.\"\"\"\n\n    def __init__(self, csv_dataframe, root_dir, transform=None):\n        \"\"\"\n        Args:\n            csv_file (string): Path to the csv file with annotations.\n            root_dir (string): Directory with all the images.\n            transform (callable, optional): Optional transform to be applied\n                on a sample.\n        \"\"\"\n        self.csv_dataframe = csv_dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.csv_dataframe)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        img_name = os.path.join(self.root_dir,\n                                self.csv_dataframe.iloc[idx, 0]) + '.jpg'\n        image_name = self.csv_dataframe.iloc[idx, 0]\n        image = io.imread(img_name)\n        csv_data = self.csv_dataframe.iloc[idx, 1:4]\n        csv_data = np.array([csv_data])\n        csv_data = csv_data.astype('double').reshape(-1, 3).flatten()\n        labels = self.csv_dataframe.iloc[idx, 4]\n        labels = labels.reshape(-1, 1).flatten()\n        sample = {'image': image, 'csv_data': csv_data, 'labels': labels, 'image_name': image_name }\n\n        if self.transform:\n            sample = self.transform(sample)\n\n        return sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"melanoma_dataset = MelanomaCancerDataset(csv_dataframe=train_csv,\n                                    root_dir='./data/train/')\n\nfig = plt.figure()\n\nfor i in range(len(melanoma_dataset)):\n    sample = melanoma_dataset[i]\n\n    print(i, sample['image'].shape, sample['csv_data'].shape, sample['labels'].shape)\n\n    ax = plt.subplot(1, 4, i + 1)\n    plt.tight_layout()\n    ax.set_title('Sample #{}'.format(i))\n    ax.axis('off')\n    plt.imshow(sample['image'])\n\n    if i == 3:\n        plt.show()\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class Rescale(object):\n    \"\"\"Rescale the image in a sample to a given size.\n\n    Args:\n        output_size (tuple or int): Desired output size. If tuple, output is\n            matched to output_size. If int, smaller of image edges is matched\n            to output_size keeping aspect ratio the same.\n    \"\"\"\n\n    def __init__(self, output_size):\n        assert isinstance(output_size, (int, tuple))\n        self.output_size = output_size\n\n    def __call__(self, sample):\n        image, csv_data, labels = sample['image'], sample['csv_data'], sample['labels']\n\n        h, w = image.shape[:2]\n        if isinstance(self.output_size, int):\n            if h > w:\n                new_h, new_w = self.output_size * h / w, self.output_size\n            else:\n                new_h, new_w = self.output_size, self.output_size * w / h\n        else:\n            new_h, new_w = self.output_size\n\n        new_h, new_w = int(new_h), int(new_w)\n\n        img = transform.resize(image, (new_h, new_w))\n\n        # h and w are swapped for landmarks because for images,\n        # x and y axes are axis 1 and 0 respectively\n#         landmarks = landmarks * [new_w / w, new_h / h]\n\n        return {'image': img, 'csv_data': csv_data, 'labels': labels}\n\n\nclass RandomCrop(object):\n    \"\"\"Crop randomly the image in a sample.\n\n    Args:\n        output_size (tuple or int): Desired output size. If int, square crop\n            is made.\n    \"\"\"\n\n    def __init__(self, output_size):\n        assert isinstance(output_size, (int, tuple))\n        if isinstance(output_size, int):\n            self.output_size = (output_size, output_size)\n        else:\n            assert len(output_size) == 2\n            self.output_size = output_size\n\n    def __call__(self, sample):\n        image, csv_data, labels = sample['image'], sample['csv_data'], sample['labels']\n\n        h, w = image.shape[:2]\n        new_h, new_w = self.output_size\n\n        top = np.random.randint(0, h - new_h)\n        left = np.random.randint(0, w - new_w)\n\n        image = image[top: top + new_h,\n                      left: left + new_w]\n\n#         landmarks = landmarks - [left, top]\n\n        return {'image': image, 'csv_data': csv_data, 'labels': labels}\n\n\nclass ToTensor(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n\n    def __call__(self, sample):\n        image, csv_data, labels, image_name = sample['image'], sample['csv_data'], sample['labels'], sample['image_name']\n\n        # swap color axis because\n        # numpy image: H x W x C\n        # torch image: C X H X W\n        image = image.transpose((2, 0, 1))\n        return {'image': torch.from_numpy(image),\n                'csv_data': torch.from_numpy(csv_data), 'labels': torch.from_numpy(labels), 'image_name': image_name }","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n# crop = RandomCrop(1000)\n# scale = Rescale(128)\n# composed = transforms.Compose([Rescale(1024),\n#                                RandomCrop(128)])\n\n# # Apply each of the above transforms on sample.\n# fig = plt.figure()\n# sample = melanoma_dataset[65]\n# for i, tsfrm in enumerate([scale, crop, composed]):\n#     transformed_sample = tsfrm(sample)\n\n#     ax = plt.subplot(1, 3, i + 1)\n#     plt.tight_layout()\n#     ax.set_title(type(tsfrm).__name__)\n#     plt.imshow(sample['image'])\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"transformed_dataset = MelanomaCancerDataset(csv_dataframe=train_csv,\n                                            root_dir='./data/train/',\n#                                              transform=transforms.Compose([\n#                                                 transforms.Resize(64),  # resize the image to 64x64 (remove if images are already 64x64)\n#                                                 transforms.RandomHorizontalFlip(),  # randomly flip image horizontally\n#                                                 ToTensor()]))\n                                           transform=transforms.Compose([\n#                                                Rescale(256),\n#                                                RandomCrop(64),\n                                               ToTensor()\n                                           ]))\nval_transformed_dataset = MelanomaCancerDataset(csv_dataframe=val_csv,\n                                            root_dir='./data/val/',\n#                                              transform=transforms.Compose([\n#                                                 transforms.Resize(64),  # resize the image to 64x64 (remove if images are already 64x64)\n#                                                 transforms.RandomHorizontalFlip(),  # randomly flip image horizontally\n#                                                 ToTensor()]))\n                                           transform=transforms.Compose([\n#                                                Rescale(256),\n#                                                RandomCrop(64),\n                                               ToTensor()\n                                           ]))\n# print(transformed_dataset[0])\nfor i in range(len(transformed_dataset)):\n    sample = transformed_dataset[i]\n\n    print(i, sample['image'].size(), sample['csv_data'].size(), sample['labels'].size())\n\n    if i == 3:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dataloader = DataLoader(transformed_dataset, batch_size=512,\n                        shuffle=True, pin_memory=True)\nval_dataloader = DataLoader(val_transformed_dataset, batch_size=512,\n                        shuffle=True, pin_memory=True)\n## 512 batch size takes long time\n##1024 batch size mem not enough\n#, num_workers=4\n\n\n# Helper function to show a batch\ndef show_batch(sample_batched):\n    \"\"\"Show image for a batch of samples.\"\"\"\n    images_batch, csv_data = \\\n            sample_batched['image'], sample_batched['csv_data']\n    batch_size = len(images_batch)\n    im_size = images_batch.size(2)\n    grid_border_size = 2\n\n    grid = utils.make_grid(images_batch)\n    plt.imshow(grid.numpy().transpose((1, 2, 0)))\n\n    for i in range(batch_size):\n        print('heree')\n#         plt.scatter(landmarks_batch[i, :, 0].numpy() + i * im_size + (i + 1) * grid_border_size,\n#                     landmarks_batch[i, :, 1].numpy() + grid_border_size,\n#                     s=10, marker='.', c='r')\n\n#         plt.title('Batch from dataloader')\n\n# for i_batch, sample_batched in enumerate(dataloader):\n#     print(i_batch, sample_batched['image'].size(),\n#           sample_batched['csv_data'].size())\n\n#     # observe 4th batch and stop.\n#     if i_batch == 3:\n#         plt.figure()\n#         show_batch(sample_batched)\n#         plt.axis('off')\n#         plt.ioff()\n#         plt.show()\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass imageModel(nn.Module):\n\n    def __init__(self):\n        super(imageModel, self).__init__()\n        # 1 input image channel, 6 output channels, 3x3 square convolution\n        # kernel\n        self.conv1 = nn.Conv2d(3, 32, 5)\n        self.conv2 = nn.Conv2d(32, 64, 5)\n        self.conv3 = nn.Conv2d(64, 128, 5)\n        self.pool1 = nn.MaxPool2d(3,1)\n        self.pool2 = nn.MaxPool2d(3,3)\n        # an affine operation: y = Wx + b\n        self.fc1 = nn.Linear(2048, 64)  # 6*6 from image dimension\n#         self.fc2 = nn.Linear(120, 64)\n#         self.fc3 = nn.Linear(64, 1)\n\n    def forward(self, x):\n        # Max pooling over a (2, 2) window\n#         print(x.shape)\n        x = self.pool1(F.relu(self.conv1(x)))\n#         print(x.shape)\n        # If the size is a square you can only specify a single number\n        x = self.pool2(F.relu(self.conv2(x)))\n#         print(x.shape)\n        x = self.pool2(F.relu(self.conv3(x)))\n#         print(x.shape)\n        x = x.view(-1, self.num_flat_features(x))\n#         print('flat features')\n#         print(x.shape)\n        x = F.relu(self.fc1(x))\n#         x = F.relu(self.fc2(x))\n#         x = F.sigmoid(x)\n        return x\n\n    def num_flat_features(self, x):\n#         return 32768\n        size = x.size()[1:]  # all dimensions except the batch dimension\n        num_features = 1\n        for s in size:\n            num_features *= s\n        return num_features\n\n\nimageModel = imageModel()\nprint(imageModel)\nclass melanomaModel(nn.Module):\n    def __init__(self):\n        super(melanomaModel, self).__init__()\n        self.cnn =imageModel\n#         self.cnn.fc = nn.Linear(\n#             self.cnn.fc.in_features, 20)\n        self.pi = 0.01\n        self.bias_init = np.log10((1-self.pi)/self.pi)\n        print('self.bias_init: ', self.bias_init)\n        self.fc1 = nn.Linear(64 + 3, 32)\n        self.fc2 = nn.Linear(32, 1)\n#         self.fc2.bias.data.fill_(self.bias_init)\n#         self.fc2 = nn.Linear(60, 5)\n        \n    def forward(self, image, data):\n        x1 = self.cnn(image)\n        x2 = data\n#         print('x1.shape')\n#         print(x1.shape)\n#         print('x2.shape')\n#         print(x2.shape)\n        \n        x = torch.cat((x1, x2), dim=1)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n#         x = F.sigmoid(x)\n#         print(x.shape)\n        return x\n        \n\n# model = melanomaModel()\n\n# batch_size = 2\n# image = torch.randn(batch_size, 3, 299, 299)\n# data = torch.randn(batch_size, 10)\n\n# output = model(image, data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"malignant_weight = 0.5\nbenign_weight = 0.95\npos_weight = torch.tensor([494])\ndef weighted_binary_cross_entropy(output, target, weights=torch.tensor([malignant_weight, benign_weight])):\n        \n    if weights is not None:\n        assert len(weights) == 2\n        \n        loss = weights[1] * (target * torch.log(output)) + weights[0] * ((1 - target) * torch.log(1 - output))\n    else:\n        loss = target * torch.log(output) + (1 - target) * torch.log(1 - output)\n\n    return torch.neg(torch.mean(loss))\n\ndef focal_loss(inputs, targets, alpha=0.8, gamma=2, logits=False, reduce=True):\n#     print('------------- inside loss')\n#     print(inputs.shape)\n#     print(inputs)\n    if logits:\n#         print('------heree in if')\n        BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n    else:\n#         print('------heree in elseeeeeeeeeeeeeeeeeeeeeee')\n        BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n#     print('BCELOSS', BCE_loss)\n    pt = torch.exp(-BCE_loss)\n#     print('pt', pt)\n    F_loss = alpha * (1-pt)**gamma * BCE_loss\n#     print(F_loss)\n\n    if reduce:\n        return torch.mean(F_loss)\n    else:\n        return F_loss\n    \n\nclass RunningAverage():\n    \"\"\"A simple class that maintains the running average of a quantity\n    \n    Example:\n    ```\n    loss_avg = RunningAverage()\n    loss_avg.update(2)\n    loss_avg.update(4)\n    loss_avg() = 3\n    ```\n    \"\"\"\n    def __init__(self):\n        self.steps = 0\n        self.total = 0\n    \n    def update(self, val):\n        self.total += val\n        self.steps += 1\n    \n    def __call__(self):\n        return self.total/float(self.steps)\n\ndef train(model, optimizer, dataloader, metrics, params):\n    \"\"\"Train the model on `num_steps` batches\n\n    Args:\n        model: (torch.nn.Module) the neural network\n        optimizer: (torch.optim) optimizer for parameters of model\n        loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch\n        dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches training data\n        metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch\n        params: (Params) hyperparameters\n        num_steps: (int) number of batches to train on, each of size params.batch_size\n    \"\"\"\n\n    # set model to training mode\n    model.train()\n\n    # summary for current training loop and a running average object for loss\n    summ = []\n    loss_avg = RunningAverage()\n#     criterion = weighted_binary_cross_entropy\n    criterion = nn.BCEWithLogitsLoss()\n\n    # Use tqdm for progress bar\n    with tqdm(total=len(dataloader)) as t:\n        for i, sample_batched in enumerate(dataloader):\n            # move to GPU if available\n            image, csv_data, labels_batch = sample_batched['image'].float(), sample_batched['csv_data'].float(), sample_batched['labels'].float()\n            if params.cuda:\n                image, csv_data, labels_batch = image.cuda(non_blocking=True), csv_data.cuda(non_blocking=True), labels_batch.cuda(non_blocking=True)\n            # convert to torch Variables\n#             image, csv_data, labels = Variable(\n#                 image), Variable(csv_data), Variable(labels_batch)\n\n            # compute model output and loss\n            output_batch = model(image, csv_data)\n            loss = criterion(output_batch, labels_batch) \n\n            # clear previous gradients, compute gradients of all variables wrt loss\n            optimizer.zero_grad()\n            loss.backward()\n\n            # performs updates using calculated gradients\n            optimizer.step()\n\n            # Evaluate summaries only once in a while\n            if i % params.save_summary_steps == 0:\n                # extract data from torch Variable, move to cpu, convert to numpy arrays\n                output_batch = output_batch.data.cpu().numpy()\n                labels_batch = labels_batch.data.cpu().numpy()\n\n                # compute all metrics on this batch\n                summary_batch = {metric: metrics[metric](output_batch, labels_batch)\n                                 for metric in metrics}\n                summary_batch['loss'] = loss.item()\n                summ.append(summary_batch)\n\n            # update the average loss\n            loss_avg.update(loss.item())\n\n            t.set_postfix(loss='{:05.3f}'.format(loss_avg()))\n            t.update()\n\n    # compute mean of all metrics in summary\n    metrics_mean = {metric: np.mean([x[metric]\n                                     for x in summ]) for metric in summ[0]}\n    metrics_string = \" ; \".join(\"{}: {:05.3f}\".format(k, v)\n                                for k, v in metrics_mean.items())\n    print(\"- Train metrics: \" + metrics_string)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def save_dict_to_json(d, json_path):\n    \"\"\"Saves dict of floats in json file\n\n    Args:\n        d: (dict) of float-castable values (np.float, int, float, etc.)\n        json_path: (string) path to json file\n    \"\"\"\n    with open(json_path, 'w') as f:\n        # We need to convert the values to float for json (it doesn't accept np.array, np.float, )\n        d = {k: float(v) for k, v in d.items()}\n        json.dump(d, f, indent=4)\n\ndef train_and_evaluate(model, train_dataloader, val_dataloader, optimizer, metrics, params, model_dir,\n                       restore_file=None):\n    \"\"\"Train the model and evaluate every epoch.\n\n    Args:\n        model: (torch.nn.Module) the neural network\n        train_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches training data\n        val_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches validation data\n        optimizer: (torch.optim) optimizer for parameters of model\n        loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch\n        metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch\n        params: (Params) hyperparameters\n        model_dir: (string) directory containing config, weights and log\n        restore_file: (string) optional- name of file to restore from (without its extension .pth.tar)\n    \"\"\"\n    # reload weights from restore_file if specified\n    if restore_file is not None:\n        print('restored file')\n        checkpoint = torch.load(params.PATH)\n        model.load_state_dict(checkpoint['model_state_dict'])\n        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n#         restore_path = os.path.join(\n#             args.model_dir, args.restore_file + '.pth.tar')\n#         logging.info(\"Restoring parameters from {}\".format(restore_path))\n#         utils.load_checkpoint(restore_path, model, optimizer)\n\n    best_val_acc = 0.0\n\n    for epoch in range(params.num_epochs):\n        # Run one epoch\n        print(\"Epoch {}/{}\".format(epoch + 1, params.num_epochs))\n\n        # compute number of batches in one epoch (one full pass over the training set)\n        train(model, optimizer, train_dataloader, metrics, params)\n\n        # Evaluate for one epoch on validation set\n#         val_metrics = evaluate(model, val_dataloader, metrics, params)\n\n#         val_acc = val_metrics['accuracy']\n#         is_best = val_acc >= best_val_acc\n\n        # Save weights\n        torch.save({\n            'epoch': epoch+1,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n#             'loss': loss\n#             is_best=is_best\n            }, params.PATH)\n#         utils.save_checkpoint({'epoch': epoch + 1,\n#                                'state_dict': model.state_dict(),\n#                                'optim_dict': optimizer.state_dict()},\n#                               is_best=is_best,\n#                               checkpoint=model_dir)\n\n        # If best_eval, best_save_path\n#         if is_best:\n#             logging.info(\"- Found new best accuracy\")\n#             best_val_acc = val_acc\n\n#             # Save best val metrics in a json file in the model directory\n#             best_json_path = os.path.join(\n#                 model_dir, \"metrics_val_best_weights.json\")\n#             save_dict_to_json(val_metrics, best_json_path)\n\n#         # Save latest val metrics in a json file in the model directory\n#         last_json_path = os.path.join(\n#             model_dir, \"metrics_val_last_weights.json\")\n#         save_dict_to_json(val_metrics, last_json_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\nimport torch.optim as optim\nimport gc\ndef accuracy(outputs, labels):\n    \"\"\"\n    Compute the accuracy, given the outputs and labels for all images.\n\n    Args:\n        outputs: (np.ndarray) dimension batch_size x 6 - log softmax output of the model\n        labels: (np.ndarray) dimension batch_size, where each element is a value in [0, 1, 2, 3, 4, 5]\n\n    Returns: (float) accuracy in [0,1]\n    \"\"\"\n    outputs = np.argmax(outputs, axis=1)\n    return np.sum(outputs==labels)/float(labels.size)\n\ndef binary_acc(y_pred, y_test):\n    y_pred_tag = np.round(y_pred)\n#     y_pred_tag=y_pred\n#     print(y_pred)\n    print('------------------')\n#     print(y_pred_tag)\n\n    correct_results_sum = float((y_pred_tag == y_test).sum())\n    actual_one_results_sum = (y_test==1).sum()\n    actual_zero_results_sum = (y_test==0).sum()\n    predicted_one_results_sum = (y_pred_tag == 1).sum()\n    predicted_zero_results_sum = (y_pred_tag == 0).sum()\n    acc = correct_results_sum/y_test.shape[0]\n    acc = np.round(acc * 100)\n    print('correct_results_sum: ', correct_results_sum)\n    print('actual_one_results_sum: ', actual_one_results_sum)\n    print('actual_zero_results_sum: ', actual_zero_results_sum)\n    print('predicted_one_results_sum: ', predicted_one_results_sum)\n    print('predicted_zero_results_sum: ', predicted_zero_results_sum)\n    print('y_test.shape[0]: ', y_test.shape[0])\n#     acc=-1\n    \n    return acc\n\nclass dotdict(dict):\n    \"\"\"dot.notation access to dictionary attributes\"\"\"\n    __getattr__ = dict.get\n    __setattr__ = dict.__setitem__\n    __delattr__ = dict.__delitem__\n\nmetrics = {\n    'accuracy': binary_acc,\n    # could add more metrics such as accuracy for each token type\n}\nparams = {\n    \"learning_rate\": 1e-3,\n    \"batch_size\": 128,\n    \"num_epochs\": 150,\n    \"dropout_rate\":0.8, \n    \"num_channels\": 32,\n    \"save_summary_steps\": 200,\n    \"num_workers\": 4,\n    \"PATH\": r'\\trained_weights\\weights.pth'\n}\nparams['cuda'] = torch.cuda.is_available()\nparams = dotdict(params)\nprint(params.learning_rate)\nmodel = melanomaModel().cuda() if params.cuda else melanomaModel()\n# optimizer = optim.Adam(model.parameters(), lr=params.learning_rate)\n# optimizer = optim.SGD(model.parameters(), params.learning_rate)\noptimizer = optim.SGD(model.parameters(), lr=0.00001, momentum=0.5)\ntrain_and_evaluate(model, dataloader, val_dataloader, optimizer, metrics, params, 'ML\\\\Melanoma Classification\\\\',\n                       'yes')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_csv = pd.read_csv(DATA_DIR + 'test.csv')\ntest_csv.head()\nprint(test_csv.shape)\ntest_csv.anatom_site_general_challenge.value_counts()\ndel test_csv['patient_id']\ntest_csv[\"anatom_site_general_challenge\"].fillna(\"na\", inplace = True) \ntest_csv.isnull().sum()\nencoder = {\"sex\":     {\"male\": 0, \"female\": 1},\n \"anatom_site_general_challenge\": {\"head/neck\":0, \"upper extremity\":1, \"lower extremity\":2, \"torso\":3, \"palms/soles\":4, \"oral/genital\":5, \"na\":6}}\ntest_csv.replace(encoder, inplace=True)\n\ntest_csv.isnull().sum()\ntest_csv['target'] = -1\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_csv.anatom_site_general_challenge.value_counts()\ntransformed_test_dataset = MelanomaCancerDataset(csv_dataframe=test_csv,\n                                            root_dir='./data/test/',\n#                                              transform=transforms.Compose([\n#                                                 transforms.Resize(64),  # resize the image to 64x64 (remove if images are already 64x64)\n#                                                 transforms.RandomHorizontalFlip(),  # randomly flip image horizontally\n#                                                 ToTensor()]))\n                                           transform=transforms.Compose([\n#                                                Rescale(256),\n#                                                RandomCrop(64),\n                                               ToTensor()\n                                           ]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dataloader = DataLoader(transformed_test_dataset,batch_size=32)\n\nPATH = r'\\Melanoma Classification\\trained_weights\\weights.pth'\ncheckpoint = torch.load(PATH)\nmodel = melanomaModel().cuda() \nmodel.load_state_dict(checkpoint['model_state_dict'])\nvalues = []\nfor i, sample_batched in enumerate(dataloader):\n#     print(sample_batched)\n    image, csv_data, labels_batch, image_names = sample_batched['image'].float(), sample_batched['csv_data'].float(), sample_batched['labels'].float(), sample_batched['image_name']\n    if params.cuda:\n        image, csv_data, labels_batch = image.cuda(non_blocking=True), csv_data.cuda(non_blocking=True), labels_batch.cuda(non_blocking=True)\n    outputs = model(image, csv_data)\n    outputs = outputs.data.cpu().numpy()\n    t=[]\n    t.append(image_names)\n    image_names = np.asarray(t).transpose()\n#     print(image_names.shape)\n#     print(outputs.shape)\n    final_values = np.concatenate([image_names, outputs], axis=1)\n    for j in range(final_values.shape[0]):\n        values.append(final_values[j].tolist())\nsample_submission = pd.DataFrame(values, columns=['image_name', 'target'])\nsample_submission.to_csv('submission.csv', index=False)\n# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(sample_submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# print(device)\n# torch.cuda.empty_cache()\n# torch.cuda.memory_summary(device=None, abbreviated=False)\nmodel = melanomaModel()\ncheckpoint = torch.load(PATH)\nmodel.load_state_dict(checkpoint['model_state_dict'])\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#  IMG_HEIGHT = 350\n# IMG_WIDTH = 300\n# IMG_SHAPE = (IMG_HEIGHT, IMG_WIDTH, 3)\n# base_model = tf.keras.applications.VGG19(\n#     include_top=False,\n#     weights=\"imagenet\",\n#     input_tensor=None,\n#     input_shape=(1024, 1024, 3)\n# )\n# base_model.trainable = False\n# for layer in base_model.layers:\n#     layer.trainable = False\n#     if isinstance(layer, tf.keras.layers.BatchNormalization):\n#         layer._per_input_updates = {}\n\n# image_model = models.Sequential()\n# image_model.add(layers.Conv2D(32, (5, 5), activation='relu', input_shape=(32, 32, 3)))\n# image_model.add(layers.MaxPooling2D((3, 3)))\n# image_model.add(layers.Conv2D(64, (5, 5), activation='relu'))\n# image_model.add(layers.MaxPooling2D((3, 3)))\n# image_model.add(layers.Conv2D(128, (5, 5), activation='relu'))\n# image_model.add(layers.MaxPooling2D((3, 3)))\n# image_model.add(layers.Flatten())\n# image_model.add(layers.Dense(64, activation='relu'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"torch.cuda.is_available() =\", torch.cuda.is_available())\nprint(\"torch.cuda.device_count() =\", torch.cuda.device_count())\nprint(\"torch.cuda.device('cuda') =\", torch.cuda.device('cuda'))\nprint(\"torch.cuda.current_device() =\", torch.cuda.current_device())\nprint(\"torch.cuda.get_device_name(0) =\",torch.cuda.get_device_name(0))\nprint(\"torch.cuda.get_device_name() =\",torch.cuda.get_device_name())\nprint(torch.cuda.device(0))\n# cudnn.benchmark = True\n#Additional Info when using cuda\nif device.type == 'cuda':\n    print(torch.cuda.get_device_name(0))\n    print('Memory Usage:')\n    print('Allocated:',torch.cuda.memory_allocated(0))\n    print('Cached: ', round(torch.cuda.memory_cached(0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# pip install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### layer = tf.keras.layers.Flatten()\n# dense_layer = tf.keras.layers.Dense(1024, activation='relu')\n# prediction_layer = tf.keras.layers.Dense(8, activation='softmax')\n# model = tf.keras.Sequential([\n#   base_model,\n#   layer,\n#   dense_layer,\n#   prediction_layer\n# ])\n# base_learning_rate = 0.0001\n# model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=base_learning_rate),\n#               loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False),\n#               metrics=['accuracy'])\n# model.summary()"},{"metadata":{"trusted":false},"cell_type":"code","source":"!python --version","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n+++++-","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}