{"cells":[{"metadata":{},"cell_type":"markdown","source":"Steps \n1. Extract the images into a different folder in jpg format\n2. Write the dataset class\n3. Write the model\n4. Write a fitting function\n5. Fit the model and plot trainng stats\n6. Generate insights from the model"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nimport shutil \nimport pydicom\nfrom PIL import Image\nimport cv2\n\n# def extract_dicom_images(rsna_dir, ):\n#     '''\n#     This function extracts jpg images from the dicom dataset(both train and test) and \n#     stores it in the ./data folder\n#     Params:\n#     rsna_dir: directory of the rsna dataset\n#     '''\n#     train_dir = os.path.join(rsna_dir, 'stage_2_train_images')\n#     try:  \n#         os.mkdir('data')  \n#     except OSError as error:  \n#         print('The directory already exists deleting the contents of the directiry')\n#         shutil.rmtree('./data')\n#         os.mkdir('data') \n#     outdir = './data/'\n#     for file in tqdm(os.listdir(train_dir)):\n#         file_path = os.path.join(train_dir, file)\n#         ds = pydicom.read_file(file_path) # read dicom image\n#         img = ds.pixel_array # get image array\n#         img = cv2.resize(img, (300,300), interpolation = cv2.INTER_AREA)\n#         img_mem = Image.fromarray(img) # Creates an image memory from an object exporting the array interface\n#         img_mem.save(outdir + file.replace('.dcm','.png'))\n        \n        \n        \n    \n    \n        \n    \n    \n    \n# extract_dicom_images(rsna_dir = '../input/rsna-pneumonia-detection-challenge')\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport pandas as pd\ndef get_labeller(df_dir):\n    '''\n    Returns a dictionary which maps patiendId to labels\n    0: ND & 1:D and bounding boxes\n    Params:\n    df_dir: directory of the df with this info\n    '''\n    df = pd.read_csv(df_dir)\n    df = df.set_index('patientId')\n    return df.T.to_dict()\n\n# get_labeller('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\ndef get_all_files():\n    labeller = get_labeller('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\n    files = list(labeller.keys())\n    print(f'Found {len(files)} files')\n    random.shuffle(files)\n    train_files = files[0:int(len(files) *0.80)]\n    test_files = files[int(len(files) *0.80):]\n    return train_files, test_files\n# train, test = get_all_files()\n# print(len(train), len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, ConcatDataset\nfrom torchvision import transforms\nfrom PIL import Image\nclass DS(Dataset):\n    def __init__(self, labeller, files, mode, base_path):\n        '''\n        Params:\n        labeller: dict to map patID to target\n        files: list of files to be trained on\n        mode = train or test\n        '''\n        self.labeller = labeller\n        self.files = files\n        self.mode = mode\n        self.trans_tr = transforms.Compose([\n            transforms.Resize(256),\n            transforms.ColorJitter(),\n            transforms.RandomCrop(224),\n            transforms.RandomHorizontalFlip(),\n            transforms.Resize(128),\n            transforms.ToTensor(),\n            transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225]) \n        ])\n        self.trans_test = transforms.Compose([\n            transforms.Resize((128,128)),\n            transforms.ToTensor(),\n            transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225]) \n        ])\n        self.trans = None\n        if self.mode == 'train':\n            self.trans = self.trans_tr\n        else:\n            self.trans = self.trans_test\n        self.base_path = base_path\n        \n    def __len__(self):\n        return  len(self.files)\n    \n    def __getitem__(self, idx):\n        img_name = self.files[idx]\n        path = os.path.join(self.base_path, img_name+'.png')\n        img =  Image.open(path).convert('RGB')\n        img = self.trans(img)\n        img = img.numpy()\n        return img.astype('float32'), self.labeller[img_name]['Target']\n    \n    \ndef get_dataloaders():\n    labeller = get_labeller('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\n    base_dir = '../input/rsna-pneumionia-detection/data/'\n    files_train, files_test = get_all_files()\n    \n    train_ds = DS(labeller, files_train, 'train', base_dir)\n    test_ds = DS(labeller, files_test, 'test', base_dir)\n    dl_train = DataLoader(train_ds, batch_size = 32, shuffle=True, num_workers=4)\n    dl_test = DataLoader(test_ds, batch_size = 32, shuffle=True, num_workers=4)\n    return dl_train, dl_test\n# a, b = get_dataloaders()  \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a, b = get_dataloaders()  \nimport  matplotlib.pyplot as plt\nimport torchvision\nimport numpy as np\nsamples, labels = iter(b).next()\nplt.figure(figsize=(16,24))\ngrid_imgs = torchvision.utils.make_grid(samples[:24])\nnp_grid_imgs = grid_imgs.numpy()\n# in tensor, image is (batch, width, height), so you have to transpose it to (width, height, batch) in numpy to show it.\nplt.imshow(np.transpose(np_grid_imgs, (1,2,0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\ndef get_model():\n    device = 'cuda'\n    model = torchvision.models.densenet121(pretrained=True)\n#     for param in model.parameters():\n#         param.requires_grad = False\n        \n    num_ftrs = model.classifier.in_features\n    model.classifier = nn.Sequential(\n        nn.Linear(num_ftrs, 256),\n        nn.ReLU(),\n#         nn.Dropout(0.4),\n        nn.Linear(256, 2)\n    )\n    model = model.to(device)\n    \n    return model\n# get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\ndef train(model, epochs, dataloader, test_dl):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters())\n#     scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[500,1000,1500], gamma=0.5)\n    loss_list = []\n    acc_list = []\n    val_loss_list = []\n    acc_val_list = []\n    model.train()\n    device = 'cuda'\n    for epoch in (range(epochs)):\n        total_loss = 0\n        num_batch = 0\n        total_acc = 0\n        print(f'STARTED EPOCH {epoch}')\n        for samples, labels in tqdm(dataloader):\n            samples, labels = samples.to(device), labels.to(device)\n            labels = labels.long()\n            optimizer.zero_grad()\n            output = model(samples)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n#             scheduler.step()\n            op = nn.Softmax(dim=1)(output)\n            pred = torch.argmax(op, dim=1)\n            correct = pred.eq(labels)\n            acc = torch.mean(correct.float())\n            total_acc += acc.item()\n            num_batch += 1\n            \n\n        print(f'Loss is {total_loss / num_batch}')\n        print(f'Accuracy is {total_acc / num_batch}')\n        loss_val, acc_val = get_validation_stats(model, test_dl)\n        acc_list.append(total_acc / num_batch)\n        loss_list.append(total_loss / num_batch)\n        val_loss_list.append(loss_val)\n        acc_val_list.append(acc_val)\n        if epoch%10 == 0:\n            torch.save(model, f'model-v3-epoch{epoch}.pt')\n        \n    fig, axs = plt.subplots(4)\n    fig.suptitle('Training Stats')\n    axs[0].plot(loss_list)\n    axs[1].plot(acc_list)\n    axs[2].plot(val_loss_list)\n    axs[3].plot(acc_val_list)\n    \n    \ndef get_validation_stats(model, test_dl):\n    with torch.no_grad():\n        model.eval()\n        total_loss = 0\n        device = 'cuda'\n        criterion = nn.CrossEntropyLoss()\n        total_loss = 0\n        num_batch = 0\n        total_acc = 0\n        for samples, labels in tqdm(test_dl):\n            samples, labels = samples.to(device), labels.to(device)\n            labels = labels.long()\n            output = model(samples)\n            output = nn.Softmax(dim=1)(output)\n            loss = criterion(output, labels)\n            total_loss += loss.item()\n#             scheduler.step()\n            pred = torch.argmax(output, dim=1)\n            correct = pred.eq(labels)\n            acc = torch.mean(correct.float())\n            total_acc += acc.item()\n            num_batch += 1\n\n        print(f'Val Loss is {total_loss / num_batch}')\n        print(f'Val Accuracy is {total_acc / num_batch}')\n        return (total_loss / num_batch), (total_acc / num_batch)\n\n    \nmodel = get_model()\ntrain_dl, test_dl = get_dataloaders()  \n\ntrain(model, 120, train_dl, test_dl)\nget_validation_stats(model, test_dl)\ntorch.save(model, 'modelv3.pt')\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def plot(losses, accs, val_losses, val_accs):\n#     plt.figure(figsize=(16, 9))\n#     plt.plot(history.epoch, history.history['acc'])\n#     plt.title('Model Accuracy')\n#     plt.legend(['train'], loc='upper left')\n#     plt.show()\n\n#     plt.figure(figsize=(16, 9))\n#     plt.plot(history.epoch, history.history['loss'])\n#     plt.title('Model Loss')\n#     plt.legend(['train'], loc='upper left')\n#     plt.show()\n\n#     plt.figure(figsize=(16, 9))\n#     plt.plot(history.epoch, history.history['val_acc'])\n#     plt.title('Model Validation Accuracy')\n#     plt.legend(['train'], loc='upper left')\n#     plt.show()\n\n#     plt.figure(figsize=(16, 9))\n#     plt.plot(history.epoch, history.history['val_loss'])\n#     plt.title('Model Validation Loss')\n#     plt.legend(['train'], loc='upper left')\n#     plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samples, labels = iter(test_dl).next()\ndevice = 'cuda'\nsamples = samples.to(device)\nfig = plt.figure(figsize=(24, 16))\nfig.tight_layout()\nmodel.eval()\noutput = nn.Softmax(dim=1)(model(samples[:24]))\npred = torch.argmax(output, dim=1)\npred = [p.item() for p in pred]\nreal = [p.item() for p in labels]\nad = {0:'No', 1:'Yes'}\nfor num, sample in enumerate(samples[:24]):\n    plt.subplot(4,6,num+1)\n    plt.title(f'{ad[pred[num]]}*{ad[real[num]]}')\n    plt.axis('off')\n    sample = sample.cpu().numpy()\n    plt.imshow(np.transpose(sample, (1,2,0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_validation_stats(model, test_dl):\n#     with torch.no_grad():\n#         model.eval()\n#         total_loss = 0\n#         device = 'cuda'\n#         criterion = nn.CrossEntropyLoss()\n#         for samples, labels in tqdm(test_dl):\n#             samples, labels = samples.to(device), labels.to(device)\n#             labels = labels.long()\n#             output = model(samples)\n#             loss = criterion(output, labels)\n#             total_loss += loss.item()\n#         print(f'Loss is {total_loss / 32}')\n#         pred = torch.argmax(output, dim=1)\n#         correct = pred.eq(labels)\n#         pos = torch.mean(1*(pred == 1))\n#         acc = torch.mean(correct.float())\n#         print(f'Accuracy is {acc}')\n#         print(f'Fraction of positiive cases {pos}')\n\n# get_validation_stats(model, test_dl)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}