{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport torch\nimport torchvision\nfrom PIL import Image\nfrom pathlib import Path\nfrom tqdm.notebook import trange, tqdm\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import datasets, models, transforms\n\nimport matplotlib.pyplot as plt\nimport time\nimport copy\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # TensorBoard\n# from torch.utils.tensorboard import SummaryWriter\n# import time\n# writer  = SummaryWriter('runs/ranzcr_Dropout_0.5_{}{}{}'.format(time.localtime().tm_mday,\n#                                               time.localtime().tm_hour,time.localtime().tm_min))\n# # writer.add_image('test_image',img_grid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sgd\n# args = {\"initial_lr\": 0.001, \"momentum\":0.9, \"optimizer\":\"Adam\", \"step_size\":7, \"gamma\":0.1,\n#        \"batch\":128,\"epochs\":5, \"workers\":4, \"network\":\"resnetManual\"}\n\n# adam\nargs = {\"initial_lr\": 0.001, \"optimizer\":\"Adam\", \"weight_decay\" :1e-4,\n        \"max_lr\": 0.008,\"batch\":128,\"epochs\":28, \"workers\":4, \"shape\":128,\"Dropout\":0.5,\n        \"grad_clip\": 1}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install clearml\n\n# from clearml import Task\n# from clearml import StorageManager\n\n# task = Task.init(project_name='pytorch multilabel TransLear', \n#                  task_name='resnet_epochs30_lr_0.001_max_lr0.008_Dropout_0.5_d{}_h{}_m{}'.format(\n# time.localtime().tm_mday,time.localtime().tm_hour,time.localtime().tm_min))\n# logger = task.get_logger()\n\n# task.connect(args)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/ranzcr-clip-catheter-line-classification/'\nlist_train = os.listdir(data_dir+\"train/\")\nprint(list_train[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_csv = pd.read_csv(data_dir+\"train.csv\")\n# train_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(data_dir+\"sample_submission.csv\")\nsample_submission.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Shuffle the data\nnp.random.seed(42)\n#Create random indicies\ninds = np.random.choice(train_csv.shape[0], train_csv.shape[0], replace=False)\ntrain_csv = train_csv.iloc[inds]\ntrain_csv.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"!!!!!!!!!!!!!! Train!!!!!!!!!!!!!!!!!!!!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_csv.shape\n# train_csv = train_csv[0:1500]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create ImageDataset class\nclass ImageDataset_train(Dataset):\n    def __init__(self, csv, train, test):\n        self.csv = csv\n        self.train = train\n        self.test = test\n        self.all_image_names = self.csv[:]['StudyInstanceUID']\n        self.all_labels = np.array(self.csv.drop(['StudyInstanceUID', 'PatientID'], axis=1))\n        self.train_ratio = int(0.85 * len(self.csv))\n        self.valid_ratio = len(self.csv) - self.train_ratio\n        # set the training data images and labels\n        if self.train == True:\n            print(f\"Number of training images: {self.train_ratio}\")\n            self.image_names = list(self.all_image_names[:self.train_ratio])\n            self.labels = list(self.all_labels[:self.train_ratio])\n            # define the training transforms\n            self.transform = transforms.Compose([\n#                 transforms.ToPILImage(),\n                transforms.CenterCrop(2800),\n                transforms.Resize((args['shape'], args['shape'])),\n#                 transforms.RandomAffine(degrees=10),\n#                 transforms.RandomHorizontalFlip(p=0.3),\n#                 transforms.RandomCrop(size=32),\n#                 transforms.RandomErasing(p=0.2, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False),\n#                 transforms.RandomRotation(degrees=),\n                transforms.ToTensor(),\n                transforms.Normalize(*stats)\n            ])\n            \n            # set the validation data images and labels\n        elif self.train == False and self.test == False:\n            print(f\"Number of validation images: {self.valid_ratio}\")\n#             self.image_names = list(self.all_image_names[-self.valid_ratio:-10])\n            self.image_names = list(self.all_image_names[-self.valid_ratio:])\n            self.labels = list(self.all_labels[-self.valid_ratio:])\n            # define the validation transforms\n            self.transform = transforms.Compose([\n#                 transforms.ToPILImage(),\n                transforms.CenterCrop(2800),\n                transforms.Resize((args['shape'], args['shape'])),\n                transforms.ToTensor(),\n                transforms.Normalize(*stats)\n                ])\n            \n            \n    \n    def __len__(self):\n        return len(self.image_names)\n    \n    def __getitem__(self, index):\n        image = Image.open(f\"../input/ranzcr-clip-catheter-line-classification/train/{self.image_names[index]}.jpg\")  \n        image = self.transform(image)\n        targets = self.labels[index]\n        \n        return {\n            'image': torch.tensor(image, dtype=torch.float32),\n            'label': torch.tensor(targets, dtype=torch.float32)\n        }\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(2):\n    image = Image.open(f\"../input/ranzcr-clip-catheter-line-classification/train/\"+list_train[i]) \n    print(np.array(image).shape)\n    print(image)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Test dataset\nclass ImageDataset_test(Dataset):\n\n    def __init__(self, root, image_dir, transform=None):\n        self.root = root\n        self.image_dir = image_dir\n        self.image_files = os.listdir(image_dir)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_files)\n\n    def __getitem__(self, index):\n        image_name = os.path.join(self.image_dir, self.image_files[index])  \n        image = Image.open(image_name)\n        #label = self.data[index]\n        if self.transform:\n            image = self.transform(image)\n        return image\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stats = (0.4454318881034851, 0.25035765767097473)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test dataset "},{"metadata":{"trusted":true},"cell_type":"code","source":"root = data_dir\nimage_dir = root+'test/'\ntransform_img = transforms.Compose([ \n#     transforms.ToPILImage(),\n                                    transforms.CenterCrop(2800),\n                                    transforms.Resize((args['shape'], args['shape'])),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize(*stats)\n])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = ImageDataset_test(root,image_dir, transform = transform_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loader = DataLoader(test_ds,1,shuffle=False,num_workers = args['workers'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in test_loader:\n    \n    print(i)\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" ### train dataset "},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_ds\ntrain_ds = ImageDataset_train(train_csv, train=True, test=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### validation dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"#val_ds\nval_ds = ImageDataset_train(train_csv, train=False, test=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = args['batch']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train data loader\n# train_loader = DataLoader(train_ds, batch_size=batch_size,shuffle=True)\n# validation data loader\n# valid_loader = DataLoader(val_ds,batch_size=batch_size,shuffle=False)\n# test data loader\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds[0]['image'].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lab_name = sample_submission.columns.drop(['StudyInstanceUID'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plot an example\n\ndef show_example(dataset):\n    print(\"labels: {}\".format(dataset['label']))\n    print(lab_name[dataset['label']==1])\n    plt.imshow(dataset['image'].squeeze())\n    plt.show\n      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_example(train_ds[15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# means = []\n# stds = []\n# for img in tqdm(train_ds):\n#     means.append(torch.mean(img['image']))\n#     stds.append(torch.std(img['image']))\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mean = torch.mean(torch.tensor(means))\n# std = torch.mean(torch.tensor(stds))\n# stats = mean.item(), std.item()\n# stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create dictinary with train and val\nimage_datasets = {'train':train_ds, 'val':val_ds}\n#batch_size\nbatch = batch_size\n#Create dictionary with train dataloader and val dataloader\ndataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size = batch, shuffle=True,num_workers = 4) \n               for x in ['train', 'val']}\n#Dataset sizes\ndataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n#label names\nclass_names = lab_name\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(dataloaders['train'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in dataloaders['train']:\n    print(i['image'].shape)\n    \n#     print(label['label'])\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.utils import make_grid\n\ndef show_batch(dl):\n    for images in tqdm(dl, desc = \"Progress\"):\n        fig, ax = plt.subplots(figsize = (22,18))\n        ax.set_xticks([]); ax.set_yticks([])\n        ax.imshow(make_grid(images['image'],10).permute(1,2,0))\n        break\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_batch(dataloaders['train'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in dataloaders['train']:\n    print(i['image'])\n    print(i['label'])\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef train_model(model, criterion, optimizer, scheduler=None, num_epochs=25, grad_clip=None):\n\n   \n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_loss = 10.0\n\n    for epoch in tqdm(range(num_epochs)):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n        \n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            \n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for i in dataloaders[phase]:\n                inputs = i[\"image\"].to(device)\n                labels = i[\"label\"].to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    # apply sigmoid activation to get all the outputs between 0 and 1\n                    outputs = torch.sigmoid(outputs)\n                    loss = criterion(outputs, labels)\n                    \n                    \n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        #Gradiend clipping\n                        if grad_clip:\n                            nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n                            \n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n#                 running_corrects += torch.sum(outputs == labels.data)\n            if phase == 'train':\n                \n                if scheduler:\n                    scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            \n            \n#             epoch_acc = running_corrects.double() / dataset_sizes[phase]\n            print('{} Loss: {:.4f} '.format(phase, epoch_loss))\n                \n#             writer.add_scalar('epoch_loss',epoch_loss, epoch)  \n#             logger.report_scalar(title = phase, series='epoch_loss',iteration = epoch, value = epoch_loss)\n#             logger.report_scalar(title =phase, series='epoch_acc',iteration = epoch, value = epoch_acc)\n\n            # deep copy the model\n            if phase == 'val' and epoch_loss < best_loss:\n                print(phase)\n                best_loss = epoch_loss\n                \n#                 print(\"best_acc_val:\" , best_loss)\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val loss: {:4f}'.format(best_loss))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Conv block function\ndef conv_block(in_channels, out_channels, pool = False):\n    layers = [nn.Conv2d(in_channels, out_channels, kernel_size = 3, padding=1),\n             nn.BatchNorm2d(out_channels),\n             nn.ReLU(inplace = True)]\n    \n    if pool: layers.append(nn.MaxPool2d(2))\n   \n   \n    return nn.Sequential(*layers) #asterki gia na parei olo to dictionary\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Resnet \nclass ResNet(nn.Module):\n    def __init__(self, in_channels, num_classes):\n        super().__init__()  # 1*128*128 = 16384\n        self.conv1 = conv_block(in_channels, 64)  #128\n        self.conv2 = conv_block(64,128,pool=True) # 64\n        self.res1 = nn.Sequential(conv_block(128,128),conv_block(128,128))\n        self.conv3 = conv_block(128,128,pool=True) #32\n        self.conv4 = conv_block(128,256,pool=True)# 16\n        self.res2 = nn.Sequential(conv_block(256,256),conv_block(256,256))\n        self.conv5 = conv_block(256,512,pool=True) # 8\n        self.conv6 = conv_block(512,512,pool=True) # 4\n        self.res3 = nn.Sequential(conv_block(512,512),conv_block(512,512))\n#         self.conv7 = conv_block(256,512,pool=True) # 8\n#         self.conv8 = conv_block(512,512,pool=True) # 4\n#         self.res4 = nn.Sequential(conv_block(512,512),conv_block(512,512))\n#         self.conv9 = conv_block(512,512,pool=True) # 2\n#         self.conv10 = conv_block(512,512,pool=True) # 1\n        self.classifier = nn.Sequential(nn.MaxPool2d(4),\n                \n                                        nn.Flatten(),\n                                        nn.Dropout(args['Dropout']),\n                                        nn.Linear(512,num_classes)\n        )\n                                            \n                                       \n        \n    def forward(self,xb):\n        out = self.conv1(xb)\n        out = self.conv2(out)\n        out = self.res1(out) + out\n        out = self.conv3(out)\n        out = self.conv4(out)\n        out = self.res2(out) + out\n        out = self.conv5(out) \n        out = self.conv6(out)\n        out = self.res3(out) + out\n#         out = self.conv7(out)\n#         out = self.conv8(out)\n#         out = self.res4(out) + out\n#         out = self.conv9(out)\n#         out = self.conv10(out)\n        out = self.classifier(out)\n        \n        return out\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_ft = ResNet(1,11)\nmodel_ft","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Pretrained\n# model_ft = models.resnet18(pretrained=True)\n# num_ftrs = model_ft.fc.in_features\n# model_ft.conv1=nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n# model_ft.fc = nn.Linear(num_ftrs, 11)\n\n\nmodel_ft = model_ft.to(device)\n\ncriterion = nn.BCELoss() #binary cross entropy loss\n# Observe that all parameters are being optimized\noptimizer_ft = torch.optim.Adam(model_ft.parameters(), \n                                lr=args['initial_lr'], \n                                weight_decay = args['weight_decay'])\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.OneCycleLR(optimizer_ft,\n                                           max_lr = args['max_lr'],\n                                           epochs = args['epochs'],\n                                           steps_per_epoch = len(dataloaders['train']),\n                                           verbose =True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,\n                       num_epochs=args['epochs'], grad_clip = args['grad_clip'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.save(model_ft, \"model_ft_manualResnet11.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Model class must be defined somewhere\n# model_ft = torch.load(\"../input/ranzcr-clip-catheter-line-classification/model_ft.pth\")\n# # model1.eval()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test DataSet"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_ft.to(device)\nmodel_ft.eval()\n\nStudyInstanceUID = pd.DataFrame({\"StudyInstanceUID\":os.listdir(root+'test')})\n# print(StudyInstanceUID)\n\nsubmission = pd.DataFrame()\nfor img in tqdm(test_loader):\n\n    output = model_ft(img.to(device))\n\n    output = torch.sigmoid(output)\n    output = np.array(output.detach().cpu())\n#     print(output)\n    kouvas = pd.DataFrame(data = output)\n    submission = pd.concat([submission, kouvas],axis = 0)\n#     break\n\nsubmission.head()    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Restet index in submission\nsubmission = submission.reset_index(drop=True)\n# submission.index\nsubmission = pd.concat([StudyInstanceUID,submission],axis=1)\nsubmission['StudyInstanceUID']=submission['StudyInstanceUID'].str.replace(\".jpg\",\"\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.columns = sample_submission.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}