{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install \"../input/pretrained-models/pretrained-models.pytorch-master\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\nimport time\nimport random\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport albumentations as albu\nimport numpy as np\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2\n\n\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = \"../input/ranzcr-clip-catheter-line-classification/train\"\ntrain_files = os.listdir(train_path)\n\ntest_path = \"../input/ranzcr-clip-catheter-line-classification/test\"\ntest_files = os.listdir(test_path)\n\ntrain_df = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\nsubmission = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, valid = train_test_split(\n    train_df, \n    test_size=0.15, \n    random_state=42,\n)\n\n# reset index on both dataframes\ntrain = train.reset_index(drop=True)\nvalid = valid.reset_index(drop=True)\n\nprint(train.shape)\nprint(valid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Some basic augs\n#Try more augs and other img_size for better results\n\nimg_size = 256\n\ntrain_augs = albu.Compose([\n    albu.Resize(height=img_size, width=img_size, p=1.0),\n    #albu.Cutout(p=0.1, num_holes=10,  max_h_size=8, max_w_size=8),\n    albu.Normalize(    \n        mean=[0.5],\n        std=[0.5],),\n    ToTensorV2(),\n])\n\nvalid_augs = albu.Compose([\n    albu.Resize(height=img_size, width=img_size, p=1.0),\n    albu.Normalize(\n        mean=[0.5],\n        std=[0.5],),\n    ToTensorV2(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RanzcrDataset(Dataset):\n    \n    def __init__(self, files_folder_path, df, transfroms = None):\n        self.files_folder_path = files_folder_path\n        self.df = df\n        self.transforms = transfroms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        image_id = self.df.StudyInstanceUID.values[idx]\n        image = cv2.imread(os.path.join(self.files_folder_path, image_id + \".jpg\" ), 0)\n    \n        if self.transforms:\n            image = self.transforms(image=image)['image']\n            \n        labels = self.df[self.df.StudyInstanceUID == image_id].values.tolist()[0][1:-1]\n        labels = torch.tensor(labels,dtype= torch.float32) #.view(1,-1)\n        \n        return image, labels\n    \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainset = RanzcrDataset(train_path,  train, train_augs)\nvalidset = RanzcrDataset(train_path,  valid, valid_augs)\n\n#Try diff params for Dataloader too\ntrainloader = DataLoader(trainset, batch_size = 32, num_workers = 4, shuffle = True)\nvalidloader = DataLoader(validset, batch_size = 32, num_workers = 4, shuffle = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To work with gray images we need to patch first conv layer from 3 input channels to 1. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def patch_first_conv(model, in_channels):\n    \"\"\"Change first convolution layer input channels.\n    In case:\n        in_channels == 1 or in_channels == 2 -> reuse original weights\n        in_channels > 3 -> make random kaiming normal initialization\n    \"\"\"\n\n    # get first conv\n    for module in model.modules():\n        if isinstance(module, nn.Conv2d):\n            break\n\n    # change input channels for first conv\n    module.in_channels = in_channels\n    weight = module.weight.detach()\n    reset = False\n\n    if in_channels == 1:\n        weight = weight.sum(1, keepdim=True)\n    elif in_channels == 2:\n        weight = weight[:, :2] * (3.0 / 2.0)\n    else:\n        reset = True\n        weight = torch.Tensor(\n            module.out_channels,\n            module.in_channels // module.groups,\n            *module.kernel_size\n        )\n\n    module.weight = nn.parameter.Parameter(weight)\n    if reset:\n        module.reset_parameters()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load our pretrained model. Try to use different models like efficientnet or resnest.\n\nmodel = pretrainedmodels.__dict__['se_resnext50_32x4d']( pretrained=None)\nmodel.load_state_dict(torch.load(\"../input/seresnext50/se_resnext50_32x4d-a260b3a4.pth\"))\n\n#if you want to freeze layers than uncomment next 2 rows:\n\n#for param in model_conv.parameters():\n#    param.requires_grad = False\nmodel.avg_pool = nn.AdaptiveAvgPool2d(output_size=1)\n\n#we have 11 classes\nmodel.last_linear = nn.Linear(2048, 11)\npatch_first_conv(model, 1)\n#learning params\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=2, )\ncriterion = nn.BCEWithLogitsLoss()\nnum_epochs=10\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model_conv, train_loader, valid_loader, criterion, optimizer, n_epochs=num_epochs, attempt=1):\n    model_conv.to(device)\n    valid_loss_min = np.Inf\n    patience = 5\n    # current number of epochs, where validation loss didn't increase\n    p = 0\n    # whether training should be stopped\n    stop = False\n\n    # number of epochs to train the model\n    for epoch in range(1, n_epochs+1):\n        print(time.ctime(), 'Epoch:', epoch)\n\n        train_loss = []\n        train_auc = []\n\n        for batch_i, (data, target) in tqdm(enumerate(train_loader)):\n\n            data, target = data.cuda(), target.cuda()\n\n            optimizer.zero_grad()\n            output = model_conv(data)\n            loss = criterion(output, target.float())\n            train_loss.append(loss.item())\n            loss.backward()\n            optimizer.step()\n\n        model_conv.eval()\n        val_loss = []\n        for batch_i, (data, target) in tqdm(enumerate(valid_loader)):\n            data, target = data.cuda(), target.cuda()\n            output = model_conv(data)\n            loss = criterion(output, target.float())\n            val_loss.append(loss.item()) \n            \n        print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}.')\n\n        valid_loss = np.mean(val_loss)\n        scheduler.step(valid_loss)\n        if valid_loss <= valid_loss_min:\n            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n            valid_loss_min,\n            valid_loss))\n            torch.save(model_conv.state_dict(), 'model_{}.pt'.format(attempt))\n            valid_loss_min = valid_loss\n            p = 0\n\n        # check if validation loss didn't improve\n        if valid_loss > valid_loss_min:\n            p += 1\n            print(f'{p} epochs of increasing val loss')\n            if p > patience:\n                print('Stopping training')\n                stop = True\n                break        \n\n        if stop:\n            break\n    return model_conv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Train it! \n#feel free to change optimizer, scheduler and other params\n#dont forget to increase epochs number!!\nmodel_seresnext = train_model(model, trainloader, validloader, criterion = criterion, \n                              optimizer = optimizer, n_epochs=2, attempt=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RanzcrTestDataset(Dataset):\n    \n    def __init__(self, files_folder_path, test_files, transfroms = None):\n        self.files_folder_path = files_folder_path\n        self.test_files = test_files\n        self.transforms = transfroms\n        \n    def __len__(self):\n        return len(self.test_files)\n    \n    def __getitem__(self, idx):\n        \n        image = cv2.imread(os.path.join(self.files_folder_path, self.test_files[idx] ), 0)\n    \n        if self.transforms:\n            image = self.transforms(image=image)['image']\n            \n        return image\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testset = RanzcrTestDataset(test_path, test_files, valid_augs)\ntestloader = DataLoader(testset, batch_size = 64, num_workers = 0, shuffle = False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_seresnext.eval()\npreds = []\ntk0 = tqdm(enumerate(testloader), total=len(testloader))\n\nfor i, images in tk0:\n\n    images = images.to(device)\n\n    with torch.no_grad():\n        y_preds = model_seresnext(images)\n\n    preds.append(torch.sigmoid(y_preds).to('cpu').numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.concatenate(preds)\ntarget_cols = train.columns[1:-1]\n\nfor i, row in enumerate(predictions):\n    submission.loc[i, target_cols] = row\n    \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}