{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm --quiet\n!pip install -U albumentations --quiet\n!pip install transformers --quiet\n!pip install adamp --quiet\nimport gc\nimport os\nimport psutil\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm \n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import precision_recall_fscore_support\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.metrics import accuracy_score,roc_auc_score\n\nfrom adamp import AdamP\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\nimport gc\nfrom tqdm.auto import tqdm\nimport cv2\nimport warnings\nwarnings.simplefilter('ignore')\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:38:46.642308Z","iopub.execute_input":"2021-07-15T07:38:46.642690Z","iopub.status.idle":"2021-07-15T07:39:11.318576Z","shell.execute_reply.started":"2021-07-15T07:38:46.642653Z","shell.execute_reply":"2021-07-15T07:39:11.317561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    INPUT_ROOT = \"../input/siim-covid19-resized-to-512px-png/train\"\n    SEED = 42\n    train_BATCH_SIZE = 12\n    val_BATCH_SIZE = 64\n    MODEL_NAME = \"tf_efficientnetv2_m_in21ft1k\"\n    LEARNING_RATE = 1e-3\n    NUM_EPOCHS = 25","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:25.718380Z","iopub.execute_input":"2021-07-15T07:39:25.718751Z","iopub.status.idle":"2021-07-15T07:39:25.724562Z","shell.execute_reply.started":"2021-07-15T07:39:25.718720Z","shell.execute_reply":"2021-07-15T07:39:25.722378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nset_seed(config.SEED)","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:33.431050Z","iopub.execute_input":"2021-07-15T07:39:33.431367Z","iopub.status.idle":"2021-07-15T07:39:33.440869Z","shell.execute_reply.started":"2021-07-15T07:39:33.431338Z","shell.execute_reply":"2021-07-15T07:39:33.439955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nann_df=pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:37.152929Z","iopub.execute_input":"2021-07-15T07:39:37.153264Z","iopub.status.idle":"2021-07-15T07:39:37.213327Z","shell.execute_reply.started":"2021-07-15T07:39:37.153231Z","shell.execute_reply":"2021-07-15T07:39:37.212282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id1=[]\nfor x in range(len(df)):\n    id1.append(df['id'].iloc[x].split(sep='_')[0])\ndf['StudyInstanceUID']=id1\ndf.drop(columns=['id'],inplace=True)\npath=[]\nann_df=ann_df.merge(df,on='StudyInstanceUID')\nfor x in range(len(ann_df)):\n    ann_df['id'].iloc[x]=ann_df['id'].iloc[x].split(sep='_')[0]\nfor instance_id in tqdm(ann_df['id']):\n    path.append(os.path.join(config.INPUT_ROOT, instance_id +'.png'))\nann_df['path'] = path","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:37.843900Z","iopub.execute_input":"2021-07-15T07:39:37.844220Z","iopub.status.idle":"2021-07-15T07:39:39.466359Z","shell.execute_reply.started":"2021-07-15T07:39:37.844190Z","shell.execute_reply":"2021-07-15T07:39:39.465521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique=ann_df['StudyInstanceUID'].unique()\nindex_comp=[]\nfor x in range(len(unique)):\n    index_val=ann_df[ann_df.StudyInstanceUID==unique[x]].index\n    index_comp.append(index_val[0])\nann_df=ann_df.iloc[index_comp].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:40.883930Z","iopub.execute_input":"2021-07-15T07:39:40.884336Z","iopub.status.idle":"2021-07-15T07:39:48.718126Z","shell.execute_reply.started":"2021-07-15T07:39:40.884292Z","shell.execute_reply":"2021-07-15T07:39:48.717284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_array=ann_df[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']].values\nlabel=np.argmax(label_array,axis=1)\nann_df['target']=label","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:48.719588Z","iopub.execute_input":"2021-07-15T07:39:48.719910Z","iopub.status.idle":"2021-07-15T07:39:48.726097Z","shell.execute_reply.started":"2021-07-15T07:39:48.719875Z","shell.execute_reply":"2021-07-15T07:39:48.725156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=ann_df[['path','Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance','StudyInstanceUID','target']]\nprint(len(df))","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:48.727974Z","iopub.execute_input":"2021-07-15T07:39:48.728676Z","iopub.status.idle":"2021-07-15T07:39:48.739342Z","shell.execute_reply.started":"2021-07-15T07:39:48.728635Z","shell.execute_reply":"2021-07-15T07:39:48.738235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SIIM_Train_Dataset(torch.utils.data.Dataset):\n    def __init__(self, X, y,transforms):\n        self.X = X\n        self.y = y\n        self.transforms=transforms\n        \n    def __len__(self):\n        return len(self.y)\n\n    def __getitem__(self, idx):\n        fname = self.X[idx]\n        label = self.y[idx]\n        img = Image.open(fname).convert('RGB')\n        img = np.array(img)\n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n        return img,torch.tensor(label,dtype=torch.float32)","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:48.740773Z","iopub.execute_input":"2021-07-15T07:39:48.741191Z","iopub.status.idle":"2021-07-15T07:39:48.749031Z","shell.execute_reply.started":"2021-07-15T07:39:48.741150Z","shell.execute_reply":"2021-07-15T07:39:48.748148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_tfms():\n      return A.Compose([\n              A.Resize(384, 384,always_apply=True,p=1.0),     \n              A.HorizontalFlip(p=0.5),\n              A.RandomBrightness(limit=0.1, p=0.75),\n              A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=30, border_mode=0, p=0.75),\n              A.Cutout(max_h_size=int(384 * 0.4), max_w_size=int(384 * 0.4), num_holes=1, p=0.75),    #512 for 384\n              A.Normalize(\n                    mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225],\n                    always_apply=True,p=1.0\n              ),\n              ToTensorV2(always_apply=True,p=1.0)\n      ])\n    \ndef valid_tfms():\n      return A.Compose([\n              A.Resize(384,384,always_apply=True,p=1.0),\n              A.Normalize(\n                    mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225],\n                    always_apply=True,p=1.0\n              ),\n              ToTensorV2(always_apply=True,p=1.0)\n      ])","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:52.116479Z","iopub.execute_input":"2021-07-15T07:39:52.116842Z","iopub.status.idle":"2021-07-15T07:39:52.126608Z","shell.execute_reply.started":"2021-07-15T07:39:52.116811Z","shell.execute_reply":"2021-07-15T07:39:52.125254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_loop(train_data_loader, model, optimizer,criterion):\n    losses = []\n    model.train()\n    optimizer.zero_grad()\n    for n_iter, (X, y) in tqdm(enumerate(train_data_loader), total=len(train_data_loader)):\n        X, y = X.to(device).float(), y.to(device)\n        outputs = model(X)\n        loss = criterion(outputs,y)\n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        losses.append(loss.item())\n        del X,y\n    return losses\n\ndef valid_loop(valid_dataloader,model,criterion):\n    losses = []\n    predicts = []\n    model.eval()\n    for i,(x,y) in tqdm(enumerate(valid_dataloader),total=len(valid_dataloader)):\n        x , y = x.to(device).float() , y.to(device)\n        with torch.no_grad():\n            outputs = model(x)\n            loss = criterion(outputs , y)\n        losses.append(loss.item())\n        predicts.extend(outputs.detach().cpu().numpy())\n        del x,y\n    return losses,predicts","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:54.723991Z","iopub.execute_input":"2021-07-15T07:39:54.724310Z","iopub.status.idle":"2021-07-15T07:39:54.736027Z","shell.execute_reply.started":"2021-07-15T07:39:54.724280Z","shell.execute_reply":"2021-07-15T07:39:54.735094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SIIMNet(nn.Module):\n    def __init__(self, model_name):\n        super(SIIMNet, self).__init__()\n        self.model_name = model_name\n        self.base_model = timm.create_model(model_name, pretrained=True,num_classes=4)\n    def forward(self, x):\n        x=self.base_model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:39:57.407402Z","iopub.execute_input":"2021-07-15T07:39:57.407771Z","iopub.status.idle":"2021-07-15T07:39:57.417672Z","shell.execute_reply.started":"2021-07-15T07:39:57.407728Z","shell.execute_reply":"2021-07-15T07:39:57.413685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nskf = GroupKFold(n_splits=5)\nfor fold, (train_index, valid_index) in enumerate(skf.split(df,groups=df['StudyInstanceUID'].values)):\n    if fold==3:\n        print(f\"### FOLD-{fold} ###\")\n        set_seed(config.SEED)\n\n        train_df=df.iloc[train_index].reset_index(drop=True)\n        valid_df=df.iloc[valid_index].reset_index(drop=True)\n        Id_train = train_df['path'].values\n        label_train = train_df[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']].values\n        Id_valid = valid_df['path'].values\n        label_valid = valid_df[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']].values\n        train_dset = SIIM_Train_Dataset(Id_train,label_train,train_tfms())\n        train_data_loader = torch.utils.data.DataLoader(train_dset, batch_size=config.train_BATCH_SIZE, shuffle=True, num_workers=2,drop_last=True,pin_memory=True)\n        valid_dset = SIIM_Train_Dataset(Id_valid,label_valid,valid_tfms())\n        valid_dataloader = torch.utils.data.DataLoader(valid_dset, batch_size=config.val_BATCH_SIZE, shuffle=False, num_workers=2,pin_memory=True)\n        model = SIIMNet(config.MODEL_NAME).to(device)\n        optimizer = AdamP(model.parameters(), lr = config.LEARNING_RATE)\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.7, patience=2, threshold=0.00004, min_lr=1e-5, verbose=True) # (0.75,0.00004,1e-5) (0.7)\n        best_score = 0\n        results = []\n        criterion_train =  nn.BCEWithLogitsLoss().to(device)\n        criterion_val = nn.BCEWithLogitsLoss().to(device)\n        for epoch in range(config.NUM_EPOCHS):\n            print(f\"epoch={epoch}\")\n            train_losses = train_loop(train_data_loader, model, optimizer,criterion_train)\n            valid_losses, valid_predicts = valid_loop(valid_dataloader, model,criterion_val)\n            \n            y_true = valid_df[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']].values\n            roc_auc = roc_auc_score(y_true, valid_predicts)\n\n            t_loss, v_loss = np.array(train_losses).mean(), np.array(valid_losses).mean()\n            scheduler.step(v_loss)\n            res = {\"t_loss\": t_loss, \"v_loss\": v_loss, \"roc_auc_score\":roc_auc}\n            if best_score<roc_auc:\n                best_score=roc_auc\n                torch.save(model.state_dict(), f\"SIIMnet_f{fold}_best_1model_{config.MODEL_NAME}_512_resolution.pth\")\n            print(res)\n            results.append(res)\n            torch.save({'model_state_dict':model.state_dict(),'optimizer_state_dict':optimizer.state_dict(),\n                          'scheduler_state_dict':scheduler.state_dict(),'train_loss':t_loss,'valid_loss':v_loss,\n                          'roc_auc_score':roc_auc}\n                          , f\"SIIMnet_f{fold}_last_model_{config.MODEL_NAME}_512_resolution.pth\")\n            with open(f'file_f{fold}_{config.MODEL_NAME}SIIM_512_resolution.txt','a') as file1:\n                file1.write(f'Epoch:{epoch} Train_loss:{t_loss} Valid_loss:{v_loss} roc_auc_score:{roc_auc}\\n')\n            gc.collect()\n            torch.cuda.empty_cache()\n        del model,optimizer,scheduler,train_data_loader,valid_dataloader,criterion_train,criterion_val,train_losses,valid_losses,valid_predicts\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-07-15T07:40:13.158745Z","iopub.execute_input":"2021-07-15T07:40:13.159069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}