{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# setting kernel type\n\nto use tpu  set training = 'tpu'   for gpu  training = 'gpu'   for cpu training = 'cpu'\n\n# **Note for TPU\n**while these notebook is ok to work in tpu  ,  the xla for pytorch As per they have announced, its still under development and so for operations not supported they automatically switch to CPU so it might get even slower than GPU.\n\nso for some reson in these notebook when using this data sets where  we have  large size of images when  image resize and transformations  it will use cpu for that transformation and do training in tpu and these will cost memory to work around with these either you save images after transformation and read them diractly into the model or  i just skip val phase and also  make batches very small in tpu \nto  use tpu in these dataset\njust set profile='low'  other  can set profile=''high'","metadata":{}},{"cell_type":"code","source":"training = 'gpu'\nprofile =  'high'  # for this dtata set to use tpu set these var to low\norginal = False # work on orginal data or on the preprocessed jpg data","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:14:59.863713Z","iopub.execute_input":"2022-08-11T14:14:59.864008Z","iopub.status.idle":"2022-08-11T14:14:59.869991Z","shell.execute_reply.started":"2022-08-11T14:14:59.863980Z","shell.execute_reply":"2022-08-11T14:14:59.868747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# install lighting and xla for tpu","metadata":{}},{"cell_type":"code","source":"if training =='tpu':\n    !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n    !python pytorch-xla-env-setup.py --version 1.7 --apt-packages libomp5 libopenblas-dev","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:00.500733Z","iopub.execute_input":"2022-08-11T14:15:00.501593Z","iopub.status.idle":"2022-08-11T14:15:00.510320Z","shell.execute_reply.started":"2022-08-11T14:15:00.501540Z","shell.execute_reply":"2022-08-11T14:15:00.508645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install pytorch-lightning==1.1.8\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-11T14:15:00.912144Z","iopub.execute_input":"2022-08-11T14:15:00.913152Z","iopub.status.idle":"2022-08-11T14:15:10.093129Z","shell.execute_reply.started":"2022-08-11T14:15:00.913119Z","shell.execute_reply":"2022-08-11T14:15:10.091984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:10.097236Z","iopub.execute_input":"2022-08-11T14:15:10.097588Z","iopub.status.idle":"2022-08-11T14:15:18.930617Z","shell.execute_reply.started":"2022-08-11T14:15:10.097546Z","shell.execute_reply":"2022-08-11T14:15:18.929354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rasterio","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:18.933673Z","iopub.execute_input":"2022-08-11T14:15:18.934683Z","iopub.status.idle":"2022-08-11T14:15:27.866229Z","shell.execute_reply.started":"2022-08-11T14:15:18.934633Z","shell.execute_reply":"2022-08-11T14:15:27.864861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# library  needed by lightining","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.nn import functional as F\nfrom torch import nn\nfrom pytorch_lightning.core import LightningModule\nfrom sklearn.metrics import label_ranking_average_precision_score\n\nprint(torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.870161Z","iopub.execute_input":"2022-08-11T14:15:27.871039Z","iopub.status.idle":"2022-08-11T14:15:27.878580Z","shell.execute_reply.started":"2022-08-11T14:15:27.871006Z","shell.execute_reply":"2022-08-11T14:15:27.877471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# other libraarys","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nfrom torchvision import datasets, models, transforms\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torchvision import datasets, models, transforms  \nfrom torch.utils.data.sampler import SubsetRandomSampler  \nfrom torch.utils.data import Dataset, DataLoader\n\nimport pandas as pd\nimport torch.nn.functional as F\n\nimport time\nimport os\nimport time\nimport random\nfrom datetime import datetime\nimport tifffile\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\nimport cv2\nimport timm\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint , EarlyStopping\nfrom PIL import Image, ImageSequence\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.880656Z","iopub.execute_input":"2022-08-11T14:15:27.881041Z","iopub.status.idle":"2022-08-11T14:15:27.893727Z","shell.execute_reply.started":"2022-08-11T14:15:27.881006Z","shell.execute_reply":"2022-08-11T14:15:27.892179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# seeding everthing","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if training == 'gpu':\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\nseed_everything(3407)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.895819Z","iopub.execute_input":"2022-08-11T14:15:27.897031Z","iopub.status.idle":"2022-08-11T14:15:27.912731Z","shell.execute_reply.started":"2022-08-11T14:15:27.896991Z","shell.execute_reply":"2022-08-11T14:15:27.911567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# configartions","metadata":{}},{"cell_type":"code","source":"DATA_PATH = \"../input/mayo-clinic-strip-ai/\"\n\nif orginal:\n    TRAIN_PATH = \"../input/mayo-clinic-strip-ai/train/\"\nelse:\n    TRAIN_PATH = \"../input/jpg-images-strip-ai/train/\"\nTEST_PATH = \"../input/mayo-clinic-strip-ai/test/\"\n\n\n\n\nIMG_SIZE = 384\nif profile =='low':\n    BATCH_SIZE = 2 \n    val_BATCH_SIZE = 2\n    valchecking = 0\n    epoch = 3\n    monit = 'val_loss'\nelse:\n    BATCH_SIZE = 8\n    val_BATCH_SIZE = 8\n    valchecking = 1\n    epoch = 10\n    monit = 'val_loss'","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.914660Z","iopub.execute_input":"2022-08-11T14:15:27.915517Z","iopub.status.idle":"2022-08-11T14:15:27.924747Z","shell.execute_reply.started":"2022-08-11T14:15:27.915478Z","shell.execute_reply":"2022-08-11T14:15:27.923558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# read csv file with pandas","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.926517Z","iopub.execute_input":"2022-08-11T14:15:27.927638Z","iopub.status.idle":"2022-08-11T14:15:27.962434Z","shell.execute_reply.started":"2022-08-11T14:15:27.927600Z","shell.execute_reply":"2022-08-11T14:15:27.961466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.964284Z","iopub.execute_input":"2022-08-11T14:15:27.964945Z","iopub.status.idle":"2022-08-11T14:15:27.989270Z","shell.execute_reply.started":"2022-08-11T14:15:27.964907Z","shell.execute_reply":"2022-08-11T14:15:27.988142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:27.994182Z","iopub.execute_input":"2022-08-11T14:15:27.994500Z","iopub.status.idle":"2022-08-11T14:15:28.005585Z","shell.execute_reply.started":"2022-08-11T14:15:27.994472Z","shell.execute_reply":"2022-08-11T14:15:28.004152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts().plot(kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.007659Z","iopub.execute_input":"2022-08-11T14:15:28.008293Z","iopub.status.idle":"2022-08-11T14:15:28.268851Z","shell.execute_reply.started":"2022-08-11T14:15:28.008216Z","shell.execute_reply":"2022-08-11T14:15:28.267651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.270687Z","iopub.execute_input":"2022-08-11T14:15:28.271679Z","iopub.status.idle":"2022-08-11T14:15:28.294274Z","shell.execute_reply.started":"2022-08-11T14:15:28.271633Z","shell.execute_reply":"2022-08-11T14:15:28.293075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# split data","metadata":{}},{"cell_type":"code","source":"train_df, valid_df = model_selection.train_test_split(\n    df, test_size=0.1, random_state=999\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.296102Z","iopub.execute_input":"2022-08-11T14:15:28.296593Z","iopub.status.idle":"2022-08-11T14:15:28.304349Z","shell.execute_reply.started":"2022-08-11T14:15:28.296552Z","shell.execute_reply":"2022-08-11T14:15:28.303133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data set class for getting data from path","metadata":{}},{"cell_type":"code","source":"\n    \nclass TaskDataset(torch.utils.data.Dataset):\n    \"\"\"\n    Helper Class to create the pytorch dataset\n    \"\"\"\n\n    def __init__(self, df, data_path=DATA_PATH, mode=\"train\",  transforms=None):\n        super().__init__()\n        self.df_data = df\n        self.data_path = data_path\n        self.transforms = transforms\n        self.mode = mode\n        self.data_dir = TRAIN_PATH if mode == \"train\" else TEST_PATH\n\n    def __len__(self):\n        return len(self.df_data)\n\n    def __getitem__(self, index):\n        img_name = self.df_data.image_id.values[index]\n        if orginal:\n            img_path = os.path.join(self.data_dir, img_name + \".tif\" )\n            img  =  tifffile.imread(img_path)  \n            img = Image.fromarray(np.uint8(img)).resize((IMG_SIZE, IMG_SIZE))\n        else:\n            img_path = os.path.join(self.data_dir, img_name + \".jpg\" )\n            img =   Image.open(img_path).convert(\"RGB\")\n        if self.mode == \"train\":\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df_data.iloc[index].label]\n            label = torch.tensor(label,dtype= torch.float32) \n            label = label.type(torch.LongTensor) \n        if self.transforms is not None:\n            image = self.transforms(img)\n        if self.mode == \"train\":\n            return image, label\n        del img\n        return image","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.305932Z","iopub.execute_input":"2022-08-11T14:15:28.306494Z","iopub.status.idle":"2022-08-11T14:15:28.322511Z","shell.execute_reply.started":"2022-08-11T14:15:28.306457Z","shell.execute_reply":"2022-08-11T14:15:28.320379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **  transforming data doing augmention **","metadata":{}},{"cell_type":"code","source":"# Imagenet means and stds\nmean = [0.485, 0.456, 0.406]\nstd  = [0.229, 0.224, 0.225] \n        \ntransforms_train = transforms.Compose([\n                            transforms.Resize((IMG_SIZE, IMG_SIZE)),\n                         transforms.RandomRotation(45),\n                            transforms.RandomHorizontalFlip(),\n                            transforms.RandomVerticalFlip(),\n                            transforms.RandomResizedCrop(IMG_SIZE),\n                            transforms.ToTensor(),\n                          transforms.Normalize(mean=mean, std=std),\n                           # transforms.Grayscale(num_output_channels=1)\n                        ]\n                    )\n\ntransforms_valid = transforms.Compose(\n                        [\n                            transforms.Resize((IMG_SIZE, IMG_SIZE)),\n                            transforms.ToTensor(),\n                          transforms.Normalize(mean=mean, std=std),\n                            #transforms.Grayscale(num_output_channels=1)\n                        ]\n                    )        \n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.324036Z","iopub.execute_input":"2022-08-11T14:15:28.324978Z","iopub.status.idle":"2022-08-11T14:15:28.341762Z","shell.execute_reply.started":"2022-08-11T14:15:28.324888Z","shell.execute_reply":"2022-08-11T14:15:28.340490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  make datasets and data loaders","metadata":{}},{"cell_type":"code","source":"train_dataset = TaskDataset(train_df, transforms=transforms_train)\nvalid_dataset = TaskDataset(valid_df, transforms=transforms_valid)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.343663Z","iopub.execute_input":"2022-08-11T14:15:28.344074Z","iopub.status.idle":"2022-08-11T14:15:28.360070Z","shell.execute_reply.started":"2022-08-11T14:15:28.344026Z","shell.execute_reply":"2022-08-11T14:15:28.358738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''del df\ndel train_df\ndel valid_df'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.361563Z","iopub.execute_input":"2022-08-11T14:15:28.362842Z","iopub.status.idle":"2022-08-11T14:15:28.377208Z","shell.execute_reply.started":"2022-08-11T14:15:28.362805Z","shell.execute_reply":"2022-08-11T14:15:28.375829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader =  DataLoader( dataset=train_dataset,\n        batch_size=BATCH_SIZE, \n      \n        drop_last=True,\n         shuffle=True)\nvalid_loader =    DataLoader(  dataset=valid_dataset,\n        batch_size=val_BATCH_SIZE ,\n        \n        drop_last=True,\n         shuffle=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.379293Z","iopub.execute_input":"2022-08-11T14:15:28.379833Z","iopub.status.idle":"2022-08-11T14:15:28.389376Z","shell.execute_reply.started":"2022-08-11T14:15:28.379776Z","shell.execute_reply":"2022-08-11T14:15:28.388110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs, classes = next(iter(train_loader))  \nprint(classes)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.391642Z","iopub.execute_input":"2022-08-11T14:15:28.392081Z","iopub.status.idle":"2022-08-11T14:15:28.631309Z","shell.execute_reply.started":"2022-08-11T14:15:28.392038Z","shell.execute_reply":"2022-08-11T14:15:28.629222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(inputs[0].shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.633787Z","iopub.execute_input":"2022-08-11T14:15:28.634954Z","iopub.status.idle":"2022-08-11T14:15:28.642000Z","shell.execute_reply.started":"2022-08-11T14:15:28.634912Z","shell.execute_reply":"2022-08-11T14:15:28.640808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(inputs[1].view(IMG_SIZE,IMG_SIZE,3).numpy())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.643847Z","iopub.execute_input":"2022-08-11T14:15:28.645153Z","iopub.status.idle":"2022-08-11T14:15:28.928118Z","shell.execute_reply.started":"2022-08-11T14:15:28.645111Z","shell.execute_reply":"2022-08-11T14:15:28.927045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(inputs[1].view(IMG_SIZE,IMG_SIZE,3).numpy())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:28.929517Z","iopub.execute_input":"2022-08-11T14:15:28.930601Z","iopub.status.idle":"2022-08-11T14:15:29.201454Z","shell.execute_reply.started":"2022-08-11T14:15:28.930559Z","shell.execute_reply":"2022-08-11T14:15:29.200302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# in these code we defin pytorchlightning model\n\n\n","metadata":{}},{"cell_type":"code","source":"gpu = torch.cuda.is_available()\ndevice = torch.device(\"cuda\" if gpu else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:29.203554Z","iopub.execute_input":"2022-08-11T14:15:29.203861Z","iopub.status.idle":"2022-08-11T14:15:29.209666Z","shell.execute_reply.started":"2022-08-11T14:15:29.203821Z","shell.execute_reply":"2022-08-11T14:15:29.208606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning.loggers import TensorBoardLogger\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:29.211508Z","iopub.execute_input":"2022-08-11T14:15:29.212277Z","iopub.status.idle":"2022-08-11T14:15:29.223770Z","shell.execute_reply.started":"2022-08-11T14:15:29.212220Z","shell.execute_reply":"2022-08-11T14:15:29.222494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg = []\ntr_avg = []","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:29.225597Z","iopub.execute_input":"2022-08-11T14:15:29.225928Z","iopub.status.idle":"2022-08-11T14:15:29.238824Z","shell.execute_reply.started":"2022-08-11T14:15:29.225878Z","shell.execute_reply":"2022-08-11T14:15:29.237583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mamonmodel(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        i = 0\n        self.lr=1e-3\n        self.wd = 1e-6\n        self.schd = 'CosineAnnealingLR'\n        #self.best_model_wts = copy.deepcopy(model.state_dict())\n        self.best_loss = np.Inf\n        #self.logger = TensorBoardLogger()\n        basemodel = torch.hub.load('facebookresearch/deit:main', 'deit_base_patch16_384', pretrained=True)\n        for child in basemodel.blocks.children():\n            if i < 5:\n                for param in child.parameters():\n                    param.requires_grad = False\n            else:\n                for param in child.parameters():\n                    param.requires_grad = True\n            i +=1\n        print(i)\n        basemodel.head  =  nn.Sequential( \n                            \n             nn.Linear(768, 2) \n                    )\n        self.model = basemodel\n        \n    def forward(self,x):\n        x = self.model(x)\n        return x\n    def training_step(self,batch,batch_idx):\n        x , y = batch\n        y_hat = self.model(x)\n        loss =  F.cross_entropy(y_hat, y) # F.binary_cross_entropy_with_logits(y_hat , y)\n        #scor = label_ranking_average_precision_score(y.cpu(), torch.sigmoid(y_hat).cpu())\n        self.log('train_loss', loss)\n        #self.log('train_score', scor)\n        return  loss   \n    def validation_step(self,batch,batch_idx):\n        x , y = batch\n        y_hat = self.model(x)\n        loss = F.cross_entropy(y_hat, y) # F.binary_cross_entropy_with_logits(y_hat , y)\n        #scor = label_ranking_average_precision_score(y.cpu(), torch.sigmoid(y_hat).cpu())\n        #self.log('val_score', scor)\n        self.log('val_loss', loss) \n        if self.best_loss > loss:\n            self.best_loss = loss\n            torch.save(self.model.state_dict(), 'Deit-bestwigh-ep-'+str(self.current_epoch)+'.pth')\n            torch.save(self.model, 'Deit-best-model.pth')\n        return {'val_loss': loss}\n    def training_epoch_end(self, outputs):\n        #print(outputs)\n        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n        self.logger.experiment.add_scalar('train_loss',avg_loss, self.current_epoch)\n        tr_avg.append(avg_loss)\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n        self.logger.experiment.add_scalar('loss',avg_loss, self.current_epoch)\n        avg.append(avg_loss)\n        return {'av_loss' : avg_loss}\n    def configure_optimizers(self):\n        optimizer = optim.AdamW(\n            self.model.parameters(), lr=self.lr, weight_decay=self.wd\n        )\n        if self.schd == 'ReduceLROnPlateau':\n            scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.9, patience=3,\n                                          verbose=True)\n        elif self.schd == 'CosineAnnealingLR':\n            scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)\n        elif self.schd == 'CosineAnnealingWarmRestarts':\n            scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=0.001, last_epoch=-1) \n\n        return {'optimizer': optimizer, 'lr_scheduler': scheduler  ,  'monitor':monit }\n        ","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:29.240764Z","iopub.execute_input":"2022-08-11T14:15:29.241494Z","iopub.status.idle":"2022-08-11T14:15:29.528088Z","shell.execute_reply.started":"2022-08-11T14:15:29.241427Z","shell.execute_reply":"2022-08-11T14:15:29.526738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# init model\nmemomodel = Mamonmodel()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:29.531719Z","iopub.execute_input":"2022-08-11T14:15:29.532299Z","iopub.status.idle":"2022-08-11T14:15:32.200736Z","shell.execute_reply.started":"2022-08-11T14:15:29.532254Z","shell.execute_reply":"2022-08-11T14:15:32.199565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n{val_loss:.2f}\n'''\nearly_stop_callback = EarlyStopping(\n   monitor='val_loss',\n    min_delta=0.0000,\n   patience=3,\n    verbose=True,\n   mode='min'\n )\nif training == 'gpu':\n    checkpoint_callback = ModelCheckpoint(filename='{epoch}-gpumodel.pth', \n    verbose=True,\n    monitor='val_loss',\n    mode='min')\n    trainer = pl.Trainer(min_epochs=1 , max_epochs=epoch, check_val_every_n_epoch=valchecking,\n            \n                        gpus=-1,callbacks=[checkpoint_callback , early_stop_callback]\n    ) \n    \nelif training == 'tpu':\n    print('now we n tpu')\n    checkpoint_callback = ModelCheckpoint(filename='{epoch}-tpumodel.pth',  \n    verbose=True,\n    monitor=monit,\n    mode='min')\n    trainer = pl.Trainer(tpu_cores=8,min_epochs=1,num_sanity_val_steps=0 , check_val_every_n_epoch=valchecking , max_epochs=epoch,  callbacks=[checkpoint_callback])\nelse:\n    checkpoint_callback = ModelCheckpoint(filename='{epoch}-cpumodel.pth',  \n    verbose=True,\n    monitor='val_loss',\n    mode='min')\n    trainer = pl.Trainer(min_epochs=1 , max_epochs=epoch,check_val_every_n_epoch=valchecking,  callbacks=[checkpoint_callback , early_stop_callback])\n\n    \n\n\ntrainer.fit(memomodel, train_loader , valid_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:15:32.205942Z","iopub.execute_input":"2022-08-11T14:15:32.206758Z","iopub.status.idle":"2022-08-11T14:21:24.266322Z","shell.execute_reply.started":"2022-08-11T14:15:32.206727Z","shell.execute_reply":"2022-08-11T14:21:24.265188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = {'val_avg' : [ x.item() for x in  avg[1:] ]  , 'train_avg ': [ xx.item() for xx in  tr_avg ] } \nout_res = pd.DataFrame.from_dict(all_data)\nout_res.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:21:24.268385Z","iopub.execute_input":"2022-08-11T14:21:24.268777Z","iopub.status.idle":"2022-08-11T14:21:24.288392Z","shell.execute_reply.started":"2022-08-11T14:21:24.268736Z","shell.execute_reply":"2022-08-11T14:21:24.287100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference ","metadata":{}},{"cell_type":"code","source":" MODEL_PATH = (\n    \"./Deit-best-model.pth\"\n)\nBATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:40.555909Z","iopub.execute_input":"2022-08-09T11:29:40.556788Z","iopub.status.idle":"2022-08-09T11:29:40.561251Z","shell.execute_reply.started":"2022-08-09T11:29:40.556743Z","shell.execute_reply":"2022-08-09T11:29:40.560350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dftest = pd.read_csv(os.path.join(DATA_PATH, \"test.csv\"))\ndftest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:41.359170Z","iopub.execute_input":"2022-08-09T11:29:41.359410Z","iopub.status.idle":"2022-08-09T11:29:41.388484Z","shell.execute_reply.started":"2022-08-09T11:29:41.359382Z","shell.execute_reply":"2022-08-09T11:29:41.387740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntransforms_test = transforms.Compose(\n    [\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:44.216312Z","iopub.execute_input":"2022-08-09T11:29:44.216893Z","iopub.status.idle":"2022-08-09T11:29:44.223610Z","shell.execute_reply.started":"2022-08-09T11:29:44.216855Z","shell.execute_reply":"2022-08-09T11:29:44.221071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = '../input/jpg-images-strip-ai/test/'","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:44.999352Z","iopub.execute_input":"2022-08-09T11:29:44.999849Z","iopub.status.idle":"2022-08-09T11:29:45.004517Z","shell.execute_reply.started":"2022-08-09T11:29:44.999817Z","shell.execute_reply":"2022-08-09T11:29:45.003184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = TaskDataset(dftest,data_path=TEST_PATH,  mode=\"test\" , transforms=transforms_test )\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:45.777689Z","iopub.execute_input":"2022-08-09T11:29:45.778112Z","iopub.status.idle":"2022-08-09T11:29:45.784302Z","shell.execute_reply.started":"2022-08-09T11:29:45.778082Z","shell.execute_reply":"2022-08-09T11:29:45.783361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, pin_memory=False) \n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:46.943319Z","iopub.execute_input":"2022-08-09T11:29:46.944058Z","iopub.status.idle":"2022-08-09T11:29:46.949301Z","shell.execute_reply.started":"2022-08-09T11:29:46.944023Z","shell.execute_reply":"2022-08-09T11:29:46.948427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"memomodel.eval()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:48.094885Z","iopub.execute_input":"2022-08-09T11:29:48.095168Z","iopub.status.idle":"2022-08-09T11:29:48.106557Z","shell.execute_reply.started":"2022-08-09T11:29:48.095137Z","shell.execute_reply":"2022-08-09T11:29:48.105779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"memomodel.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:49.636660Z","iopub.execute_input":"2022-08-09T11:29:49.637230Z","iopub.status.idle":"2022-08-09T11:29:49.796263Z","shell.execute_reply.started":"2022-08-09T11:29:49.637198Z","shell.execute_reply":"2022-08-09T11:29:49.795351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport tqdm.notebook as tq\npreds_torch = []\nwith torch.no_grad():\n    test_tqdm  = tq.tqdm(test_loader, total=len(test_loader), desc=\"Testing\", position=0, leave=True)\n    for images in test_tqdm:\n        images = images.to(device)\n        preds_torch.extend(memomodel(images).detach().cpu().numpy()) ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:51.499767Z","iopub.execute_input":"2022-08-09T11:29:51.501732Z","iopub.status.idle":"2022-08-09T11:29:51.717252Z","shell.execute_reply.started":"2022-08-09T11:29:51.501673Z","shell.execute_reply":"2022-08-09T11:29:51.716465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_torch","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:54.448495Z","iopub.execute_input":"2022-08-09T11:29:54.448775Z","iopub.status.idle":"2022-08-09T11:29:54.460945Z","shell.execute_reply.started":"2022-08-09T11:29:54.448745Z","shell.execute_reply":"2022-08-09T11:29:54.459993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_out = pd.DataFrame(preds_torch, columns = ['CE','LAA'])\ndf_out","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:58.265950Z","iopub.execute_input":"2022-08-09T11:29:58.266807Z","iopub.status.idle":"2022-08-09T11:29:58.279186Z","shell.execute_reply.started":"2022-08-09T11:29:58.266760Z","shell.execute_reply":"2022-08-09T11:29:58.277497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:30:03.824970Z","iopub.execute_input":"2022-08-09T11:30:03.825265Z","iopub.status.idle":"2022-08-09T11:30:03.842034Z","shell.execute_reply.started":"2022-08-09T11:30:03.825232Z","shell.execute_reply":"2022-08-09T11:30:03.841275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['CE'] = df_out['CE']\nsub['LAA'] = df_out['LAA']\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:30:07.111134Z","iopub.execute_input":"2022-08-09T11:30:07.111551Z","iopub.status.idle":"2022-08-09T11:30:07.129430Z","shell.execute_reply.started":"2022-08-09T11:30:07.111518Z","shell.execute_reply":"2022-08-09T11:30:07.128611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index = False )\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:30:15.731215Z","iopub.execute_input":"2022-08-09T11:30:15.731544Z","iopub.status.idle":"2022-08-09T11:30:15.747625Z","shell.execute_reply.started":"2022-08-09T11:30:15.731513Z","shell.execute_reply":"2022-08-09T11:30:15.746892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}