{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is Resnet34 baseline model using [Pytorch-lightning](https://github.com/williamFalcon/pytorch-lightning).\n\npytorch-lightning is a very lightweight wrapper on Pytorch and you can write a training loop, validation loop, etc very easily.\n\nThis is my first public kernel. Thanks in advance !!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# install pytorch-lightning\n!pip install pytorch-lightning","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport os\nimport random as rn\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom PIL import Image, ImageFile\n\nimport torch\nfrom torch.nn import functional as F\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms as T\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fix random seed\nos.environ['PYTHONHASHSEED'] = '0'\nnp.random.seed(42)\nrn.seed(12345)\ntorch.manual_seed(2019)\ntorch.cuda.manual_seed(2019)\ntorch.cuda.manual_seed_all(2019)\ntorch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ref: https://www.kaggle.com/yhn112/resnet18-baseline-pytorch-ignite\nclass RCICDataset(Dataset):\n    def __init__(\n        self,\n        df,\n        img_dir,\n        mode='train',\n        site=1,\n        debug=False\n        ):\n\n        self.df = df\n        if debug:\n            self.df = df[:100]\n        self.records = self.df.to_records(index=False)\n        self.channels = [1,2,3,4,5,6]\n        self.site = site\n        self.mode = mode\n        self.debug = debug\n        self.img_dir = str(img_dir)\n        self.len = self.df.shape[0]\n        self.size = 256\n\n    @staticmethod\n    def _load_img_as_tensor(file_name):\n        with Image.open(file_name) as img:\n            return T.ToTensor()(img)\n#             return np.array(img)\n\n    def _get_img_path(self, index, channel):\n        experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate\n        return '/'.join([self.img_dir,\n                         self.mode,\n                         experiment,\n                         'Plate{}'.format(plate),\n                         '{}_s{}_w{}.png'.format(well,\n                                                 self.site,\n                                                 channel)])\n\n    def __getitem__(self, index):\n        paths = [self._get_img_path(index, ch) for ch in self.channels]\n        img = torch.cat([self._load_img_as_tensor(img_path)\n                         for img_path in paths])\n        if self.mode == 'train':\n            return img, self.records[index].sirna\n        else:\n            return img, self.records[index].id_code\n\n    def __len__(self):\n        \"\"\"\n        Total number of samples in the dataset\n        \"\"\"\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define variables\nepoch = 10\nn_class = 1108\nDATA_DIR = '../input/recursion-cellular-image-classification'\ndebug = False\nbatchsize = 64\nnum_workers = 4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If you use Pytorch-lightning, you need to do 2 things below.\n\n1. Define a LightningModule\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class RCICSystem(pl.LightningModule):\n\n    def __init__(self, train_loader, val_loader, model):\n        super(RCICSystem, self).__init__()\n        # not the best model...\n        self.train_loader = train_loader\n        self.val_loader = val_loader\n        self.model = model\n        self.criteria = nn.CrossEntropyLoss()\n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_nb):\n        # REQUIRED\n        x, y = batch\n        y_hat = self.forward(x)\n        loss = self.criteria(y_hat, y)\n        loss = loss.unsqueeze(dim=-1)\n        return {'loss': loss}\n\n    def validation_step(self, batch, batch_nb):\n        # OPTIONAL\n        x, y = batch\n        y_hat = self.forward(x)\n        val_loss = self.criteria(y_hat, y)\n        val_loss = val_loss.unsqueeze(dim=-1)\n        return {'val_loss': val_loss}\n\n    def validation_end(self, outputs):\n        # OPTIONAL\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        return {'avg_val_loss': avg_loss}\n\n    def configure_optimizers(self):\n        # REQUIRED\n        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.001)\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)\n        return [optimizer], [scheduler]\n\n    @pl.data_loader\n    def tng_dataloader(self):\n        # REQUIRED\n        return self.train_loader\n\n    @pl.data_loader\n    def val_dataloader(self):\n        # OPTIONAL\n        return self.val_loader\n    @pl.data_loader\n    def test_dataloader(self):\n        # OPTIONAL\n        pass\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))\ntrain_df, val_df = train_test_split(train_df, stratify=train_df['sirna'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# you can also define a checkpoint callback to save best model like keras.\ncheckpoint_callback = ModelCheckpoint(\n    filepath='../working',\n    save_best_only=True,\n    verbose=True,\n    monitor='avg_val_loss',\n    mode='min'\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get resnet34 model with 6 channels\nfrom torchvision.models import resnet34\ndef get_model(pretrained=False):\n    model = resnet34(pretrained=pretrained)\n    model.fc = nn.Linear(512, n_class)\n    trained_kernel = model.conv1.weight\n    new_conv = nn.Conv2d(6, 64, kernel_size=7, stride=2, padding=3, bias=False)\n    with torch.no_grad():\n        new_conv.weight[:,:] = torch.stack([torch.mean(trained_kernel, 1)]*6, dim=1)\n    model.conv1 = new_conv\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = RCICDataset(train_df, DATA_DIR, debug=debug)\ntrain_loader = DataLoader(train, batch_size=batchsize, pin_memory=True,\n                          shuffle=True)\nval = RCICDataset(val_df, DATA_DIR)\nval_loader = DataLoader(val, batch_size=batchsize, pin_memory=True,\n                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. Fit with Trainer"},{"metadata":{"trusted":true},"cell_type":"code","source":"\npl_model = RCICSystem(train_loader, val_loader, model)\n\n# set gpus, epoch and callbacks\ntrainer = Trainer(gpus=[0], max_nb_epochs=epoch,\n                  checkpoint_callback=checkpoint_callback)\n# fit model !\ntrainer.fit(pl_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\ndef load_pytorch_model(state_dict, *args, **kwargs):\n    new_state_dict = OrderedDict()\n    for k, v in state_dict.items():\n        name = k\n        if name.startswith('model.'):\n            name = name.replace('model.', '') # remove `model.`\n        new_state_dict[name] = v\n    model = get_model(pretrained=False)\n    model.load_state_dict(new_state_dict)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load best model\nfrom pathlib import Path\nckpt_path = list(Path('../working').glob('*.ckpt'))[0]\nckpt_dict = torch.load(ckpt_path)\nbest_model = load_pytorch_model(ckpt_dict['state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, dataloader, n_class, device, tta=1):\n    model.eval()\n    model.to(device)\n    preds = np.zeros([0, n_class])\n    for data, _ in dataloader:\n        data = data.to(device)\n        with torch.no_grad():\n            y_pred = model(data).detach()\n        #y_pred = F.softmax(y_pred, dim=1).cpu().numpy()\n        y_pred = y_pred.cpu().numpy()\n        preds = np.concatenate([preds, y_pred])\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculate validation accuracy\nval_preds = predict(best_model, val_loader, n_class=n_class, device=device)\nval_acc = accuracy_score(val_df.sirna, np.argmax(val_preds, axis=1))\nprint(f'val acc: {val_acc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))\ntest_dataset = RCICDataset(test_df, DATA_DIR, mode='test')\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batchsize, \n                                          shuffle=False, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict test data\ntest_preds = predict(best_model, test_loader, n_class=n_class, device=device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save submission csv\nsubmission_df = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'))\nsubmission_df.sirna = np.argmax(test_preds, axis=1)\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}