{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Load libraries"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data as D\nfrom torch.optim.lr_scheduler import ExponentialLR\n\nfrom torchvision import models, transforms as T\n\nfrom ignite.engine import Events, create_supervised_evaluator, create_supervised_trainer\nfrom ignite.metrics import Loss, Accuracy\nfrom ignite.contrib.handlers.tqdm_logger import ProgressBar\nfrom ignite.handlers import  EarlyStopping, ModelCheckpoint\n\nfrom tqdm import tqdm_notebook\n\nfrom sklearn.model_selection import train_test_split\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define dataset and model"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"path_data = '../input'\ndevice = 'cuda'\nbatch_size = 32\ntorch.manual_seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImagesDS(D.Dataset):\n    def __init__(self, df, img_dir, mode='train', site=1, channels=[1,2,3,4,5,6]):\n        self.records = df.to_records(index=False)\n        self.channels = channels\n        self.site = site\n        self.mode = mode\n        self.img_dir = img_dir\n        self.len = df.shape[0]\n        \n    @staticmethod\n    def _load_img_as_tensor(file_name):\n        with Image.open(file_name) as img:\n            return T.ToTensor()(img)\n\n    def _get_img_path(self, index, channel):\n        experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate\n        return '/'.join([self.img_dir,self.mode,experiment,f'Plate{plate}',f'{well}_s{self.site}_w{channel}.png'])\n        \n    def __getitem__(self, index):\n        paths = [self._get_img_path(index, ch) for ch in self.channels]\n        img = torch.cat([self._load_img_as_tensor(img_path) for img_path in paths])\n        if self.mode == 'train':\n            return img, int(self.records[index].sirna)\n        else:\n            return img, self.records[index].id_code\n\n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path_data+'/train.csv')\ndf_train, df_val = train_test_split(df, test_size = 0.025, random_state=42)\ndf_test = pd.read_csv(path_data+'/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = ImagesDS(df_train, path_data, mode='train')\nds_val = ImagesDS(df_val, path_data, mode='train')\nds_test = ImagesDS(df_test, path_data, mode='test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = 1108\nmodel = models.resnet18(pretrained=True)\nnum_ftrs = model.fc.in_features\nmodel.fc = torch.nn.Linear(num_ftrs, classes)\n\n# let's make our model work with 6 channels\ntrained_kernel = model.conv1.weight\nnew_conv = nn.Conv2d(6, 64, kernel_size=7, stride=2, padding=3, bias=False)\nwith torch.no_grad():\n    new_conv.weight[:,:] = torch.stack([torch.mean(trained_kernel, 1)]*6, dim=1)\nmodel.conv1 = new_conv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=4)\nval_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=4)\ntloader = D.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0003)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ignite magic starts here"},{"metadata":{},"cell_type":"markdown","source":"Let's define which metrics we will use and create magic objects to train and validate our model"},{"metadata":{"trusted":true},"cell_type":"code","source":"metrics = {\n    'loss': Loss(criterion),\n    'accuracy': Accuracy(),\n}\n\n\ntrainer = create_supervised_trainer(model, optimizer, criterion, device=device)\nval_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Attach to our trainer a function to run a validator at the end of each epoch"},{"metadata":{"trusted":true},"cell_type":"code","source":"@trainer.on(Events.EPOCH_COMPLETED)\ndef compute_and_display_val_metrics(engine):\n    epoch = engine.state.epoch\n    metrics = val_evaluator.run(val_loader).metrics\n    print(\"Validation Results - Epoch: {}  Average Loss: {:.4f} | Accuracy: {:.4f} \"\n          .format(engine.state.epoch, \n                      metrics['loss'], \n                      metrics['accuracy']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In this problem I think it's better not to use the same learning rate during all the training, so let's make it decrease after each epoch"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_scheduler = ExponentialLR(optimizer, gamma=0.95)\n\n@trainer.on(Events.EPOCH_COMPLETED)\ndef update_lr_scheduler(engine):\n    lr_scheduler.step()\n    lr = float(optimizer.param_groups[0]['lr'])\n    print(\"Learning rate: {}\".format(lr))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We definitely need early stopping, I don't want to tune the number of epochs by hand"},{"metadata":{"trusted":true},"cell_type":"code","source":"handler = EarlyStopping(patience=6, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer)\nval_evaluator.add_event_handler(Events.COMPLETED, handler)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's warmup our last linear layer by freezing all the other layers for a couple of epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"@trainer.on(Events.EPOCH_STARTED)\ndef turn_on_layers(engine):\n    epoch = engine.state.epoch\n    if epoch == 1:\n        for name, child in model.named_children():\n            if name == 'fc':\n                pbar.log_message(name + ' is unfrozen')\n                for param in child.parameters():\n                    param.requires_grad = True\n            else:\n                pbar.log_message(name + ' is frozen')\n                for param in child.parameters():\n                    param.requires_grad = False\n    if epoch == 3:\n        pbar.log_message(\"Turn on all the layers\")\n        for name, child in model.named_children():\n            for param in child.parameters():\n                param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also, let's save our model's weights after some epochs to be able to use them later"},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoints = ModelCheckpoint('models', 'Model', save_interval=3, n_saved=3, create_dir=True)\ntrainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoints, {'ResNet18': model})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And we obviously need beautiful tqdm-based progress bars for our training process"},{"metadata":{"trusted":true},"cell_type":"code","source":"pbar = ProgressBar(bar_format='')\npbar.attach(trainer, output_transform=lambda x: {'loss': x})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's log some interesting information about our learning process to Tensorboard\n(Does not work in kaggle kernels, you need to have TensorboadX installed)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nif not 'KAGGLE_WORKING_DIR' in os.environ:  #  If we are not on kaggle server\n    from ignite.contrib.handlers.tensorboard_logger import *\n    tb_logger = TensorboardLogger(\"board/ResNet18\")\n    tb_logger.attach(trainer, log_handler=OutputHandler(tag=\"training\", output_transform=lambda loss: {'loss': loss}),\n                     event_name=Events.ITERATION_COMPLETED)\n\n    tb_logger.attach(val_evaluator, log_handler=OutputHandler(tag=\"validation\", metric_names=[\"accuracy\", \"loss\"],\n                     another_engine=trainer),event_name=Events.EPOCH_COMPLETED)\n\n    tb_logger.attach(trainer, log_handler=OptimizerParamsHandler(optimizer), event_name=Events.ITERATION_STARTED)\n\n    tb_logger.attach(trainer, log_handler=GradsHistHandler(model), event_name=Events.EPOCH_COMPLETED)\n    tb_logger.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally, we are ready to start training"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.run(loader, max_epochs=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction for test"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    preds = np.empty(0)\n    for x, _ in tqdm_notebook(tloader): \n        x = x.to(device)\n        output = model(x)\n        idx = output.max(dim=-1)[1].cpu().numpy()\n        preds = np.append(preds, idx, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(path_data + '/test.csv')\nsubmission['sirna'] = preds.astype(int)\nsubmission.to_csv('submission.csv', index=False, columns=['id_code','sirna'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"2b62ae829edc4d60acf1d9a9e1d598d8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"7740dfb227e54da8b1510dac2d094406":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"921a9c670b6e4a2db86c75a7ff5d9ee6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9dfcb7497f8842af817750eec565b8b9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_921a9c670b6e4a2db86c75a7ff5d9ee6","placeholder":"​","style":"IPY_MODEL_2b62ae829edc4d60acf1d9a9e1d598d8","value":" 94% 2151/2283 [22:45<01:23,  1.58it/s]"}},"d2df0eb5abab4e3895ec792681cfa8d2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"e3ff3ae302394523bb5b28ee009842d5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ff74a4321a59419cb24e116db9dd1e3e","IPY_MODEL_9dfcb7497f8842af817750eec565b8b9"],"layout":"IPY_MODEL_7740dfb227e54da8b1510dac2d094406"}},"fad7703039454db7af5d7fb4bce65003":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ff74a4321a59419cb24e116db9dd1e3e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"","description":"Loss: 128.54232788085938","description_tooltip":null,"layout":"IPY_MODEL_fad7703039454db7af5d7fb4bce65003","max":2283,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d2df0eb5abab4e3895ec792681cfa8d2","value":2151}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}