{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"! nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\npip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p ../working/models ../working/logs ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! cp -r ../input/trainingstuff .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls ../working","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(action='ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np \nfrom tqdm.notebook import tqdm\n\nimport sklearn\nfrom sklearn.model_selection import train_test_split\n\nimport torch as th \nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torchvision\nfrom torchvision import transforms\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import seed_everything, Trainer\nfrom pytorch_lightning.metrics.functional import accuracy\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, GPUStatsMonitor\nfrom pytorch_lightning.loggers import TensorBoardLogger\n\n# Personal training stuff\nfrom trainingstuff.dataset import LeafDataset, DataModule\nfrom trainingstuff.model import Model\n\nprint(f'[INFO] using pytorch version : {th.__version__}')\nprint(f'[INFO] using pytorch lightning version : {pl.__version__}')\nprint(f'[INFO] using torchvision version : {torchvision.__version__}')\nprint(f'[INFO] using pandas version : {pd.__version__}')\nprint(f'[INFO] using numpy version : {np.__version__}')\nprint(f'[INFO] using scikit-learn version : {sklearn.__version__}')\n\n\n\n%load_ext tensorboard","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Experiment Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    data_dir = os.path.abspath('../input/cassava-leaf-disease-classification')\n    models_dir = os.path.abspath('../working/models')\n    logs_dir = os.path.abspath('../working/logs')\n    train_data_dir = os.path.abspath('../input/cassava-leaf-disease-classification/train_images')\n    test_data_dir = os.path.abspath('../input/cassava-leaf-disease-classification/test_images')\n    num_epochs = 15\n    lr = 2e-2\n    resize = 600\n    img_h = 512\n    img_w = 512\n    weight_decay = .01\n    eps = 1e-8\n    train_batch_size = 32\n    test_batch_size = 32\n    base_model = 'resnet34'\n    seed_val = 2021\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = seed_everything(seed =  Config.seed_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config_dict = Config.__dict__.items()\nconfig_dict = dict([item for item in config_dict if '__' not in item[0]])\nconfig_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset & data loader pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"ls {Config.test_data_dir}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# load train.csv file\ntrain_df = pd.read_csv(os.path.join(Config.data_dir, 'train.csv'))\n\n# create a basic transformations pipeline\ndata_transform = {\n    'train': transforms.Compose([\n        transforms.Resize(size=(Config.resize, Config.resize)),\n        transforms.RandomHorizontalFlip(p=.7),\n        transforms.RandomVerticalFlip(p=.3),\n        transforms.RandomRotation(degrees=25),\n        transforms.CenterCrop(size=(Config.img_h, Config.img_w)),\n        transforms.ColorJitter(brightness=(0.4, 1), contrast=.2, saturation=0, hue=0),\n        transforms.GaussianBlur(kernel_size=3)\n    ]),\n    \n    'validation':transforms.Compose([\n        transforms.Resize(size=(Config.resize, Config.resize)),\n        transforms.RandomRotation(degrees=25),\n        transforms.CenterCrop(size=(Config.img_h, Config.img_w)),\n        transforms.ColorJitter(brightness=(0.45, 1), \n                               contrast=.1, \n                               saturation=.1, \n                               hue=0.1),\n        transforms.GaussianBlur(kernel_size=3)\n    ]), \n    \n    'test':transforms.Compose([\n        transforms.Resize(size=(Config.img_h, Config.img_w)),\n        transforms.RandomRotation(degrees=25),\n    ])\n    \n}\n\n\n# make our datamodtule from trainingstuff codes (https://www.kaggle.com/cedricmanouan11/trainingstuff)\ndm = DataModule(config=Config, \n                 train_data_dir=Config.train_data_dir, \n                 test_data_dir=Config.test_data_dir, \n                 train_df=train_df,\n                 data_transform=data_transform,\n                 validation_split=.2,\n                 train_frac = 1)\ndm.setup()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Model(config=config_dict)\nmodel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configure training pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n\nckpt_cb = ModelCheckpoint(\n    monitor='val_acc', \n    mode='max', \n    dirpath=Config.models_dir, \n    filename=f\"{Config.base_model}-\"+'leaf_disease_classifier-{val_acc:.5f}-{val_loss:.5f}'\n)\n\ngpu_stats = GPUStatsMonitor(\n    memory_utilization=True, \n    gpu_utilization=True, \n    fan_speed=True, \n    temperature=True\n)\nes = EarlyStopping(\n    monitor='val_acc', \n    patience=5, \n    mode='max'\n)\n\nLogger = TensorBoardLogger(\n    save_dir=Config.logs_dir, \n    name='cassava_leaf_disease'\n)\n\nCallbacks = [es, ckpt_cb, gpu_stats]\n\ntrainer = pl.Trainer(\n    gpus=-1, \n    max_epochs=Config.num_epochs, \n    precision=16,\n    callbacks=Callbacks,\n    logger=Logger,\n    #fast_dev_run=True\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train and save model "},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrainer.fit(model=model, datamodule=dm)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load & export previously trained model for inference\n* Load model checkpoint (.ckpt file)\n* convert lightning module to torchScript (.pt/.pth/.bin etc.)\n* save Script module "},{"metadata":{},"cell_type":"markdown","source":"## Some utilities to convert Lightning module to torch.jit format"},{"metadata":{"trusted":true},"cell_type":"code","source":"def Load_model(ckpt_path, device=None):\n    config_dict = Config.__dict__.items()\n    config_dict = dict([item for item in config_dict if '__' not in item[0]])\n    \n    loaded_model = Model(config=config_dict)\n    \n    loaded_model = loaded_model.load_from_checkpoint(ckpt_path)\n    \n    if device is None:\n        loaded_model = loaded_model.cpu()\n    else:\n        loaded_model = loaded_model.cuda()\n        \n    return loaded_model.eval()\n\n\ndef convert_to_script(model:Model, save=True):\n    scriptModule = th.jit.script(obj=model)\n    if save:\n        fname = os.path.join(Config.models_dir, f'{Config.base_model}-cassava-leaf.pt')\n        th.jit.save(\n            m=scriptModule, \n            f=fname\n        )\n        print(f'[INFO] Script module saved as {fname}')\n        \n        return fname","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(Config.models_dir)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fn = os.listdir(Config.models_dir)[0]\npath = os.path.join(\n    Config.models_dir, fn\n)\ntrained_model = Load_model(ckpt_path=path, device='cuda')\n\nmodel_path = convert_to_script(model=trained_model, save=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use ScriptModule to evaluate model before inference\n\ndef evaluate_model(model:Model, dataloader:DataLoader):\n    model.eval()\n    model.cuda()\n    accs = []\n    bar = tqdm(dataloader, desc='Evaluating')\n    with th.no_grad():\n        for data in bar:\n            xs, ys = data['img'], data['targets']\n            logits = model(xs.cuda())\n            preds = F.log_softmax(logits, dim=1)\n            acc = accuracy(pred=preds.detach().cpu(), target=ys.detach().cpu())\n            accs.append(acc.item())\n            bar.set_postfix({\n                \"accuracy\" : acc.item()\n            })\n            bar.refresh()\n    \n    return np.array(accs).mean()\n\n\ndef predict(dataset:LeafDataset, model:Model, tta=None):\n\n    test_dl = DataLoader(dataset=test_ds, batch_size=4)\n\n    predictions = []\n\n    with th.no_grad():\n        model.eval()\n        model.cuda()\n        for data in tqdm(test_dl, desc='Predicting'):\n            xs = data['img']\n            preds = model(xs.cuda())\n            predictions+=(preds.argmax(dim=1).detach().cpu().numpy().tolist())\n\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inf_model = th.jit.load(f=model_path)\ninf_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluate model\navg_acc = evaluate_model(model=inf_model, dataloader=dm.val_dataloader())\nprint(f'[INFO] Average accuracy : {avg_acc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run evaluation on test data\ntest_df = pd.read_csv(os.path.join(Config.data_dir, 'sample_submission.csv'))\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_test_df = pd.concat([test_df] * 25, ignore_index=True)\n\ntest_ds = LeafDataset(\n    data_dir=Config.test_data_dir, \n    df=fake_test_df, \n    transform=data_transform['test'], \n    task='test'\n)\n\npredictions = predict(\n    dataset=test_ds, \n    model=inf_model\n)\n\nfake_test_df['label'] = predictions\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}