{"cells":[{"metadata":{"papermill":{"duration":0.013021,"end_time":"2020-11-20T07:49:33.328421","exception":false,"start_time":"2020-11-20T07:49:33.3154","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Reduce training time by using npy files"},{"metadata":{},"cell_type":"markdown","source":"This notebook is an attempt at reducing training times by using npy image files instead of raw jpg images. The notebook itself is mostly adapted from \npestipeti wonderful [starter notebook](https://www.kaggle.com/pestipeti/cassava-pytorch-starter-train).\n\n\n## Training time comparison\n\n### cv2 2-fold training \n- fold0 - train_set(4:03 mins), val_set(2:01 mins)\n- fold1 - train_set(3:47 mins), val_set(1:51 mins)\n\n### npy 2-fold training\n- fold0 - train_set(1:28 mins), val_set(1:55 mins)\n- fold1 - train_set(1:05 mins), val_set(1:09 mins)\n\n\nAny feedback on the method and dataset would be most welcome.`"},{"metadata":{},"cell_type":"markdown","source":"## References\n\n1. [plot_confusion_matrix](https://deeplizard.com/learn/video/0LhiS6yu2qQ)\n2. [sklearn metrics example](https://towardsdatascience.com/confusion-matrix-for-your-multi-class-machine-learning-model-ff9aa3bf7826)\n3. [multi_class_classification](https://towardsdatascience.com/multi-class-classification-extracting-performance-metrics-from-the-confusion-matrix-b379b427a872)"},{"metadata":{},"cell_type":"markdown","source":"## Library imports"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-11-20T07:49:33.361273Z","iopub.status.busy":"2020-11-20T07:49:33.360454Z","iopub.status.idle":"2020-11-20T07:49:38.340877Z","shell.execute_reply":"2020-11-20T07:49:38.340239Z"},"papermill":{"duration":5.000532,"end_time":"2020-11-20T07:49:38.340989","exception":false,"start_time":"2020-11-20T07:49:33.340457","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# basic imports\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport itertools\nfrom tqdm.notebook import tqdm\nimport math\n\n# augumentations library\nfrom albumentations.pytorch import ToTensorV2\nimport albumentations as A\nimport cv2\n\n# DL library imports\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\n# metrics calculation\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\n# basic plotting library\nimport matplotlib.pyplot as plt\n\n# interactive plots\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport warnings  \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"temp = torch.Tensor([4.0, 30.0, 23.0])\ndevice = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device('cpu')\ntemp = temp.to(device)\nnp.save('temp.npy', temp.cpu().data.numpy())\na = np.load('temp.npy')"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!rm -rf /output/kaggle/working/*","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config files"},{"metadata":{"trusted":true},"cell_type":"code","source":"pipeline = {'train' : True, 'lr_find' : False, 'test' : False}\nmodel_cfg = {'model_architecture': 'resnet50', 'model_name': 'cv2_npy_test_v1',\n             'init_lr': 3e-4, 'weight_path': '../input/cassava-pytorch-r50-baseline'}\n\ntrain_cfg = {'batch_size': 16, 'shuffle': False, 'num_workers': 4, 'checkpt_every' : 1 }\nvalid_cfg = {'batch_size': 16, 'shuffle': False, 'num_workers': 4, 'validate_every' : 1 }\ntest_cfg  = {'batch_size': 16, 'shuffle': False, 'num_workers': 4}\n\nDIR_INPUT = '../input/cassava-leaf-disease-classification'\nNPY_FOLDER = '../input/cassava-npy-train-images/train_npy_images'\nSEED = 42\nN_FOLDS = 2 #if pipeline['DEBUG'] else 5\nN_EPOCHS = 1 #if pipeline['DEBUG'] else 10\nBATCH_SIZE = 32\nSIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index_label_map = {\n                0: \"Cassava Bacterial Blight (CBB)\", \n                1: \"Cassava Brown Streak Disease (CBSD)\",\n                2: \"Cassava Green Mottle (CGM)\", \n                3: \"Cassava Mosaic Disease (CMD)\", \n                4: \"Healthy\"\n                }\n\nclass_names = [value for key,value in index_label_map.items()]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_no_of_trainable_params(model):\n    total_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n    #print(total_trainable_params)\n    return total_trainable_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    \n#RANDOM_STATE = 42\nset_seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    #print(cm)\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    #plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt), horizontalalignment=\"center\", color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.578506Z","iopub.status.busy":"2020-11-20T07:49:38.577513Z","iopub.status.idle":"2020-11-20T07:49:38.655931Z","shell.execute_reply":"2020-11-20T07:49:38.657324Z"},"papermill":{"duration":0.140183,"end_time":"2020-11-20T07:49:38.657516","exception":false,"start_time":"2020-11-20T07:49:38.517333","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'{DIR_INPUT}/train.csv')\ntrain_df[['cls0', 'cls1', 'cls2', 'cls3', 'cls4']] = train_labels = pd.get_dummies(train_df.iloc[:, 1])\ntrain_df['npy_image_id'] = train_df['image_id'].str.replace('jpg', 'npy')\ntrain_labels = train_df.iloc[:, 1].values\nprint(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.405054Z","iopub.status.busy":"2020-11-20T07:49:38.404138Z","iopub.status.idle":"2020-11-20T07:49:38.407181Z","shell.execute_reply":"2020-11-20T07:49:38.406559Z"},"papermill":{"duration":0.024286,"end_time":"2020-11-20T07:49:38.407297","exception":false,"start_time":"2020-11-20T07:49:38.383011","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    \n    def __init__(self, df, dataset='train', transforms=None):\n        self.df = df\n        self.transforms=transforms\n        self.dataset=dataset\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):        \n        #image_src = f'{DIR_INPUT}/{self.dataset}_images/{self.df.loc[idx, \"image_id\"]}'\n        #image = cv2.imread(image_src, cv2.IMREAD_COLOR)\n        #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = np.load(f'{NPY_FOLDER}/{self.df.loc[idx, \"npy_image_id\"]}')\n\n        if self.dataset == 'train':\n            labels = self.df.loc[idx, ['cls0', 'cls1', 'cls2', 'cls3', 'cls4']].values\n            labels = torch.from_numpy(labels.astype(np.int8))\n            labels = labels.unsqueeze(-1)\n        \n        else:\n            labels = torch.Tensor(1)\n        \n        if self.transforms:\n            transformed = self.transforms(image=image)\n            image = transformed['image']\n\n        return image, labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transforms for Augumentations"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.496406Z","iopub.status.busy":"2020-11-20T07:49:38.495298Z","iopub.status.idle":"2020-11-20T07:49:38.500954Z","shell.execute_reply":"2020-11-20T07:49:38.500432Z"},"papermill":{"duration":0.032811,"end_time":"2020-11-20T07:49:38.501054","exception":false,"start_time":"2020-11-20T07:49:38.468243","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"transforms_train = A.Compose([\n    A.RandomResizedCrop(height=SIZE, width=SIZE, p=1.0),\n    #A.Flip(),\n    #A.ShiftScaleRotate(rotate_limit=1.0, p=0.8),\n    A.Normalize([0.4303133, 0.49675637, 0.3135656], \n                         [0.2379062, 0.24065569, 0.22874062], p=1.0),\n    ToTensorV2(p=1.0),\n])\n\ntransforms_valid = A.Compose([\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    A.Normalize([0.4303133, 0.49675637, 0.3135656], \n                         [0.2379062, 0.24065569, 0.22874062], p=1.0),\n    ToTensorV2(p=1.0),\n])\n\ntransforms_test = A.Compose([\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    A.Normalize([0.4303133, 0.49675637, 0.3135656], \n                         [0.2379062, 0.24065569, 0.22874062], p=1.0),\n    ToTensorV2(p=1.0),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CV strategy"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.711579Z","iopub.status.busy":"2020-11-20T07:49:38.710679Z","iopub.status.idle":"2020-11-20T07:49:38.713044Z","shell.execute_reply":"2020-11-20T07:49:38.712347Z"},"papermill":{"duration":0.031306,"end_time":"2020-11-20T07:49:38.713168","exception":false,"start_time":"2020-11-20T07:49:38.681862","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\noof_preds = np.zeros((train_df.shape[0],))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model class"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.44003Z","iopub.status.busy":"2020-11-20T07:49:38.439273Z","iopub.status.idle":"2020-11-20T07:49:38.445623Z","shell.execute_reply":"2020-11-20T07:49:38.443219Z"},"papermill":{"duration":0.027733,"end_time":"2020-11-20T07:49:38.445758","exception":false,"start_time":"2020-11-20T07:49:38.418025","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassavaModel(nn.Module):\n    \n    def __init__(self, num_classes=5, use_pretrained_weights=False):\n        super().__init__()\n        self.backbone = torchvision.models.resnet18(pretrained=use_pretrained_weights)\n        in_features = self.backbone.fc.in_features\n        self.logit = nn.Linear(in_features, num_classes)\n        \n    def forward(self, x):\n        batch_size, C, H, W = x.shape\n        \n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n        \n        x = F.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        x = F.dropout(x, 0.25, self.training)\n\n        x = self.logit(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.755438Z","iopub.status.busy":"2020-11-20T07:49:38.754653Z","iopub.status.idle":"2020-11-20T07:49:38.759335Z","shell.execute_reply":"2020-11-20T07:49:38.760049Z"},"papermill":{"duration":0.030221,"end_time":"2020-11-20T07:49:38.760202","exception":false,"start_time":"2020-11-20T07:49:38.729981","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class DenseCrossEntropy(nn.Module):\n\n    def __init__(self):\n        super(DenseCrossEntropy, self).__init__()\n        \n        \n    def forward(self, logits, labels):\n        logits = logits.float()\n        labels = labels.float()\n        \n        logprobs = F.log_softmax(logits, dim=-1)\n        \n        loss = -labels * logprobs\n        loss = loss.sum(-1)\n\n        return loss.mean()\n\n# creating loss function instance\ncriterion = DenseCrossEntropy()\n#criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Device as cpu or tpu\ndevice = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device('cpu')\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lr_find"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_lr_finder_results(lr_finder): \n    # Create subplot grid\n    fig = make_subplots(rows=1, cols=2)\n    # layout ={'title': 'Lr_finder_result'}\n    \n    # Create a line (trace) for the lr vs loss, gradient of loss\n    trace0 = go.Scatter(x=lr_finder['log_lr'], y=lr_finder['smooth_loss'],name='log_lr vs smooth_loss')\n    trace1 = go.Scatter(x=lr_finder['log_lr'], y=lr_finder['grad_loss'],name='log_lr vs loss gradient')\n\n    # Add subplot trace & assign to each grid\n    fig.add_trace(trace0, row=1, col=1);\n    fig.add_trace(trace1, row=1, col=2);\n    #iplot(fig, show_link=False)\n    fig.write_html(model_cfg['model_name'] + '_lr_find.html');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_lr(model, data_loader, optimizer, init_value = 1e-8, final_value=100.0, beta = 0.98, num_batches = 200):\n    assert(num_batches > 0)\n    mult = (final_value / init_value) ** (1/num_batches)\n    lr = init_value\n    optimizer.param_groups[0]['lr'] = lr\n    batch_num = 0\n    avg_loss = 0.0\n    best_loss = 0.0\n    smooth_losses = []\n    raw_losses = []\n    log_lrs = []\n    dataloader_it = iter(data_loader)\n    progress_bar = tqdm(range(num_batches))\n        \n    for idx in progress_bar:\n        batch_num += 1\n        try:\n            images, labels = next(dataloader_it)\n            #print(images.shape)\n        except:\n            dataloader_it = iter(data_loader)\n            images, labels = next(dataloader_it)\n\n        # Move input and label tensors to the default device\n        images = images.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n        \n        # handle exception in criterion\n        try:\n            # Forward pass\n            log_ps = model(images)\n            loss = criterion(log_ps, labels.squeeze(-1))\n        except:\n            if len(smooth_losses) > 1:\n                grad_loss = np.gradient(smooth_losses)\n            else:\n                grad_loss = 0.0\n            lr_finder_results = {'log_lr':log_lrs, 'raw_loss':raw_losses, \n                                 'smooth_loss':smooth_losses, 'grad_loss': grad_loss}\n            return lr_finder_results \n                    \n        #Compute the smoothed loss\n        avg_loss = beta * avg_loss + (1-beta) *loss.item()\n        smoothed_loss = avg_loss / (1 - beta**batch_num)\n        \n        #Stop if the loss is exploding\n        if batch_num > 1 and smoothed_loss > 50 * best_loss:\n            if len(smooth_losses) > 1:\n                grad_loss = np.gradient(smooth_losses)\n            else:\n                grad_loss = 0.0\n            lr_finder_results = {'log_lr':log_lrs, 'raw_loss':raw_losses, \n                                 'smooth_loss':smooth_losses, 'grad_loss': grad_loss}\n            return lr_finder_results\n        \n        #Record the best loss\n        if smoothed_loss < best_loss or batch_num==1:\n            best_loss = smoothed_loss\n        \n        #Store the values\n        raw_losses.append(loss.item())\n        smooth_losses.append(smoothed_loss)\n        log_lrs.append(math.log10(lr))\n        \n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        # print info\n        progress_bar.set_description(f\"loss: {loss.item()},smoothed_loss: {smoothed_loss},lr : {lr}\")\n\n        #Update the lr for the next step\n        lr *= mult\n        optimizer.param_groups[0]['lr'] = lr\n    \n    grad_loss = np.gradient(smooth_losses)\n    lr_finder_results = {'log_lr':log_lrs, 'raw_loss':raw_losses, \n                         'smooth_loss':smooth_losses, 'grad_loss': grad_loss}\n    return lr_finder_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if pipeline['lr_find'] == True:\n    # create Dataset\n    temp_train_dataset = CassavaDataset(df=train_df, dataset='train', transforms=transforms_train)\n    temp_train_dataloader = DataLoader(temp_train_dataset, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)\n    \n    # create model instance\n    model = CassavaModel(num_classes=5, use_pretrained_weights=True)\n    model.to(device)\n    optimizer = optim.Adam(model.parameters(), lr=model_cfg['init_lr'])\n    \n    lr_finder_results = find_lr(model, temp_train_dataloader, optimizer)\n    plot_lr_finder_results(lr_finder_results)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## One fold train and validation function"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.80578Z","iopub.status.busy":"2020-11-20T07:49:38.804867Z","iopub.status.idle":"2020-11-20T07:49:38.824741Z","shell.execute_reply":"2020-11-20T07:49:38.825868Z"},"papermill":{"duration":0.048245,"end_time":"2020-11-20T07:49:38.826028","exception":false,"start_time":"2020-11-20T07:49:38.777783","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def train_one_fold(i_fold, model, optimizer, dataloader_train, dataloader_valid):\n    \n    train_fold_results = []\n\n    for epoch in range(N_EPOCHS):\n        print('  Epoch {}/{}'.format(epoch + 1, N_EPOCHS))\n        model.train()\n        tr_loss = 0\n        lr_list = []\n        \n        # training iterator\n        tr_iterator = iter(dataloader_train)\n        train_progress_bar = tqdm(range(len(dataloader_train)))\n    \n        for idx in train_progress_bar:\n            try:\n                images, labels = next(tr_iterator)\n                #print(images.shape)\n            except StopIteration:\n                tr_iterator = iter(dataloader_train)\n                images, labels = next(tr_iterator)\n\n            images = images.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n            \n            # Forward pass\n            outputs = model(images)\n\n            # Backward pass\n            loss = criterion(outputs, labels.squeeze(-1))                \n            tr_loss += loss.item()\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n            \n            lr_list.append(optimizer.state_dict()[\"param_groups\"][0]['lr'])\n\n            # print to console\n            train_progress_bar.set_description(f\"Train_loss: {tr_loss} loss(avg): {tr_loss/(idx+1)}\")\n        \n        lr_list = np.array(lr_list)\n        np.save(model_cfg['model_name'] + '_' + str(i_fold) + 'fold_lr_list.npy', lr_list)\n\n        # Validate\n        model.eval()\n        val_loss = 0.0\n        val_preds = None\n        val_labels = None\n        \n        valid_iterator = iter(dataloader_valid)\n        valid_progress_bar = tqdm(range(len(dataloader_valid)))\n\n        for idx in valid_progress_bar:\n            try:\n                images, labels = next(valid_iterator)\n            except StopIteration:\n                tr_iterator = iter(dataloader_valid)\n                images, labels = next(valid_iterator)\n\n            if val_labels is None:\n                val_labels = labels.clone().squeeze(-1)\n            else:\n                val_labels = torch.cat((val_labels, labels.squeeze(-1)), dim=0)\n\n            images = images.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n\n            with torch.no_grad():\n                outputs = model(images)\n\n                loss = criterion(outputs, labels.squeeze(-1))\n                val_loss += loss.item()\n\n                preds = torch.softmax(outputs, dim=1).data.cpu()\n\n                if val_preds is None:\n                    val_preds = preds\n                else:\n                    val_preds = torch.cat((val_preds, preds), dim=0)\n            \n            # print to console\n            valid_progress_bar.set_description(f\"val_loss: {val_loss} loss(avg): {val_loss/(idx+1)}\")\n\n        \n        val_preds = torch.argmax(val_preds, dim=1)\n        val_labels = torch.argmax(val_labels, dim=1)\n        \n        np.save(model_cfg['model_name'] + '_val_preds_' + str(i_fold) + '.npy', val_preds.cpu().data.numpy())\n        np.save(model_cfg['model_name'] + '_val_labels_' + str(i_fold) + '.npy', val_labels.cpu().data.numpy())\n\n        train_fold_results.append({\n            'fold': i_fold,\n            'epoch': epoch,\n            'train_loss': tr_loss / len(dataloader_train),\n            'valid_loss': val_loss / len(dataloader_valid),\n            'valid_score': accuracy_score(val_labels, val_preds)\n        })\n\n    return val_preds, train_fold_results","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training and validation function calls"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-20T07:49:38.870498Z","iopub.status.busy":"2020-11-20T07:49:38.869688Z","iopub.status.idle":"2020-11-20T11:00:03.095416Z","shell.execute_reply":"2020-11-20T11:00:03.096102Z"},"papermill":{"duration":11424.253057,"end_time":"2020-11-20T11:00:03.096334","exception":false,"start_time":"2020-11-20T07:49:38.843277","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if pipeline['train'] == True:\n    submissions = None\n    train_results = []\n\n    for i_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df, train_labels)):\n        print(\"Fold {}/{}\".format(i_fold + 1, N_FOLDS))\n\n        valid = train_df.iloc[valid_idx]\n        valid.reset_index(drop=True, inplace=True)\n\n        train = train_df.iloc[train_idx]\n        train.reset_index(drop=True, inplace=True)    \n\n        dataset_train = CassavaDataset(df=train, dataset='train', transforms=transforms_train)\n        dataset_valid = CassavaDataset(df=valid, dataset='train', transforms=transforms_valid)\n\n        dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)\n        dataloader_valid = DataLoader(dataset_valid, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)\n\n        model = CassavaModel(num_classes=5, use_pretrained_weights=True)\n        model.to(device)\n\n        plist = [{'params': model.parameters(), 'lr': 1e-4}]\n        optimizer = optim.Adam(plist, lr=model_cfg['init_lr'])\n\n        val_preds, train_fold_results = train_one_fold(i_fold, model, optimizer, \n                                                       dataloader_train, dataloader_valid)\n        oof_preds[valid_idx] = val_preds.numpy()\n        train_results = train_results + train_fold_results\n\n        torch.save({\n            'fold': i_fold,\n            'lr': optimizer.state_dict()[\"param_groups\"][0]['lr'],\n            'model_state_dict': model.cpu().state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            # 'scheduler_state_dict'\n            # 'scaler_state_dict'\n        }, f\"{model_cfg['model_name']}_fold_{i_fold}.pth\")\n\n    print(\"{}-Folds CV score: {:.4f}\".format(N_FOLDS, accuracy_score(train_labels, oof_preds)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot training results"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_training_results():\n    fig = make_subplots(rows=2, cols=1)\n\n    colors = [\n        ('#d32f2f', '#ef5350'),\n        ('#303f9f', '#5c6bc0'),\n        ('#00796b', '#26a69a'),\n        ('#fbc02d', '#ffeb3b'),\n        ('#5d4037', '#8d6e63'),\n    ]\n\n    for i in range(N_FOLDS):\n        data = train_results[train_results['fold'] == i]\n\n        fig.add_trace(go.Scatter(x=data['epoch'].values,\n                                 y=data['train_loss'].values,\n                                 mode='lines',\n                                 visible='legendonly' if i > 0 else True,\n                                 line=dict(color=colors[i][0], width=2),\n                                 name='Train loss - Fold #{}'.format(i)),\n                     row=1, col=1)\n\n        fig.add_trace(go.Scatter(x=data['epoch'],\n                                 y=data['valid_loss'].values,\n                                 mode='lines+markers',\n                                 visible='legendonly' if i > 0 else True,\n                                 line=dict(color=colors[i][1], width=2),\n                                 name='Valid loss - Fold #{}'.format(i)),\n                     row=1, col=1)\n\n        fig.add_trace(go.Scatter(x=data['epoch'].values,\n                                 y=data['valid_score'].values,\n                                 mode='lines+markers',\n                                 line=dict(color=colors[i][0], width=2),\n                                 name='Valid score - Fold #{}'.format(i),\n                                 showlegend=False),\n                     row=2, col=1)\n\n    fig.update_layout({\n      \"annotations\": [\n        {\n          \"x\": 0.225, \n          \"y\": 1.0, \n          \"font\": {\"size\": 16}, \n          \"text\": \"Train / valid losses\", \n          \"xref\": \"paper\", \n          \"yref\": \"paper\", \n          \"xanchor\": \"center\", \n          \"yanchor\": \"bottom\", \n          \"showarrow\": False\n        }, \n        {\n          \"x\": 0.775, \n          \"y\": 1.0, \n          \"font\": {\"size\": 16}, \n          \"text\": \"Validation scores\", \n          \"xref\": \"paper\", \n          \"yref\": \"paper\", \n          \"xanchor\": \"center\", \n          \"yanchor\": \"bottom\", \n          \"showarrow\": False\n        }, \n      ]\n    })\n\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"val_preds_0 = np.load('./R18_imagenet_v2_val_preds_0.npy')\nval_labels_0 = np.load('./R18_imagenet_v2_val_labels_0.npy')\n\ncm = confusion_matrix(val_labels_0, val_preds_0)\nprint(cm)\nplt.figure(figsize=(8,8))\nplot_confusion_matrix(cm, classes=class_names, normalize=True)"},{"metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-11-20T11:00:03.254336Z","iopub.status.busy":"2020-11-20T11:00:03.253434Z","iopub.status.idle":"2020-11-20T11:00:03.279422Z","shell.execute_reply":"2020-11-20T11:00:03.276855Z"},"papermill":{"duration":0.121489,"end_time":"2020-11-20T11:00:03.27957","exception":false,"start_time":"2020-11-20T11:00:03.158081","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if pipeline['train'] == True:\n    train_results = pd.DataFrame(train_results)\n    print(train_results.head(10))\n    \n    final_results = train_results[train_results['epoch']==train_results['epoch'].max()]\n    print(final_results)\n    print(final_results['valid_score'].mean(), final_results['valid_score'].std())\n    plot_training_results()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Testing function"},{"metadata":{"trusted":true},"cell_type":"code","source":"if pipeline[\"test\"] == True:\n    # read submission file\n    submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')\n    submission_df.iloc[:, 1] = 4\n    #print(submission_df.head())\n    submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"if pipeline[\"test\"] == True:\n    # read submission file\n    submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')\n    submission_df.iloc[:, 1] = 0\n    #print(submission_df.head())\n\n\n    # just for debugging purporse, adding 1 more row\n    if submission_df.shape[0] == 1:\n        submission_df = pd.DataFrame([{'image_id': '2216849948.jpg', 'label': 0},{'image_id': '2216849948.jpg', 'label': 0}])\n        submission_df.reset_index(drop=True, inplace=True)\n    #print(submission_df.head())\n\n\n    # Creating test dataset and dataloaders\n    dataset_test = CassavaDataset(df=submission_df, dataset='test', transforms=transforms_test)\n    dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)\n    \n    \n    # placeholder for final submission csv\n    submissions = None\n\n    \"\"\"\n    1. Iterate and store predictions (one-hot encoded format) of N-folds of model \n    2. Average the predictions of all folds\n    3. argmax of mean one-hot encoded prediction is output\n    \"\"\"\n    for i_fold in range(N_FOLDS):\n        print(f'Inference for {i_fold}th fold')\n        model = CassavaModel(num_classes=5, use_pretrained_weights=False)\n        model.to(device)\n\n        checkpoint = torch.load(f\"{model_cfg['weight_path']}/{model_cfg['model_name']}_fold_{i_fold}.pth\", map_location=device)\n        model.load_state_dict(checkpoint['model_state_dict'], strict=True)\n        model.eval()\n        test_preds = None\n\n        for step, (images, _) in enumerate(dataloader_test):\n            images = images.to(device, dtype=torch.float)\n            with torch.no_grad():\n                outputs = model(images)\n                preds = torch.softmax(outputs, dim=1).data.cpu()\n                if test_preds is None:\n                    test_preds = preds\n                else:\n                    test_preds = torch.cat((test_preds, preds), dim=0)\n\n        # submission_df[['label']] = test_preds.argmax(test_preds, dim=1)\n        # submission_df.to_csv('submission_fold_{}.csv'.format(i_fold), index=False)\n\n        # logits avg\n        if submissions is None:\n            submissions = test_preds / N_FOLDS\n        else:\n            submissions += test_preds / N_FOLDS\n            \n        \n    #print(submissions[:10])\n    # argmax of predictions and write to csv\n    submission_df['label'] = torch.argmax(submissions, dim=1)\n    submission_df.to_csv('submission.csv', index=False)\n    #print(submission_df.head())"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}