{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Library Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader as DL\nfrom torch.nn.utils import weight_norm as WN\nimport torch.nn.functional as F\n\nimport gc\nfrom time import time\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\nseed = 42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def breaker():\n    print(\"\\n\" + 50*\"-\" + \"\\n\")\n\ndef head(x, no_of_ele=5):\n    print(x[:no_of_ele])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Handling"},{"metadata":{},"cell_type":"markdown","source":"**Loading Image Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"images = np.load(\"../input/rccl-1x144x144/images_1x144x144.npy\")\nlabels = np.load(\"../input/rccl-1x144x144/labels_1x144x144.npy\")\n\ntr_images, va_images, tr_labels, va_labels = train_test_split(images, \n                                                              labels, \n                                                              test_size=0.2, \n                                                              shuffle=True, \n                                                              random_state=seed)\n\ndel images, labels\n\nbreaker()\nprint(\"Garbage Collected : {}\".format(gc.collect()))\nbreaker()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Dataset Template**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(Dataset):\n    def __init__(this, X=None, y=None, mode=\"train\"):\n        this.mode = mode\n        this.X = X\n        if mode == \"train\":\n            this.y = y\n            \n    def __len__(this):\n        return this.X.shape[0]\n    \n    def __getitem__(this, idx):\n        if this.mode == \"train\":\n            return torch.FloatTensor(this.X[idx]), torch.FloatTensor(this.y[idx])\n        else:\n            return torch.FloatTensor(this.X[idx])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CNN Configuration and Setup"},{"metadata":{},"cell_type":"markdown","source":"**Config**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG():\n    tr_batch_size = 128 # Alos va_batch_size\n    ts_batch_size = 128\n    \n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    in_channels = 1\n    OL = 11\n    \n    def __init__(this, filter_sizes=[64, 128, 256, 512], HL=[4096, 4096], epochs=50, n_folds=5):\n        this.filter_sizes = filter_sizes\n        this.HL = HL\n        this.epochs = epochs\n        this.n_folds = n_folds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Setup**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(this, in_channels=1, filter_sizes=None, HL=None, OL=None, use_DP=False, DP1=0.2, DP2=0.5):\n        super(CNN, this).__init__()\n        \n        this.use_DP = use_DP\n        \n        this.DP1 = nn.Dropout(p=0.2)\n        this.DP2 = nn.Dropout(p=0.5)\n        \n        this.MP_ = nn.MaxPool2d(kernel_size=2)\n        \n        this.CN1 = nn.Conv2d(in_channels=in_channels, out_channels=filter_sizes[0], kernel_size=3, stride=1, padding=1)\n        this.BN1 = nn.BatchNorm2d(num_features=filter_sizes[0], eps=1e-5)\n        \n        this.CN2 = nn.Conv2d(in_channels=filter_sizes[0], out_channels=filter_sizes[1], kernel_size=3, stride=1, padding=1)\n        this.BN2 = nn.BatchNorm2d(num_features=filter_sizes[1], eps=1e-5)\n        \n        this.CN3 = nn.Conv2d(in_channels=filter_sizes[1], out_channels=filter_sizes[2], kernel_size=3, stride=1, padding=1)\n        this.BN3 = nn.BatchNorm2d(num_features=filter_sizes[2], eps=1e-5)\n    \n        this.CN4 = nn.Conv2d(in_channels=filter_sizes[2], out_channels=filter_sizes[3], kernel_size=3, stride=1, padding=1)\n        this.BN4 = nn.BatchNorm2d(num_features=filter_sizes[3], eps=1e-5)\n        \n        this.CN5 = nn.Conv2d(in_channels=filter_sizes[3], out_channels=filter_sizes[3], kernel_size=3, stride=1, padding=1)\n        this.BN5 = nn.BatchNorm2d(num_features=filter_sizes[3], eps=1e-5)\n        \n        this.CN6 = nn.Conv2d(in_channels=filter_sizes[3], out_channels=filter_sizes[3], kernel_size=3, stride=1, padding=1)\n        this.BN6 = nn.BatchNorm2d(num_features=filter_sizes[3], eps=1e-5)\n        \n        this.FC1 = nn.Linear(in_features=filter_sizes[3]*2*2, out_features=HL[0])\n        this.FC2 = nn.Linear(in_features=HL[0], out_features=HL[1])\n        this.FC3 = nn.Linear(in_features=HL[1], out_features=OL)\n        \n    def getOptimizer(this, A_S=True, lr=1e-3, wd=0):\n        if A_S:\n            return optim.Adam(this.parameters(), lr=lr, weight_decay=wd)\n        else:\n            return optim.SGD(this.parameters(), lr=lr, momentum=0.9, weight_decay=wd)\n\n    def getStepLR(this, optimizer=None, step_size=5, gamma=0.1):\n        return optim.lr_scheduler.StepLR(optimizer=optimizer, step_size=step_size, gamma=gamma)\n\n    def getMultiStepLR(this, optimizer=None, milestones=None, gamma=0.1):\n        return optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=milestones, gamma=gamma)\n        \n    def getPlateauLR(this, optimizer=None, patience=5, eps=1e-6):\n        return optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, patience=patience, eps=eps, verbose=True)\n    \n    def forward(this, x):\n        if not this.use_DP:\n            x = F.relu(this.MP_(this.BN1(this.CN1(x))))\n            x = F.relu(this.MP_(this.BN2(this.CN2(x))))\n            x = F.relu(this.MP_(this.BN3(this.CN3(x))))\n            x = F.relu(this.MP_(this.BN4(this.CN4(x))))\n            x = F.relu(this.MP_(this.BN5(this.CN5(x))))\n            x = F.relu(this.MP_(this.BN6(this.CN6(x))))\n            \n            x = x.view(x.shape[0], -1)\n            \n            x = F.relu(this.FC1(x))\n            x = F.relu(this.FC2(x))\n            x = this.FC3(x)\n            \n            return x\n        else:\n            x = F.relu(this.MP_(this.BN1(this.CN1(x))))\n            x = F.relu(this.MP_(this.BN2(this.CN2(x))))\n            x = F.relu(this.MP_(this.BN3(this.CN3(x))))\n            x = F.relu(this.MP_(this.BN4(this.CN4(x))))\n            x = F.relu(this.MP_(this.BN5(this.CN5(x))))\n            x = F.relu(this.MP_(this.BN6(this.CN6(x))))\n            \n            x = x.view(x.shape[0], -1)\n            \n            x = F.relu(this.DP2(this.FC1(x)))\n            x = F.relu(this.DP2(this.FC2(x)))\n            x = this.FC3(x)\n            \n            return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Train Function**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_(model=None, optimizer=None, scheduler=None, epochs=None, early_stopping_patience=5, \n         trainloader=None, validloader=None, criterion=None, device=None, verbose=False):\n    \n    breaker()\n    print(\"Training ...\")\n    breaker()\n    \n    # model.to(device)\n    Losses = []\n\n    DLS = {\"train\" : trainloader, \"valid\" : validloader}\n    bestLoss = {\"train\" : np.inf, \"valid\" : np.inf}\n    \n    start_time = time()\n    for e in range(epochs):\n        e_st = time()\n        epochLoss = {\"train\" : 0.0, \"valid\" : 0.0}\n        \n        for phase in [\"train\", \"valid\"]:\n            if phase == \"train\":\n                model.train()\n            else:\n                model.eval()\n                \n            lossPerPass  = []\n            \n            for X, y in DLS[phase]:\n                X, y = X.to(device), y.to(device)\n                \n                optimizer.zero_grad()\n                with torch.set_grad_enabled(phase == \"train\"):\n                    output = model(X)\n                    loss = criterion(output, y)\n                    if phase == \"train\":\n                        loss.backward()\n                        optimizer.step()\n                lossPerPass.append(loss.item())\n            epochLoss[phase] = np.mean(np.array(lossPerPass))\n        Losses.append(epochLoss)\n        \n        \"\"\"if epochLoss[\"valid\"] < bestLoss[\"valid\"]:\n            bestLoss = epochLoss\n            name = \"./Epoch_{}.pt\".format(e+1)\n            torch.save(model.state_dict(), name)\n            early_stopping_step = 0\n            bestEpoch = e+1\n        else:\n            early_stopping_step += 1\n            if early_stopping_step > early_stopping_patience:\n                breaker()\n                print(\"Early Stopping at Epoch {} - Best Valid Loss {:.5f} at Epoch {}\".format(e+1, bestLoss[\"valid\"], bestEpoch))\n                break\"\"\"\n        \n        torch.save(model.state_dict(), \"./Epoch_{}.pt\".format(e+1))\n        \n        if epochLoss[\"valid\"] < bestLoss[\"valid\"]:\n            bestLoss = epochLoss\n            bestEpoch = e+1\n\n        if scheduler:\n            # scheduler.step()\n            scheduler.step(epochLoss[\"valid\"])\n            \n        if verbose:\n            print(\"Epoch: {} | Train Loss: {:.5f} | Valid Loss: {:.5f} | Time: {:.2f} seconds\".format(e+1, epochLoss[\"train\"], epochLoss[\"valid\"], time()-e_st))\n\n    breaker()\n    print(\"-----> Best Validation Loss at Epoch {}\".format(bestEpoch))\n    breaker()\n    print(\"Time Taken [{} Epochs] : {:.2f} minutes\".format(epochs, (time()-start_time)/60))\n    breaker()\n    print(\"Training Complete\")\n    breaker()\n\n    return Losses, bestEpoch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training**"},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = CFG(filter_sizes=[64, 128, 256, 512], HL=[4096, 4096], epochs=50, n_folds=5)\n\ntr_data_setup = Dataset(tr_images, tr_labels)\nva_data_setup = Dataset(va_images, va_labels)\n\ntr_data = DL(tr_data_setup, batch_size=cfg.tr_batch_size, shuffle=True, generator=torch.manual_seed(seed))\nva_data = DL(va_data_setup, batch_size=cfg.tr_batch_size, shuffle=False)\n\ndel tr_data_setup, va_data_setup\n\nbreaker()\nprint(\"Garbage Collected : {}\".format(gc.collect()))\n\ntorch.manual_seed(seed)\n\nmodel = CNN(filter_sizes=cfg.filter_sizes, HL=cfg.HL, OL=cfg.OL, use_DP=True).to(cfg.device)\noptimizer = model.getOptimizer(lr=1e-3, wd=1e-5)\n\n# scheduler = model.getStepLr(optimizer=optimizer, step_size=5, gamma=0.1)\n# scheduler = model.getMultiStepLR(optimizer=optimizer, milestones=[10, 20, 30, 40], gamma=0.1)\nscheduler = model.getPlateauLR(optimizer=optimizer, patience=5, eps=1e-8)\n\nLosses, bestEpoch = fit_(model=model, optimizer=optimizer, scheduler=scheduler, epochs=cfg.epochs, early_stopping_patience=5,\n                         trainloader=tr_data, validloader=va_data, criterion=nn.BCEWithLogitsLoss(), device=cfg.device,\n                         verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Loss Plot**"},{"metadata":{"trusted":true},"cell_type":"code","source":"LT = []\nLV = []\n\nfor i in range(len(Losses)):\n    LT.append(Losses[i][\"train\"])\n    LV.append(Losses[i][\"valid\"])\n\nplt.figure(figsize=(8, 6))\nplt.plot([i+1 for i in range(len(LT))], LT, \"r\", label=\"Training Loss\")\nplt.plot([i+1 for i in range(len(LV))], LV, \"b--\", label=\"Validation Loss\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}