{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport torch\nimport numpy as np\nimport pandas as pd\nimport albumentations as A\nimport pytorch_lightning as pl\nimport matplotlib.pyplot as plt\nfrom ray import tune\nfrom torch import nn, optim\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models import mobilenet_v3_small\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nseed = 1234\nnp.random.seed(seed)\ntorch.manual_seed(seed)\nplt.style.use('_mpl-gallery')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set Hyperparameter","metadata":{}},{"cell_type":"code","source":"os.mkdir(\"./output\")\nPROJECT_NAME = \"digit-recognizer\"\nDATA_PATH = os.path.join(os.getcwd()[:-8], \"input\", PROJECT_NAME)\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nBATCH_SIZE = 64\nEPOCH = 50\nPATIENCE = 10","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, x, y=None, transform=None):\n        self.x = x\n        if y is not None:\n            self.y = torch.LongTensor(y)\n        else:\n            self.y = None\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.x)\n\n    def __getitem__(self, idx):\n        image = self.transform(image=self.x[idx])[\"image\"]\n        if self.y is None:\n            return image\n        else:\n            return image, self.y[idx]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataloader(only_valid_data=False, transforms=None):\n    train_data = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\n\n    x_train = train_data.iloc[:,1:].to_numpy(dtype=np.float32).reshape(-1, 28, 28, 1)\n    y_train = train_data.label.to_numpy()\n    x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, \n        test_size=0.2, random_state=1234)\n    # Feature engineering\n    x_train, x_valid = x_train/255, x_valid/255\n    # Dataset and DataLoader\n    train_transform, valid_transform = transforms\n    valid_ds = CustomDataset(x_valid, y_valid, transform=valid_transform)\n    valid_dataloader = DataLoader(valid_ds, batch_size=BATCH_SIZE, pin_memory=True)\n    if only_valid_data:\n        return valid_dataloader\n    else:\n        train_ds = CustomDataset(x_train, y_train, transform=train_transform)\n        train_dataloader = DataLoader(train_ds, batch_size=BATCH_SIZE, pin_memory=True)\n        return train_dataloader, valid_dataloader","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_dataloader(transform=None):\n    test_data  = pd.read_csv(os.path.join(DATA_PATH, \"test.csv\"))\n\n    x_test = test_data.to_numpy(dtype=np.float32).reshape(-1, 28, 28, 1)\n    # Feature engineering\n    x_test = x_test/255\n    # Dataset and DataLoader\n    test_ds  = CustomDataset(x_test, None, transform=transform)\n    test_dataloader  = DataLoader(test_ds, batch_size=BATCH_SIZE, pin_memory=True)\n\n    return test_dataloader","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"train_transform = A.Compose([\n        A.ShiftScaleRotate(shift_limit=0.15, \n                           scale_limit=0.2, \n                           rotate_limit=25, \n                           border_mode=cv2.BORDER_CONSTANT, p=0.75),\n        ToTensorV2(),\n    ])\n\nvalid_transform = A.Compose([\n        ToTensorV2(),\n    ])\n\ntest_transform = A.Compose([\n        ToTensorV2(),\n    ])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class NeuralNetwork(nn.Module):\n    def __init__(self):\n        super(NeuralNetwork, self).__init__()\n        self.upsample = nn.Upsample(scale_factor=8, mode=\"bilinear\")\n        self.base_model = mobilenet_v3_small(pretrained=True).features\n        self.avgpool2d = nn.AdaptiveAvgPool2d((1, 1))\n        self.flatten = nn.Flatten()\n        self.dropout1 = nn.Dropout(0.4)\n        self.fc = nn.Linear(576, 10)\n\n    def forward(self, x):\n        x = torch.tile(self.upsample(x), (1, 3, 1, 1))\n        x = self.base_model(x)\n        x = self.fc(self.flatten(self.avgpool2d(x)))\n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"def train(dataloader, model, loss_fn, optimizer):\n    size = len(dataloader.dataset)\n    num_batches = len(dataloader)\n    model.train()\n    train_loss, correct = 0, 0\n    for X, y in dataloader:\n        X, y = X.to(DEVICE), y.to(DEVICE)\n        pred = model(X)\n        loss = loss_fn(pred, y)\n        for param in model.parameters():\n            param.grad = None\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n    train_loss /= num_batches\n    correct /= size\n    return train_loss, correct","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(dataloader, model, loss_fn):\n    size = len(dataloader.dataset)\n    num_batches = len(dataloader)\n    model.eval()\n    test_loss, correct = 0, 0\n    with torch.no_grad():\n        for X, y in dataloader:\n            X, y = X.to(DEVICE), y.to(DEVICE)\n            pred = model(X)\n            test_loss += loss_fn(pred, y).item()\n            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n    test_loss /= num_batches\n    correct /= size\n    return test_loss, correct","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(config):\n    transforms = train_transform, valid_transform\n\n    train_dataloader, valid_dataloader = get_train_dataloader(transforms=transforms)\n    \n    model = NeuralNetwork().to(DEVICE)\n    loss_fn = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=config[\"lr\"])\n\n    current_val_accuracy = 0\n    patience_count = 0\n\n    for t in range(EPOCH):\n        loss, accuracy = train(train_dataloader, model, loss_fn, optimizer)\n        val_loss, val_accuracy = test(valid_dataloader, model, loss_fn)\n        if patience_count == PATIENCE+1:\n            break\n        tune.report(\n            accuracy=accuracy,\n            loss=loss,\n            val_accuracy=val_accuracy, \n            val_loss=val_loss)\n        if current_val_accuracy < val_accuracy:\n            current_val_accuracy = val_accuracy\n            torch.save(model.state_dict(), \"./model.pth\")\n            patience_count = 0\n        else:\n            patience_count += 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameter Tuning","metadata":{}},{"cell_type":"code","source":"search_space = {\n    \"lr\": tune.uniform(0.0001, 0.01)\n    }\n\nanalysis = tune.run(\n    train_model,\n    config=search_space,\n    num_samples=8,\n    resources_per_trial={'gpu': 1},\n    local_dir='./output/',\n    verbose=1,\n    fail_fast=True\n    )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best hyperparameters found were:\")\nbest_trial = analysis.get_best_trial(\"val_accuracy\", \"max\", \"last\")\n\nfor k, v in analysis.get_all_configs().items():\n    if v == best_trial.config:\n        keys = k\n        break\nprint(best_trial.config)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_list = []\nfor i in analysis.get_all_configs().keys():\n    config_list.append(analysis.get_all_configs()[i])\n\naccury_list = []\ndfs = analysis.trial_dataframes\nfor i, d in enumerate(dfs.values()):\n    accury_list.append(d.val_accuracy.max())\n# Top 3\nindex = sorted(range(len(accury_list)), key=lambda i: accury_list[i])[-3:]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 4))\nfor i in index:\n    dfs[list(dfs.keys())[i]].val_accuracy.plot(label=str(config_list[i]))\nplt.legend()\nplt.tight_layout()\nplt.grid(axis='x')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Vaildate the best model","metadata":{}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4))\nax1.set_title('Accuracy')\nax1.plot(dfs[keys]['accuracy'], 'r', label='accuracy')\nax1.plot(dfs[keys]['val_accuracy'], 'g', label='val_accuracy')\nax1.legend()\nax1.grid(axis='x')\n\nax2.set_title('Loss')\nax2.plot(dfs[keys]['loss'], 'r', label='loss')\nax2.plot(dfs[keys]['val_loss'], 'g', label='val_loss')\nax2.legend()\nax2.grid(axis='x')\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logdir = analysis.get_best_logdir(\"val_accuracy\", mode=\"max\")\nstate_dict = torch.load(os.path.join(logdir, \"model.pth\"))\nmodel = NeuralNetwork().to(DEVICE)\nmodel.load_state_dict(state_dict)\nmodel.eval()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = train_transform, valid_transform\n\nvalid_dataloader = get_train_dataloader(only_valid_data=True, transforms=transforms)\ny_valid = []\ny_valid_pred = []\n\nwith torch.no_grad():\n    for X, y in valid_dataloader:\n        X = X.to(DEVICE)\n        for i in y.numpy():\n            y_valid.append(i)\n        pred = model(X)\n        for i in pred.argmax(1).cpu().numpy():\n            y_valid_pred.append(i)\ny_valid = np.array(y_valid)\ny_valid_pred = np.array(y_valid_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix = confusion_matrix(y_valid, y_valid_pred)\n\nplt.figure(figsize=(5, 5))\nplt.imshow(matrix, vmin=0, vmax=20)\nfor i in range(10):\n    for j in range(10):\n        plt.text(x=i, y=j, s=matrix[j, i], horizontalalignment='center')\nplt.title(\"Confusion Matrix\")\nplt.xticks(range(10))\nplt.yticks(range(10))\nplt.xlabel(\"Predicted Class\")\nplt.ylabel(\"True Class\")\nplt.grid(0)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit the prediction of test dataset","metadata":{}},{"cell_type":"code","source":"test_dataloader = get_test_dataloader(transform=test_transform)\n\ny_test_pred = []\nwith torch.no_grad():\n    for X in test_dataloader:\n        X = X.to(DEVICE)\n        pred = model(X)\n        for i in pred.argmax(1).cpu().numpy():\n            y_test_pred.append(i)\ny_test_pred = np.array(y_test_pred)\n\noutput = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\noutput['Label'] = y_test_pred\noutput.to_csv('./digit-recognizer.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}