{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-13T06:25:12.589385Z","iopub.execute_input":"2022-07-13T06:25:12.590110Z","iopub.status.idle":"2022-07-13T06:25:12.604232Z","shell.execute_reply.started":"2022-07-13T06:25:12.590016Z","shell.execute_reply":"2022-07-13T06:25:12.602761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import division\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport numpy as np\nimport os \n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn \nimport torch.utils.data\nfrom torchvision import transforms\nfrom torchvision.datasets import MNIST\nfrom torch.utils.data import DataLoader","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:12.612355Z","iopub.execute_input":"2022-07-13T06:25:12.612892Z","iopub.status.idle":"2022-07-13T06:25:13.579695Z","shell.execute_reply.started":"2022-07-13T06:25:12.612855Z","shell.execute_reply":"2022-07-13T06:25:13.578721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/digit-recognizer/train.csv',dtype = np.float32)\ntest_df = pd.read_csv('../input/digit-recognizer/test.csv',dtype = np.float32)\n\n\ny = torch.tensor(train_df.label.values)\nx = torch.tensor(train_df.loc[:,train_df.columns != \"label\"].values/255)\n\n\n\n\nx_tr, x_ts, y_tr, y_ts = train_test_split(x,y, test_size = 0.1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:13.581034Z","iopub.execute_input":"2022-07-13T06:25:13.581694Z","iopub.status.idle":"2022-07-13T06:25:18.991286Z","shell.execute_reply.started":"2022-07-13T06:25:13.581654Z","shell.execute_reply":"2022-07-13T06:25:18.990217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = torch.utils.data.TensorDataset(x_tr,y_tr)\ntest = torch.utils.data.TensorDataset(x_ts,y_ts)\n\ntrain_dataloader = DataLoader(train, batch_size = 124, shuffle = True)\ntest_dataloader = DataLoader(test, batch_size = 1, shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:18.996387Z","iopub.execute_input":"2022-07-13T06:25:18.996677Z","iopub.status.idle":"2022-07-13T06:25:19.002166Z","shell.execute_reply.started":"2022-07-13T06:25:18.996650Z","shell.execute_reply":"2022-07-13T06:25:19.000968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:19.003847Z","iopub.execute_input":"2022-07-13T06:25:19.004221Z","iopub.status.idle":"2022-07-13T06:25:19.044885Z","shell.execute_reply.started":"2022-07-13T06:25:19.004181Z","shell.execute_reply":"2022-07-13T06:25:19.043873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Autoencoder(nn.Module):\n    def __init__(self, device='cpu'):\n        super().__init__()\n        self.device = device\n        self.l1 = nn.Linear(784, 200) # encoder\n        self.l2 = nn.Linear(200, 784) # decoder\n\n    def forward(self, x):\n        h = self.l1(x)       # input to encoder\n        h = torch.relu(h)    # use ReLU\n\n        h = self.l2(h)       # input to decoder\n        y = torch.sigmoid(h) # use sigmoid\n\n        return y","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:19.046400Z","iopub.execute_input":"2022-07-13T06:25:19.046801Z","iopub.status.idle":"2022-07-13T06:25:19.057975Z","shell.execute_reply.started":"2022-07-13T06:25:19.046771Z","shell.execute_reply":"2022-07-13T06:25:19.056884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optimizers\n\n# generate AutoEncder\nmodel = Autoencoder(device=device).to(device)\n# The loss function is the binary cross-entropy error\ncriterion = nn.BCELoss()\n# set Adam as optimizer\noptimizer = optimizers.Adam(model.parameters())","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:19.059736Z","iopub.execute_input":"2022-07-13T06:25:19.060662Z","iopub.status.idle":"2022-07-13T06:25:20.945360Z","shell.execute_reply.started":"2022-07-13T06:25:19.060617Z","shell.execute_reply":"2022-07-13T06:25:20.944194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of epoch\nepochs = 10\n\n# training\nfor epoch in range(epochs):\n    train_loss = 0.\n    # loop of minibatch\n    for (x, _) in train_dataloader:\n        x = x.to(device) # use CPU\n        model.train()    # train mode\n        preds = model(x) # get output\n        loss = criterion(preds, x) # Get error between input x and reconstructed preds\n        optimizer.zero_grad()      # Initialize gradient to 0\n        loss.backward()  # calculate the gradient of the error\n        optimizer.step() # Update Parameters\n        train_loss += loss.item() # Update loss\n    # loss per epoch\n    train_loss /= len(train_dataloader)\n    # print loss per epoch\n    print('Epoch({}) -- Loss: {:.3f}'.format(\n        epoch+1,\n        train_loss\n    ))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:20.947027Z","iopub.execute_input":"2022-07-13T06:25:20.947655Z","iopub.status.idle":"2022-07-13T06:25:28.070592Z","shell.execute_reply.started":"2022-07-13T06:25:20.947614Z","shell.execute_reply":"2022-07-13T06:25:28.069577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, _ = next(iter(test_dataloader))\nx = x.to(device)\n\n# evaluation mode\nmodel.eval()\n# input test data and get results\nx_rec = model(x)\n\n# visulalization\nfor i, image in enumerate([x, x_rec]):\n    image = image.view(28, 28).detach().cpu().numpy()\n    plt.subplot(1, 2, i+1)\n    plt.imshow(image, cmap='binary_r')\n    plt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:28.072296Z","iopub.execute_input":"2022-07-13T06:25:28.072733Z","iopub.status.idle":"2022-07-13T06:25:28.198019Z","shell.execute_reply.started":"2022-07-13T06:25:28.072693Z","shell.execute_reply":"2022-07-13T06:25:28.196740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make Enocoder class\nclass VariationalEncoder(nn.Module):\n    \n    #initialization\n    def __init__(self, device='cpu'):\n        super().__init__()\n        self.device = device\n        # input:784 output:200\n        self.l1 = nn.Linear(784, 200)\n        # input:200 output:10 for calculate mean\n        self.l_mean = nn.Linear(200, 10)\n        # input:200 output:10 for calculate variance\n        self.l_var = nn.Linear(200, 10)\n\n        \n    def forward(self, x):\n        # input:784 output:200\n        h = self.l1(x)\n        # use Relu\n        h = torch.relu(h)\n        # input:200 output:10 for calculate mean\n        mean = self.l_mean(h)\n        # input:200 output:10 for calculate variance\n        var = self.l_var(h)\n        # use softplus\n        var = F.softplus(var)\n        \n        #get parameters for distribution\n        return mean, var","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:28.203749Z","iopub.execute_input":"2022-07-13T06:25:28.204149Z","iopub.status.idle":"2022-07-13T06:25:28.219245Z","shell.execute_reply.started":"2022-07-13T06:25:28.204103Z","shell.execute_reply":"2022-07-13T06:25:28.218060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make Decoder class\nclass VariationalDecoder(nn.Module):\n\n    #initialization\n    def __init__(self, device='cpu'):\n        super().__init__()\n        self.device = device\n        # input:10 output:200\n        self.l1 = nn.Linear(10, 200)\n        # input:200 output:784(original dimension)\n        self.l2 = nn.Linear(200, 784)\n\n    def forward(self, x):\n        # input:10 output:200\n        h = self.l1(x)\n        # use Relu\n        h = torch.relu(h)\n       # input:200 output:784(original dimension)\n        h = self.l2(h)\n        # use sigmoid\n        y = torch.sigmoid(h)\n\n        #get reconstructed image\n        return y","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:28.220995Z","iopub.execute_input":"2022-07-13T06:25:28.221362Z","iopub.status.idle":"2022-07-13T06:25:28.231578Z","shell.execute_reply.started":"2022-07-13T06:25:28.221327Z","shell.execute_reply":"2022-07-13T06:25:28.230169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VAE(nn.Module):\n\n    def __init__(self, device='cpu'):\n      \n        super().__init__()\n        self.device = device\n        self.encoder = VariationalEncoder(device=device)\n        self.decoder = VariationalDecoder(device=device)\n\n    def forward(self, x):\n      \n        # use encoder to get mean and variance\n        mean, var = self.encoder(x)\n        # make latent variable using mean and variance\n        z = self.latent_variable(mean, var)\n        # use decoder to reconstruct\n        y = self.decoder(z)\n        # get reconstructed image and latent variable\n        return y, z\n\n    def latent_variable(self, mean, var):\n   \n        # sampling from standard normal distribution for \"Reparameterization trick\"\n        eps = torch.randn(mean.size()).to(self.device)\n        \n        # calculate latent variable\n        z = mean + torch.sqrt(var)*eps\n        return z\n\n    def lower_bound(self, x):\n        \n        # use encoder to get mean and variance\n        mean, var = self.encoder(x)\n        # make latent variable using mean and variance\n        z = self.latent_variable(mean, var)\n        # use decoder to reconstruct\n        y = self.decoder(z)\n        \n        # calculate the mean of binary cross-entropy error as reconstructed error\n        reconst_loss =  - torch.mean(\n            torch.sum(x * torch.log(y) + (1 - x)*torch.log(1 - y),\n                      dim=1))\n        \n        # calculate latent loss\n        latent_loss = - 1/2 * torch.mean(\n            torch.sum(1 + torch.log(var) - mean**2 - var,\n                      dim=1))\n        \n        # get loss function\n        loss =  reconst_loss + latent_loss\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:28.233299Z","iopub.execute_input":"2022-07-13T06:25:28.233666Z","iopub.status.idle":"2022-07-13T06:25:28.250980Z","shell.execute_reply.started":"2022-07-13T06:25:28.233631Z","shell.execute_reply":"2022-07-13T06:25:28.250041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optimizers\n\n# generate vae model \nmodel = VAE(device=device).to(device)\n# set loss function\ncriterion = model.lower_bound\n# set optimizer (Adam)\noptimizer = optimizers.Adam(model.parameters())\n\n# see the archtecture of model\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:28.255530Z","iopub.execute_input":"2022-07-13T06:25:28.255813Z","iopub.status.idle":"2022-07-13T06:25:28.272065Z","shell.execute_reply.started":"2022-07-13T06:25:28.255788Z","shell.execute_reply":"2022-07-13T06:25:28.270889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# epochs\nepochs = 10\n\n# start training\nfor epoch in range(epochs):\n    train_loss = 0.\n    # バッチサイズのループ\n    for (x, _) in train_dataloader:\n        x = x.to(device) # use CPU\n        model.train()    # train mode\n        preds = model(x) # get output\n        loss = criterion(x) # Get error between input x and reconstructed preds\n        optimizer.zero_grad()      # Initialize gradient to 0\n        loss.backward()  # calculate the gradient of the error\n        optimizer.step() # Update Parameters\n        train_loss += loss.item() # Update loss\n    # loss per epoch\n    train_loss /= len(train_dataloader)\n    # print loss per epoch\n    print('Epoch({}) -- Loss: {:.3f}'.format(\n        epoch+1,\n        train_loss\n    ))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:28.273499Z","iopub.execute_input":"2022-07-13T06:25:28.274112Z","iopub.status.idle":"2022-07-13T06:25:40.622960Z","shell.execute_reply.started":"2022-07-13T06:25:28.274071Z","shell.execute_reply":"2022-07-13T06:25:40.621926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, _ = next(iter(test_dataloader))\nx = x.to(device)\n\n# evaluation mode\nmodel.eval()\n# input test data and get results\nx_rec,_ = model(x)\n# visulalization\nfor i, image in enumerate([x, x_rec]):\n    image = image.view(28, 28).detach().cpu().numpy()\n    plt.subplot(1, 2, i+1)\n    plt.imshow(image, cmap='binary_r')\n    plt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T06:25:40.624584Z","iopub.execute_input":"2022-07-13T06:25:40.624990Z","iopub.status.idle":"2022-07-13T06:25:40.739800Z","shell.execute_reply.started":"2022-07-13T06:25:40.624952Z","shell.execute_reply":"2022-07-13T06:25:40.738717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}