{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# How train a simple Denoising Autoencoder.\n* Implementation borrowed [here](https://www.kaggle.com/code/debarshichanda/pytorch-dae-starter). Please upvote original work by @Debarshi Chanda\n\n","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport time\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n\nfrom sklearn.preprocessing import QuantileTransformer\n\nfrom tqdm import tqdm\nfrom collections import defaultdict","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:45.697074Z","iopub.execute_input":"2022-07-05T04:10:45.69813Z","iopub.status.idle":"2022-07-05T04:10:45.706276Z","shell.execute_reply.started":"2022-07-05T04:10:45.69808Z","shell.execute_reply":"2022-07-05T04:10:45.705128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"epochs\": 10,\n    \"train_batch_size\": 1024,\n    \"learning_rate\": 1e-3,\n    \"T_max\": 2000,\n    \"min_lr\": 1e-5,\n    \"cat_weight\": 1./3,\n    \"cont_weight\": 2./3,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:45.746703Z","iopub.execute_input":"2022-07-05T04:10:45.747937Z","iopub.status.idle":"2022-07-05T04:10:45.755735Z","shell.execute_reply.started":"2022-07-05T04:10:45.747848Z","shell.execute_reply":"2022-07-05T04:10:45.754943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:45.787024Z","iopub.execute_input":"2022-07-05T04:10:45.788233Z","iopub.status.idle":"2022-07-05T04:10:46.71291Z","shell.execute_reply.started":"2022-07-05T04:10:45.788176Z","shell.execute_reply":"2022-07-05T04:10:46.711829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:46.714926Z","iopub.execute_input":"2022-07-05T04:10:46.715278Z","iopub.status.idle":"2022-07-05T04:10:46.724468Z","shell.execute_reply.started":"2022-07-05T04:10:46.715246Z","shell.execute_reply":"2022-07-05T04:10:46.722834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:46.727537Z","iopub.execute_input":"2022-07-05T04:10:46.728099Z","iopub.status.idle":"2022-07-05T04:10:46.754788Z","shell.execute_reply.started":"2022-07-05T04:10:46.728032Z","shell.execute_reply":"2022-07-05T04:10:46.753838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = [col for col in df.columns if col not in ['id']]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:46.757215Z","iopub.execute_input":"2022-07-05T04:10:46.758205Z","iopub.status.idle":"2022-07-05T04:10:46.763801Z","shell.execute_reply.started":"2022-07-05T04:10:46.758169Z","shell.execute_reply":"2022-07-05T04:10:46.762909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = [col for col in feature_cols if df[col].nunique() < 100]\ncont_cols = [col for col in feature_cols if df[col].nunique() > 100]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:46.764935Z","iopub.execute_input":"2022-07-05T04:10:46.765879Z","iopub.status.idle":"2022-07-05T04:10:46.995092Z","shell.execute_reply.started":"2022-07-05T04:10:46.765842Z","shell.execute_reply":"2022-07-05T04:10:46.993846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"epochs\": 10,\n    \"train_batch_size\": 1024,\n    \"learning_rate\": 1e-3,\n    \"T_max\": 2000,\n    \"min_lr\": 1e-5,\n    \"cat_weight\": 1./3,\n    \"cont_weight\": 2./3,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:46.996829Z","iopub.execute_input":"2022-07-05T04:10:46.99719Z","iopub.status.idle":"2022-07-05T04:10:47.004085Z","shell.execute_reply.started":"2022-07-05T04:10:46.997158Z","shell.execute_reply":"2022-07-05T04:10:47.002859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG[\"seed\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.005403Z","iopub.execute_input":"2022-07-05T04:10:47.006069Z","iopub.status.idle":"2022-07-05T04:10:47.020237Z","shell.execute_reply.started":"2022-07-05T04:10:47.006031Z","shell.execute_reply":"2022-07-05T04:10:47.018797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TPSJulyDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        self.cat_features = df[cat_cols].values\n        self.cont_features = df[cont_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        X_cat = self.cat_features[index]\n        X_cont = self.cont_features[index]\n        \n        return X_cat, X_cont","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.021846Z","iopub.execute_input":"2022-07-05T04:10:47.02219Z","iopub.status.idle":"2022-07-05T04:10:47.03373Z","shell.execute_reply.started":"2022-07-05T04:10:47.022158Z","shell.execute_reply":"2022-07-05T04:10:47.032768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DenoisingAutoEncoder(nn.Module):\n    def __init__(self):\n        super(DenoisingAutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(len(cat_cols) + len(cont_cols), 100),\n            nn.BatchNorm1d(100),\n            nn.ReLU(),\n            nn.Linear(100, 200)\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(200, 100),\n            nn.BatchNorm1d(100),\n            nn.ReLU(),\n        )\n        self.decoder_cat_head = nn.Linear(100, len(cat_cols))\n        self.decoder_cont_head = nn.Linear(100, len(cont_cols))\n        \n    def extract(self, x):\n        features = self.encoder(x)\n        return features\n        \n    def forward(self, x):\n        features = self.encoder(x)\n        output = self.decoder(F.relu(features))\n        cat_output = self.decoder_cat_head(output)\n        cont_output = self.decoder_cont_head(output)\n        \n        return cat_output, cont_output\n    \nmodel = DenoisingAutoEncoder()\nmodel.to(CONFIG['device']);","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.035268Z","iopub.execute_input":"2022-07-05T04:10:47.035881Z","iopub.status.idle":"2022-07-05T04:10:47.051753Z","shell.execute_reply.started":"2022-07-05T04:10:47.035848Z","shell.execute_reply":"2022-07-05T04:10:47.050806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cat_criterion(cat_outputs, cat_targets):\n    return nn.BCEWithLogitsLoss()(cat_outputs, cat_targets)\n\ndef cont_criterion(cont_outputs, cont_targets):\n    return nn.MSELoss()(cont_outputs.view(-1), cont_targets.view(-1))","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.055206Z","iopub.execute_input":"2022-07-05T04:10:47.055906Z","iopub.status.idle":"2022-07-05T04:10:47.066784Z","shell.execute_reply.started":"2022-07-05T04:10:47.055873Z","shell.execute_reply":"2022-07-05T04:10:47.065804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_swap_noise(X, ratio=.15, return_mask=False):\n    obfuscation_mask = torch.bernoulli(ratio * torch.ones(X.shape)).to(X.device)\n    obfuscated_X = torch.where(obfuscation_mask == 1, X[torch.randperm(X.shape[0])], X)\n    \n    if return_mask:\n        return obfuscated_X, obfuscation_mask\n    \n    return obfuscated_X","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.068634Z","iopub.execute_input":"2022-07-05T04:10:47.069322Z","iopub.status.idle":"2022-07-05T04:10:47.078622Z","shell.execute_reply.started":"2022-07-05T04:10:47.069288Z","shell.execute_reply":"2022-07-05T04:10:47.077743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_swap_noise():\n    X_rand = torch.randn(6, 8)\n    print(\"Original Array\")\n    print(X_rand)\n    \n    X_noise = add_swap_noise(X_rand)\n    print(\"Array after noise\")\n    print(X_noise)\n    \ntest_swap_noise()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.080098Z","iopub.execute_input":"2022-07-05T04:10:47.080673Z","iopub.status.idle":"2022-07-05T04:10:47.096841Z","shell.execute_reply.started":"2022-07-05T04:10:47.080638Z","shell.execute_reply":"2022-07-05T04:10:47.095764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    scaler = amp.GradScaler()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, (X_cat, X_cont) in bar: \n       \n        X_cat = X_cat.to(device, dtype=torch.float)\n        X_cont = X_cont.to(device, dtype=torch.float)\n\n        batch_size = X_cat.size(0)\n        \n        X_cat_noise = add_swap_noise(X_cat)\n    \n        X_cat_noise = X_cat_noise.to(device, dtype=torch.float)\n        X_cont_noise = add_swap_noise(X_cont)\n      \n        X_cont_noise = X_cont_noise.to(device, dtype=torch.float)\n        \n        with amp.autocast(enabled=True):\n            X_noise = torch.cat([X_cat_noise, X_cont_noise], dim=1)\n            cat_outputs, cont_outputs = model(X_noise)\n            cat_loss = cat_criterion(cat_outputs, X_cat)\n            cont_loss = cont_criterion(cont_outputs, X_cont)\n            loss = CONFIG['cat_weight']*cat_loss + CONFIG['cont_weight']*cont_loss\n\n     \n        scaler.scale(loss).backward()\n        \n        scaler.step(optimizer)\n        scaler.update()\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n        \n        if scheduler is not None:\n            scheduler.step()\n                \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])\n    gc.collect()\n    \n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.098451Z","iopub.execute_input":"2022-07-05T04:10:47.099015Z","iopub.status.idle":"2022-07-05T04:10:47.114907Z","shell.execute_reply.started":"2022-07-05T04:10:47.098982Z","shell.execute_reply":"2022-07-05T04:10:47.113762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n      \n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    \n    for epoch in range(1, num_epochs + 1):\n            train_epoch_loss = train_one_epoch(model, optimizer, scheduler, \n                                               train_loader, device, epoch)\n            print()\n            \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    \n    torch.save(model.state_dict(), 'model.bin')\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.116655Z","iopub.execute_input":"2022-07-05T04:10:47.117302Z","iopub.status.idle":"2022-07-05T04:10:47.132761Z","shell.execute_reply.started":"2022-07-05T04:10:47.117266Z","shell.execute_reply":"2022-07-05T04:10:47.131796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TPSJulyDataset(df)\ntrain_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                          num_workers=2, shuffle=True, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.134137Z","iopub.execute_input":"2022-07-05T04:10:47.134758Z","iopub.status.idle":"2022-07-05T04:10:47.156132Z","shell.execute_reply.started":"2022-07-05T04:10:47.134717Z","shell.execute_reply":"2022-07-05T04:10:47.154786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'])\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CONFIG['T_max'], eta_min=CONFIG['min_lr'])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.157847Z","iopub.execute_input":"2022-07-05T04:10:47.158743Z","iopub.status.idle":"2022-07-05T04:10:47.166003Z","shell.execute_reply.started":"2022-07-05T04:10:47.158701Z","shell.execute_reply":"2022-07-05T04:10:47.164756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = run_training(model, optimizer, scheduler, \n                     device=CONFIG['device'], \n                     num_epochs=CONFIG['epochs'])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T04:10:47.167465Z","iopub.execute_input":"2022-07-05T04:10:47.168329Z","iopub.status.idle":"2022-07-05T04:11:11.855428Z","shell.execute_reply.started":"2022-07-05T04:10:47.168293Z","shell.execute_reply":"2022-07-05T04:11:11.854104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}