{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training Notebook is [here](https://www.kaggle.com/code/hasangam/pytorch-denoise-autoencoder-starter)","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport time\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n\nfrom sklearn.preprocessing import QuantileTransformer\n\nfrom tqdm import tqdm\nfrom collections import defaultdict","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T14:27:18.543117Z","iopub.execute_input":"2022-07-06T14:27:18.543598Z","iopub.status.idle":"2022-07-06T14:27:19.607632Z","shell.execute_reply.started":"2022-07-06T14:27:18.543490Z","shell.execute_reply":"2022-07-06T14:27:19.606515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:19.609774Z","iopub.execute_input":"2022-07-06T14:27:19.610389Z","iopub.status.idle":"2022-07-06T14:27:21.039683Z","shell.execute_reply.started":"2022-07-06T14:27:19.610352Z","shell.execute_reply":"2022-07-06T14:27:21.038549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = [col for col in df.columns if col not in ['id']]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.040989Z","iopub.execute_input":"2022-07-06T14:27:21.041295Z","iopub.status.idle":"2022-07-06T14:27:21.046495Z","shell.execute_reply.started":"2022-07-06T14:27:21.041267Z","shell.execute_reply":"2022-07-06T14:27:21.045399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = [col for col in feature_cols if df[col].nunique() < 100]\ncont_cols = [col for col in feature_cols if df[col].nunique() > 100]\nid_cols = [\"id\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.048821Z","iopub.execute_input":"2022-07-06T14:27:21.049149Z","iopub.status.idle":"2022-07-06T14:27:21.262381Z","shell.execute_reply.started":"2022-07-06T14:27:21.049120Z","shell.execute_reply":"2022-07-06T14:27:21.261169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"epochs\": 10,\n    \"train_batch_size\": 1024,\n    \"test_batch_size\": 1024,\n    \"learning_rate\": 1e-3,\n    \"T_max\": 2000,\n    \"min_lr\": 1e-5,\n    \"cat_weight\": 1./3,\n    \"cont_weight\": 2./3,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.263752Z","iopub.execute_input":"2022-07-06T14:27:21.264068Z","iopub.status.idle":"2022-07-06T14:27:21.270464Z","shell.execute_reply.started":"2022-07-06T14:27:21.264039Z","shell.execute_reply":"2022-07-06T14:27:21.269457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG[\"seed\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.271763Z","iopub.execute_input":"2022-07-06T14:27:21.272054Z","iopub.status.idle":"2022-07-06T14:27:21.287885Z","shell.execute_reply.started":"2022-07-06T14:27:21.272027Z","shell.execute_reply":"2022-07-06T14:27:21.286705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DenoisingAutoEncoder(nn.Module):\n    def __init__(self):\n        super(DenoisingAutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(len(cat_cols) + len(cont_cols), 100),\n            nn.BatchNorm1d(100),\n            nn.ReLU(),\n            nn.Linear(100, 200)\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(200, 100),\n            nn.BatchNorm1d(100),\n            nn.ReLU(),\n        )\n        self.decoder_cat_head = nn.Linear(100, len(cat_cols))\n        self.decoder_cont_head = nn.Linear(100, len(cont_cols))\n        \n    def extract(self, x):\n        features = self.encoder(x)\n        return features\n        \n    def forward(self, x):\n        features = self.encoder(x)\n        output = self.decoder(F.relu(features))\n        cat_output = self.decoder_cat_head(output)\n        cont_output = self.decoder_cont_head(output)\n        \n        return cat_output, cont_output\n    \nmodel = DenoisingAutoEncoder()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.289178Z","iopub.execute_input":"2022-07-06T14:27:21.289675Z","iopub.status.idle":"2022-07-06T14:27:21.307669Z","shell.execute_reply.started":"2022-07-06T14:27:21.289621Z","shell.execute_reply":"2022-07-06T14:27:21.306571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = torch.load(\"../input/training-denoise-autoencoder-starter/model.bin\")\nmodel.load_state_dict(net)\nmodel.to(CONFIG['device'])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.308766Z","iopub.execute_input":"2022-07-06T14:27:21.309553Z","iopub.status.idle":"2022-07-06T14:27:21.327773Z","shell.execute_reply.started":"2022-07-06T14:27:21.309509Z","shell.execute_reply":"2022-07-06T14:27:21.326583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_cat = torch.tensor(df[cat_cols].values)\nX_cont = torch.tensor(df[cont_cols].values)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.329072Z","iopub.execute_input":"2022-07-06T14:27:21.329468Z","iopub.status.idle":"2022-07-06T14:27:21.351728Z","shell.execute_reply.started":"2022-07-06T14:27:21.329439Z","shell.execute_reply":"2022-07-06T14:27:21.350732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = torch.cat([X_cat, X_cont], dim=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.356469Z","iopub.execute_input":"2022-07-06T14:27:21.356914Z","iopub.status.idle":"2022-07-06T14:27:21.375603Z","shell.execute_reply.started":"2022-07-06T14:27:21.356879Z","shell.execute_reply":"2022-07-06T14:27:21.374371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TPSJulyTestDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        self.id = df[id_cols].values\n        self.cat_features = df[cat_cols].values\n        self.cont_features = df[cont_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        X_id = self.id[index]\n        X_cat = self.cat_features[index]\n        X_cont = self.cont_features[index]\n        \n        return X_id,X_cat, X_cont","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.377095Z","iopub.execute_input":"2022-07-06T14:27:21.377757Z","iopub.status.idle":"2022-07-06T14:27:21.386626Z","shell.execute_reply.started":"2022-07-06T14:27:21.377712Z","shell.execute_reply":"2022-07-06T14:27:21.385204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TPSJulyTestDataset(df)\ntest_loader = DataLoader(test_dataset, batch_size=CONFIG['test_batch_size'], \n                          num_workers=2, shuffle=False, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.388376Z","iopub.execute_input":"2022-07-06T14:27:21.388857Z","iopub.status.idle":"2022-07-06T14:27:21.403037Z","shell.execute_reply.started":"2022-07-06T14:27:21.388802Z","shell.execute_reply":"2022-07-06T14:27:21.401945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean_full_tensor = torch.empty((98000, 30), dtype=torch.float64)\nclean_data_arr = []\n#clean_full_tensor = torch.zeros(98000, 30)\nfor  (X_id , X_cat, X_cont) in test_loader:\n    #with amp.autocast(enabled=True):\n    X_cat = X_cat.to(CONFIG[\"device\"], dtype=torch.float)\n    X_cont = X_cont.to(CONFIG[\"device\"], dtype=torch.float)\n    X = torch.cat([X_cat , X_cont], dim=1)       \n    cat_outputs, cont_outputs = model(X)\n    temp = torch.cat([X_id , cat_outputs , cont_outputs], dim=1) \n    clean_data_arr.append(temp)\n\n   \n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:21.404146Z","iopub.execute_input":"2022-07-06T14:27:21.404547Z","iopub.status.idle":"2022-07-06T14:27:22.835525Z","shell.execute_reply.started":"2022-07-06T14:27:21.404509Z","shell.execute_reply":"2022-07-06T14:27:22.834223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_full_tensor = torch.cat(clean_data_arr , dim = 0)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.837461Z","iopub.execute_input":"2022-07-06T14:27:22.837883Z","iopub.status.idle":"2022-07-06T14:27:22.850975Z","shell.execute_reply.started":"2022-07-06T14:27:22.837825Z","shell.execute_reply":"2022-07-06T14:27:22.849967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_full_np = clean_full_tensor.detach().numpy()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.852618Z","iopub.execute_input":"2022-07-06T14:27:22.853392Z","iopub.status.idle":"2022-07-06T14:27:22.858198Z","shell.execute_reply.started":"2022-07-06T14:27:22.853359Z","shell.execute_reply":"2022-07-06T14:27:22.856982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = id_cols + cat_cols +  cont_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.859934Z","iopub.execute_input":"2022-07-06T14:27:22.860257Z","iopub.status.idle":"2022-07-06T14:27:22.868327Z","shell.execute_reply.started":"2022-07-06T14:27:22.860220Z","shell.execute_reply":"2022-07-06T14:27:22.867138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(clean_full_np ,columns = col)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.869763Z","iopub.execute_input":"2022-07-06T14:27:22.870829Z","iopub.status.idle":"2022-07-06T14:27:22.879189Z","shell.execute_reply.started":"2022-07-06T14:27:22.870783Z","shell.execute_reply":"2022-07-06T14:27:22.878226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = id_cols + feature_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.880839Z","iopub.execute_input":"2022-07-06T14:27:22.881445Z","iopub.status.idle":"2022-07-06T14:27:22.889478Z","shell.execute_reply.started":"2022-07-06T14:27:22.881414Z","shell.execute_reply":"2022-07-06T14:27:22.888130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[cols] ","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.890911Z","iopub.execute_input":"2022-07-06T14:27:22.891322Z","iopub.status.idle":"2022-07-06T14:27:22.909841Z","shell.execute_reply.started":"2022-07-06T14:27:22.891289Z","shell.execute_reply":"2022-07-06T14:27:22.908687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.astype({\"id\": int}, errors='raise') ","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.911736Z","iopub.execute_input":"2022-07-06T14:27:22.912299Z","iopub.status.idle":"2022-07-06T14:27:22.942936Z","shell.execute_reply.started":"2022-07-06T14:27:22.912249Z","shell.execute_reply":"2022-07-06T14:27:22.941750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"tps_july_2022_denoise.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:27:22.944342Z","iopub.execute_input":"2022-07-06T14:27:22.945179Z","iopub.status.idle":"2022-07-06T14:27:26.962968Z","shell.execute_reply.started":"2022-07-06T14:27:22.945143Z","shell.execute_reply":"2022-07-06T14:27:26.961699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}