{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introduction","metadata":{}},{"cell_type":"markdown","source":"**Recommendation Using Item to Item GNN Embedding**\n\n1. [Create dataset](https://www.kaggle.com/code/cafelatte1/otto-create-dataset-gnn-embedding)\n2. [Training](https://www.kaggle.com/cafelatte1/otto-training-gnn-embedding)\n3. [Inference](https://www.kaggle.com/cafelatte1/otto-inference-gnn-embedding)","metadata":{}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"!nvcc -V","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:39:16.101217Z","iopub.execute_input":"2023-01-17T01:39:16.102054Z","iopub.status.idle":"2023-01-17T01:39:17.118910Z","shell.execute_reply.started":"2023-01-17T01:39:16.101960Z","shell.execute_reply":"2023-01-17T01:39:17.117734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install -q --no-cache-dir merlin-dataloader==0.0.2","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:39:17.121270Z","iopub.execute_input":"2023-01-17T01:39:17.122043Z","iopub.status.idle":"2023-01-17T01:39:17.127472Z","shell.execute_reply.started":"2023-01-17T01:39:17.121995Z","shell.execute_reply":"2023-01-17T01:39:17.126235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# UPGRADE TORCH VERSION TO FIT WITH TORCH GEOMETRIC GPU\n!pip install -q --no-cache-dir torch==1.12.1+cu102 --extra-index-url https://download.pytorch.org/whl/cu102","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:39:17.128803Z","iopub.execute_input":"2023-01-17T01:39:17.129806Z","iopub.status.idle":"2023-01-17T01:40:42.339505Z","shell.execute_reply.started":"2023-01-17T01:39:17.129738Z","shell.execute_reply":"2023-01-17T01:40:42.338058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GLOBAL_SEED = 42\n\nimport os\nos.environ[\"PYTHONIOENCODING\"] = \"utf8\"\nos.environ['PYTHONHASHSEED'] = str(GLOBAL_SEED)\nimport sys\nfrom glob import glob\n\nimport pandas as pd\nimport numpy as np\nfrom numpy import random as np_rnd\nimport random as rnd\nimport shutil\nimport gc\nimport datetime\nfrom collections import defaultdict, Counter\nfrom tqdm import tqdm\nfrom multiprocessing import Pool, cpu_count\nimport time\nimport pickle\n\nimport sklearn as skl\nfrom sklearn import model_selection\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW, Adam, SparseAdam\nfrom transformers import get_polynomial_decay_schedule_with_warmup\n\n# from merlin.io import Dataset\n# from merlin.loader.torch import Loader\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ntorch.__version__\n\nif torch.cuda.is_available():\n    import cudf\n    import cuml","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:40:42.343650Z","iopub.execute_input":"2023-01-17T01:40:42.343973Z","iopub.status.idle":"2023-01-17T01:40:51.750752Z","shell.execute_reply.started":"2023-01-17T01:40:42.343944Z","shell.execute_reply":"2023-01-17T01:40:51.749739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in glob(\"/kaggle/input/pytorch-geometric/pytorch_geometric_gpu/*\" if torch.cuda.is_available() else \"/kaggle/input/pytorch-geometric/pytorch_geometric_cpu/*\") :\n    !pip install -q --no-cache-dir {i}\n    \n!pip install -q --no-cache-dir torch-geometric\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.utils import coalesce, is_undirected, to_undirected, sort_edge_index\nfrom torch_geometric.sampler import BaseSampler\nfrom torch_geometric.nn import GCNConv","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:40:51.752197Z","iopub.execute_input":"2023-01-17T01:40:51.752896Z","iopub.status.idle":"2023-01-17T01:41:58.762192Z","shell.execute_reply.started":"2023-01-17T01:40:51.752858Z","shell.execute_reply":"2023-01-17T01:41:58.761029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # python random\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n    # tf random\n    try:\n        tf_rnd.set_seed(seed)\n    except:\n        pass\n    # RAPIDS random\n    try:\n        cupy.random.seed(seed)\n    except:\n        pass\n    # pytorch random\n    try:\n        torch.manual_seed(seed)\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n    except:\n        pass\n\ndef pickleIO(obj, src, op=\"w\"):\n    if op == \"w\":\n        with open(src, op + \"b\") as f:\n            pickle.dump(obj, f)\n    elif op == \"r\":\n        with open(src, op + \"b\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    else:\n        print(\"unknown operation\")\n        return obj\n    \ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\n\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n        \ndef create_submission(df):\n    df = df.reset_index()\n    df[\"type\"] = df[\"type\"].map(CFG.contentType_mapper)\n    df[\"session_type\"] = df[\"session\"].astype(\"str\") + \"_\" + df[\"type\"].astype(\"str\") + \"s\"\n    df = df[[\"session_type\", \"prediction\"]].rename({\"prediction\": \"labels\"}, axis=1)\n    return df\n\ndef create_get_ts(ts):\n    return int((ts.replace(tzinfo=CFG.tz) - CFG.ts_zero).total_seconds())\n\ndef visualize_graph(G, color):\n    plt.figure(figsize=(7,7))\n    plt.xticks([])\n    plt.yticks([])\n    nx.draw_networkx(G, pos=nx.spring_layout(G, seed=42), with_labels=False, node_color=color, cmap=\"Set2\")\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-17T01:41:58.763723Z","iopub.execute_input":"2023-01-17T01:41:58.764345Z","iopub.status.idle":"2023-01-17T01:41:58.782935Z","shell.execute_reply.started":"2023-01-17T01:41:58.764288Z","shell.execute_reply":"2023-01-17T01:41:58.780545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    local = False\n    debug = False\n    tz = datetime.timezone.utc\n    ts_zero = datetime.datetime(1970, 1, 1, tzinfo=tz)\n    contentType_mapper = pd.Series([\"clicks\", \"carts\", \"orders\"], index=[0, 1, 2])\n    target_weight = (0.1, 0.3, 0.6)\n    \n    n_folds = 5\n    batch_size = 1024\n    epochs = 1000\n    early_stopping_rounds = epochs // 5\n    eta = 5e-4\n    weight_decay = 1e-4\n    max_grad_norm = 1e+2\n    embed_dim = 32\n    \nif CFG.local:\n    CFG.folder_path = \"./dataset/\"\nelse:\n    CFG.folder_path = \"/kaggle/input/\"\n\nif CFG.debug:\n    CFG.batch_size = 1024 * 64\nelse:\n    CFG.batch_size = 1024 * 256","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:41:58.784559Z","iopub.execute_input":"2023-01-17T01:41:58.784924Z","iopub.status.idle":"2023-01-17T01:41:58.934354Z","shell.execute_reply.started":"2023-01-17T01:41:58.784887Z","shell.execute_reply":"2023-01-17T01:41:58.933204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"node_feature = pickleIO(None, \"/kaggle/input/create-dataset-gnn-embedding/node_feature.pkl\", \"r\")\nnode_feature = node_feature.to(device)\nn_aids = node_feature.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:41:58.937592Z","iopub.execute_input":"2023-01-17T01:41:58.938037Z","iopub.status.idle":"2023-01-17T01:42:01.141972Z","shell.execute_reply.started":"2023-01-17T01:41:58.938001Z","shell.execute_reply":"2023-01-17T01:42:01.140752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sampling train & valid edge (memory issue)\nseed_everything()\n\nedge_index = torch.tensor(pd.read_parquet(\"/kaggle/input/create-dataset-gnn-embedding/train_edge.parquet\").values, dtype=torch.int64).T\nshuffled_idx = torch.randperm(edge_index.shape[1])\nedge_train = edge_index[:, shuffled_idx[: ((1024 * 10000) * 1)]]\npickleIO(edge_train, \"sampled_edge_train.pkl\", \"w\")\nedge_train = edge_train.to(device)\n\nedge_index = torch.tensor(pd.read_parquet(\"/kaggle/input/create-dataset-gnn-embedding/valid_edge.parquet\").values, dtype=torch.int64).T\nshuffled_idx = torch.randperm(edge_index.shape[1])\nedge_valid = edge_index[:, shuffled_idx[: ((1024 * 2000) * 1)]]\npickleIO(edge_valid, \"sampled_edge_valid.pkl\", \"w\")\nedge_valid = edge_valid.to(device)\n\ndel edge_index, shuffled_idx\ntorch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T02:01:50.204321Z","iopub.execute_input":"2023-01-17T02:01:50.205048Z","iopub.status.idle":"2023-01-17T02:02:11.670351Z","shell.execute_reply.started":"2023-01-17T02:01:50.205012Z","shell.execute_reply":"2023-01-17T02:02:11.669217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Model & Helper functions","metadata":{}},{"cell_type":"code","source":"def get_optimizer_params(model, eta, weight_decay):\n    param_optimizer = list(model.named_parameters())\n    no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n    optimizer_parameters = [\n        # apply weight decay\n        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n         'lr': eta, 'weight_decay': weight_decay},\n        # don't apply weight decay for bias & specific layer\n        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n         'lr': eta, 'weight_decay': 0.0},\n    ]\n    return optimizer_parameters\n\ndef get_scheduler(optimizer, num_warmup_steps, num_training_steps, power=0.5):\n    scheduler = get_polynomial_decay_schedule_with_warmup(\n        optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps, power=power, lr_end=1e-7\n    )\n    return scheduler\n\n# class CustomDataset(Dataset):\n#     def __init__(self, features):\n#         self.features = features\n#     def __len__(self):\n#         return len(self.features)\n#     def __getitem__(self, idx):\n#         features = self.features[idx]\n#         return features\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-17T01:57:22.099632Z","iopub.execute_input":"2023-01-17T01:57:22.100148Z","iopub.status.idle":"2023-01-17T01:57:22.118990Z","shell.execute_reply.started":"2023-01-17T01:57:22.100101Z","shell.execute_reply":"2023-01-17T01:57:22.117806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GCN(torch.nn.Module):\n    def __init__(self, n_aids, embed_dim):\n        super().__init__()\n        self.aid_factors = nn.Embedding(n_aids, embed_dim, sparse=False)\n        self.gcn = GCNConv(embed_dim, embed_dim)\n        self.gcn_act = nn.ReLU()\n        self.lin = nn.Linear(embed_dim, embed_dim)\n    def forward(self, x, edge_index):\n        x = self.aid_factors(x)\n        x = self.gcn(x, edge_index=edge_index)\n        x = self.gcn_act(x)\n        x = self.lin(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-01-17T01:57:23.449930Z","iopub.execute_input":"2023-01-17T01:57:23.450330Z","iopub.status.idle":"2023-01-17T01:57:23.457507Z","shell.execute_reply.started":"2023-01-17T01:57:23.450277Z","shell.execute_reply":"2023-01-17T01:57:23.456367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def train_fn(fold, model, criterion, optimizer, scheduler, grad_scaler):\n    model.train()\n    metrics = {\n        \"loss\": AverageMeter(),\n        \"accuracy\": AverageMeter(),\n    }\n    \n    with torch.cuda.amp.autocast():\n        # Get embedding from model with train edge index for backpropagation\n        output = model(node_feature, edge_train)\n        # Calculate loss & metric with embedding vector\n        output_neg = (output[edge_train[0]] * output[edge_train[1]][torch.randperm(len(edge_train[1]))]).sum(dim=-1)\n        output_pos = (output[edge_train[0]] * output[edge_train[1]]).sum(dim=-1)\n        loss = criterion(torch.cat([output_pos, output_neg]), torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_neg)]))\n    # initialization gradients to zero\n    optimizer.zero_grad()\n    # get scaled gradients by float16 (default)\n    grad_scaler.scale(loss).backward()\n    # apply original gradients (unscaling) to parameters\n    # if these gradients do not contain infs or NaNs, optimizer.step() is then called.\n    # otherwise, optimizer.step() is skipped.\n    grad_scaler.step(optimizer)\n    grad_scaler.update()\n    scheduler.step()\n\n    metrics[\"loss\"].update(loss.item())\n    metrics[\"accuracy\"].update(torch.cat([output_pos.flatten().sigmoid() >= 0.5, output_neg.flatten().sigmoid() < 0.5]).float().mean().item())        \n\n    return metrics\n\n\ndef valid_fn(fold, model, criterion):\n    model.eval()\n    losses = AverageMeter()\n    metrics = {\n        \"loss\": AverageMeter(),\n        \"accuracy\": AverageMeter(),\n    }   \n    \n    with torch.no_grad():\n        # Get embedding from trained model with train edge index\n        output = model(node_feature, edge_train)\n        # Calculate loss & metric with embedding vector\n        output_neg = (output[edge_valid[0]] * output[edge_valid[1]][torch.randperm(len(edge_valid[1]))]).sum(dim=-1)\n        output_pos = (output[edge_valid[0]] * output[edge_valid[1]]).sum(dim=-1)\n        loss = criterion(torch.cat([output_pos, output_neg]), torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_neg)]))\n\n    metrics[\"loss\"].update(loss.item())\n    metrics[\"accuracy\"].update(torch.cat([output_pos.flatten().sigmoid() >= 0.5, output_neg.flatten().sigmoid() < 0.5]).float().mean().item())        \n\n    return metrics\n\n\ndef infer_fn(data, model, force_to_cpu=True):\n    model.to(\"cpu\") if force_to_cpu else model.to(device)\n    model.eval()\n  \n    with torch.no_grad():\n        predictions = model(data.x, data.edge_index)\n\n    return predictions\n\n\ndef do_fold_training(fold):\n    seed_everything(fold)\n    model = GCN(n_aids=n_aids, embed_dim=CFG.embed_dim).to(device)\n    # set optimizer\n    optimizer_parameters = get_optimizer_params(\n        model,\n        eta=CFG.eta,\n        weight_decay=CFG.weight_decay\n    )\n    optimizer = AdamW(optimizer_parameters, lr=CFG.eta, weight_decay=CFG.weight_decay)\n    scheduler = get_scheduler(\n        optimizer,\n        num_warmup_steps=0,\n        num_training_steps=CFG.epochs\n    )\n    grad_scaler = torch.cuda.amp.GradScaler() if torch.cuda.is_available() else None\n    criterion = nn.BCEWithLogitsLoss(reduction=\"mean\")\n    best_score = np.inf\n    \n    early_stopping_cnt = 0\n    for epoch in range(CFG.epochs):\n        epoch_start_time = time.time()\n        # training\n        train_metrics = train_fn(fold, model, criterion, optimizer, scheduler, grad_scaler)\n        # evaluation\n        valid_metrics = valid_fn(fold, model, criterion)\n\n        score = valid_metrics[\"loss\"].avg\n        print(\"Epoch[{0}/{1}]\\n train loss : {2}\\n valid loss : {3}\\n train accuracy : {4}\\n valid accuracy : {5}\\n eta : {6}\\n Elapsed : {7}\\n\"\n              .format(\n                  epoch+1, CFG.epochs,\n                  round(train_metrics[\"loss\"].avg, 5), round(valid_metrics[\"loss\"].avg, 5),\n                  round(train_metrics[\"accuracy\"].avg, 5), round(valid_metrics[\"accuracy\"].avg, 5),\n                  round(scheduler.get_lr()[0], 5), round(time.time() - epoch_start_time, 3)\n              )\n        )\n        \n        if score < best_score:\n            best_score = score\n            return_score_dic = {\n                \"fold\": fold,\n                \"train_loss\": train_metrics[\"loss\"].avg,\n                \"valid_loss\": valid_metrics[\"loss\"].avg,\n                \"train_accuracy\": train_metrics[\"accuracy\"].avg,\n                \"valid_accuracy\": valid_metrics[\"accuracy\"].avg,\n            }\n            model_save_dic = {'model': model.state_dict()}\n            early_stopping_cnt = 0\n        else:\n            early_stopping_cnt += 1\n        \n        if early_stopping_cnt == CFG.early_stopping_rounds:\n            print(\"INFO : Early Stopped ! (Epoch[{0}/{1}])\".format(epoch+1, CFG.epochs))\n            break\n\n    torch.save(\n        model_save_dic,\n        f\"./model_fold{fold}_best.pth\",\n    )\n    score_list.append(return_score_dic)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-17T01:57:25.353761Z","iopub.execute_input":"2023-01-17T01:57:25.354189Z","iopub.status.idle":"2023-01-17T01:57:25.379960Z","shell.execute_reply.started":"2023-01-17T01:57:25.354156Z","shell.execute_reply":"2023-01-17T01:57:25.378795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nscore_list = []\n\ndo_fold_training(0)\n\ntorch.cuda.empty_cache()\ngc.collect()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-17T02:02:11.672569Z","iopub.execute_input":"2023-01-17T02:02:11.672951Z","iopub.status.idle":"2023-01-17T02:04:17.588348Z","shell.execute_reply.started":"2023-01-17T02:02:11.672913Z","shell.execute_reply":"2023-01-17T02:04:17.587345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(score_list[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T02:04:17.590361Z","iopub.execute_input":"2023-01-17T02:04:17.591070Z","iopub.status.idle":"2023-01-17T02:04:17.598444Z","shell.execute_reply.started":"2023-01-17T02:04:17.591031Z","shell.execute_reply":"2023-01-17T02:04:17.597459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}