{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a Bert4Rec model where W2Vec embeddings are used. Training (even with only week 1 training data) takes very long.\n\nThe output dimension of the final layer is high (1.2 mio), finding a subset with many sessions and few aids would be good.","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport glob\nimport torch\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom datetime import datetime\nfrom collections import Counter\nfrom sklearn import preprocessing\nfrom tqdm import tqdm\n                       \nfrom torch.utils.data import DataLoader\nimport pytorch_lightning as pl\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nimport torch\nimport torch.nn as nn\nfrom torch.nn import Linear\nfrom torch.nn import functional as F\n\n\nimport gensim\nfrom gensim.models import Word2Vec","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_and_concatenate_parquet_files(pattern):\n    \"\"\"Read and concatenate parquet files matching given pattern.\"\"\"\n    parquet_files = {}\n    for file in glob.glob(pattern):\n        parquet_files[file] = pd.read_parquet(file)\n    return pd.concat(parquet_files.values())\n\n# Read and concatenate training data from first week\ntrain_w1 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/train_w0_part*.parquet\")\n\n# Read and concatenate label data from first week\nlabel_w1 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/label_w0_part*.parquet\")\n\n# Read and concatenate training data from fourth week for validation\ntrain_w4 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-training-wo-split/train_w3_part*.parquet\")\n\n# Read and concatenate label data from fourth week for validation\nlabel_w4 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/label_w3_part*.parquet\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#only use week 1 as training data for now\n\n# # Read and concatenate training data from 2nd week\n# train_w2 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/train_w1_part*.parquet\")\n\n# # Read and concatenate label data from 2nd week\n# label_w2 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/label_w1_part*.parquet\")\n\n\n# # Read and concatenate training data from 3rd week\n# train_w3 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/train_w2_part*.parquet\")\n\n# # Read and concatenate label data from 3rd week\n# label_w3 = read_and_concatenate_parquet_files(\"/kaggle/input/otto-prep-4-weeks/label_w2_part*.parquet\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_column_mapping(df, col_name):\n    # Read in the data from the parquet file\n    all_aid = pd.read_parquet('/kaggle/input/otto-full-optimized-memory-footprint/train.parquet')\n    \n    # Get a sorted list of the unique values in the specified column\n    aid_sorted = sorted(list(all_aid[col_name].unique()))\n    \n    # Delete the dataframe to save memory\n    del all_aid\n    \n    # Create a mapping from the unique values to integers, starting from 2\n    mapping = {k: i + 2 for i, k in enumerate(aid_sorted)}\n    \n    # Create an inverse mapping from the integers back to the unique values\n    inverse_mapping = {v: k for k, v in mapping.items()}\n    \n    return mapping, inverse_mapping\n\ndef map_column(df, col_name, mapping):\n    # Replace the values in the specified column with their corresponding integer values\n    df[col_name] = df[col_name].map(mapping)\n    return df\n\n# Create the mapping and inverse mapping for the 'aid' column\nmapping, inverse_mapping = create_column_mapping(train_w1, 'aid')\n\n# Replace the values in the 'aid' column of the train_w1 dataframe with their corresponding integer values\ntrain_w1 = map_column(train_w1, 'aid', mapping)\n\n# Replace the values in the 'aid' column of the train_w4 dataframe with their corresponding integer values\ntrain_w4 = map_column(train_w4, 'aid', mapping)\n\n\n#discard sessions with NaN aid (in 4th week but not in first)\ntrain_w4 = train_w4[~train_w4.session.isin(train_w4[train_w4['aid'].isna()]['session'].unique())]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_merged_sessions(session_df, label_session_df):\n    \"\"\"\n    Merge the session data and label data by session.\n\n    \"\"\"\n    input_df = pd.DataFrame(session_df.groupby('session')['aid'].unique().agg(list))\n    label_df = pd.DataFrame(label_session_df.groupby('session')['aid'].unique().agg(list))\n    \n    # Rename the 'aid' column to 'input'\n    input_df = input_df.rename(columns={'aid': 'input'})\n    \n    # Rename the 'aid' column to 'label'\n    label_df = label_df.rename(columns={'aid': 'label'})\n\n    # Merge the input and label dataframes on the 'session' column, using 'session' as the index\n    merged_df = input_df.merge(label_df, left_index=True, right_index=True)\n    \n    return merged_df\n\ntrain_w1 = get_merged_sessions(train_w1, label_w1)\ntrain_w4 = get_merged_sessions(train_w4, label_w4)\n\n# Delete the label dataframes to save memory\ndel label_w1, label_w4\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pre-trained Word2Vec model.\nw2v = gensim.models.Word2Vec.load(\"/kaggle/input/otto-w2vec/word2vec.model\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBEDDING_LENGTH = 32\nTARGET_LENGTH = 20\nSEQUENCE_LENGTH = 100\n\nclass Dataset(torch.utils.data.Dataset):\n    def __init__(self, sessions, input_length = SEQUENCE_LENGTH, target_length = TARGET_LENGTH):\n        self.sessions = sessions\n        self.input_length = input_length\n        self.target_length = target_length\n\n    def __len__(self):\n        return len(self.sessions)\n    \n    def pad_items(self, session, length, input_item = True):\n        if len(session)< length:\n            session = session + list((length - len(session)) * [0])\n        else: \n            if input_item:\n                session = session[-length:]\n            else: session = session[:length]\n        return session\n\n\n\n    def __getitem__(self, idx):\n        input_tokens = self.sessions.iloc[idx, self.sessions.columns.get_loc(\"input\")]\n        \n        length_input = len(input_tokens)\n        input_tokens = self.pad_items(input_tokens, self.input_length)\n            \n        target = self.sessions.iloc[idx, self.sessions.columns.get_loc(\"label\")]\n        target = self.pad_items(target, self.target_length, input_item = False)\n        \n        target_mask = [1 if item != 0 else item for item in target ]\n        \n        \n        target = input_tokens +target\n        input_tokens = input_tokens + target_mask \n        \n \n        \n        input_tokens_orig = torch.tensor(input_tokens)\n        input_tokens = w2v.wv[input_tokens]\n        \n        input_tokens = torch.tensor(input_tokens, dtype = torch.float )\n        target = torch.tensor(target, dtype = torch.long )\n\n        \n        \n        mask = torch.eq(input_tokens_orig, 0).type(torch.bool).unsqueeze(0).permute(1,0)\n        \n       \n        return input_tokens, target, mask, input_tokens_orig\n                                     ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def masked_ce(y_pred, y_true, mask):\n\n    loss = F.cross_entropy(y_pred, y_true, reduction=\"none\")\n\n    loss = loss * mask\n\n    return loss.sum() / (mask.sum() + 1e-8)\n\n\ndef masked_accuracy(y_pred: torch.Tensor, y_true: torch.Tensor, mask: torch.Tensor):\n\n    _, predicted = torch.max(y_pred, 1)\n\n    y_true = torch.masked_select(y_true, mask)\n    predicted = torch.masked_select(predicted, mask)\n\n    acc = (y_true == predicted).double().mean()\n\n    return acc\n\n\n\nclass Recommender(pl.LightningModule):\n    def __init__(\n        self,\n        out = len(mapping)+2,\n        channels=EMBEDDING_LENGTH,\n        dropout=0.2,\n        lr=1e-4,\n        word2vec = w2v\n    ):\n        super().__init__()\n        \n        self.lr = lr\n        self.dropout = dropout\n        self.out = out\n        \n        self.item_embeddings = w2v\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=channels, nhead=4, dropout=self.dropout, batch_first = True\n        )\n\n        self.encoder = torch.nn.TransformerEncoder(encoder_layer, num_layers=10)\n\n        self.linear_out = Linear(channels, self.out)\n\n        self.do = nn.Dropout(p=self.dropout)\n\n\n    def forward(self, input_items, mask):\n\n        mask = mask.squeeze(-1)\n        \n        if torch.cuda.is_available():\n            input_items = input_items.to(\"cuda:0\")\n            mask = mask.to(\"cuda:0\")\n        \n        \n        input_items = self.encoder(input_items, src_key_padding_mask =mask)\n        \n        out = self.linear_out(input_items)\n\n        return out\n    \n\n\n    def training_step(self, batch, batch_idx):\n        \n        \n        input_items, y_true, mask, input_tokens_orig = batch\n\n        y_pred = self(input_items, mask)\n        \n\n        if torch.cuda.is_available():\n            y_pred = y_pred.to(\"cuda:0\")\n            y_true = y_true.to(\"cuda:0\")\n        \n        \n        y_pred = y_pred.view(-1, y_pred.size(2))\n        y_true = y_true.view(-1)\n\n        \n        \n        input_tokens_orig = input_tokens_orig.view(-1)\n        loss_mask = input_tokens_orig == 1\n        \n        loss = masked_ce(y_pred=y_pred, y_true=y_true, mask=loss_mask)\n       \n        #accuracy = self.masked_accuracy(y_pred=y_pred, y_true=y_true, mask=loss_mask)\n\n\n        self.log(\"train_loss\", loss, prog_bar=True, on_step=False, on_epoch=True)\n        #self.log(\"train_accuracy\", accuracy)\n\n        return loss   \n    \n\n\n    def validation_step(self, batch, batch_idx):\n        input_items, y_true, mask, input_tokens_orig = batch\n\n        y_pred = self(input_items, mask)\n\n        y_pred = y_pred.view(-1, y_pred.size(2))\n        y_true = y_true.view(-1)\n\n        input_tokens_orig = input_tokens_orig.view(-1)\n        loss_mask = input_tokens_orig == 1\n        \n        loss = masked_ce(y_pred=y_pred, y_true=y_true, mask=loss_mask)   \n        \n        #accuracy = self.masked_accuracy(y_pred=y_pred, y_true=y_true, mask=loss_mask)\n        \n        self.log('valid_loss', loss, prog_bar=True, on_step=False, on_epoch=True)\n        #self.log(\"valid_accuracy\", accuracy)\n\n        return loss\n\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer, patience=10, factor=0.1\n        )\n        return {\n            \"optimizer\": optimizer,\n            \"lr_scheduler\": scheduler,\n            \"monitor\": \"valid_loss\",\n        }\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 3\n\n\ntrain_data = Dataset(\n    sessions=train_w1\n)\n\nval_data = Dataset(\nsessions = train_w4\n)\n\nprint(\"len(train_data)\", len(train_data))\nprint(\"len(val_data)\", len(val_data))\n\n\ntrain_loader = DataLoader(\n    train_data,\n    batch_size=batch_size,\n    num_workers=2,\n    shuffle=True,\n)\nval_loader = DataLoader(\n    val_data,\n    batch_size=batch_size,\n    num_workers=2,\n    shuffle=False,\n)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train\n\nepochs = 10\n\nmodel = Recommender()\ntrainer = pl.Trainer(\n    max_epochs=epochs,\n    gpus=1\n)\n\nlogger = TensorBoardLogger(\n    save_dir='/kaggle/working/',\n    )\n\ncheckpoint_callback = ModelCheckpoint(\n    monitor=\"valid_loss\",\n    mode=\"min\",\n    dirpath='/kaggle/working/',\n    filename=\"recommender\",\n    )\n\ntrainer.fit(model, train_loader, val_loader)\n\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}