{"metadata":{"colab":{"collapsed_sections":["LX4M190s4pxo","TBIrKxhRGGQR"],"provenance":[]},"interpreter":{"hash":"eee4fdb9ca52ce5d5f0a2a1c2a1d0a4896d6b735579ddf3d9c0ee93e21b97ca8"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":9293917,"sourceType":"datasetVersion","datasetId":5626760},{"sourceId":9321925,"sourceType":"datasetVersion","datasetId":5646823}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install -U lightgbm==3.3.2","metadata":{"id":"Ej-kIjP5d_zw","outputId":"82247b5c-15ab-4205-df00-710da0f168a4","executionInfo":{"status":"ok","timestamp":1725503795078,"user_tz":-420,"elapsed":9276,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.copytree('/kaggle/input/method-45th-kaggle-hm/H-M-Fashion-RecSys', '/kaggle/working/H-M-Fashion-RecSys')","metadata":{"execution":{"iopub.status.busy":"2024-09-05T03:20:47.597942Z","iopub.execute_input":"2024-09-05T03:20:47.599074Z","iopub.status.idle":"2024-09-05T03:20:48.115927Z","shell.execute_reply.started":"2024-09-05T03:20:47.599023Z","shell.execute_reply":"2024-09-05T03:20:48.114553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pip install implicit","metadata":{"id":"yVREKx6JJqx1","outputId":"8160326a-ba67-4d28-e1c1-f140d5aea872","executionInfo":{"status":"ok","timestamp":1725503799389,"user_tz":-420,"elapsed":4315,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:20:58.785859Z","iopub.execute_input":"2024-09-05T03:20:58.786299Z","iopub.status.idle":"2024-09-05T03:21:18.423289Z","shell.execute_reply.started":"2024-09-05T03:20:58.786258Z","shell.execute_reply":"2024-09-05T03:21:18.421794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from google.colab import drive\ndrive.mount('/content/drive')","metadata":{"id":"tlJ51VFz8Kjw","executionInfo":{"status":"ok","timestamp":1725503805874,"user_tz":-420,"elapsed":6497,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"4dee2f9f-b67f-4777-c2f4-eb392ecc8652"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install fast-transformer","metadata":{"id":"lPbHTdGqXzl5","executionInfo":{"status":"ok","timestamp":1725409814748,"user_tz":-420,"elapsed":14270,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"da3ac252-99f6-4d4c-e731-1a2e1cc755af","collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\n# from fast_transformer import FastTransformer\n\n# mask = tf.ones([1, 4096], dtype=tf.bool)\n# model = FastTransformer(\n#     num_tokens = 20000,\n#     dim = 512,\n#     depth = 2,\n#     max_seq_len = 4096,\n#     absolute_pos_emb = True, # Absolute positional embeddings\n#     mask = mask\n# )\n# x = tf.experimental.numpy.random.randint(0, 20000, (1, 4096))","metadata":{"id":"md2fbE_RYlz4","executionInfo":{"status":"ok","timestamp":1725409828863,"user_tz":-420,"elapsed":9243,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"85889a22-b3fc-4fc5-9c04-fc36a7a839a9","collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# logits = model(x) # (1, 4096, 20000)\n# logits.shape","metadata":{"id":"4_3rGbekbpK9","executionInfo":{"status":"error","timestamp":1725409867812,"user_tz":-420,"elapsed":570,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"00dcb2ef-ee7f-43a8-a364-17fd8fe5943a","collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"id":"rzC8WBFDexeU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"\n\nfrom tensorflow import keras\nfrom keras.utils import to_categorical\nfrom keras.layers import *\nfrom keras.models import load_model\nfrom keras import backend as K\nfrom keras import Model\nfrom sklearn.metrics import *\nfrom keras.optimizers import *","metadata":{"id":"eg7o5G2beagj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Fastformer(Layer):\n\n    def __init__(self, nb_head, size_per_head, **kwargs):\n        self.nb_head = nb_head\n        self.size_per_head = size_per_head\n        self.output_dim = nb_head*size_per_head\n        self.now_input_shape=None\n        super(Fastformer, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.now_input_shape=input_shape\n        self.WQ = self.add_weight(name='WQ',\n                                  shape=(input_shape[0][-1], self.output_dim),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n        self.WK = self.add_weight(name='WK',\n                                  shape=(input_shape[1][-1], self.output_dim),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n        self.Wq = self.add_weight(name='Wq',\n                                  shape=(self.output_dim,self.nb_head),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n        self.Wk = self.add_weight(name='Wk',\n                                  shape=(self.output_dim,self.nb_head),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n\n        self.WP = self.add_weight(name='WP',\n                                  shape=(self.output_dim,self.output_dim),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n\n\n        super(Fastformer, self).build(input_shape)\n\n    def call(self, x):\n        if len(x) == 2:\n            Q_seq,K_seq = x\n        elif len(x) == 4:\n            Q_seq,K_seq,Q_mask,K_mask = x #different mask lengths, reserved for cross attention\n\n        Q_seq = K.dot(Q_seq, self.WQ)\n        Q_seq_reshape = K.reshape(Q_seq, (-1, self.now_input_shape[0][1], self.nb_head*self.size_per_head))\n\n        Q_att=  K.permute_dimensions(K.dot(Q_seq_reshape, self.Wq),(0,2,1))/ self.size_per_head**0.5\n\n        if len(x)  == 4:\n            Q_att = Q_att-(1-K.expand_dims(Q_mask,axis=1))*1e8\n\n        Q_att = K.softmax(Q_att)\n        Q_seq = K.reshape(Q_seq, (-1,self.now_input_shape[0][1], self.nb_head, self.size_per_head))\n        Q_seq = K.permute_dimensions(Q_seq, (0,2,1,3))\n\n        K_seq = K.dot(K_seq, self.WK)\n        K_seq = K.reshape(K_seq, (-1,self.now_input_shape[1][1], self.nb_head, self.size_per_head))\n        K_seq = K.permute_dimensions(K_seq, (0,2,1,3))\n\n        Q_att = Lambda(lambda x: K.repeat_elements(K.expand_dims(x,axis=3),self.size_per_head,axis=3))(Q_att)\n        global_q = K.sum(multiply([Q_att, Q_seq]),axis=2)\n\n        global_q_repeat = Lambda(lambda x: K.repeat_elements(K.expand_dims(x,axis=2), self.now_input_shape[1][1],axis=2))(global_q)\n\n        QK_interaction = multiply([K_seq, global_q_repeat])\n        QK_interaction_reshape = K.reshape(QK_interaction, (-1, self.now_input_shape[0][1], self.nb_head*self.size_per_head))\n        K_att = K.permute_dimensions(K.dot(QK_interaction_reshape, self.Wk),(0,2,1))/ self.size_per_head**0.5\n\n        if len(x)  == 4:\n            K_att = K_att-(1-K.expand_dims(K_mask,axis=1))*1e8\n\n        K_att = K.softmax(K_att)\n\n        K_att = Lambda(lambda x: K.repeat_elements(K.expand_dims(x,axis=3),self.size_per_head,axis=3))(K_att)\n\n        global_k = K.sum(multiply([K_att, QK_interaction]),axis=2)\n\n        global_k_repeat = Lambda(lambda x: K.repeat_elements(K.expand_dims(x,axis=2), self.now_input_shape[0][1],axis=2))(global_k)\n        #Q=V\n        QKQ_interaction = multiply([global_k_repeat, Q_seq])\n        QKQ_interaction = K.permute_dimensions(QKQ_interaction, (0,2,1,3))\n        QKQ_interaction = K.reshape(QKQ_interaction, (-1,self.now_input_shape[0][1], self.nb_head*self.size_per_head))\n        QKQ_interaction = K.dot(QKQ_interaction, self.WP)\n        QKQ_interaction = K.reshape(QKQ_interaction, (-1,self.now_input_shape[0][1], self.nb_head,self.size_per_head))\n        QKQ_interaction = K.permute_dimensions(QKQ_interaction, (0,2,1,3))\n        QKQ_interaction = QKQ_interaction+Q_seq\n        QKQ_interaction = K.permute_dimensions(QKQ_interaction, (0,2,1,3))\n        QKQ_interaction = K.reshape(QKQ_interaction, (-1,self.now_input_shape[0][1], self.nb_head*self.size_per_head))\n\n        #many operations can be optimized if higher versions are used.\n\n        return QKQ_interaction\n\n    def compute_output_shape(self, input_shape):\n        return (input_shape[0][0], input_shape[0][1], self.output_dim)","metadata":{"id":"Thrj96rKZ4Fd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom pandas.api.types import CategoricalDtype\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\n\nimport pickle\nfrom tqdm import tqdm\nimport gc\nfrom pathlib import Path","metadata":{"id":"gboISq82XUN1","executionInfo":{"status":"ok","timestamp":1725503842634,"user_tz":-420,"elapsed":5459,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"81c4f178-97f7-417e-f2a6-3fb84e853271","execution":{"iopub.status.busy":"2024-09-05T03:22:31.022477Z","iopub.execute_input":"2024-09-05T03:22:31.022989Z","iopub.status.idle":"2024-09-05T03:22:33.939490Z","shell.execute_reply.started":"2024-09-05T03:22:31.022943Z","shell.execute_reply":"2024-09-05T03:22:33.938423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nimport sys\nfrom IPython.core.interactiveshell import InteractiveShell\n\nwarnings.filterwarnings(\"ignore\")\nsys.path.append(\"/kaggle/working/H-M-Fashion-RecSys\") # path to the `src`` folder\nInteractiveShell.ast_node_interactivity = \"all\"\ntqdm.pandas()","metadata":{"id":"F6WO2VtEXefj","executionInfo":{"status":"ok","timestamp":1725503842635,"user_tz":-420,"elapsed":3,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:23:05.673936Z","iopub.execute_input":"2024-09-05T03:23:05.674679Z","iopub.status.idle":"2024-09-05T03:23:05.682035Z","shell.execute_reply.started":"2024-09-05T03:23:05.674635Z","shell.execute_reply":"2024-09-05T03:23:05.680813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/H-M-Fashion-RecSys","metadata":{"id":"PTlAM5lJASc1","executionInfo":{"status":"ok","timestamp":1725503842635,"user_tz":-420,"elapsed":3,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"84f572a3-df10-4886-c208-ad2aa606a8fa","execution":{"iopub.status.busy":"2024-09-05T03:23:15.205021Z","iopub.execute_input":"2024-09-05T03:23:15.206016Z","iopub.status.idle":"2024-09-05T03:23:15.213596Z","shell.execute_reply.started":"2024-09-05T03:23:15.205967Z","shell.execute_reply":"2024-09-05T03:23:15.212171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from src.data import DataHelper\nfrom src.data.metrics import map_at_k, hr_at_k, recall_at_k\n\nfrom src.retrieval.rules import (\n    OrderHistory,\n    OrderHistoryDecay,\n    ItemPair,\n    UserGroupTimeHistory,\n    UserGroupSaleTrend,\n    TimeHistory,\n    TimeHistoryDecay,\n    SaleTrend,\n    OutOfStock,\n)\nfrom src.retrieval.collector import RuleCollector\n\nfrom src.features import full_sale, week_sale, repurchase_ratio, popularity, period_sale\n\nfrom src.utils import (\n    calc_valid_date,\n    merge_week_data,\n    reduce_mem_usage,\n    calc_embd_similarity,\n)","metadata":{"id":"zxhDyWVbJR3w","executionInfo":{"status":"ok","timestamp":1725503842635,"user_tz":-420,"elapsed":2,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:23:18.110915Z","iopub.execute_input":"2024-09-05T03:23:18.111768Z","iopub.status.idle":"2024-09-05T03:23:18.194043Z","shell.execute_reply.started":"2024-09-05T03:23:18.111720Z","shell.execute_reply":"2024-09-05T03:23:18.192798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path(\"/kaggle/working/H-M-Fashion-RecSys/data/\")\nmodel_dir = Path(\"/kaggle/working/H-M-Fashion-RecSys/models/\")","metadata":{"id":"lL6IRSJ2JR3x","executionInfo":{"status":"ok","timestamp":1725503843767,"user_tz":-420,"elapsed":2,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:23:34.017048Z","iopub.execute_input":"2024-09-05T03:23:34.017785Z","iopub.status.idle":"2024-09-05T03:23:34.023390Z","shell.execute_reply.started":"2024-09-05T03:23:34.017697Z","shell.execute_reply":"2024-09-05T03:23:34.022125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_WEEK_NUM = 2\nWEEK_NUM = TRAIN_WEEK_NUM + 2\n\nVERSION_NAME = \"Recall 1\"\nTEST = False # * Set as `False` when do local experiments to save time","metadata":{"id":"AakTfKUQJR3x","executionInfo":{"status":"ok","timestamp":1725503845019,"user_tz":-420,"elapsed":3,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:33:30.696488Z","iopub.execute_input":"2024-09-05T03:33:30.696965Z","iopub.status.idle":"2024-09-05T03:33:30.702736Z","shell.execute_reply.started":"2024-09-05T03:33:30.696923Z","shell.execute_reply":"2024-09-05T03:33:30.701413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(\"/kaggle/working/H-M-Fashion-RecSys/data/\")\nos.mkdir(\"/kaggle/working/H-M-Fashion-RecSys/data/raw\")\nos.mkdir(\"/kaggle/working/H-M-Fashion-RecSys/data/external\")\nos.mkdir(\"/kaggle/working/H-M-Fashion-RecSys/data/interim\")\nos.mkdir(\"/kaggle/working/H-M-Fashion-RecSys/data/processed\")\n!cp /kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv /kaggle/working/H-M-Fashion-RecSys/data/raw\n!cp /kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv /kaggle/working/H-M-Fashion-RecSys/data/raw\n!cp /kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv /kaggle/working/H-M-Fashion-RecSys/data/raw\n!cp /kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv /kaggle/working/H-M-Fashion-RecSys/data/raw","metadata":{"execution":{"iopub.status.busy":"2024-09-05T03:31:41.700648Z","iopub.execute_input":"2024-09-05T03:31:41.701299Z","iopub.status.idle":"2024-09-05T03:32:18.986085Z","shell.execute_reply.started":"2024-09-05T03:31:41.701226Z","shell.execute_reply":"2024-09-05T03:32:18.984304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nif not os.path.exists(data_dir/\"interim\"/VERSION_NAME):\n    os.mkdir(data_dir/\"interim\"/VERSION_NAME)\nif not os.path.exists(data_dir/\"processed\"/VERSION_NAME):\n    os.mkdir(data_dir/\"processed\"/VERSION_NAME)","metadata":{"id":"RHu8rsZMJR3y","executionInfo":{"status":"ok","timestamp":1725503845019,"user_tz":-420,"elapsed":2,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:33:10.303386Z","iopub.execute_input":"2024-09-05T03:33:10.303945Z","iopub.status.idle":"2024-09-05T03:33:10.313188Z","shell.execute_reply.started":"2024-09-05T03:33:10.303888Z","shell.execute_reply":"2024-09-05T03:33:10.311747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pepare data: encoding ids and preprocessing","metadata":{"id":"Mazy6bP0JR30"}},{"cell_type":"code","source":"dh = DataHelper(data_dir)","metadata":{"id":"2BPv2lcWGGQJ","executionInfo":{"status":"ok","timestamp":1725503847531,"user_tz":-420,"elapsed":3,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:33:13.584533Z","iopub.execute_input":"2024-09-05T03:33:13.585790Z","iopub.status.idle":"2024-09-05T03:33:13.591434Z","shell.execute_reply.started":"2024-09-05T03:33:13.585685Z","shell.execute_reply":"2024-09-05T03:33:13.590169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = dh.preprocess_data(save=True, name=\"encoded_full\") # * run only once, processed data will be saved","metadata":{"id":"kiIB1uvjJR33","executionInfo":{"status":"ok","timestamp":1725504085403,"user_tz":-420,"elapsed":237874,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"0e9a9935-8383-4367-cf07-381a78083f09","execution":{"iopub.status.busy":"2024-09-05T03:33:45.923687Z","iopub.execute_input":"2024-09-05T03:33:45.924787Z","iopub.status.idle":"2024-09-05T03:36:53.644384Z","shell.execute_reply.started":"2024-09-05T03:33:45.924700Z","shell.execute_reply":"2024-09-05T03:36:53.643107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = dh.load_data(name=\"encoded_full\")","metadata":{"id":"i3ngtb7NUGbH","executionInfo":{"status":"ok","timestamp":1725504089329,"user_tz":-420,"elapsed":3928,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:36:57.036660Z","iopub.execute_input":"2024-09-05T03:36:57.037809Z","iopub.status.idle":"2024-09-05T03:37:00.123340Z","shell.execute_reply.started":"2024-09-05T03:36:57.037747Z","shell.execute_reply":"2024-09-05T03:37:00.122025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uid2idx = pickle.load(open(data_dir/\"index_id_map/user_id2index.pkl\", \"rb\"))\nsubmission = pd.read_csv(data_dir/\"raw\"/'sample_submission.csv')\nsubmission['customer_id'] = submission['customer_id'].map(uid2idx)","metadata":{"id":"7ecaPObIJR34","executionInfo":{"status":"ok","timestamp":1725504099255,"user_tz":-420,"elapsed":9931,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:37:03.768832Z","iopub.execute_input":"2024-09-05T03:37:03.769770Z","iopub.status.idle":"2024-09-05T03:37:09.494570Z","shell.execute_reply.started":"2024-09-05T03:37:03.769690Z","shell.execute_reply":"2024-09-05T03:37:09.492768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retrieval\n","metadata":{"id":"LX4M190s4pxo"}},{"cell_type":"markdown","source":"Generate candidates for each week","metadata":{"id":"fcShMOQPJR35"}},{"cell_type":"code","source":"listBin = [-1, 19, 29, 39, 49, 59, 69, 119]\ndata['user']['age_bins'] = pd.cut(data['user']['age'], listBin)","metadata":{"id":"J7gu-mwjJR36","executionInfo":{"status":"ok","timestamp":1725504099256,"user_tz":-420,"elapsed":46,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"execution":{"iopub.status.busy":"2024-09-05T03:37:12.603333Z","iopub.execute_input":"2024-09-05T03:37:12.603837Z","iopub.status.idle":"2024-09-05T03:37:12.662264Z","shell.execute_reply.started":"2024-09-05T03:37:12.603791Z","shell.execute_reply":"2024-09-05T03:37:12.661023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST = True","metadata":{"execution":{"iopub.status.busy":"2024-09-05T04:26:43.173966Z","iopub.execute_input":"2024-09-05T04:26:43.175054Z","iopub.status.idle":"2024-09-05T04:26:43.181148Z","shell.execute_reply.started":"2024-09-05T04:26:43.174998Z","shell.execute_reply":"2024-09-05T04:26:43.179839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# * use the threshold in week 1 to generate candidates for test data, see the log in the upper cell\nif TEST:\n    week = 0\n    trans = data[\"inter\"]\n\n    start_date, end_date = calc_valid_date(week)\n    print(f\"Week {week}: [{start_date}, {end_date})\")\n\n    train, valid = dh.split_data(trans, start_date, end_date)\n    train = train.merge(data['user'][['customer_id','age_bins']], on='customer_id', how='left')\n\n    last_week_start = pd.to_datetime(start_date) - pd.Timedelta(days=7)\n    last_week_start = last_week_start.strftime(\"%Y-%m-%d\")\n    last_week = train.loc[train.t_dat >= last_week_start]\n\n    last_3day_start = pd.to_datetime(start_date) - pd.Timedelta(days=3)\n    last_3day_start = last_3day_start.strftime(\"%Y-%m-%d\")\n    last_3days = train.loc[train.t_dat >= last_3day_start]\n\n    customer_list = submission['customer_id'].values\n\n    # * ========================== Retrieval Strategies ==========================\n\n    candidates = RuleCollector().collect(\n        week_num = week,\n        trans_df = trans,\n        customer_list=customer_list,\n        rules=[\n            OrderHistory(train, days=3, name='1'),\n            OrderHistory(train, days=7, name='2'),\n            OrderHistoryDecay(train, days=3, n=50, name='1'),\n            OrderHistoryDecay(train, days=7, n=50, name='2'),\n            ItemPair(OrderHistory(train, days=3).retrieve(), name='1'),\n            ItemPair(OrderHistory(train, days=7).retrieve(), name='2'),\n            ItemPair(OrderHistoryDecay(train, 3, n=50).retrieve(), name='3'),\n            ItemPair(OrderHistoryDecay(train, 7, n=50).retrieve(), name='4'),\n            UserGroupTimeHistory(data, customer_list, last_week, ['age_bins'], n=15, name='1'),\n            UserGroupTimeHistory(data, customer_list, last_3days, ['age_bins'], n=20.5, name='2'),\n            UserGroupSaleTrend(data, customer_list, train, ['age_bins'], days=7, n=2),\n            TimeHistory(customer_list, last_week, n=9, name='1'),\n            TimeHistory(customer_list, last_3days, n=16, name='2'),\n            TimeHistoryDecay(customer_list, train, days=3, n=12, name='1'),\n            TimeHistoryDecay(customer_list, train, days=7, n=8, name='2'),\n            SaleTrend(customer_list, train, days=7, n=2),\n        ],\n        filters=[OutOfStock(trans)],\n        min_pos_rate=0.006,\n        compress=False,\n    )\n\n    candidates, _ = reduce_mem_usage(candidates)\n    candidates = (\n        pd.pivot_table(\n            candidates,\n            values=\"score\",\n            index=[\"customer_id\", \"article_id\"],\n            columns=[\"method\"],\n            aggfunc=np.sum,\n        )\n        .reset_index()\n    )\n\n    candidates.to_parquet(data_dir/\"interim\"/VERSION_NAME/f\"week{week}_candidate.pqt\")\n    valid.to_parquet(data_dir/\"processed\"/VERSION_NAME/f\"week{week}_label.pqt\")","metadata":{"id":"3wqDho1ZJR38","execution":{"iopub.status.busy":"2024-09-05T04:26:45.194760Z","iopub.execute_input":"2024-09-05T04:26:45.195798Z","iopub.status.idle":"2024-09-05T04:42:01.349963Z","shell.execute_reply.started":"2024-09-05T04:26:45.195743Z","shell.execute_reply":"2024-09-05T04:42:01.348428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, valid, last_week, customer_list, candidates\ngc.collect()","metadata":{"id":"gQEog9JJJR39","executionInfo":{"status":"error","timestamp":1725335077559,"user_tz":-420,"elapsed":4,"user":{"displayName":"Nghia Quang","userId":"04079879198526107387"}},"outputId":"11963b99-cde6-4d72-e54b-d48df6be0ec0","execution":{"iopub.status.busy":"2024-09-05T04:43:16.156558Z","iopub.execute_input":"2024-09-05T04:43:16.157039Z","iopub.status.idle":"2024-09-05T04:43:16.482644Z","shell.execute_reply.started":"2024-09-05T04:43:16.156998Z","shell.execute_reply":"2024-09-05T04:43:16.481539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature engineering\n","metadata":{"id":"TBIrKxhRGGQR"}},{"cell_type":"code","source":"user = data[\"user\"]\nitem = data[\"item\"]\ninter = data[\"inter\"]","metadata":{"id":"9pF-sOMSGGQP","execution":{"iopub.status.busy":"2024-09-05T04:43:20.971180Z","iopub.execute_input":"2024-09-05T04:43:20.972461Z","iopub.status.idle":"2024-09-05T04:43:20.977223Z","shell.execute_reply.started":"2024-09-05T04:43:20.972402Z","shell.execute_reply":"2024-09-05T04:43:20.976163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate week number\ninter['week'] = (pd.to_datetime('2020-09-29') - pd.to_datetime(inter['t_dat'])).dt.days // 7","metadata":{"id":"heKIAtoRdtIk","execution":{"iopub.status.busy":"2024-09-05T04:43:38.703626Z","iopub.execute_input":"2024-09-05T04:43:38.704664Z","iopub.status.idle":"2024-09-05T04:43:45.123681Z","shell.execute_reply.started":"2024-09-05T04:43:38.704606Z","shell.execute_reply":"2024-09-05T04:43:45.122200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge full candidates to transaction data (avoid feature missing in training data)\nfull_candidates = []\nfor i in tqdm(range(WEEK_NUM)):\n    candidate = pd.read_parquet(data_dir/\"interim\"/VERSION_NAME/f\"week{i}_candidate.pqt\")\n    full_candidates += candidate['article_id'].values.tolist()\nfull_candidates = list(set(full_candidates))\ndel candidate\ngc.collect()\n\nnum_candidates = len(full_candidates)\nfull_candidates = np.array(full_candidates)\nfull_candidates = np.tile(full_candidates, WEEK_NUM + 1)\nweeks = np.repeat(np.arange(1,WEEK_NUM+2), num_candidates)\nfull_candidates = pd.DataFrame({'article_id':full_candidates, 'week':weeks})\n\ninter['valid'] = 1\nin_train = inter[inter['week']<=WEEK_NUM + 1]\nout_train = inter[inter['week']>WEEK_NUM + 1]\n\nin_train = in_train.merge(full_candidates, on=['article_id','week'], how='right')\nin_train['valid'] = in_train['valid'].fillna(0)\ninter = pd.concat([in_train, out_train], ignore_index=True)\ninter = inter.sort_values([\"valid\"], ascending=False).reset_index(drop=True)","metadata":{"id":"kaEwbD6KY5R8","outputId":"f71ff8a2-e5e1-4aa8-b1ce-91462fcaa87c","execution":{"iopub.status.busy":"2024-09-05T04:45:36.711838Z","iopub.execute_input":"2024-09-05T04:45:36.712386Z","iopub.status.idle":"2024-09-05T04:46:11.812914Z","shell.execute_reply.started":"2024-09-05T04:45:36.712341Z","shell.execute_reply":"2024-09-05T04:46:11.811789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge `product_code`\ninter = inter.merge(item[[\"article_id\", \"product_code\"]], on=\"article_id\", how=\"left\")","metadata":{"id":"QsW8gPPUUUg_","execution":{"iopub.status.busy":"2024-09-05T04:56:02.374342Z","iopub.execute_input":"2024-09-05T04:56:02.375102Z","iopub.status.idle":"2024-09-05T04:56:07.258013Z","shell.execute_reply.started":"2024-09-05T04:56:02.375042Z","shell.execute_reply":"2024-09-05T04:56:07.256677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter.shape","metadata":{"id":"Hz4m4zsiieqn","outputId":"f4855f51-d2ea-4c3b-f1f2-a59ecbce5000","execution":{"iopub.status.busy":"2024-09-05T04:56:50.975784Z","iopub.execute_input":"2024-09-05T04:56:50.977001Z","iopub.status.idle":"2024-09-05T04:56:50.984500Z","shell.execute_reply.started":"2024-09-05T04:56:50.976939Z","shell.execute_reply":"2024-09-05T04:56:50.983193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, inter[\"i_1w_sale_rank\"], inter[\"i_1w_sale_norm\"] = period_sale(\n    inter, [\"article_id\"], days=14, rank=True, norm=True, week_num=WEEK_NUM\n)\n_, inter[\"p_1w_sale_rank\"], inter[\"p_1w_sale_norm\"] = period_sale(\n    inter, [\"product_code\"], days=14, rank=True, norm=True, week_num=WEEK_NUM\n)\ninter[\"i_2w_sale\"], inter[\"i_2w_sale_rank\"], inter[\"i_2w_sale_norm\"] = period_sale(\n    inter, [\"article_id\"], days=14, rank=True, norm=True, week_num=WEEK_NUM\n)\ninter[\"p_2w_sale\"], inter[\"p_2w_sale_rank\"], inter[\"p_2w_sale_norm\"] = period_sale(\n    inter, [\"product_code\"], days=14, rank=True, norm=True, week_num=WEEK_NUM\n)","metadata":{"id":"kYdOqlyldvNM","execution":{"iopub.status.busy":"2024-09-05T04:56:57.835811Z","iopub.execute_input":"2024-09-05T04:56:57.836282Z","iopub.status.idle":"2024-09-05T04:57:53.865416Z","shell.execute_reply.started":"2024-09-05T04:56:57.836235Z","shell.execute_reply":"2024-09-05T04:57:53.864084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter[\"i_3w_sale\"], inter[\"i_3w_sale_rank\"], inter[\"i_3w_sale_norm\"] = period_sale(\n    inter, [\"article_id\"], days=21, rank=True, norm=True, week_num=WEEK_NUM\n)\ninter[\"p_3w_sale\"], inter[\"p_3w_sale_rank\"], inter[\"p_3w_sale_norm\"] = period_sale(\n    inter, [\"product_code\"], days=21, rank=True, norm=True, week_num=WEEK_NUM\n)\ninter[\"i_4w_sale\"], inter[\"i_4w_sale_rank\"], inter[\"i_4w_sale_norm\"] = period_sale(\n    inter, [\"article_id\"], days=28, rank=True, norm=True, week_num=WEEK_NUM\n)\ninter[\"p_4w_sale\"], inter[\"p_4w_sale_rank\"], inter[\"p_4w_sale_norm\"] = period_sale(\n    inter, [\"product_code\"], days=28, rank=True, norm=True, week_num=WEEK_NUM\n)","metadata":{"id":"7Lv1FtoOeI9n","execution":{"iopub.status.busy":"2024-09-05T05:00:14.543406Z","iopub.execute_input":"2024-09-05T05:00:14.543995Z","iopub.status.idle":"2024-09-05T05:01:20.197006Z","shell.execute_reply.started":"2024-09-05T05:00:14.543944Z","shell.execute_reply":"2024-09-05T05:01:20.195836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter.shape","metadata":{"id":"2Yqb3KaWinZM","outputId":"06dc54f9-2c38-4fa5-f242-f8d42ad7a5cc","execution":{"iopub.status.busy":"2024-09-05T05:05:31.332677Z","iopub.execute_input":"2024-09-05T05:05:31.333253Z","iopub.status.idle":"2024-09-05T05:05:31.341480Z","shell.execute_reply.started":"2024-09-05T05:05:31.333209Z","shell.execute_reply":"2024-09-05T05:05:31.340224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter['i_repurchase_ratio'] = repurchase_ratio(inter, ['article_id'], week_num=WEEK_NUM)\ninter['p_repurchase_ratio'] = repurchase_ratio(inter, ['product_code'], week_num=WEEK_NUM)","metadata":{"id":"FdlZqq5QzbMR","outputId":"dab8fbf6-0167-42d6-9e74-ecc826ec84c1","execution":{"iopub.status.busy":"2024-09-05T05:05:40.290047Z","iopub.execute_input":"2024-09-05T05:05:40.290599Z","iopub.status.idle":"2024-09-05T05:10:26.646512Z","shell.execute_reply.started":"2024-09-05T05:05:40.290554Z","shell.execute_reply":"2024-09-05T05:10:26.645160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter.shape","metadata":{"id":"Oh3Otv_Cipmi","outputId":"0224c3f9-8534-4658-edca-2f0dc887f728","execution":{"iopub.status.busy":"2024-09-05T05:37:56.772626Z","iopub.execute_input":"2024-09-05T05:37:56.773641Z","iopub.status.idle":"2024-09-05T05:37:56.784959Z","shell.execute_reply.started":"2024-09-05T05:37:56.773567Z","shell.execute_reply":"2024-09-05T05:37:56.783260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter, _ = reduce_mem_usage(inter)","metadata":{"id":"riX-F5H5lbMu","execution":{"iopub.status.busy":"2024-09-05T05:38:07.778258Z","iopub.execute_input":"2024-09-05T05:38:07.778801Z","iopub.status.idle":"2024-09-05T05:38:28.597579Z","shell.execute_reply.started":"2024-09-05T05:38:07.778751Z","shell.execute_reply":"2024-09-05T05:38:28.595663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter[\"i_sale\"] = week_sale(inter, [\"article_id\"], week_num=WEEK_NUM)\ninter[\"p_sale\"] = week_sale(inter, [\"product_code\"], week_num=WEEK_NUM)\ninter[\"i_sale_uni\"] = week_sale(inter, [\"article_id\"], True, week_num=WEEK_NUM)\ninter[\"p_sale_uni\"] = week_sale(inter, [\"product_code\"], True, week_num=WEEK_NUM)\ninter[\"lw_i_sale\"] = week_sale(inter, [\"article_id\"], step=1, week_num=WEEK_NUM) # * last week sale\ninter[\"lw_p_sale\"] = week_sale(inter, [\"product_code\"], step=1, week_num=WEEK_NUM)\ninter[\"lw_i_sale_uni\"] = week_sale(inter, [\"article_id\"], True, step=1, week_num=WEEK_NUM)\ninter[\"lw_p_sale_uni\"] = week_sale(inter, [\"product_code\"], True, step=1, week_num=WEEK_NUM)\n\ninter[\"i_sale_ratio\"] = inter[\"i_sale\"] / (inter[\"p_sale\"] + 1e-6)\ninter[\"i_sale_uni_ratio\"] = inter[\"i_sale_uni\"] / (inter[\"p_sale_uni\"] + 1e-6)\ninter[\"lw_i_sale_ratio\"] = inter[\"lw_i_sale\"] / (inter[\"lw_p_sale\"] + 1e-6)\ninter[\"lw_i_sale_uni_ratio\"] = inter[\"lw_i_sale_uni\"] / (inter[\"lw_p_sale_uni\"] + 1e-6)\n\ninter[\"i_uni_ratio\"] = inter[\"i_sale\"] / (inter[\"i_sale_uni\"] + 1e-6)\ninter[\"p_uni_ratio\"] = inter[\"p_sale\"] / (inter[\"p_sale_uni\"] + 1e-6)\ninter[\"lw_i_uni_ratio\"] = inter[\"lw_i_sale\"] / (inter[\"lw_i_sale_uni\"] + 1e-6)\ninter[\"lw_p_uni_ratio\"] = inter[\"lw_p_sale\"] / (inter[\"lw_p_sale_uni\"] + 1e-6)\n\ninter[\"i_sale_trend\"] = (inter[\"i_sale\"] - inter[\"lw_i_sale\"]) / (inter[\"lw_i_sale\"] + 1e-6)\ninter[\"p_sale_trend\"] = (inter[\"p_sale\"] - inter[\"lw_p_sale\"]) / (inter[\"lw_p_sale\"] + 1e-6)\n\nitem_feats = [\n    \"product_type_no\",\n    # \"product_group_name\",\n    # \"graphical_appearance_no\",\n    # \"colour_group_code\",\n    # \"perceived_colour_value_id\",\n    # \"perceived_colour_master_id\",\n]\ninter = inter.merge(item[[\"article_id\", *item_feats]], on=\"article_id\", how=\"left\")\n\nfor f in tqdm(item_feats):\n    inter[f\"{f}_sale\"] = week_sale(inter, [f], f\"{f}_sale\", week_num=WEEK_NUM)\n    inter[f\"lw_{f}_sale\"] = week_sale(inter, [f], f\"{f}_sale\", step=1, week_num=WEEK_NUM)\n    inter[f\"{f}_sale_trend\"] = (inter[f\"{f}_sale\"] - inter[f\"lw_{f}_sale\"]) / (inter[f\"lw_{f}_sale\"] + 1e-6)","metadata":{"id":"70okKvWfzgip","outputId":"a1a2392a-c88d-49f3-b4be-d3aebecb7761","execution":{"iopub.status.busy":"2024-09-05T05:43:11.827104Z","iopub.execute_input":"2024-09-05T05:43:11.827942Z","iopub.status.idle":"2024-09-05T05:43:12.294901Z","shell.execute_reply.started":"2024-09-05T05:43:11.827894Z","shell.execute_reply":"2024-09-05T05:43:12.293245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter.shape","metadata":{"id":"K4hO0oT3isNu","outputId":"bf229999-43ac-4e2d-ee40-eb93d608911e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# * Date related\ncurr_date_dict = {x:calc_valid_date(x-1)[0] for x in range(100)}\ncurrent_dat = inter['week'].map(curr_date_dict)\nmask = inter['valid']==0\ninter.loc[mask, 't_dat'] = inter.loc[mask, 'week'].map(curr_date_dict)\nfirst_date = inter.groupby('article_id')['t_dat'].min().reset_index(name='first_dat')\ninter = pd.merge(inter, first_date, on='article_id', how='left')\n# df = pd.merge(df, last_date, on='article_id', how='left')\ninter['first_dat'] = (pd.to_datetime(current_dat)-pd.to_datetime(inter['first_dat'])).dt.days","metadata":{"id":"mQkGNyjaBbi-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter.shape","metadata":{"id":"sZHdSXtAi_zK","outputId":"2a802998-74c8-4695-95fd-d134d3d7d549"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter['i_full_sale'] = full_sale(inter, ['article_id'], week_num=WEEK_NUM)\ninter['p_full_sale'] = full_sale(inter, ['product_code'], week_num=WEEK_NUM)\n\ninter['i_daily_sale'] = inter['i_full_sale'] / inter['first_dat']\ninter['p_daily_sale'] = inter['p_full_sale'] / inter['first_dat']\ninter['i_daily_sale_ratio'] = inter['i_daily_sale'] / inter['p_daily_sale']\ninter['i_w_full_sale_ratio'] = inter['i_sale'] / inter['i_full_sale']\n\ninter['i_2w_full_sale_ratio'] = inter['i_2w_sale'] / inter['i_full_sale']\ninter['p_w_full_sale_ratio'] = inter['p_sale'] / inter['p_full_sale']\ninter['p_2w_full_sale_ratio'] = inter['p_2w_sale'] / inter['p_full_sale']\n\ninter['i_week_above_daily_sale'] = inter['i_sale'] / 7 - inter['i_daily_sale']\ninter['p_week_above_full_sale'] = inter['p_sale'] / 7 - inter['i_full_sale']\ninter['i_2w_week_above_daily_sale'] = inter['i_2w_sale'] / 14 - inter['i_daily_sale']\ninter['p_2w_week_above_daily_sale'] = inter['p_2w_sale'] / 14 - inter['p_daily_sale']","metadata":{"id":"OC8SIYK_Ca-g"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"id":"334plAMZ5Ics","outputId":"d0d6230d-ba07-43d5-9640-ca8c97e60b7a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in tqdm(item_feats):\n    inter[f'{f}_full_sale'] = full_sale(inter, [f], week_num=WEEK_NUM)\n    f_first_date = inter.groupby(f)['t_dat'].min().reset_index(name=f'{f}_first_dat')\n    inter = inter.merge(f_first_date, on=f, how='left')\n    inter[f'{f}_daily_sale'] = inter[f'{f}_full_sale'] / (pd.to_datetime(current_dat) - pd.to_datetime(inter[f'{f}_first_dat'])).dt.days\n    inter[f'i_{f}_daily_sale_ratio'] = inter['i_daily_sale'] / inter[f'{f}_daily_sale']\n    inter[f'p_{f}_daily_sale_ratio'] = inter['p_daily_sale'] / inter[f'{f}_daily_sale']\n    del inter[f'{f}_full_sale'], inter[f'{f}_first_dat']\n    gc.collect()","metadata":{"id":"z4HdQYbZ5CC-","outputId":"daf9bf66-b53d-4504-bd05-b9a385f1fe49"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in item_feats + ['i_full_sale','p_full_sale']:\n    del inter[f]","metadata":{"id":"_fhlw8z4z5hD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter['i_pop'] = popularity(inter, 'article_id', week_num=WEEK_NUM)\ninter['p_pop'] = popularity(inter, 'product_code', week_num=WEEK_NUM)","metadata":{"id":"ms4s0YUJznFU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter = inter.loc[inter['week'] <= WEEK_NUM + 2]","metadata":{"id":"ip6AdvV0kS1y"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inter.to_parquet(data_dir / \"processed/processed_inter.pqt\")","metadata":{"id":"huYRSLE91PJH"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge Features\n","metadata":{"id":"jywhBCAGGGQS"}},{"cell_type":"code","source":"inter = pd.read_parquet(data_dir / \"processed/processed_inter.pqt\")\ninter = inter[inter['week'] <= WEEK_NUM + 2]","metadata":{"id":"YwbQ4jyD4MxX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* embeddings from DSSM model\ndssm_user_embd = np.load(data_dir / \"external/dssm_user_embd.npy\", allow_pickle=True)\ndssm_item_embd = np.load(data_dir / \"external/dssm_item_embd.npy\", allow_pickle=True)\n# * embeddings from YouTubeDNN model\nyt_user_embd = np.load(data_dir / \"external/yt_user_embd.npy\", allow_pickle=True)\nyt_item_embd = np.load(data_dir / \"external/yt_item_embd.npy\", allow_pickle=True)\n# * embeddings from Word2Vector model\nw2v_user_embd = np.load(data_dir/'external'/'w2v_user_embd.npy', allow_pickle=True)\nw2v_item_embd = np.load(data_dir/'external'/'w2v_item_embd.npy', allow_pickle=True)","metadata":{"id":"-7YlHho_JR4F"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in inter.columns:\n    inter[col] = np.nan_to_num(inter[col])","metadata":{"id":"IveB8V0VJGLm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(WEEK_NUM)):\n    if i == 0 and not TEST:\n        continue\n    candidate = pd.read_parquet(data_dir/\"interim\"/VERSION_NAME/f\"week{i}_candidate.pqt\")\n    if i == 0:\n        chunk_size = int(candidate.shape[0] * 0.5)\n        for chunk,batch in enumerate(range(0, candidate.shape[0], chunk_size)):\n            sub_candidate = candidate.iloc[batch:batch+chunk_size-1]\n            # * merge features\n            sub_candidate = merge_week_data(data, inter, i, sub_candidate)\n            sub_candidate['article_id'] = sub_candidate['article_id'].astype(int)\n            sub_candidate['customer_id'] = sub_candidate['customer_id'].astype(int)\n            # * merge DSSM user and item embeddings\n            sub_candidate[\"dssm_similarity\"] = calc_embd_similarity(sub_candidate, dssm_user_embd, dssm_item_embd)\n            # * merge YouTubeDNN user and item embeddings\n            sub_candidate[\"yt_similarity\"] = calc_embd_similarity(sub_candidate, yt_user_embd, yt_item_embd)\n            # * merge Word2Vector user and item embeddings\n            sub_candidate[\"wv_similarity\"] = calc_embd_similarity(sub_candidate, w2v_user_embd, w2v_item_embd, sub=False)\n            print(f\"Chunk {chunk} done...\")\n            sub_candidate.to_parquet(data_dir/\"processed\"/VERSION_NAME/f\"week{i}_candidate_{chunk}.pqt\")\n    else:\n        # * merge features\n        candidate = merge_week_data(data, inter, i, candidate)\n        print(candidate['week'].unique())\n        # * merge DSSM user and item embeddings\n        candidate[\"dssm_similarity\"] = calc_embd_similarity(candidate, dssm_user_embd, dssm_item_embd)\n        # * merge YouTubeDNN user and item embeddings\n        candidate[\"yt_similarity\"] = calc_embd_similarity(candidate, yt_user_embd, yt_item_embd)\n        candidate[\"wv_similarity\"] = calc_embd_similarity(candidate, w2v_user_embd, w2v_item_embd, sub=False)\n    candidate.to_parquet(data_dir/\"processed\"/VERSION_NAME/f\"week{i}_candidate.pqt\")","metadata":{"id":"HXmOMDkO33sr","outputId":"2e393829-09ab-4fd6-9372-edc7dfeda088"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del dssm_user_embd, dssm_item_embd, yt_user_embd, yt_item_embd\ngc.collect()","metadata":{"id":"sWyKQBAt3x--","outputId":"fbf329ec-67aa-4074-f106-f90285bf1d58"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ranking\n","metadata":{"id":"7Zml-cxryoU_"}},{"cell_type":"code","source":"candidates = {}\nlabels = {}\nfor i in tqdm(range(1, WEEK_NUM)):\n    candidates[i] = pd.read_parquet(data_dir/\"processed\"/VERSION_NAME/f\"week{i}_candidate.pqt\")\n    labels[i] = pd.read_parquet(data_dir/\"processed\"/VERSION_NAME/f\"week{i}_label.pqt\")","metadata":{"id":"PsmE6rgjdtIr","outputId":"33501621-403f-4a60-f351-2a112cb1dfdb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = [\n    x\n    for x in candidates[1].columns\n    if x\n    not in [\n        \"label\",\n        \"sales_channel_id\",\n        \"t_dat\",\n        \"week\",\n    ]\n]\ncat_features = [\n    \"customer_id\",\n    \"article_id\",\n    \"product_code\",\n    \"FN\",\n    \"Active\",\n    \"club_member_status\",\n    \"fashion_news_frequency\",\n    \"age\",\n    \"product_type_no\",\n    \"product_group_name\",\n    \"graphical_appearance_no\",\n    \"colour_group_code\",\n    \"perceived_colour_value_id\",\n    \"perceived_colour_master_id\",\n\n    \"user_gender\",\n    \"article_gender\",\n    \"season_type\"\n]","metadata":{"id":"xWqYwCbsdtIr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# * Convert categorical featues as `CategoricalDtype`\ncate_dict = {}\nfor feat in tqdm(cat_features):\n    if feat in data['user'].columns:\n        value_set = set(data['user'][feat].unique())\n    elif feat in data['item'].columns:\n        value_set = set(data['item'][feat].unique())\n    else:\n        value_set = set(data['inter'][feat].unique())\n    cate_dict[feat] = CategoricalDtype(categories=value_set)","metadata":{"id":"cweXHQMrJR4I","outputId":"4b89aacb-d149-4284-f0f4-1bd3cf472500"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_data = pd.concat([candidates[i] for i in range(1, WEEK_NUM)], ignore_index=True)","metadata":{"id":"G-fVjg3JT_OE"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extra Features","metadata":{"id":"aLttJ3CtJ39a"}},{"cell_type":"code","source":"inter = data['inter']\ninter = inter[inter['t_dat']<'2020-08-19'] # * start date of the last valid week\ninter['week'] = (pd.to_datetime('2020-09-29') - pd.to_datetime(inter['t_dat'])).dt.days // 7\ninter = inter.merge(data['item'][[\"article_id\", \"product_code\"]], on=\"article_id\", how=\"left\")","metadata":{"id":"lcsSONupJ9zO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = inter.groupby('article_id').week.mean()\nfull_data['article_time_mean'] = full_data['article_id'].map(tmp)\n\ntmp = inter.groupby('customer_id').week.nth(-1)\nfull_data['customer_id_last_time'] = full_data['customer_id'].map(tmp)\n\ntmp = inter.groupby('customer_id').week.nth(0)\nfull_data['customer_id_first_time'] = full_data['customer_id'].map(tmp)\n\ntmp = inter.groupby('customer_id').week.mean()\nfull_data['customer_id_time_mean'] = full_data['customer_id'].map(tmp)\n\nfull_data['customer_id_gap'] = full_data['customer_id_first_time'] - full_data['customer_id_last_time']","metadata":{"id":"Z3vaGE2nJ-zT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats += [\n    'article_time_mean',\n    'customer_id_last_time',\n    'customer_id_first_time',\n    'customer_id_time_mean',\n    'customer_id_gap'\n]","metadata":{"id":"RYEfzdHmczFn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del tmp\ngc.collect()","metadata":{"id":"psZK0985KCnl","outputId":"7d67a323-97ec-4435-ec61-3cb53bc634e5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train\n","metadata":{"id":"3RojtbK9GGQU"}},{"cell_type":"code","source":"for feat in tqdm(cat_features):\n    full_data[feat] = full_data[feat].astype(cate_dict[feat])","metadata":{"id":"4UeLfwqRc4iQ","outputId":"78f0b549-b400-4773-be89-c53272c85c37"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = full_data.loc[full_data['week']>1]\nvalid = full_data.loc[full_data['week']==1]\n\ndel full_data\ngc.collect()","metadata":{"id":"rJIW8LACUCmg","outputId":"8f48a53a-eedf-4e01-cc6c-4230bcac7111"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    \"objective\": \"binary\",#\"lambdarank\",\n    \"boosting_type\": \"gbdt\",\n    \"metric\": \"auc\",#\"map\",\n    \"max_depth\": 8,\n    \"num_leaves\": 128,\n    \"learning_rate\": 0.03,\n\n    \"verbose\": -1,\n    \"eval_at\": 12,\n}","metadata":{"id":"nTm9bz6vR6T4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_rank_model(train, valid, train_group, valid_group):\n\n    train_set = lgb.Dataset(\n        data=train[feats],\n        label=train[\"label\"],\n        group=train_group,\n        feature_name=feats,\n        categorical_feature=cat_features,\n        params=params,\n    )\n\n    valid_set = lgb.Dataset(\n        data=valid[feats],\n        label=valid[\"label\"],\n        group=valid_group,\n        feature_name=feats,\n        categorical_feature=cat_features,\n        params=params,\n    )\n\n    ranker = lgb.train(\n        params,\n        train_set,\n        num_boost_round=300,\n        valid_sets=[valid_set],\n        early_stopping_rounds=30,\n        verbose_eval=10,\n    )\n    ranker.save_model(\n        model_dir / f\"lgb_small_ranker.model\",\n        num_iteration=ranker.best_iteration,\n    )\n    return ranker","metadata":{"id":"G2wtl4vo9F8J"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_binary_model(train, valid):\n\n    train_set = lgb.Dataset(\n        data=train[feats],\n        label=train[\"label\"],\n        feature_name=feats,\n        categorical_feature=cat_features,\n        params=params,\n    )\n\n    valid_set = lgb.Dataset(\n        data=valid[feats],\n        label=valid[\"label\"],\n        feature_name=feats,\n        categorical_feature=cat_features,\n        params=params,\n    )\n\n    ranker = lgb.train(\n        params,\n        train_set,\n        num_boost_round=300,\n        valid_sets=[valid_set],\n        early_stopping_rounds=30,\n        verbose_eval=10,\n    )\n    ranker.save_model(\n        model_dir / f\"lgb_small_binary.model\",\n        num_iteration=ranker.best_iteration,\n    )\n    return ranker","metadata":{"id":"01ZUO87Gg_-e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del candidates\ngc.collect()","metadata":{"id":"vw2PmIZwPdFf","outputId":"596f7eee-2618-4311-eebd-b62c0fd9c635"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train positive rate:\", train.label.mean())","metadata":{"id":"F-wxWYRFPd9O","outputId":"7f5a0882-8a44-4533-abee-3ace8d20f93e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.sort_values(by=[\"week\", \"customer_id\"], ascending=True).reset_index(drop=True)\nvalid = valid.sort_values(by=[\"customer_id\"], ascending=True).reset_index(drop=True)","metadata":{"id":"O9JQdMFFPfov"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_group = train[[\"customer_id\", \"article_id\", \"week\"]]\ntrain_group = train_group.astype(\"int32\")  # * convert to int to avoid `0` in groupby count result\ntrain_group = (train_group.groupby([\"week\", \"customer_id\"]).size().values)","metadata":{"id":"aj70p1WDPgfQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_group = valid[[\"customer_id\", \"article_id\"]]\nvalid_group = valid_group.astype(\"int32\")  # * convert to int to avoid `0` in groupby count result\nvalid_group = valid_group.groupby([\"customer_id\"]).size().values","metadata":{"id":"lYOh4wBaPhaN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[feats+['label']]\nvalid = valid[feats+['label']]","metadata":{"id":"Z1LrjrLKPiYA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"id":"lm8JNIuqPjMN","outputId":"cdfc2f52-28b9-47e7-dc54-0be7187fb16d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ranker = train_rank_model(train, valid, train_group, valid_group)\nranker = train_binary_model(train, valid)","metadata":{"id":"1FxwqE1cJR4L","outputId":"8d510452-faa1-473e-9240-38d6657fc4e5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference","metadata":{"id":"KbFdGrYRdtIt"}},{"cell_type":"code","source":"# ranker = lgb.Booster(model_file=model_dir / \"lgb_small_binary.model\")\nranker = lgb.Booster(model_file=model_dir / \"lgb_small_ranker.model\")","metadata":{"id":"hl3DPTaBGGQV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_importance = pd.DataFrame(\n    {\"feature\": feats, \"importance\": ranker.feature_importance()}\n).sort_values(by=\"importance\", ascending=False)\nplt.figure(figsize=(8, 22))\nsns.barplot(y=\"feature\", x=\"importance\", data=feat_importance)","metadata":{"id":"OG9AwSntGGQV"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Validate","metadata":{"id":"lLWs-Z5CGGQV"}},{"cell_type":"code","source":"val_candidates = valid.reset_index(drop=True)","metadata":{"id":"SSpgfI0mdtIv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(ranker, candidates, batch_size = 5_000_000):\n    probs = np.zeros(candidates.shape[0])\n    for batch in range(0, candidates.shape[0], batch_size):\n        outputs = ranker.predict(candidates.loc[batch : batch + batch_size - 1, feats])\n        probs[batch : batch + batch_size] = outputs\n    candidates[\"prob\"] = probs\n    pred_lgb = candidates[['customer_id','article_id','prob']]\n    pred_lgb = pred_lgb.sort_values(by=[\"customer_id\",\"prob\"], ascending=False).reset_index(drop=True)\n    pred_lgb.rename(columns={'article_id':'prediction'}, inplace=True)\n    pred_lgb = pred_lgb.drop_duplicates(['customer_id', 'prediction'], keep='first')\n    pred_lgb['customer_id'] = pred_lgb['customer_id'].astype(int)\n    pred_lgb = pred_lgb.groupby(\"customer_id\")[\"prediction\"].progress_apply(list).reset_index()\n    return pred_lgb","metadata":{"id":"pEhET3KZJR4N"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predict(ranker, val_candidates)","metadata":{"id":"Ifgoemo6JR4N","outputId":"f1accf63-1aeb-4c3e-bf0b-c0edf1daeb26"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = labels[1]\nlabel = pd.merge(label, pred, on=\"customer_id\", how=\"left\")","metadata":{"id":"G1Qh3k2aGGQW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_at_k(label[\"article_id\"], label[\"prediction\"], k=12)\n\n# 0.029813727108367518 ranker\n# 0.029791925075924913 binary","metadata":{"id":"XrFnQ2evGGQW","outputId":"505a0a6f-a7cf-4866-b305-abca1658aed5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 5_000_000\nprobs = np.zeros(val_candidates.shape[0])\nfor batch in range(0, val_candidates.shape[0], batch_size):\n    outputs = ranker.predict(val_candidates.loc[batch : batch + batch_size - 1, feats])\n    probs[batch : batch + batch_size] = outputs\nval_candidates[\"prob\"] = probs\npred_lgb = val_candidates[['customer_id','article_id','prob']]\npred_lgb = pred_lgb.sort_values(by=[\"customer_id\",\"prob\"], ascending=False).reset_index(drop=True)\npred_lgb.rename(columns={'article_id':'prediction'}, inplace=True)\npred_lgb = pred_lgb.drop_duplicates(['customer_id', 'prediction'], keep='first')\npred_lgb['customer_id'] = pred_lgb['customer_id'].astype(int)","metadata":{"id":"SEpG-KVUJR4O"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_lgb.to_parquet(data_dir/\"processed\"/\"small_binary_valid.pqt\")","metadata":{"id":"UILjptYKJR4O"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test","metadata":{"id":"_5_faqlKJR4R"}},{"cell_type":"code","source":"del candidates\ngc.collect()","metadata":{"id":"qFaPtoF7JR4R","outputId":"8817b209-c405-4874-9102-81fdbfab1ad4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = []\nfor chunk in range(2):\n    print(f\"Chunk {chunk}\")\n    test_candidates = pd.read_parquet(data_dir/\"processed\"/VERSION_NAME/f\"week0_candidate_{chunk}.pqt\")\n    for feat in cat_features:\n        test_candidates[feat] = test_candidates[feat].astype(cate_dict[feat])\n\n    # * Extra Features ===================================\n\n    tmp = inter.groupby('article_id').week.mean()\n    test_candidates['article_time_mean'] = test_candidates['article_id'].map(tmp)\n\n    tmp = inter.groupby('customer_id').week.nth(-1)\n    test_candidates['customer_id_last_time'] = test_candidates['customer_id'].map(tmp)\n\n    tmp = inter.groupby('customer_id').week.nth(0)\n    test_candidates['customer_id_first_time'] = test_candidates['customer_id'].map(tmp)\n\n    tmp = inter.groupby('customer_id').week.mean()\n    test_candidates['customer_id_time_mean'] = test_candidates['customer_id'].map(tmp)\n\n    test_candidates['customer_id_gap'] = test_candidates['customer_id_first_time'] - test_candidates['customer_id_last_time']\n\n    gc.collect()\n    # * ==================================================\n\n    batch_size = 5_000_000\n    probs = np.zeros(test_candidates.shape[0])\n    for batch in tqdm(range(0, test_candidates.shape[0], batch_size)):\n        outputs = ranker.predict(test_candidates.loc[batch : batch + batch_size - 1, feats])\n        probs[batch : batch + batch_size] = outputs\n    test_candidates[\"prob\"] = probs\n    pred_lgb = test_candidates[['customer_id','article_id','prob']]\n    pred_lgb = pred_lgb.sort_values(by=[\"customer_id\",\"prob\"], ascending=False).reset_index(drop=True)\n    pred_lgb.rename(columns={'article_id':'prediction'}, inplace=True)\n    pred_lgb = pred_lgb.drop_duplicates(['customer_id', 'prediction'], keep='first')\n    pred_lgb['customer_id'] = pred_lgb['customer_id'].astype(int)\n    test_pred.append(pred_lgb)\n    del test_candidates\n    gc.collect()","metadata":{"id":"GxTa-32qPMNh","outputId":"98bc03f6-32d4-4747-bec4-0a3024e5c09c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_lgb = pd.concat(test_pred, ignore_index=True)","metadata":{"id":"S1D0jEFqPzng"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_lgb.to_parquet(data_dir/\"processed\"/\"small_binary_test.pqt\")\npred_lgb.to_parquet(data_dir/\"processed\"/\"small_rank_test.pqt\")","metadata":{"id":"ZpdiPY58JR4R"},"execution_count":null,"outputs":[]}]}