{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-27T12:59:18.138021Z","iopub.execute_input":"2023-12-27T12:59:18.138434Z","iopub.status.idle":"2023-12-27T12:59:18.589002Z","shell.execute_reply.started":"2023-12-27T12:59:18.138400Z","shell.execute_reply":"2023-12-27T12:59:18.587811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget https://raw.githubusercontent.com/benhamner/Metrics/master/Python/ml_metrics/average_precision.py","metadata":{"execution":{"iopub.status.busy":"2023-12-27T12:59:18.591082Z","iopub.execute_input":"2023-12-27T12:59:18.591540Z","iopub.status.idle":"2023-12-27T12:59:19.976400Z","shell.execute_reply.started":"2023-12-27T12:59:18.591509Z","shell.execute_reply":"2023-12-27T12:59:19.975032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade implicit\nimport os; os.environ['OPENBLAS_NUM_THREADS']='1'\nimport numpy as np\nimport pandas as pd\nimport implicit\nfrom scipy.sparse import coo_matrix\nfrom implicit.evaluation import mean_average_precision_at_k","metadata":{"execution":{"iopub.status.busy":"2023-12-27T12:59:19.978199Z","iopub.execute_input":"2023-12-27T12:59:19.978562Z","iopub.status.idle":"2023-12-27T12:59:36.775475Z","shell.execute_reply.started":"2023-12-27T12:59:19.978528Z","shell.execute_reply":"2023-12-27T12:59:36.773956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = '../input/h-and-m-personalized-fashion-recommendations/'\ncsv_train = f'{base_path}transactions_train.csv'\ncsv_sub = f'{base_path}sample_submission.csv'\ncsv_users = f'{base_path}customers.csv'\ncsv_items = f'{base_path}articles.csv'\n\ntransactions = pd.read_csv(csv_train, dtype={'article_id': str}, parse_dates=['t_dat'])\ndf_sub = pd.read_csv(csv_sub)\ncustomers = pd.read_csv(csv_users)\narticles = pd.read_csv(csv_items, dtype={'article_id': str})","metadata":{"execution":{"iopub.status.busy":"2023-12-27T12:59:36.778694Z","iopub.execute_input":"2023-12-27T12:59:36.779316Z","iopub.status.idle":"2023-12-27T13:01:17.148961Z","shell.execute_reply.started":"2023-12-27T12:59:36.779279Z","shell.execute_reply":"2023-12-27T13:01:17.147439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:01:17.150878Z","iopub.execute_input":"2023-12-27T13:01:17.151400Z","iopub.status.idle":"2023-12-27T13:01:17.178444Z","shell.execute_reply.started":"2023-12-27T13:01:17.151336Z","shell.execute_reply":"2023-12-27T13:01:17.177151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helper functions\nfrom sklearn.base import BaseEstimator, TransformerMixin\nimport numpy as np\nfrom average_precision import apk\n\n# https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/308635\ndef customer_hex_id_to_int(series):\n    return series.str[-16:].apply(hex_id_to_int)\n\ndef hex_id_to_int(str):\n    return int(str[-16:], 16)\n\ndef article_id_str_to_int(series):\n    return series.astype('int32')\n\ndef article_id_int_to_str(series):\n    return '0' + series.astype('str')\n\nclass Categorize(BaseEstimator, TransformerMixin):\n    def __init__(self, min_examples=0):\n        self.min_examples = min_examples\n        self.categories = []\n        \n    def fit(self, X):\n        for i in range(X.shape[1]):\n            vc = X.iloc[:, i].value_counts()\n            self.categories.append(vc[vc > self.min_examples].index.tolist())\n        return self\n\n    def transform(self, X):\n        data = {X.columns[i]: pd.Categorical(X.iloc[:, i], categories=self.categories[i]).codes for i in range(X.shape[1])}\n        return pd.DataFrame(data=data)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:01:17.182147Z","iopub.execute_input":"2023-12-27T13:01:17.182528Z","iopub.status.idle":"2023-12-27T13:01:17.664220Z","shell.execute_reply.started":"2023-12-27T13:01:17.182497Z","shell.execute_reply":"2023-12-27T13:01:17.662764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['customer_id'] = customer_hex_id_to_int(transactions['customer_id'])","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:01:17.665919Z","iopub.execute_input":"2023-12-27T13:01:17.666572Z","iopub.status.idle":"2023-12-27T13:02:17.459959Z","shell.execute_reply.started":"2023-12-27T13:01:17.666528Z","shell.execute_reply":"2023-12-27T13:02:17.457697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.t_dat = pd.to_datetime(transactions.t_dat, format='%Y-%m-%d')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:17.462750Z","iopub.execute_input":"2023-12-27T13:02:17.463234Z","iopub.status.idle":"2023-12-27T13:02:18.421869Z","shell.execute_reply.started":"2023-12-27T13:02:17.463191Z","shell.execute_reply":"2023-12-27T13:02:18.420403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['week'] = 104 - (transactions.t_dat.max() - transactions.t_dat).dt.days // 7","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:18.423616Z","iopub.execute_input":"2023-12-27T13:02:18.424246Z","iopub.status.idle":"2023-12-27T13:02:19.645437Z","shell.execute_reply.started":"2023-12-27T13:02:18.424212Z","shell.execute_reply":"2023-12-27T13:02:19.643708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.article_id = article_id_str_to_int(transactions.article_id)\narticles.article_id = article_id_str_to_int(articles.article_id)\n\ntransactions.week = transactions.week.astype('int8')\ntransactions.sales_channel_id = transactions.sales_channel_id.astype('int8')\ntransactions.price = transactions.price.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:19.650909Z","iopub.execute_input":"2023-12-27T13:02:19.651358Z","iopub.status.idle":"2023-12-27T13:02:26.429965Z","shell.execute_reply.started":"2023-12-27T13:02:19.651322Z","shell.execute_reply":"2023-12-27T13:02:26.428839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.customer_id = customer_hex_id_to_int(customers.customer_id)\nfor col in ['FN', 'Active', 'age']:\n    customers[col].fillna(-1, inplace=True)\n    customers[col] = customers[col].astype('int8')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:26.431676Z","iopub.execute_input":"2023-12-27T13:02:26.432136Z","iopub.status.idle":"2023-12-27T13:02:28.652799Z","shell.execute_reply.started":"2023-12-27T13:02:26.432094Z","shell.execute_reply":"2023-12-27T13:02:28.651614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.club_member_status = Categorize().fit_transform(customers[['club_member_status']]).club_member_status\ncustomers.postal_code = Categorize().fit_transform(customers[['postal_code']]).postal_code\ncustomers.fashion_news_frequency = Categorize().fit_transform(customers[['fashion_news_frequency']]).fashion_news_frequency","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:28.654494Z","iopub.execute_input":"2023-12-27T13:02:28.655504Z","iopub.status.idle":"2023-12-27T13:02:31.484900Z","shell.execute_reply.started":"2023-12-27T13:02:28.655447Z","shell.execute_reply":"2023-12-27T13:02:31.483724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in articles.columns:\n    if articles[col].dtype == 'object':\n        articles[col] = Categorize().fit_transform(articles[[col]])[col]","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:31.486687Z","iopub.execute_input":"2023-12-27T13:02:31.487304Z","iopub.status.idle":"2023-12-27T13:02:32.214466Z","shell.execute_reply.started":"2023-12-27T13:02:31.487240Z","shell.execute_reply":"2023-12-27T13:02:32.213123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in articles.columns:\n    if articles[col].dtype == 'int64':\n        articles[col] = articles[col].astype('int32')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:32.215649Z","iopub.execute_input":"2023-12-27T13:02:32.215976Z","iopub.status.idle":"2023-12-27T13:02:32.234313Z","shell.execute_reply.started":"2023-12-27T13:02:32.215948Z","shell.execute_reply":"2023-12-27T13:02:32.233000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.sort_values(['t_dat', 'customer_id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:32.235972Z","iopub.execute_input":"2023-12-27T13:02:32.236470Z","iopub.status.idle":"2023-12-27T13:02:43.815619Z","shell.execute_reply.started":"2023-12-27T13:02:32.236416Z","shell.execute_reply":"2023-12-27T13:02:43.814171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef apk(actual, predicted, k=10):\n    \"\"\"\n    Computes the average precision at k.\n\n    This function computes the average prescision at k between two lists of\n    items.\n\n    Parameters\n    ----------\n    actual : list\n             A list of elements that are to be predicted (order doesn't matter)\n    predicted : list\n                A list of predicted elements (order does matter)\n    k : int, optional\n        The maximum number of predicted elements\n\n    Returns\n    -------\n    score : double\n            The average precision at k over the input lists\n\n    \"\"\"\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=10):\n    \"\"\"\n    Computes the mean average precision at k.\n\n    This function computes the mean average prescision at k between two lists\n    of lists of items.\n\n    Parameters\n    ----------\n    actual : list\n             A list of lists of elements that are to be predicted \n             (order doesn't matter in the lists)\n    predicted : list\n                A list of lists of predicted elements\n                (order matters in the lists)\n    k : int, optional\n        The maximum number of predicted elements\n\n    Returns\n    -------\n    score : double\n            The mean average precision at k over the input lists\n\n    \"\"\"\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:43.817381Z","iopub.execute_input":"2023-12-27T13:02:43.817825Z","iopub.status.idle":"2023-12-27T13:02:43.832880Z","shell.execute_reply.started":"2023-12-27T13:02:43.817786Z","shell.execute_reply":"2023-12-27T13:02:43.831322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\ndef customer_hex_id_to_int(series):\n    return series.str[-16:].apply(hex_id_to_int)\n\ndef hex_id_to_int(str):\n    return int(str[-16:], 16)\n\ndef article_id_str_to_int(series):\n    return series.astype('int32')\n\ndef article_id_int_to_str(series):\n    return '0' + series.astype('str')\n\nclass Categorize(BaseEstimator, TransformerMixin):\n    def __init__(self, min_examples=0):\n        self.min_examples = min_examples\n        self.categories = []\n        \n    def fit(self, X):\n        for i in range(X.shape[1]):\n            vc = X.iloc[:, i].value_counts()\n            self.categories.append(vc[vc > self.min_examples].index.tolist())\n        return self\n\n    def transform(self, X):\n        data = {X.columns[i]: pd.Categorical(X.iloc[:, i], categories=self.categories[i]).codes for i in range(X.shape[1])}\n        return pd.DataFrame(data=data)\n\n\ndef calculate_apk(list_of_preds, list_of_gts):\n    # for fast validation this can be changed to operate on dicts of {'cust_id_int': [art_id_int, ...]}\n    # using 'data/val_week_purchases_by_cust.pkl'\n    apks = []\n    for preds, gt in zip(list_of_preds, list_of_gts):\n        apks.append(apk(gt, preds, k=12))\n    return np.mean(apks)\n\ndef eval_sub(sub_csv, skip_cust_with_no_purchases=True):\n    sub=pd.read_csv(sub_csv)\n    validation_set=pd.read_parquet('data/validation_ground_truth.parquet')\n\n    apks = []\n\n    no_purchases_pattern = []\n    for pred, gt in zip(sub.prediction.str.split(), validation_set.prediction.str.split()):\n        if skip_cust_with_no_purchases and (gt == no_purchases_pattern): continue\n        apks.append(apk(gt, pred, k=12))\n    return np.mean(apks)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:43.835008Z","iopub.execute_input":"2023-12-27T13:02:43.835564Z","iopub.status.idle":"2023-12-27T13:02:43.860586Z","shell.execute_reply.started":"2023-12-27T13:02:43.835517Z","shell.execute_reply":"2023-12-27T13:02:43.858927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:43.862755Z","iopub.execute_input":"2023-12-27T13:02:43.863310Z","iopub.status.idle":"2023-12-27T13:02:43.894116Z","shell.execute_reply.started":"2023-12-27T13:02:43.863266Z","shell.execute_reply":"2023-12-27T13:02:43.892764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:43.896107Z","iopub.execute_input":"2023-12-27T13:02:43.896957Z","iopub.status.idle":"2023-12-27T13:02:43.953016Z","shell.execute_reply.started":"2023-12-27T13:02:43.896914Z","shell.execute_reply":"2023-12-27T13:02:43.951906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:43.954941Z","iopub.execute_input":"2023-12-27T13:02:43.955757Z","iopub.status.idle":"2023-12-27T13:02:43.980721Z","shell.execute_reply.started":"2023-12-27T13:02:43.955714Z","shell.execute_reply":"2023-12-27T13:02:43.979334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_week = transactions.week.max() + 1\ntransactions = transactions[transactions.week > transactions.week.max() - 10]","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:43.982206Z","iopub.execute_input":"2023-12-27T13:02:43.982531Z","iopub.status.idle":"2023-12-27T13:02:44.194403Z","shell.execute_reply.started":"2023-12-27T13:02:43.982503Z","shell.execute_reply":"2023-12-27T13:02:44.193143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nc2weeks = transactions.groupby('customer_id')['week'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:02:44.196019Z","iopub.execute_input":"2023-12-27T13:02:44.200787Z","iopub.status.idle":"2023-12-27T13:03:15.862603Z","shell.execute_reply.started":"2023-12-27T13:02:44.200749Z","shell.execute_reply":"2023-12-27T13:03:15.861418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.groupby('week')['t_dat'].agg(['min', 'max'])","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:15.864117Z","iopub.execute_input":"2023-12-27T13:03:15.865143Z","iopub.status.idle":"2023-12-27T13:03:15.959356Z","shell.execute_reply.started":"2023-12-27T13:03:15.865099Z","shell.execute_reply":"2023-12-27T13:03:15.958009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c2weeks","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:15.960593Z","iopub.execute_input":"2023-12-27T13:03:15.960901Z","iopub.status.idle":"2023-12-27T13:03:15.972362Z","shell.execute_reply.started":"2023-12-27T13:03:15.960875Z","shell.execute_reply":"2023-12-27T13:03:15.971219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nc2weeks2shifted_weeks = {}\n\nfor c_id, weeks in c2weeks.items():\n    c2weeks2shifted_weeks[c_id] = {}\n    for i in range(weeks.shape[0]-1):\n        c2weeks2shifted_weeks[c_id][weeks[i]] = weeks[i+1]\n    c2weeks2shifted_weeks[c_id][weeks[-1]] = test_week","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:15.973580Z","iopub.execute_input":"2023-12-27T13:03:15.973879Z","iopub.status.idle":"2023-12-27T13:03:17.163380Z","shell.execute_reply.started":"2023-12-27T13:03:15.973854Z","shell.execute_reply":"2023-12-27T13:03:17.162177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_last_purchase = transactions.copy()","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:17.167798Z","iopub.execute_input":"2023-12-27T13:03:17.168227Z","iopub.status.idle":"2023-12-27T13:03:17.196542Z","shell.execute_reply.started":"2023-12-27T13:03:17.168194Z","shell.execute_reply":"2023-12-27T13:03:17.195222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nweeks = []\nfor i, (c_id, week) in enumerate(zip(transactions['customer_id'], transactions['week'])):\n    weeks.append(c2weeks2shifted_weeks[c_id][week])\n    \ncandidates_last_purchase.week=weeks","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:17.198163Z","iopub.execute_input":"2023-12-27T13:03:17.198619Z","iopub.status.idle":"2023-12-27T13:03:49.655470Z","shell.execute_reply.started":"2023-12-27T13:03:17.198570Z","shell.execute_reply":"2023-12-27T13:03:49.654352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_price = transactions \\\n    .groupby(['week', 'article_id'])['price'].mean()","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:49.711873Z","iopub.execute_input":"2023-12-27T13:03:49.712323Z","iopub.status.idle":"2023-12-27T13:03:49.951991Z","shell.execute_reply.started":"2023-12-27T13:03:49.712289Z","shell.execute_reply":"2023-12-27T13:03:49.950479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales = transactions \\\n    .groupby('week')['article_id'].value_counts() \\\n    .groupby('week').rank(method='dense', ascending=False) \\\n    .groupby('week').head(12).rename('bestseller_rank').astype('int8')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:49.953632Z","iopub.execute_input":"2023-12-27T13:03:49.954005Z","iopub.status.idle":"2023-12-27T13:03:50.236003Z","shell.execute_reply.started":"2023-12-27T13:03:49.953973Z","shell.execute_reply":"2023-12-27T13:03:50.234668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bestsellers_previous_week = pd.merge(sales, mean_price, on=['week', 'article_id']).reset_index()\nbestsellers_previous_week.week += 1","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:50.237589Z","iopub.execute_input":"2023-12-27T13:03:50.237946Z","iopub.status.idle":"2023-12-27T13:03:50.280952Z","shell.execute_reply.started":"2023-12-27T13:03:50.237916Z","shell.execute_reply":"2023-12-27T13:03:50.280087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_transactions = transactions \\\n    .groupby(['week', 'customer_id']) \\\n    .head(1) \\\n    .drop(columns=['article_id', 'price']) \\\n    .copy()","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:50.282364Z","iopub.execute_input":"2023-12-27T13:03:50.282901Z","iopub.status.idle":"2023-12-27T13:03:50.936108Z","shell.execute_reply.started":"2023-12-27T13:03:50.282870Z","shell.execute_reply":"2023-12-27T13:03:50.934773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.drop_duplicates(['week', 'customer_id'])","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:50.937651Z","iopub.execute_input":"2023-12-27T13:03:50.938018Z","iopub.status.idle":"2023-12-27T13:03:51.239367Z","shell.execute_reply.started":"2023-12-27T13:03:50.937987Z","shell.execute_reply":"2023-12-27T13:03:51.238303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers = pd.merge(\n    unique_transactions,\n    bestsellers_previous_week,\n    on='week',\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:51.240681Z","iopub.execute_input":"2023-12-27T13:03:51.240995Z","iopub.status.idle":"2023-12-27T13:03:51.886191Z","shell.execute_reply.started":"2023-12-27T13:03:51.240967Z","shell.execute_reply":"2023-12-27T13:03:51.885163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set_transactions = unique_transactions.drop_duplicates('customer_id').reset_index(drop=True)\ntest_set_transactions.week = test_week","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:51.887590Z","iopub.execute_input":"2023-12-27T13:03:51.888173Z","iopub.status.idle":"2023-12-27T13:03:51.949862Z","shell.execute_reply.started":"2023-12-27T13:03:51.888141Z","shell.execute_reply":"2023-12-27T13:03:51.948651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers_test_week = pd.merge(\n    test_set_transactions,\n    bestsellers_previous_week,\n    on='week'\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:51.951413Z","iopub.execute_input":"2023-12-27T13:03:51.951812Z","iopub.status.idle":"2023-12-27T13:03:52.395516Z","shell.execute_reply.started":"2023-12-27T13:03:51.951778Z","shell.execute_reply":"2023-12-27T13:03:52.394163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers = pd.concat([candidates_bestsellers, candidates_bestsellers_test_week])\ncandidates_bestsellers.drop(columns='bestseller_rank', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:52.398441Z","iopub.execute_input":"2023-12-27T13:03:52.398953Z","iopub.status.idle":"2023-12-27T13:03:53.177327Z","shell.execute_reply.started":"2023-12-27T13:03:52.398893Z","shell.execute_reply":"2023-12-27T13:03:53.176167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['purchased'] = 1","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:53.178783Z","iopub.execute_input":"2023-12-27T13:03:53.179165Z","iopub.status.idle":"2023-12-27T13:03:53.187560Z","shell.execute_reply.started":"2023-12-27T13:03:53.179130Z","shell.execute_reply":"2023-12-27T13:03:53.186133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([transactions, candidates_last_purchase, candidates_bestsellers])\ndata.purchased.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:53.188895Z","iopub.execute_input":"2023-12-27T13:03:53.189243Z","iopub.status.idle":"2023-12-27T13:03:53.696339Z","shell.execute_reply.started":"2023-12-27T13:03:53.189213Z","shell.execute_reply":"2023-12-27T13:03:53.695119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop_duplicates(['customer_id', 'article_id', 'week'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:03:53.697763Z","iopub.execute_input":"2023-12-27T13:03:53.698113Z","iopub.status.idle":"2023-12-27T13:04:00.454163Z","shell.execute_reply.started":"2023-12-27T13:03:53.698078Z","shell.execute_reply":"2023-12-27T13:04:00.453030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.merge(\n    data,\n    bestsellers_previous_week[['week', 'article_id', 'bestseller_rank']],\n    on=['week', 'article_id'],\n    how='left'\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:00.455801Z","iopub.execute_input":"2023-12-27T13:04:00.456307Z","iopub.status.idle":"2023-12-27T13:04:03.094830Z","shell.execute_reply.started":"2023-12-27T13:04:00.456274Z","shell.execute_reply":"2023-12-27T13:04:03.093497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[data.week != data.week.min()]\ndata.bestseller_rank.fillna(999, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:03.096404Z","iopub.execute_input":"2023-12-27T13:04:03.096784Z","iopub.status.idle":"2023-12-27T13:04:04.726878Z","shell.execute_reply.started":"2023-12-27T13:04:03.096752Z","shell.execute_reply":"2023-12-27T13:04:04.725055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.merge(data, articles, on='article_id', how='left')\ndata = pd.merge(data, customers, on='customer_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:04.728483Z","iopub.execute_input":"2023-12-27T13:04:04.728956Z","iopub.status.idle":"2023-12-27T13:04:15.105741Z","shell.execute_reply.started":"2023-12-27T13:04:04.728914Z","shell.execute_reply":"2023-12-27T13:04:15.104649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.sort_values(['week', 'customer_id'], inplace=True)\ndata.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:15.107425Z","iopub.execute_input":"2023-12-27T13:04:15.107871Z","iopub.status.idle":"2023-12-27T13:04:22.321839Z","shell.execute_reply.started":"2023-12-27T13:04:15.107828Z","shell.execute_reply":"2023-12-27T13:04:22.320508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data[data.week != test_week]\ntest = data[data.week==test_week].drop_duplicates(['customer_id', 'article_id', 'sales_channel_id']).copy()","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:22.323211Z","iopub.execute_input":"2023-12-27T13:04:22.323572Z","iopub.status.idle":"2023-12-27T13:04:29.881766Z","shell.execute_reply.started":"2023-12-27T13:04:22.323541Z","shell.execute_reply":"2023-12-27T13:04:29.880531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_baskets = train.groupby(['week', 'customer_id'])['article_id'].count().values","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:29.883249Z","iopub.execute_input":"2023-12-27T13:04:29.883703Z","iopub.status.idle":"2023-12-27T13:04:30.751393Z","shell.execute_reply.started":"2023-12-27T13:04:29.883673Z","shell.execute_reply":"2023-12-27T13:04:30.750269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_use = ['article_id', 'product_type_no', 'graphical_appearance_no', 'colour_group_code', 'perceived_colour_value_id',\n'perceived_colour_master_id', 'department_no', 'index_code',\n'index_group_no', 'section_no', 'garment_group_no', 'FN', 'Active',\n'club_member_status', 'fashion_news_frequency', 'age', 'postal_code', 'bestseller_rank']","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:30.752792Z","iopub.execute_input":"2023-12-27T13:04:30.753200Z","iopub.status.idle":"2023-12-27T13:04:30.760178Z","shell.execute_reply.started":"2023-12-27T13:04:30.753165Z","shell.execute_reply":"2023-12-27T13:04:30.758881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_X = train[columns_to_use]\ntrain_y = train['purchased']\n\ntest_X = test[columns_to_use]","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:30.761671Z","iopub.execute_input":"2023-12-27T13:04:30.762056Z","iopub.status.idle":"2023-12-27T13:04:31.874317Z","shell.execute_reply.started":"2023-12-27T13:04:30.762025Z","shell.execute_reply":"2023-12-27T13:04:31.872970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:31.876118Z","iopub.execute_input":"2023-12-27T13:04:31.876928Z","iopub.status.idle":"2023-12-27T13:04:36.833496Z","shell.execute_reply.started":"2023-12-27T13:04:31.876884Z","shell.execute_reply":"2023-12-27T13:04:36.831942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm.sklearn import LGBMRanker","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:36.834936Z","iopub.execute_input":"2023-12-27T13:04:36.835409Z","iopub.status.idle":"2023-12-27T13:04:38.553424Z","shell.execute_reply.started":"2023-12-27T13:04:36.835366Z","shell.execute_reply":"2023-12-27T13:04:38.552249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranker = LGBMRanker(\n    objective=\"lambdarank\",\n    metric=\"ndcg\",\n    boosting_type=\"dart\",\n    n_estimators=1,\n    importance_type='gain',\n    verbose=10\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:38.555396Z","iopub.execute_input":"2023-12-27T13:04:38.556287Z","iopub.status.idle":"2023-12-27T13:04:38.563053Z","shell.execute_reply.started":"2023-12-27T13:04:38.556243Z","shell.execute_reply":"2023-12-27T13:04:38.561412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nranker = ranker.fit(\n    train_X,\n    train_y,\n    group=train_baskets,\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:38.565323Z","iopub.execute_input":"2023-12-27T13:04:38.566023Z","iopub.status.idle":"2023-12-27T13:04:49.642594Z","shell.execute_reply.started":"2023-12-27T13:04:38.565963Z","shell.execute_reply":"2023-12-27T13:04:49.641570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in ranker.feature_importances_.argsort()[::-1]:\n    print(columns_to_use[i], ranker.feature_importances_[i]/ranker.feature_importances_.sum())","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:49.644130Z","iopub.execute_input":"2023-12-27T13:04:49.644707Z","iopub.status.idle":"2023-12-27T13:04:49.651662Z","shell.execute_reply.started":"2023-12-27T13:04:49.644674Z","shell.execute_reply":"2023-12-27T13:04:49.650861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n\ntest['preds'] = ranker.predict(test_X)\n\nc_id2predicted_article_ids = test \\\n    .sort_values(['customer_id', 'preds'], ascending=False) \\\n    .groupby('customer_id')['article_id'].apply(list).to_dict()\n\nbestsellers_last_week = \\\n    bestsellers_previous_week[bestsellers_previous_week.week == bestsellers_previous_week.week.max()]['article_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:04:49.652984Z","iopub.execute_input":"2023-12-27T13:04:49.653631Z","iopub.status.idle":"2023-12-27T13:05:12.984145Z","shell.execute_reply.started":"2023-12-27T13:04:49.653598Z","shell.execute_reply":"2023-12-27T13:05:12.982754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npreds = []\nfor c_id in customer_hex_id_to_int(df_sub.customer_id):\n    pred = c_id2predicted_article_ids.get(c_id, [])\n    pred = pred + bestsellers_last_week\n    preds.append(pred[:12])","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:05:12.986012Z","iopub.execute_input":"2023-12-27T13:05:12.986497Z","iopub.status.idle":"2023-12-27T13:05:20.486130Z","shell.execute_reply.started":"2023-12-27T13:05:12.986449Z","shell.execute_reply":"2023-12-27T13:05:20.484701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = [' '.join(['0' + str(p) for p in ps]) for ps in preds]\ndf_sub.prediction = preds","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:05:20.487967Z","iopub.execute_input":"2023-12-27T13:05:20.488971Z","iopub.status.idle":"2023-12-27T13:05:27.667841Z","shell.execute_reply.started":"2023-12-27T13:05:20.488931Z","shell.execute_reply":"2023-12-27T13:05:27.666531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub","metadata":{"execution":{"iopub.status.busy":"2023-12-27T13:05:27.669469Z","iopub.execute_input":"2023-12-27T13:05:27.669801Z","iopub.status.idle":"2023-12-27T13:05:27.683481Z","shell.execute_reply.started":"2023-12-27T13:05:27.669774Z","shell.execute_reply":"2023-12-27T13:05:27.682285Z"},"trusted":true},"execution_count":null,"outputs":[]}]}