{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Radek posted about this [here](https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/309220), and linked to a GitHub repo with the code.\n\nI just transferred that code here to Kaggle notebooks, that's all.","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\ndef apk(actual, predicted, k=10):\n    \"\"\"\n    Computes the average precision at k.\n\n    This function computes the average prescision at k between two lists of\n    items.\n\n    Parameters\n    ----------\n    actual : list\n             A list of elements that are to be predicted (order doesn't matter)\n    predicted : list\n                A list of predicted elements (order does matter)\n    k : int, optional\n        The maximum number of predicted elements\n\n    Returns\n    -------\n    score : double\n            The average precision at k over the input lists\n\n    \"\"\"\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=10):\n    \"\"\"\n    Computes the mean average precision at k.\n\n    This function computes the mean average prescision at k between two lists\n    of lists of items.\n\n    Parameters\n    ----------\n    actual : list\n             A list of lists of elements that are to be predicted \n             (order doesn't matter in the lists)\n    predicted : list\n                A list of lists of predicted elements\n                (order matters in the lists)\n    k : int, optional\n        The maximum number of predicted elements\n\n    Returns\n    -------\n    score : double\n            The mean average precision at k over the input lists\n\n    \"\"\"\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:55:28.768608Z","iopub.execute_input":"2022-04-24T07:55:28.769115Z","iopub.status.idle":"2022-04-24T07:55:28.804382Z","shell.execute_reply.started":"2022-04-24T07:55:28.769027Z","shell.execute_reply":"2022-04-24T07:55:28.803584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\nimport numpy as np\n\n# https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/308635\ndef customer_hex_id_to_int(series):\n    return series.str[-16:].apply(hex_id_to_int)\n\ndef hex_id_to_int(str):\n    return int(str[-16:], 16)\n\ndef article_id_str_to_int(series):\n    return series.astype('int32')\n\ndef article_id_int_to_str(series):\n    return '0' + series.astype('str')\n\nclass Categorize(BaseEstimator, TransformerMixin):\n    def __init__(self, min_examples=0):\n        self.min_examples = min_examples\n        self.categories = []\n        \n    def fit(self, X):\n        for i in range(X.shape[1]):\n            vc = X.iloc[:, i].value_counts()\n            self.categories.append(vc[vc > self.min_examples].index.tolist())\n        return self\n\n    def transform(self, X):\n        data = {X.columns[i]: pd.Categorical(X.iloc[:, i], categories=self.categories[i]).codes for i in range(X.shape[1])}\n        return pd.DataFrame(data=data)\n\n\ndef calculate_apk(list_of_preds, list_of_gts):\n    # for fast validation this can be changed to operate on dicts of {'cust_id_int': [art_id_int, ...]}\n    # using 'data/val_week_purchases_by_cust.pkl'\n    apks = []\n    for preds, gt in zip(list_of_preds, list_of_gts):\n        apks.append(apk(gt, preds, k=12))\n    return np.mean(apks)\n\ndef eval_sub(sub_csv, skip_cust_with_no_purchases=True):\n    sub=pd.read_csv(sub_csv)\n    validation_set=pd.read_parquet('data/validation_ground_truth.parquet')\n\n    apks = []\n\n    no_purchases_pattern = []\n    for pred, gt in zip(sub.prediction.str.split(), validation_set.prediction.str.split()):\n        if skip_cust_with_no_purchases and (gt == no_purchases_pattern): continue\n        apks.append(apk(gt, pred, k=12))\n    return np.mean(apks)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:55:28.896515Z","iopub.execute_input":"2022-04-24T07:55:28.896804Z","iopub.status.idle":"2022-04-24T07:55:30.015301Z","shell.execute_reply.started":"2022-04-24T07:55:28.896773Z","shell.execute_reply":"2022-04-24T07:55:30.014513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:55:30.016882Z","iopub.execute_input":"2022-04-24T07:55:30.017596Z","iopub.status.idle":"2022-04-24T07:55:30.022045Z","shell.execute_reply.started":"2022-04-24T07:55:30.017555Z","shell.execute_reply":"2022-04-24T07:55:30.021254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntransactions = pd.read_parquet('../input/warmup/transactions_train.parquet')\ncustomers = pd.read_parquet('../input/warmup/customers.parquet')\narticles = pd.read_parquet('../input/warmup/articles.parquet')\narticles_phrases = pd.read_csv('../input/handmarticledescriptionembeddings/phrase_embeddings_pca_tiny.csv')\n# sample = 0.05\n# transactions = pd.read_parquet(f'data/transactions_train_sample_{sample}.parquet')\n# customers = pd.read_parquet(f'data/customers_sample_{sample}.parquet')\n# articles = pd.read_parquet(f'data/articles_train_sample_{sample}.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:55:30.02315Z","iopub.execute_input":"2022-04-24T07:55:30.023381Z","iopub.status.idle":"2022-04-24T07:55:38.768974Z","shell.execute_reply.started":"2022-04-24T07:55:30.023353Z","shell.execute_reply":"2022-04-24T07:55:38.767727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_week = transactions.week.max() + 1\ntransactions = transactions[transactions.week > transactions.week.max() - 10]","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:55:38.771537Z","iopub.execute_input":"2022-04-24T07:55:38.771915Z","iopub.status.idle":"2022-04-24T07:55:39.155231Z","shell.execute_reply.started":"2022-04-24T07:55:38.771871Z","shell.execute_reply":"2022-04-24T07:55:39.154473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating candidates","metadata":{}},{"cell_type":"markdown","source":"### Last purchase candidates","metadata":{}},{"cell_type":"code","source":"%%time\n\nc2weeks = transactions.groupby('customer_id')['week'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:26.175869Z","iopub.execute_input":"2022-04-24T07:56:26.176327Z","iopub.status.idle":"2022-04-24T07:56:48.562045Z","shell.execute_reply.started":"2022-04-24T07:56:26.176277Z","shell.execute_reply":"2022-04-24T07:56:48.561132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.groupby('week')['t_dat'].agg(['min', 'max'])","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:48.563919Z","iopub.execute_input":"2022-04-24T07:56:48.564374Z","iopub.status.idle":"2022-04-24T07:56:48.634705Z","shell.execute_reply.started":"2022-04-24T07:56:48.564334Z","shell.execute_reply":"2022-04-24T07:56:48.634121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c2weeks","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:48.63574Z","iopub.execute_input":"2022-04-24T07:56:48.636062Z","iopub.status.idle":"2022-04-24T07:56:48.647345Z","shell.execute_reply.started":"2022-04-24T07:56:48.636036Z","shell.execute_reply":"2022-04-24T07:56:48.646717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nc2weeks2shifted_weeks = {}\n\nfor c_id, weeks in c2weeks.items():\n    c2weeks2shifted_weeks[c_id] = {}\n    for i in range(weeks.shape[0]-1):\n        c2weeks2shifted_weeks[c_id][weeks[i]] = weeks[i+1]\n    c2weeks2shifted_weeks[c_id][weeks[-1]] = test_week","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:48.649088Z","iopub.execute_input":"2022-04-24T07:56:48.649786Z","iopub.status.idle":"2022-04-24T07:56:49.944934Z","shell.execute_reply.started":"2022-04-24T07:56:48.649738Z","shell.execute_reply":"2022-04-24T07:56:49.943887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c2weeks2shifted_weeks[28847241659200]","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:49.946263Z","iopub.execute_input":"2022-04-24T07:56:49.946515Z","iopub.status.idle":"2022-04-24T07:56:49.952933Z","shell.execute_reply.started":"2022-04-24T07:56:49.946484Z","shell.execute_reply":"2022-04-24T07:56:49.952183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_last_purchase = transactions.copy()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:49.954154Z","iopub.execute_input":"2022-04-24T07:56:49.954695Z","iopub.status.idle":"2022-04-24T07:56:49.983806Z","shell.execute_reply.started":"2022-04-24T07:56:49.954645Z","shell.execute_reply":"2022-04-24T07:56:49.982753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nweeks = []\nfor i, (c_id, week) in enumerate(zip(transactions['customer_id'], transactions['week'])):\n    weeks.append(c2weeks2shifted_weeks[c_id][week])\n    \ncandidates_last_purchase.week=weeks","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:56:49.98517Z","iopub.execute_input":"2022-04-24T07:56:49.985484Z","iopub.status.idle":"2022-04-24T07:57:02.248555Z","shell.execute_reply.started":"2022-04-24T07:56:49.98544Z","shell.execute_reply":"2022-04-24T07:57:02.247404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_last_purchase[candidates_last_purchase['customer_id']==272412481300040]","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:02.250093Z","iopub.execute_input":"2022-04-24T07:57:02.250321Z","iopub.status.idle":"2022-04-24T07:57:02.269586Z","shell.execute_reply.started":"2022-04-24T07:57:02.250294Z","shell.execute_reply":"2022-04-24T07:57:02.268489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions[transactions['customer_id']==272412481300040]","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-04-24T07:57:02.27129Z","iopub.execute_input":"2022-04-24T07:57:02.271924Z","iopub.status.idle":"2022-04-24T07:57:02.290764Z","shell.execute_reply.started":"2022-04-24T07:57:02.271878Z","shell.execute_reply":"2022-04-24T07:57:02.289885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Bestsellers candidates","metadata":{}},{"cell_type":"code","source":"mean_price = transactions \\\n    .groupby(['week', 'article_id'])['price'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:02.294258Z","iopub.execute_input":"2022-04-24T07:57:02.295002Z","iopub.status.idle":"2022-04-24T07:57:02.548021Z","shell.execute_reply.started":"2022-04-24T07:57:02.294953Z","shell.execute_reply":"2022-04-24T07:57:02.547152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_price","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:02.549102Z","iopub.execute_input":"2022-04-24T07:57:02.549322Z","iopub.status.idle":"2022-04-24T07:57:02.619413Z","shell.execute_reply.started":"2022-04-24T07:57:02.549296Z","shell.execute_reply":"2022-04-24T07:57:02.618432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales = transactions \\\n    .groupby('week')['article_id'].value_counts() \\\n    .groupby('week').rank(method='dense', ascending=False) \\\n    .groupby('week').head(12).rename('bestseller_rank').astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:02.620729Z","iopub.execute_input":"2022-04-24T07:57:02.621155Z","iopub.status.idle":"2022-04-24T07:57:03.599488Z","shell.execute_reply.started":"2022-04-24T07:57:02.62111Z","shell.execute_reply":"2022-04-24T07:57:03.598608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:03.600789Z","iopub.execute_input":"2022-04-24T07:57:03.60108Z","iopub.status.idle":"2022-04-24T07:57:03.611802Z","shell.execute_reply.started":"2022-04-24T07:57:03.601041Z","shell.execute_reply":"2022-04-24T07:57:03.610591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales.loc[95]","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:03.613233Z","iopub.execute_input":"2022-04-24T07:57:03.613611Z","iopub.status.idle":"2022-04-24T07:57:03.630809Z","shell.execute_reply.started":"2022-04-24T07:57:03.613568Z","shell.execute_reply":"2022-04-24T07:57:03.629938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bestsellers_previous_week = pd.merge(sales, mean_price, on=['week', 'article_id']).reset_index()\nbestsellers_previous_week.week += 1","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:03.634261Z","iopub.execute_input":"2022-04-24T07:57:03.63472Z","iopub.status.idle":"2022-04-24T07:57:03.689892Z","shell.execute_reply.started":"2022-04-24T07:57:03.634572Z","shell.execute_reply":"2022-04-24T07:57:03.689002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bestsellers_previous_week.pipe(lambda df: df[df['week']==96])","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:03.691064Z","iopub.execute_input":"2022-04-24T07:57:03.691298Z","iopub.status.idle":"2022-04-24T07:57:03.704273Z","shell.execute_reply.started":"2022-04-24T07:57:03.69127Z","shell.execute_reply":"2022-04-24T07:57:03.703572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_transactions = transactions \\\n    .groupby(['week', 'customer_id']) \\\n    .head(1) \\\n    .drop(columns=['article_id', 'price']) \\\n    .copy()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:03.705382Z","iopub.execute_input":"2022-04-24T07:57:03.705777Z","iopub.status.idle":"2022-04-24T07:57:04.342905Z","shell.execute_reply.started":"2022-04-24T07:57:03.705745Z","shell.execute_reply":"2022-04-24T07:57:04.341913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_transactions","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:04.344116Z","iopub.execute_input":"2022-04-24T07:57:04.344356Z","iopub.status.idle":"2022-04-24T07:57:04.360602Z","shell.execute_reply.started":"2022-04-24T07:57:04.344325Z","shell.execute_reply":"2022-04-24T07:57:04.359466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.drop_duplicates(['week', 'customer_id'])","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:04.361833Z","iopub.execute_input":"2022-04-24T07:57:04.362049Z","iopub.status.idle":"2022-04-24T07:57:04.65234Z","shell.execute_reply.started":"2022-04-24T07:57:04.362021Z","shell.execute_reply":"2022-04-24T07:57:04.651361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers = pd.merge(\n    unique_transactions,\n    bestsellers_previous_week,\n    on='week',\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:04.653667Z","iopub.execute_input":"2022-04-24T07:57:04.65398Z","iopub.status.idle":"2022-04-24T07:57:05.432314Z","shell.execute_reply.started":"2022-04-24T07:57:04.653939Z","shell.execute_reply":"2022-04-24T07:57:05.431537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set_transactions = unique_transactions.drop_duplicates('customer_id').reset_index(drop=True)\ntest_set_transactions.week = test_week","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:05.433506Z","iopub.execute_input":"2022-04-24T07:57:05.433737Z","iopub.status.idle":"2022-04-24T07:57:05.538802Z","shell.execute_reply.started":"2022-04-24T07:57:05.43371Z","shell.execute_reply":"2022-04-24T07:57:05.537615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set_transactions","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:05.540465Z","iopub.execute_input":"2022-04-24T07:57:05.540789Z","iopub.status.idle":"2022-04-24T07:57:05.558233Z","shell.execute_reply.started":"2022-04-24T07:57:05.540745Z","shell.execute_reply":"2022-04-24T07:57:05.557281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers_test_week = pd.merge(\n    test_set_transactions,\n    bestsellers_previous_week,\n    on='week'\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:05.559584Z","iopub.execute_input":"2022-04-24T07:57:05.560512Z","iopub.status.idle":"2022-04-24T07:57:06.013763Z","shell.execute_reply.started":"2022-04-24T07:57:05.560466Z","shell.execute_reply":"2022-04-24T07:57:06.012859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers = pd.concat([candidates_bestsellers, candidates_bestsellers_test_week])\ncandidates_bestsellers.drop(columns='bestseller_rank', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:06.015111Z","iopub.execute_input":"2022-04-24T07:57:06.015458Z","iopub.status.idle":"2022-04-24T07:57:06.993092Z","shell.execute_reply.started":"2022-04-24T07:57:06.015423Z","shell.execute_reply":"2022-04-24T07:57:06.99221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:06.994217Z","iopub.execute_input":"2022-04-24T07:57:06.994443Z","iopub.status.idle":"2022-04-24T07:57:07.01201Z","shell.execute_reply.started":"2022-04-24T07:57:06.994414Z","shell.execute_reply":"2022-04-24T07:57:07.011224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combining transactions and candidates / negative examples","metadata":{}},{"cell_type":"code","source":"transactions['purchased'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:07.013047Z","iopub.execute_input":"2022-04-24T07:57:07.013791Z","iopub.status.idle":"2022-04-24T07:57:07.024447Z","shell.execute_reply.started":"2022-04-24T07:57:07.013758Z","shell.execute_reply":"2022-04-24T07:57:07.023603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([transactions, candidates_last_purchase, candidates_bestsellers])\ndata.purchased.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:07.025926Z","iopub.execute_input":"2022-04-24T07:57:07.026446Z","iopub.status.idle":"2022-04-24T07:57:07.595974Z","shell.execute_reply.started":"2022-04-24T07:57:07.026399Z","shell.execute_reply":"2022-04-24T07:57:07.595051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-04-24T07:57:07.60124Z","iopub.execute_input":"2022-04-24T07:57:07.601516Z","iopub.status.idle":"2022-04-24T07:57:07.622554Z","shell.execute_reply.started":"2022-04-24T07:57:07.601484Z","shell.execute_reply":"2022-04-24T07:57:07.621552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop_duplicates(['customer_id', 'article_id', 'week'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:07.623935Z","iopub.execute_input":"2022-04-24T07:57:07.624384Z","iopub.status.idle":"2022-04-24T07:57:14.942534Z","shell.execute_reply.started":"2022-04-24T07:57:07.624338Z","shell.execute_reply":"2022-04-24T07:57:14.941725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.purchased.mean()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:14.944065Z","iopub.execute_input":"2022-04-24T07:57:14.944305Z","iopub.status.idle":"2022-04-24T07:57:15.00193Z","shell.execute_reply.started":"2022-04-24T07:57:14.944276Z","shell.execute_reply":"2022-04-24T07:57:15.001055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add bestseller information","metadata":{}},{"cell_type":"code","source":"data = pd.merge(\n    data,\n    bestsellers_previous_week[['week', 'article_id', 'bestseller_rank']],\n    on=['week', 'article_id'],\n    how='left'\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:15.0032Z","iopub.execute_input":"2022-04-24T07:57:15.003419Z","iopub.status.idle":"2022-04-24T07:57:18.842004Z","shell.execute_reply.started":"2022-04-24T07:57:15.003393Z","shell.execute_reply":"2022-04-24T07:57:18.840852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[data.week != data.week.min()]\ndata.bestseller_rank.fillna(999, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:18.843322Z","iopub.execute_input":"2022-04-24T07:57:18.84385Z","iopub.status.idle":"2022-04-24T07:57:20.967498Z","shell.execute_reply.started":"2022-04-24T07:57:18.843809Z","shell.execute_reply":"2022-04-24T07:57:20.966486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.concat([articles, articles_phrases], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:20.969156Z","iopub.execute_input":"2022-04-24T07:57:20.969632Z","iopub.status.idle":"2022-04-24T07:57:20.988354Z","shell.execute_reply.started":"2022-04-24T07:57:20.969588Z","shell.execute_reply":"2022-04-24T07:57:20.987686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.merge(data, articles, on='article_id', how='left')\ndata = pd.merge(data, customers, on='customer_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:20.989825Z","iopub.execute_input":"2022-04-24T07:57:20.99034Z","iopub.status.idle":"2022-04-24T07:57:47.998462Z","shell.execute_reply.started":"2022-04-24T07:57:20.990294Z","shell.execute_reply":"2022-04-24T07:57:47.997801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# free up some memory\ndel customers, articles_phrases\ndel articles, transactions, candidates_bestsellers","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:47.999841Z","iopub.execute_input":"2022-04-24T07:57:48.000115Z","iopub.status.idle":"2022-04-24T07:57:48.004558Z","shell.execute_reply.started":"2022-04-24T07:57:48.000077Z","shell.execute_reply":"2022-04-24T07:57:48.003717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.sort_values(['week', 'customer_id'], inplace=True)\ndata.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:48.005754Z","iopub.execute_input":"2022-04-24T07:57:48.005979Z","iopub.status.idle":"2022-04-24T07:57:55.786491Z","shell.execute_reply.started":"2022-04-24T07:57:48.005951Z","shell.execute_reply":"2022-04-24T07:57:55.78581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data[data.week != test_week]\ntest = data[data.week==test_week].drop_duplicates(['customer_id', 'article_id', 'sales_channel_id']).copy()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:57:55.787607Z","iopub.execute_input":"2022-04-24T07:57:55.788016Z","iopub.status.idle":"2022-04-24T07:58:07.940116Z","shell.execute_reply.started":"2022-04-24T07:57:55.787968Z","shell.execute_reply":"2022-04-24T07:58:07.939164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_baskets = train.groupby(['week', 'customer_id'])['article_id'].count().values","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:07.941318Z","iopub.execute_input":"2022-04-24T07:58:07.941672Z","iopub.status.idle":"2022-04-24T07:58:09.039014Z","shell.execute_reply.started":"2022-04-24T07:58:07.941628Z","shell.execute_reply":"2022-04-24T07:58:09.038186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_use = ['article_id', 'product_type_no', 'graphical_appearance_no', 'colour_group_code', 'perceived_colour_value_id',\n'perceived_colour_master_id', 'department_no', 'index_code',\n'index_group_no', 'section_no', 'garment_group_no', 'FN', 'Active',\n'club_member_status', 'fashion_news_frequency', 'age', 'postal_code', 'bestseller_rank',\n*[str(x) for x in list(range(10))]]","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:09.040302Z","iopub.execute_input":"2022-04-24T07:58:09.040553Z","iopub.status.idle":"2022-04-24T07:58:09.04578Z","shell.execute_reply.started":"2022-04-24T07:58:09.040524Z","shell.execute_reply":"2022-04-24T07:58:09.045098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:09.047023Z","iopub.execute_input":"2022-04-24T07:58:09.04746Z","iopub.status.idle":"2022-04-24T07:58:09.072993Z","shell.execute_reply.started":"2022-04-24T07:58:09.047416Z","shell.execute_reply":"2022-04-24T07:58:09.072003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_X = train[columns_to_use]\ntrain_y = train['purchased']\n\ntest_X = test[columns_to_use]","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:09.074425Z","iopub.execute_input":"2022-04-24T07:58:09.074645Z","iopub.status.idle":"2022-04-24T07:58:10.332195Z","shell.execute_reply.started":"2022-04-24T07:58:09.074618Z","shell.execute_reply":"2022-04-24T07:58:10.331169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model training","metadata":{}},{"cell_type":"code","source":"from lightgbm.sklearn import LGBMRanker","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:10.333957Z","iopub.execute_input":"2022-04-24T07:58:10.334228Z","iopub.status.idle":"2022-04-24T07:58:11.374242Z","shell.execute_reply.started":"2022-04-24T07:58:10.334197Z","shell.execute_reply":"2022-04-24T07:58:11.373118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranker = LGBMRanker(\n    objective=\"lambdarank\",\n    metric=\"ndcg\",\n    boosting_type=\"dart\",\n    n_estimators=1,\n    importance_type='gain',\n    verbose=10\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:11.37582Z","iopub.execute_input":"2022-04-24T07:58:11.376153Z","iopub.status.idle":"2022-04-24T07:58:11.380877Z","shell.execute_reply.started":"2022-04-24T07:58:11.376117Z","shell.execute_reply":"2022-04-24T07:58:11.379982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nranker = ranker.fit(\n    train_X,\n    train_y,\n    group=train_baskets,\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:11.381861Z","iopub.execute_input":"2022-04-24T07:58:11.382104Z","iopub.status.idle":"2022-04-24T07:58:29.279147Z","shell.execute_reply.started":"2022-04-24T07:58:11.382074Z","shell.execute_reply":"2022-04-24T07:58:29.278456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in ranker.feature_importances_.argsort()[::-1]:\n    print(columns_to_use[i], ranker.feature_importances_[i]/ranker.feature_importances_.sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:29.280296Z","iopub.execute_input":"2022-04-24T07:58:29.280807Z","iopub.status.idle":"2022-04-24T07:58:29.304074Z","shell.execute_reply.started":"2022-04-24T07:58:29.280765Z","shell.execute_reply":"2022-04-24T07:58:29.303111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate predictions","metadata":{}},{"cell_type":"code","source":"%time\n\ntest['preds'] = ranker.predict(test_X)\n\nc_id2predicted_article_ids = test \\\n    .sort_values(['customer_id', 'preds'], ascending=False) \\\n    .groupby('customer_id')['article_id'].apply(list).to_dict()\n\nbestsellers_last_week = \\\n    bestsellers_previous_week[bestsellers_previous_week.week == bestsellers_previous_week.week.max()]['article_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:29.305447Z","iopub.execute_input":"2022-04-24T07:58:29.305676Z","iopub.status.idle":"2022-04-24T07:58:46.419008Z","shell.execute_reply.started":"2022-04-24T07:58:29.305623Z","shell.execute_reply":"2022-04-24T07:58:46.418184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:46.420137Z","iopub.execute_input":"2022-04-24T07:58:46.420468Z","iopub.status.idle":"2022-04-24T07:58:51.672527Z","shell.execute_reply.started":"2022-04-24T07:58:46.42044Z","shell.execute_reply":"2022-04-24T07:58:51.671703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npreds = []\nfor c_id in customer_hex_id_to_int(sub.customer_id):\n    pred = c_id2predicted_article_ids.get(c_id, [])\n    pred = pred + bestsellers_last_week\n    preds.append(pred[:12])","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:51.673832Z","iopub.execute_input":"2022-04-24T07:58:51.674067Z","iopub.status.idle":"2022-04-24T07:58:59.105707Z","shell.execute_reply.started":"2022-04-24T07:58:51.67404Z","shell.execute_reply":"2022-04-24T07:58:59.104494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = [' '.join(['0' + str(p) for p in ps]) for ps in preds]\nsub.prediction = preds","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:58:59.106898Z","iopub.execute_input":"2022-04-24T07:58:59.107615Z","iopub.status.idle":"2022-04-24T07:59:05.829763Z","shell.execute_reply.started":"2022-04-24T07:58:59.107569Z","shell.execute_reply":"2022-04-24T07:59:05.828725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_name = 'basic_model_submission'\nsub.to_csv(f'{sub_name}.csv.gz', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T07:59:05.831039Z","iopub.execute_input":"2022-04-24T07:59:05.831298Z","iopub.status.idle":"2022-04-24T07:59:33.367326Z","shell.execute_reply.started":"2022-04-24T07:59:05.831268Z","shell.execute_reply":"2022-04-24T07:59:33.366525Z"},"trusted":true},"execution_count":null,"outputs":[]}]}