{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:10.783725Z","iopub.execute_input":"2022-05-24T14:08:10.784250Z","iopub.status.idle":"2022-05-24T14:08:10.812611Z","shell.execute_reply.started":"2022-05-24T14:08:10.784143Z","shell.execute_reply":"2022-05-24T14:08:10.811649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef apk(actual, predicted, k=10):\n    \"\"\"\n    Computes the average precision at k.\n\n    This function computes the average prescision at k between two lists of\n    items.\n\n    Parameters\n    ----------\n    actual : list\n             A list of elements that are to be predicted (order doesn't matter)\n    predicted : list\n                A list of predicted elements (order does matter)\n    k : int, optional\n        The maximum number of predicted elements\n\n    Returns\n    -------\n    score : double\n            The average precision at k over the input lists\n\n    \"\"\"\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=10):\n    \"\"\"\n    Computes the mean average precision at k.\n\n    This function computes the mean average prescision at k between two lists\n    of lists of items.\n\n    Parameters\n    ----------\n    actual : list\n             A list of lists of elements that are to be predicted \n             (order doesn't matter in the lists)\n    predicted : list\n                A list of lists of predicted elements\n                (order matters in the lists)\n    k : int, optional\n        The maximum number of predicted elements\n\n    Returns\n    -------\n    score : double\n            The mean average precision at k over the input lists\n\n    \"\"\"\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:37:29.102093Z","iopub.execute_input":"2022-05-24T14:37:29.102469Z","iopub.status.idle":"2022-05-24T14:37:29.112447Z","shell.execute_reply.started":"2022-05-24T14:37:29.102429Z","shell.execute_reply":"2022-05-24T14:37:29.111816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\nimport numpy as np\n\n# https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/308635\ndef customer_hex_id_to_int(series):\n    return series.str[-16:].apply(hex_id_to_int)\n\ndef hex_id_to_int(str):\n    return int(str[-16:], 16)\n\ndef article_id_str_to_int(series):\n    return series.astype('int32')\n\ndef article_id_int_to_str(series):\n    return '0' + series.astype('str')\n\nclass Categorize(BaseEstimator, TransformerMixin):\n    def __init__(self, min_examples=0):\n        self.min_examples = min_examples\n        self.categories = []\n        \n    def fit(self, X):\n        for i in range(X.shape[1]):\n            vc = X.iloc[:, i].value_counts()\n            self.categories.append(vc[vc > self.min_examples].index.tolist())\n        return self\n\n    def transform(self, X):\n        data = {X.columns[i]: pd.Categorical(X.iloc[:, i], categories=self.categories[i]).codes for i in range(X.shape[1])}\n        return pd.DataFrame(data=data)\n\n\ndef calculate_apk(list_of_preds, list_of_gts):\n    # for fast validation this can be changed to operate on dicts of {'cust_id_int': [art_id_int, ...]}\n    # using 'data/val_week_purchases_by_cust.pkl'\n    apks = []\n    for preds, gt in zip(list_of_preds, list_of_gts):\n        apks.append(apk(gt, preds, k=12))\n    return np.mean(apks)\n\ndef eval_sub(sub_csv, skip_cust_with_no_purchases=True):\n    sub=pd.read_csv(sub_csv)\n    validation_set=pd.read_parquet('data/validation_ground_truth.parquet')\n\n    apks = []\n\n    no_purchases_pattern = []\n    for pred, gt in zip(sub.prediction.str.split(), validation_set.prediction.str.split()):\n        if skip_cust_with_no_purchases and (gt == no_purchases_pattern): continue\n        apks.append(apk(gt, pred, k=12))\n    return np.mean(apks)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:37:29.725704Z","iopub.execute_input":"2022-05-24T14:37:29.726232Z","iopub.status.idle":"2022-05-24T14:37:29.742626Z","shell.execute_reply.started":"2022-05-24T14:37:29.726180Z","shell.execute_reply":"2022-05-24T14:37:29.741491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntransactions = pd.read_parquet('../input/warmup/transactions_train.parquet')\ncustomers = pd.read_parquet('../input/warmup/customers.parquet')\narticles = pd.read_parquet('../input/warmup/articles.parquet')\n\n# sample = 0.05\n# transactions = pd.read_parquet(f'data/transactions_train_sample_{sample}.parquet')\n# customers = pd.read_parquet(f'data/customers_sample_{sample}.parquet')\n# articles = pd.read_parquet(f'data/articles_train_sample_{sample}.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:10.814401Z","iopub.execute_input":"2022-05-24T14:08:10.814886Z","iopub.status.idle":"2022-05-24T14:08:18.886522Z","shell.execute_reply.started":"2022-05-24T14:08:10.814850Z","shell.execute_reply":"2022-05-24T14:08:18.885869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# adding cv_Data","metadata":{}},{"cell_type":"code","source":"cv_data = pd.read_csv('../input/ddddaaaa/checkpoint1.csv')\ncv_data = cv_data.drop('Unnamed: 0', axis=1).rename(columns={\"0\": \"article_id\"}).astype({'article_id': 'int32', '1': 'float16', '2': 'float16', '3': 'float16', '4': 'float16', '5': 'float16', '6': 'float16',\n               '7': 'float16', '8': 'float16', '9': 'float16', '10': 'float16', '11': 'float16', '12': 'float16',\n               '13': 'float16', '14': 'float16', '15': 'float16', '16': 'float16', '17': 'float16', '18': 'float16',\n               '19': 'float16', '20': 'float16'})\n\ncv_data_2 = pd.read_csv('../input/aaaaad/checkpoint2.csv')\ncv_data_2 = cv_data_2.drop('Unnamed: 0', axis=1).rename(columns={\"0\": \"article_id\", '1': '21', '2': '22', '3': '23', '4': '24',\n                                                              '5': '25', '6': '26', '7': '27', '8': '28', '9': '29', '10': '30',\n                                                              '11': '31', '12': '32', '13': '33', '14': '34', '15': '35', '16': '36',\n                                                              '17': '37', '18': '38', '19': '39', '20': '40'})\\\n.astype({'article_id': 'int32', '21': 'float16', '22': 'float16', '23': 'float16', '24': 'float16', '25': 'float16', '26': 'float16',\n               '27': 'float16', '28': 'float16', '29': 'float16', '30': 'float16', '31': 'float16', '32': 'float16',\n               '33': 'float16', '34': 'float16', '35': 'float16', '36': 'float16', '37': 'float16', '38': 'float16',\n               '39': 'float16', '40': 'float16'})\n\n\n# cv_data_3 = pd.read_csv('../input/sssssee/checkpoint3.csv')\n# cv_data_3 = cv_data_3.drop('Unnamed: 0', axis=1).rename(columns={\"0\": \"article_id\", '1': '41', '2': '42', '3': '43', '4': '44',\n#                                                               '5': '45', '6': '46', '7': '47', '8': '48', '9': '49', '10': '50',\n#                                                               '11': '51', '12': '52', '13': '53', '14': '54', '15': '55', '16': '56',\n#                                                               '17': '57', '18': '58', '19': '59', '20': '60'})\\\n# .astype({'article_id': 'int32', '41': 'float16', '42': 'float16', '43': 'float16', '44': 'float16', '45': 'float16', '46': 'float16',\n#                '47': 'float16', '48': 'float16', '49': 'float16', '50': 'float16', '51': 'float16', '52': 'float16',\n#                '53': 'float16', '54': 'float16', '55': 'float16', '56': 'float16', '57': 'float16', '58': 'float16',\n#                '59': 'float16', '60': 'float16'})\n\n# cv_data","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:18.887722Z","iopub.execute_input":"2022-05-24T14:08:18.888063Z","iopub.status.idle":"2022-05-24T14:08:21.234831Z","shell.execute_reply.started":"2022-05-24T14:08:18.888019Z","shell.execute_reply":"2022-05-24T14:08:21.233901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.merge(articles, cv_data, on='article_id', how='left').fillna(0)\narticles = pd.merge(articles, cv_data_2, on='article_id', how='left').fillna(0)\n# articles = pd.merge(articles, cv_data_3, on='article_id', how='left').fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:21.236222Z","iopub.execute_input":"2022-05-24T14:08:21.236466Z","iopub.status.idle":"2022-05-24T14:08:21.514982Z","shell.execute_reply.started":"2022-05-24T14:08:21.236438Z","shell.execute_reply":"2022-05-24T14:08:21.514030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# articles","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:21.517913Z","iopub.execute_input":"2022-05-24T14:08:21.518399Z","iopub.status.idle":"2022-05-24T14:08:21.522967Z","shell.execute_reply.started":"2022-05-24T14:08:21.518350Z","shell.execute_reply":"2022-05-24T14:08:21.522045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del cv_data","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:21.524313Z","iopub.execute_input":"2022-05-24T14:08:21.524880Z","iopub.status.idle":"2022-05-24T14:08:21.535323Z","shell.execute_reply.started":"2022-05-24T14:08:21.524826Z","shell.execute_reply":"2022-05-24T14:08:21.534407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# continue","metadata":{}},{"cell_type":"code","source":"transactions.week.max()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:21.536961Z","iopub.execute_input":"2022-05-24T14:08:21.537404Z","iopub.status.idle":"2022-05-24T14:08:21.581193Z","shell.execute_reply.started":"2022-05-24T14:08:21.537359Z","shell.execute_reply":"2022-05-24T14:08:21.580423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weeks: 49,50,51, 99,100,101\ntr_fold_1 = pd.concat([transactions[(transactions.week.max() - 52 > transactions.week) & (transactions.week > transactions.week.max() - 56)], \n         transactions[(transactions.week.max() - 2 > transactions.week) & (transactions.week > transactions.week.max() - 6)]])\ntr_fold_1_targ = transactions[transactions.week == 102]\n\n# weeks: 50,51,52, 100,101,102\ntr_fold_2 = pd.concat([transactions[(transactions.week.max() - 51 > transactions.week) & (transactions.week > transactions.week.max() - 55)], \n         transactions[(transactions.week.max() - 1 > transactions.week) & (transactions.week > transactions.week.max() - 5)]])\ntr_fold_2_targ = transactions[transactions.week == 103]\n\n# weeks: 51,52,53, 101,102,103\ntr_fold_3 = pd.concat([transactions[(transactions.week.max() - 50 > transactions.week) & (transactions.week > transactions.week.max() - 54)], \n         transactions[(transactions.week.max() > transactions.week) & (transactions.week > transactions.week.max() - 4)]])\ntr_fold_3_targ = transactions[transactions.week == 104]\n\n# weeks: 52,53,54, 102,103,104\ntr_last = pd.concat([transactions[(transactions.week.max() - 49 > transactions.week) & (transactions.week > transactions.week.max() - 53)], \n         transactions[(transactions.week.max() + 1 > transactions.week) & (transactions.week > transactions.week.max() - 3)]])\ntr_last_targ = transactions[transactions.week == 105]","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:21.582250Z","iopub.execute_input":"2022-05-24T14:08:21.582857Z","iopub.status.idle":"2022-05-24T14:08:22.998869Z","shell.execute_reply.started":"2022-05-24T14:08:21.582816Z","shell.execute_reply":"2022-05-24T14:08:22.997842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_week = transactions.week.max() + 1\n# transactions = transactions[transactions.week > transactions.week.max() - 10]","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:22.999883Z","iopub.execute_input":"2022-05-24T14:08:23.000106Z","iopub.status.idle":"2022-05-24T14:08:23.029182Z","shell.execute_reply.started":"2022-05-24T14:08:23.000078Z","shell.execute_reply":"2022-05-24T14:08:23.028384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del transactions","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:23.030415Z","iopub.execute_input":"2022-05-24T14:08:23.030664Z","iopub.status.idle":"2022-05-24T14:08:23.036663Z","shell.execute_reply.started":"2022-05-24T14:08:23.030636Z","shell.execute_reply":"2022-05-24T14:08:23.035513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_week","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:23.038188Z","iopub.execute_input":"2022-05-24T14:08:23.039079Z","iopub.status.idle":"2022-05-24T14:08:23.048718Z","shell.execute_reply.started":"2022-05-24T14:08:23.039030Z","shell.execute_reply":"2022-05-24T14:08:23.048078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating candidates","metadata":{}},{"cell_type":"markdown","source":"### Last purchase candidates","metadata":{}},{"cell_type":"code","source":"%%time\n\nc2weeks = tr_fold_3.groupby('customer_id')['week'].unique()\nc2weeks_last = tr_last.groupby('customer_id')['week'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:23.049850Z","iopub.execute_input":"2022-05-24T14:08:23.050182Z","iopub.status.idle":"2022-05-24T14:08:56.748008Z","shell.execute_reply.started":"2022-05-24T14:08:23.050153Z","shell.execute_reply":"2022-05-24T14:08:56.746576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# c2weeks","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:56.749563Z","iopub.execute_input":"2022-05-24T14:08:56.749914Z","iopub.status.idle":"2022-05-24T14:08:56.755017Z","shell.execute_reply.started":"2022-05-24T14:08:56.749869Z","shell.execute_reply":"2022-05-24T14:08:56.753681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nc2weeks2shifted_weeks = {}\n\nfor c_id, weeks in c2weeks.items():\n    c2weeks2shifted_weeks[c_id] = {}\n    for i in range(weeks.shape[0]-1):\n        c2weeks2shifted_weeks[c_id][weeks[i]] = weeks[i+1]\n    c2weeks2shifted_weeks[c_id][weeks[-1]] = 104\n    \n\nc2weeks2shifted_weeks_last = {}\n\nfor c_id, weeks in c2weeks_last.items():\n    c2weeks2shifted_weeks_last[c_id] = {}\n    for i in range(weeks.shape[0]-1):\n        c2weeks2shifted_weeks_last[c_id][weeks[i]] = weeks[i+1]\n    c2weeks2shifted_weeks_last[c_id][weeks[-1]] = 105","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:56.761233Z","iopub.execute_input":"2022-05-24T14:08:56.761570Z","iopub.status.idle":"2022-05-24T14:08:58.357428Z","shell.execute_reply.started":"2022-05-24T14:08:56.761533Z","shell.execute_reply":"2022-05-24T14:08:58.356376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_last_purchase = tr_fold_3.copy()\ncandidates_last_purchase_last = tr_last.copy()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:58.358857Z","iopub.execute_input":"2022-05-24T14:08:58.359200Z","iopub.status.idle":"2022-05-24T14:08:58.395305Z","shell.execute_reply.started":"2022-05-24T14:08:58.359153Z","shell.execute_reply":"2022-05-24T14:08:58.394356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nweeks = []\nfor i, (c_id, week) in enumerate(zip(tr_fold_3['customer_id'], tr_fold_3['week'])):\n    weeks.append(c2weeks2shifted_weeks[c_id][week])\n    \ncandidates_last_purchase.week=weeks\n\n\nweeks = []\nfor i, (c_id, week) in enumerate(zip(tr_last['customer_id'], tr_last['week'])):\n    weeks.append(c2weeks2shifted_weeks_last[c_id][week])\n    \ncandidates_last_purchase_last.week=weeks","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:08:58.397492Z","iopub.execute_input":"2022-05-24T14:08:58.397772Z","iopub.status.idle":"2022-05-24T14:09:12.702698Z","shell.execute_reply.started":"2022-05-24T14:08:58.397739Z","shell.execute_reply":"2022-05-24T14:09:12.701723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# candidates_last_purchase[candidates_last_purchase['customer_id']==272412481300040]","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:12.704102Z","iopub.execute_input":"2022-05-24T14:09:12.704527Z","iopub.status.idle":"2022-05-24T14:09:12.708962Z","shell.execute_reply.started":"2022-05-24T14:09:12.704492Z","shell.execute_reply":"2022-05-24T14:09:12.707857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tr_fold_3[tr_fold_3['customer_id']==272412481300040]","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:12.710109Z","iopub.execute_input":"2022-05-24T14:09:12.710369Z","iopub.status.idle":"2022-05-24T14:09:12.724757Z","shell.execute_reply.started":"2022-05-24T14:09:12.710332Z","shell.execute_reply":"2022-05-24T14:09:12.723932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Bestsellers candidates","metadata":{}},{"cell_type":"code","source":"mean_price = tr_fold_3 \\\n    .groupby(['week', 'article_id'])['price'].mean()\n\nmean_price_last = tr_last \\\n    .groupby(['week', 'article_id'])['price'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:12.725965Z","iopub.execute_input":"2022-05-24T14:09:12.755248Z","iopub.status.idle":"2022-05-24T14:09:13.088596Z","shell.execute_reply.started":"2022-05-24T14:09:12.755192Z","shell.execute_reply":"2022-05-24T14:09:13.087532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mean_price","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:13.089865Z","iopub.execute_input":"2022-05-24T14:09:13.090114Z","iopub.status.idle":"2022-05-24T14:09:13.093622Z","shell.execute_reply.started":"2022-05-24T14:09:13.090083Z","shell.execute_reply":"2022-05-24T14:09:13.093042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales = tr_fold_3 \\\n    .groupby('week')['article_id'].value_counts() \\\n    .groupby('week').rank(method='dense', ascending=False) \\\n    .groupby('week').head(12).rename('bestseller_rank').astype('int8')\n\nsales_last = tr_last \\\n    .groupby('week')['article_id'].value_counts() \\\n    .groupby('week').rank(method='dense', ascending=False) \\\n    .groupby('week').head(12).rename('bestseller_rank').astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:13.094654Z","iopub.execute_input":"2022-05-24T14:09:13.095145Z","iopub.status.idle":"2022-05-24T14:09:14.142648Z","shell.execute_reply.started":"2022-05-24T14:09:13.095112Z","shell.execute_reply":"2022-05-24T14:09:14.141690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sales","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:14.144020Z","iopub.execute_input":"2022-05-24T14:09:14.144284Z","iopub.status.idle":"2022-05-24T14:09:14.153381Z","shell.execute_reply.started":"2022-05-24T14:09:14.144250Z","shell.execute_reply":"2022-05-24T14:09:14.152212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bestsellers_previous_week = pd.merge(sales, mean_price, on=['week', 'article_id']).reset_index()\nbestsellers_previous_week.week += 1\n\nbestsellers_previous_week_last = pd.merge(sales_last, mean_price_last, on=['week', 'article_id']).reset_index()\nbestsellers_previous_week_last.week += 1","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:14.154748Z","iopub.execute_input":"2022-05-24T14:09:14.155073Z","iopub.status.idle":"2022-05-24T14:09:14.230389Z","shell.execute_reply.started":"2022-05-24T14:09:14.155040Z","shell.execute_reply":"2022-05-24T14:09:14.229397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_transactions = tr_fold_3 \\\n    .groupby(['week', 'customer_id']) \\\n    .head(1) \\\n    .drop(columns=['article_id', 'price']) \\\n    .copy()\n\n\nunique_transactions_last = tr_last \\\n    .groupby(['week', 'customer_id']) \\\n    .head(1) \\\n    .drop(columns=['article_id', 'price']) \\\n    .copy()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:14.231716Z","iopub.execute_input":"2022-05-24T14:09:14.232041Z","iopub.status.idle":"2022-05-24T14:09:14.988510Z","shell.execute_reply.started":"2022-05-24T14:09:14.232000Z","shell.execute_reply":"2022-05-24T14:09:14.987855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique_transactions","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:14.990004Z","iopub.execute_input":"2022-05-24T14:09:14.990495Z","iopub.status.idle":"2022-05-24T14:09:14.994492Z","shell.execute_reply.started":"2022-05-24T14:09:14.990452Z","shell.execute_reply":"2022-05-24T14:09:14.993322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers = pd.merge(\n    unique_transactions,\n    bestsellers_previous_week,\n    on='week',\n)\n\ncandidates_bestsellers_last = pd.merge(\n    unique_transactions_last,\n    bestsellers_previous_week_last,\n    on='week',\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:14.995952Z","iopub.execute_input":"2022-05-24T14:09:14.996207Z","iopub.status.idle":"2022-05-24T14:09:15.513040Z","shell.execute_reply.started":"2022-05-24T14:09:14.996176Z","shell.execute_reply":"2022-05-24T14:09:15.511876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set_transactions = unique_transactions.drop_duplicates('customer_id').reset_index(drop=True)\ntest_set_transactions.week = 104\n\n\ntest_set_transactions_last = unique_transactions_last.drop_duplicates('customer_id').reset_index(drop=True)\ntest_set_transactions.week = 105","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:15.514395Z","iopub.execute_input":"2022-05-24T14:09:15.514932Z","iopub.status.idle":"2022-05-24T14:09:15.629112Z","shell.execute_reply.started":"2022-05-24T14:09:15.514894Z","shell.execute_reply":"2022-05-24T14:09:15.628246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_set_transactions","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:15.630293Z","iopub.execute_input":"2022-05-24T14:09:15.630519Z","iopub.status.idle":"2022-05-24T14:09:15.634135Z","shell.execute_reply.started":"2022-05-24T14:09:15.630486Z","shell.execute_reply":"2022-05-24T14:09:15.633528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers_test_week = pd.merge(\n    test_set_transactions,\n    bestsellers_previous_week,\n    on='week'\n)\n\ncandidates_bestsellers_test_week_last = pd.merge(\n    test_set_transactions_last,\n    bestsellers_previous_week_last,\n    on='week'\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:15.635319Z","iopub.execute_input":"2022-05-24T14:09:15.635542Z","iopub.status.idle":"2022-05-24T14:09:15.812823Z","shell.execute_reply.started":"2022-05-24T14:09:15.635512Z","shell.execute_reply":"2022-05-24T14:09:15.811877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_bestsellers = pd.concat([candidates_bestsellers, candidates_bestsellers_test_week])\ncandidates_bestsellers.drop(columns='bestseller_rank', inplace=True)\n\ncandidates_bestsellers_last = pd.concat([candidates_bestsellers_last, candidates_bestsellers_test_week_last])\ncandidates_bestsellers_last.drop(columns='bestseller_rank', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:15.813955Z","iopub.execute_input":"2022-05-24T14:09:15.814172Z","iopub.status.idle":"2022-05-24T14:09:16.273837Z","shell.execute_reply.started":"2022-05-24T14:09:15.814144Z","shell.execute_reply":"2022-05-24T14:09:16.272845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combining transactions and candidates / negative examples","metadata":{}},{"cell_type":"code","source":"tr_fold_3['purchased'] = 1\ntr_last['purchased'] = 1\n\ntr_fold_3_targ['purchased'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:16.275211Z","iopub.execute_input":"2022-05-24T14:09:16.275677Z","iopub.status.idle":"2022-05-24T14:09:16.285322Z","shell.execute_reply.started":"2022-05-24T14:09:16.275640Z","shell.execute_reply":"2022-05-24T14:09:16.284226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([tr_fold_3, candidates_last_purchase, candidates_bestsellers])\ndata.purchased.fillna(0, inplace=True)\n \ndata_last = pd.concat([tr_last, candidates_last_purchase_last, candidates_bestsellers_last])\ndata_last.purchased.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:16.286701Z","iopub.execute_input":"2022-05-24T14:09:16.286998Z","iopub.status.idle":"2022-05-24T14:09:16.755644Z","shell.execute_reply.started":"2022-05-24T14:09:16.286964Z","shell.execute_reply":"2022-05-24T14:09:16.754611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del transactions\n# del candidates_last_purchase\n# del candidates_bestsellers","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:16.757081Z","iopub.execute_input":"2022-05-24T14:09:16.757320Z","iopub.status.idle":"2022-05-24T14:09:16.761770Z","shell.execute_reply.started":"2022-05-24T14:09:16.757286Z","shell.execute_reply":"2022-05-24T14:09:16.760427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop_duplicates(['customer_id', 'article_id', 'week'], inplace=True)\n\ndata_last.drop_duplicates(['customer_id', 'article_id', 'week'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:16.763184Z","iopub.execute_input":"2022-05-24T14:09:16.763416Z","iopub.status.idle":"2022-05-24T14:09:21.735490Z","shell.execute_reply.started":"2022-05-24T14:09:16.763389Z","shell.execute_reply":"2022-05-24T14:09:21.734426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_data\n# tr_fold_3_targ","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:21.737022Z","iopub.execute_input":"2022-05-24T14:09:21.737254Z","iopub.status.idle":"2022-05-24T14:09:21.741827Z","shell.execute_reply.started":"2022-05-24T14:09:21.737227Z","shell.execute_reply":"2022-05-24T14:09:21.740881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:21.742950Z","iopub.execute_input":"2022-05-24T14:09:21.743162Z","iopub.status.idle":"2022-05-24T14:09:21.753216Z","shell.execute_reply.started":"2022-05-24T14:09:21.743136Z","shell.execute_reply":"2022-05-24T14:09:21.752157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add bestseller information","metadata":{}},{"cell_type":"code","source":"data = pd.merge(\n    data,\n    bestsellers_previous_week[['week', 'article_id', 'bestseller_rank']],\n    on=['week', 'article_id'],\n    how='left'\n)\n\ndata_last = pd.merge(\n    data_last,\n    bestsellers_previous_week_last[['week', 'article_id', 'bestseller_rank']],\n    on=['week', 'article_id'],\n    how='left'\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:21.755227Z","iopub.execute_input":"2022-05-24T14:09:21.755763Z","iopub.status.idle":"2022-05-24T14:09:24.477657Z","shell.execute_reply.started":"2022-05-24T14:09:21.755710Z","shell.execute_reply":"2022-05-24T14:09:24.476576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[data.week != data.week.min()]\ndata.bestseller_rank.fillna(999, inplace=True)\n\n\ndata_last = data_last[data_last.week != data_last.week.min()]\ndata_last.bestseller_rank.fillna(999, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:24.479046Z","iopub.execute_input":"2022-05-24T14:09:24.479307Z","iopub.status.idle":"2022-05-24T14:09:25.663700Z","shell.execute_reply.started":"2022-05-24T14:09:24.479274Z","shell.execute_reply":"2022-05-24T14:09:25.662707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.merge(data, articles, on='article_id', how='left')\ndata = pd.merge(data, customers, on='customer_id', how='left')\n\ndata_last = pd.merge(data_last, articles, on='article_id', how='left')\ndata_last = pd.merge(data_last, customers, on='customer_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:25.665453Z","iopub.execute_input":"2022-05-24T14:09:25.665788Z","iopub.status.idle":"2022-05-24T14:09:49.459406Z","shell.execute_reply.started":"2022-05-24T14:09:25.665745Z","shell.execute_reply":"2022-05-24T14:09:49.458263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.sort_values(['week', 'customer_id'], inplace=True)\ndata.reset_index(drop=True, inplace=True)\n\ndata_last.sort_values(['week', 'customer_id'], inplace=True)\ndata_last.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:49.460653Z","iopub.execute_input":"2022-05-24T14:09:49.460962Z","iopub.status.idle":"2022-05-24T14:09:57.583792Z","shell.execute_reply.started":"2022-05-24T14:09:49.460932Z","shell.execute_reply":"2022-05-24T14:09:57.582794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# continue","metadata":{}},{"cell_type":"code","source":"train = data[data.week != 104]\ntest = data[data.week==104].drop_duplicates(['customer_id', 'article_id', 'sales_channel_id']).copy()\n\ntrain_last = data_last[data_last.week != 105]\ntest_last = data_last[data_last.week==105].drop_duplicates(['customer_id', 'article_id', 'sales_channel_id']).copy()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:09:57.591355Z","iopub.execute_input":"2022-05-24T14:09:57.591660Z","iopub.status.idle":"2022-05-24T14:10:05.844181Z","shell.execute_reply.started":"2022-05-24T14:09:57.591626Z","shell.execute_reply":"2022-05-24T14:10:05.843133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_baskets = train.groupby(['week', 'customer_id'])['article_id'].count().values\ntest_baskets = test.groupby(['week', 'customer_id'])['article_id'].count().values\ntest_baskets = [list(test_baskets)]\n\n\n\ntrain_baskets_last = train_last.groupby(['week', 'customer_id'])['article_id'].count().values","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:10:05.845908Z","iopub.execute_input":"2022-05-24T14:10:05.846260Z","iopub.status.idle":"2022-05-24T14:10:06.921550Z","shell.execute_reply.started":"2022-05-24T14:10:05.846230Z","shell.execute_reply":"2022-05-24T14:10:06.920880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del data","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:10:06.922704Z","iopub.execute_input":"2022-05-24T14:10:06.923099Z","iopub.status.idle":"2022-05-24T14:10:06.927529Z","shell.execute_reply.started":"2022-05-24T14:10:06.923031Z","shell.execute_reply":"2022-05-24T14:10:06.926620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ns = ['week']\n# s = []\nfor i in np.arange(1,41):\n    s.append(str(i))\n    \ncolumns_to_use = ['article_id', 'product_type_no', 'graphical_appearance_no', 'colour_group_code', 'perceived_colour_value_id',\n'perceived_colour_master_id', 'department_no', 'index_code',\n'index_group_no', 'section_no', 'garment_group_no', 'FN', 'Active',\n'club_member_status', 'fashion_news_frequency', 'age', 'postal_code', 'bestseller_rank'] + s","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:18.542246Z","iopub.execute_input":"2022-05-24T14:34:18.542557Z","iopub.status.idle":"2022-05-24T14:34:18.549102Z","shell.execute_reply.started":"2022-05-24T14:34:18.542517Z","shell.execute_reply":"2022-05-24T14:34:18.548184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_X = train[columns_to_use]\ntrain_y = train['purchased']\n\ntest_X = test[columns_to_use]\n\nX_test_last = test_last[columns_to_use]","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:18.551750Z","iopub.execute_input":"2022-05-24T14:34:18.552383Z","iopub.status.idle":"2022-05-24T14:34:19.988301Z","shell.execute_reply.started":"2022-05-24T14:34:18.552341Z","shell.execute_reply":"2022-05-24T14:34:19.987432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_y = pd.merge(test.drop('purchased', axis=1), tr_fold_3_targ[['article_id', 'customer_id', 'purchased']], on=['article_id', 'customer_id'], how='left')\ntest_y.purchased.fillna(0, inplace=True)\n\ntest_y.sort_values(\"purchased\", inplace = True)\ntest_y.drop_duplicates(['customer_id', 'article_id', 'sales_channel_id'], keep='last', inplace=True)\nprint(test_y['purchased'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:19.990005Z","iopub.execute_input":"2022-05-24T14:34:19.990287Z","iopub.status.idle":"2022-05-24T14:34:23.120011Z","shell.execute_reply.started":"2022-05-24T14:34:19.990254Z","shell.execute_reply":"2022-05-24T14:34:23.118987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_TEST = test_y[columns_to_use]\nY_TEST = test_y['purchased']","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:23.121230Z","iopub.execute_input":"2022-05-24T14:34:23.121453Z","iopub.status.idle":"2022-05-24T14:34:23.298203Z","shell.execute_reply.started":"2022-05-24T14:34:23.121426Z","shell.execute_reply":"2022-05-24T14:34:23.297139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_TEST","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:23.300016Z","iopub.execute_input":"2022-05-24T14:34:23.300271Z","iopub.status.idle":"2022-05-24T14:34:23.479679Z","shell.execute_reply.started":"2022-05-24T14:34:23.300239Z","shell.execute_reply":"2022-05-24T14:34:23.478884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model training","metadata":{}},{"cell_type":"code","source":"X_test_last","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:23.481381Z","iopub.execute_input":"2022-05-24T14:34:23.482430Z","iopub.status.idle":"2022-05-24T14:34:23.669608Z","shell.execute_reply.started":"2022-05-24T14:34:23.482381Z","shell.execute_reply":"2022-05-24T14:34:23.668749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm.sklearn import LGBMRanker","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:23.671091Z","iopub.execute_input":"2022-05-24T14:34:23.671943Z","iopub.status.idle":"2022-05-24T14:34:25.696436Z","shell.execute_reply.started":"2022-05-24T14:34:23.671884Z","shell.execute_reply":"2022-05-24T14:34:25.695599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranker = LGBMRanker(\n    objective=\"lambdarank\",\n    metric=\"map\",\n    boosting_type=\"dart\",\n    n_estimators=10,\n    importance_type='gain',\n    verbose=10\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:25.697637Z","iopub.execute_input":"2022-05-24T14:34:25.697871Z","iopub.status.idle":"2022-05-24T14:34:25.703153Z","shell.execute_reply.started":"2022-05-24T14:34:25.697844Z","shell.execute_reply":"2022-05-24T14:34:25.702155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nranker = ranker.fit(\n    train_X,\n    train_y,\n    group=train_baskets,\n    eval_set = [(X_TEST, Y_TEST.values)],\n    eval_group = test_baskets,\n    eval_metric = 'map'\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:34:25.704641Z","iopub.execute_input":"2022-05-24T14:34:25.705193Z","iopub.status.idle":"2022-05-24T14:35:01.202447Z","shell.execute_reply.started":"2022-05-24T14:34:25.705148Z","shell.execute_reply":"2022-05-24T14:35:01.201466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in ranker.feature_importances_.argsort()[::-1]:\n    print(columns_to_use[i], ranker.feature_importances_[i]/ranker.feature_importances_.sum())","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:35:01.210463Z","iopub.execute_input":"2022-05-24T14:35:01.211091Z","iopub.status.idle":"2022-05-24T14:35:01.238313Z","shell.execute_reply.started":"2022-05-24T14:35:01.211042Z","shell.execute_reply":"2022-05-24T14:35:01.237617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate predictions","metadata":{}},{"cell_type":"code","source":"# X_test_last","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:35:01.241494Z","iopub.execute_input":"2022-05-24T14:35:01.241734Z","iopub.status.idle":"2022-05-24T14:35:01.246229Z","shell.execute_reply.started":"2022-05-24T14:35:01.241706Z","shell.execute_reply":"2022-05-24T14:35:01.245204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n\ntest_last['preds'] = ranker.predict(X_test_last)\n\nc_id2predicted_article_ids = test_last \\\n    .sort_values(['customer_id', 'preds'], ascending=False) \\\n    .groupby('customer_id')['article_id'].apply(list).to_dict()\n\nbestsellers_last_week = \\\n    bestsellers_previous_week_last[bestsellers_previous_week_last.week == bestsellers_previous_week_last.week.max()]['article_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:35:01.247410Z","iopub.execute_input":"2022-05-24T14:35:01.248036Z","iopub.status.idle":"2022-05-24T14:35:11.244612Z","shell.execute_reply.started":"2022-05-24T14:35:01.247989Z","shell.execute_reply":"2022-05-24T14:35:11.243738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:35:11.245919Z","iopub.execute_input":"2022-05-24T14:35:11.246146Z","iopub.status.idle":"2022-05-24T14:35:16.312280Z","shell.execute_reply.started":"2022-05-24T14:35:11.246119Z","shell.execute_reply":"2022-05-24T14:35:16.311029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npreds = []\nfor c_id in customer_hex_id_to_int(sub.customer_id):\n    pred = c_id2predicted_article_ids.get(c_id, [])\n    pred = pred + bestsellers_last_week\n    preds.append(pred[:12])\n\npreds = [' '.join(['0' + str(p) for p in ps]) for ps in preds]\nsub.prediction = preds","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:37:35.163072Z","iopub.execute_input":"2022-05-24T14:37:35.164014Z","iopub.status.idle":"2022-05-24T14:37:48.999273Z","shell.execute_reply.started":"2022-05-24T14:37:35.163953Z","shell.execute_reply":"2022-05-24T14:37:48.998428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_name = 'basic_model_submission'\nsub.to_csv(f'{sub_name}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T14:37:49.000693Z","iopub.execute_input":"2022-05-24T14:37:49.000989Z","iopub.status.idle":"2022-05-24T14:38:02.112039Z","shell.execute_reply.started":"2022-05-24T14:37:49.000955Z","shell.execute_reply":"2022-05-24T14:38:02.110755Z"},"trusted":true},"execution_count":null,"outputs":[]}]}