{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Overview","metadata":{}},{"cell_type":"markdown","source":"In this notebook, I would like to present a framework for **partitioned validation** using one-week hold out.  \nAs an example of possible groupings of customers, here I choose '**online vs. offline**', which shows that it is more difficult to predict future purchases of offline uses than to predict those of online users.  \n  \nI use some tequniqus I learned from great discussions and notebooks shared in the H&M competition.  \nSo, I would really like to thank all the authors of them.","metadata":{}},{"cell_type":"markdown","source":"- **Version 6**  \nI added and fixed the code so as to make it easier to record and show results of validations on the week from 2020-9-16 to 2020-9-22.  \nIn Version 6, MAP@12 of hold-out validation is around 0.0183 for online users and 0.0298 for offline users.  ","metadata":{}},{"cell_type":"markdown","source":"### Libraries and Functions","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd, datetime as dt\nimport matplotlib.pyplot as plt; plt.style.use('ggplot')\nimport seaborn as sns\nfrom collections import defaultdict\n\ndef iter_to_str(iterable):\n    return \" \".join(map(lambda x: str(0) + str(x), iterable))\n\ndef apk(actual, predicted, k=12):\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score, nhits = 0.0, 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            nhits += 1.0\n            score += nhits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12, return_apks=False):\n    assert len(actual) == len(predicted)\n    apks = [apk(ac, pr, k) for ac, pr in zip(actual, predicted) if 0 < len(ac)]\n    if return_apks:\n        return apks\n    return np.mean(apks)\n\ndef blend(dt, w=[], k=12):\n    if len(w) == 0:\n        w = [1] * (len(dt))\n    preds = []\n    for i in range(len(w)):\n        preds.append(dt[i].split())\n    res = {}\n    for i in range(len(preds)):\n        if w[i] < 0:\n            continue\n        for n, v in enumerate(preds[i]):\n            if v in res:\n                res[v] += (w[i] / (n + 1))\n            else:\n                res[v] = (w[i] / (n + 1))    \n    res = list(dict(sorted(res.items(), key=lambda item: -item[1])).keys())\n    return ' '.join(res[:k])\n\ndef prune(pred, ok_set, k=12):\n    pred = pred.split()\n    post = []\n    for item in pred:\n        if int(item) in ok_set and not item in post:\n            post.append(item)\n    return \" \".join(post[:k])\n\ndef validation(actual, predicted, grouping, score=0, index=-1, ignore=False, figsize=(12, 6)):\n    # actual, predicted : list of lists\n    # group : pandas Series\n    # score : pandas DataFrame\n    if ignore: return\n    ap12 = mapk(actual, predicted, return_apks=True)\n    map12 = round(np.mean(ap12), 6)\n    if isinstance(score, int): score = pd.DataFrame({g:[] for g in sorted(grouping.unique().tolist())})\n    if index == -1 : index = score.shape[0]\n    score.loc[index, \"All\"] = map12\n    plt.figure(figsize=figsize)\n    plt.subplot(1, 2, 1); sns.histplot(data=ap12, log_scale=(0, 10), bins=20); plt.title(f\"MAP@12 : {map12}\")\n    for g in grouping.unique():\n        map12 = round(mapk(actual[grouping == g], predicted[grouping == g]), 6)\n        score.loc[index, g] = map12\n    plt.subplot(1, 2, 2); score[[g for g in grouping.unique()[::-1]] + ['All']].loc[index].plot.barh(); plt.title(f\"MAP@12 of Groups\")\n    vc = pd.Series(predicted).apply(len).value_counts()\n    score.loc[index, \"Fill\"] = round(1 - sum(vc[k] * (12 - k) / 12 for k in (set(range(12)) & set(vc.index))) / len(actual), 3) * 100\n    display(score)\n    return score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-25T19:37:29.830834Z","iopub.execute_input":"2022-03-25T19:37:29.831342Z","iopub.status.idle":"2022-03-25T19:37:29.858680Z","shell.execute_reply.started":"2022-03-25T19:37:29.831309Z","shell.execute_reply":"2022-03-25T19:37:29.857675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_parquet('../input/hm-parquets-of-datasets/transactions_train.parquet')\nsub = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')\ncid = pd.DataFrame(sub.customer_id.apply(lambda s: int(s[-16:], 16)))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-25T19:37:29.869955Z","iopub.execute_input":"2022-03-25T19:37:29.870378Z","iopub.status.idle":"2022-03-25T19:37:40.437948Z","shell.execute_reply.started":"2022-03-25T19:37:29.870347Z","shell.execute_reply":"2022-03-25T19:37:40.436887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Definition of Groups","metadata":{}},{"cell_type":"code","source":"group = df.groupby('customer_id').sales_channel_id.mean().round().reset_index()\\\n    .merge(cid, on='customer_id', how='right').rename(columns={'sales_channel_id':'group'})\ngrouping = group.group.fillna(1.0)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-25T19:37:40.439893Z","iopub.execute_input":"2022-03-25T19:37:40.440123Z","iopub.status.idle":"2022-03-25T19:37:44.739838Z","shell.execute_reply.started":"2022-03-25T19:37:40.440096Z","shell.execute_reply":"2022-03-25T19:37:44.738772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One-Week Hold Out","metadata":{}},{"cell_type":"code","source":"val_week = 105\n# id of week to be used in a validation; set 105 if you would like to create a submission\nval = df.loc[df.week == val_week].groupby('customer_id').article_id.apply(iter_to_str).reset_index()\\\n    .merge(cid, on='customer_id', how='right')\nactual = val.article_id.apply(lambda s: [] if pd.isna(s) else s.split())\nlast_date = df.loc[df.week < val_week].t_dat.max()","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:37:44.741085Z","iopub.execute_input":"2022-03-25T19:37:44.741403Z","iopub.status.idle":"2022-03-25T19:37:50.800605Z","shell.execute_reply.started":"2022-03-25T19:37:44.741370Z","shell.execute_reply":"2022-03-25T19:37:50.799614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Last Purchased Items","metadata":{}},{"cell_type":"code","source":"init_date = last_date - dt.timedelta(days=9999)\ntrain = df.loc[(df.t_dat >= init_date) & (df.t_dat <= last_date)].copy()\ntrain = train.merge(train.groupby('customer_id').t_dat.max().reset_index().rename(columns={'t_dat':'l_dat'}),\n                   on = 'customer_id', how='left')\ntrain['d_dat'] = (train.l_dat - train.t_dat).dt.days\ntrain = train.loc[train.d_dat < 14].sort_values(['t_dat'], ascending=False).drop_duplicates(['customer_id', 'article_id'])\nsub['last_purchase'] = train.groupby('customer_id')\\\n    .article_id.apply(iter_to_str).reset_index()\\\n    .merge(cid, on='customer_id', how='right').article_id.fillna('')\n\npredicted = sub['last_purchase'].apply(lambda s: [] if pd.isna(s) else s.split())\nscore = validation(actual, predicted, grouping, index='Last Purchase', ignore=(val_week == 105))","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:37:50.802766Z","iopub.execute_input":"2022-03-25T19:37:50.803083Z","iopub.status.idle":"2022-03-25T19:38:45.889904Z","shell.execute_reply.started":"2022-03-25T19:37:50.803054Z","shell.execute_reply":"2022-03-25T19:38:45.888992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Other Colors of Purchased Item","metadata":{}},{"cell_type":"code","source":"init_date = last_date - dt.timedelta(days=6)\ntrain = df.loc[(df.t_dat >= init_date) & (df.t_dat <= last_date)].copy()\\\n    .groupby(['article_id']).t_dat.count().reset_index()\nadf = pd.read_parquet('../input/hm-parquets-of-datasets/articles.parquet')\nadf = adf.merge(train, on='article_id', how='left').rename(columns={'t_dat':'ct'})\\\n    .sort_values('ct', ascending=False).query('ct > 0')\n\nmap_to_col = defaultdict(list)\nfor aid in adf.article_id.tolist():\n    map_to_col[aid] = list(filter(lambda x: x != aid, adf[adf.product_code == aid // 1000].article_id.tolist()))[:1]\n\ndef map_to_variation(s):\n    f = lambda item: iter_to_str(map_to_col[int(item)])\n    return ' '.join(map(f, s.split()))\nsub['other_colors'] = sub['last_purchase'].fillna('').apply(map_to_variation)\n\npredicted = sub['other_colors'].apply(lambda s: [] if pd.isna(s) else s.split())\nscore = validation(actual, predicted, grouping, score, index='Other Colors', ignore=(val_week == 105))","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:38:45.891458Z","iopub.execute_input":"2022-03-25T19:38:45.891865Z","iopub.status.idle":"2022-03-25T19:39:10.046576Z","shell.execute_reply.started":"2022-03-25T19:38:45.891820Z","shell.execute_reply":"2022-03-25T19:39:10.045594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Popular Items for Each Group","metadata":{}},{"cell_type":"code","source":"init_date = last_date - dt.timedelta(days=5 - 1)\ngroup_df = pd.concat([cid, group.group.fillna(1)], axis=1) # grouping can be changed\ngroup_df.columns = ['customer_id', 'group']\ntrain = df.loc[(df.t_dat >= init_date) & (df.t_dat <= last_date)].copy()\\\n    .merge(group_df, on='customer_id', how='left')\\\n    .groupby(['group', 'article_id']).t_dat.count().reset_index()\nitems = defaultdict(str)\nfor g in train.group.unique():\n    items[g] = iter_to_str(train.loc[train.group == g].sort_values('t_dat', ascending=False).article_id.tolist()[:12])\n\nsub['popular_items'] = group_df.group.map(items)\n\npredicted = sub['popular_items'].apply(lambda s: [] if pd.isna(s) else s.split())\nscore = validation(actual, predicted, grouping, score, index='Popular Items', ignore=(val_week == 105))","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:39:10.047838Z","iopub.execute_input":"2022-03-25T19:39:10.048089Z","iopub.status.idle":"2022-03-25T19:39:16.224492Z","shell.execute_reply.started":"2022-03-25T19:39:10.048057Z","shell.execute_reply":"2022-03-25T19:39:16.223610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Blend Predictions","metadata":{}},{"cell_type":"code","source":"init_date = last_date - dt.timedelta(days=11)\nsold_set = set(df.loc[(df.t_dat >= init_date) & (df.t_dat <= last_date)].article_id.tolist())","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:39:16.226001Z","iopub.execute_input":"2022-03-25T19:39:16.226252Z","iopub.status.idle":"2022-03-25T19:39:16.517387Z","shell.execute_reply.started":"2022-03-25T19:39:16.226223Z","shell.execute_reply":"2022-03-25T19:39:16.516389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['prediction'] = sub[['last_purchase', 'other_colors', 'popular_items']]\\\n    .apply(blend, w=[100, 10, 1], axis=1, k=32).apply(prune, ok_set=sold_set)\npredicted = sub.prediction.apply(lambda s: [] if pd.isna(s) else s.split())\nscore = validation(actual, predicted, grouping, score, index='Prediction', ignore=(val_week == 105))","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:39:16.518672Z","iopub.execute_input":"2022-03-25T19:39:16.518901Z","iopub.status.idle":"2022-03-25T19:40:30.177933Z","shell.execute_reply.started":"2022-03-25T19:39:16.518874Z","shell.execute_reply":"2022-03-25T19:40:30.176854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if val_week == 105:\n    sub[['customer_id', 'prediction']].to_csv('submission.csv', index=False)\nelse:\n    sns.barplot(data=score, x='All', y=score.index)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T19:41:23.352434Z","iopub.execute_input":"2022-03-25T19:41:23.352720Z","iopub.status.idle":"2022-03-25T19:41:36.501419Z","shell.execute_reply.started":"2022-03-25T19:41:23.352690Z","shell.execute_reply":"2022-03-25T19:41:36.500650Z"},"trusted":true},"execution_count":null,"outputs":[]}]}