{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# H&M Recommendation: Popularity Baseline\n\nBased on the work of https://www.kaggle.com/mayukh18/time-decaying-popularity-benchmark-0-0216","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.033224,"end_time":"2022-02-27T16:55:47.128998","exception":false,"start_time":"2022-02-27T16:55:47.095774","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\n#import reco\nfrom tqdm import tqdm\nimport datetime\nimport gc\nimport random","metadata":{"papermill":{"duration":0.049319,"end_time":"2022-02-27T16:55:47.211774","exception":false,"start_time":"2022-02-27T16:55:47.162455","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Forming Train Set","metadata":{"papermill":{"duration":0.030011,"end_time":"2022-02-27T16:55:47.273473","exception":false,"start_time":"2022-02-27T16:55:47.243462","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\", \n                   dtype={'article_id':str}\n                  )\ndata.head()","metadata":{"papermill":{"duration":81.466587,"end_time":"2022-02-27T16:57:08.770442","exception":false,"start_time":"2022-02-27T16:55:47.303855","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll drop everything except the last few(up for experimentation) days. The info from previous months are not coming of much use. \nWe'll keep 4 weeks as train and the last week as validation.","metadata":{"papermill":{"duration":0.028859,"end_time":"2022-02-27T16:57:08.828648","exception":false,"start_time":"2022-02-27T16:57:08.799789","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"All Transactions Date Range: {} to {}\".format(data['t_dat'].min(), data['t_dat'].max()))\n\ndata[\"t_dat\"] = pd.to_datetime(data[\"t_dat\"])\ntrain1 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,8)) & (data['t_dat'] < datetime.datetime(2020,9,16))]\ntrain2 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,1)) & (data['t_dat'] < datetime.datetime(2020,9,8))]\ntrain3 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,23)) & (data['t_dat'] < datetime.datetime(2020,9,1))]\ntrain4 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,15)) & (data['t_dat'] < datetime.datetime(2020,8,23))]\ntrain5 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,7)) & (data['t_dat'] < datetime.datetime(2020,8,15))]\n\nval = data.loc[data[\"t_dat\"] >= datetime.datetime(2020,9,16)]","metadata":{"papermill":{"duration":17.905367,"end_time":"2022-02-27T16:57:26.764202","exception":false,"start_time":"2022-02-27T16:57:08.858835","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv', \n#                           dtype={'article_id': str}\n                         )","metadata":{"papermill":{"duration":1.358247,"end_time":"2022-02-27T16:57:28.154927","exception":false,"start_time":"2022-02-27T16:57:26.79668","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### recommend last week most popular items as alternatives to current week","metadata":{}},{"cell_type":"code","source":"def get_alternate_most_popular(df_data, factor, return_orig=False):\n    \n    next_best_match = []\n    \n    df = df_data.copy()\n    df['article_count'] = df.groupby('article_id')['customer_id'].transform('count')\n    df['article_min_price'] = df.groupby('article_id')['price'].transform('min')\n    count_df = df[['article_id', 'article_count', 'article_min_price']].drop_duplicates().reset_index(drop=True)\n    \n    del df\n    \n    for article in tqdm(count_df.article_id.tolist()):\n        prodname = articles_df[articles_df.article_id==int(article)]['prod_name'].iloc[0]\n        other_article_list = articles_df[articles_df.prod_name==prodname]['article_id'].tolist()\n        other_article_list.remove(int(article))\n        k = len(other_article_list)\n        if k==1:\n            next_best_match.append(other_article_list[0])\n        if k>1:\n            if len(count_df[np.in1d(count_df['article_id'], other_article_list)])!=0:\n                next_best_match.append(count_df[np.in1d(count_df['article_id'], other_article_list)].sort_values('article_count', ascending=False)['article_id'].iloc[0])\n            else:\n                next_best_match.append(np.nan)\n        if k==0:\n            next_best_match.append(np.nan)\n\n    count_df['next_best_article'] = next_best_match\n    count_df['next_best_article'] = count_df['next_best_article'].fillna(0).astype(int)\n    count_df['next_best_article'] = np.where(count_df['next_best_article']==0, count_df['article_id'], str(0)+count_df['next_best_article'].astype(str))\n\n    right_df = count_df[['next_best_article']].copy().rename(columns={'next_best_article':'article_id'})\n\n    next_best_count = []\n    next_best_price = []\n    for article in tqdm(right_df['article_id']):\n        if len(count_df[count_df.article_id==article]['article_count'])>0:\n            next_best_count.append(count_df[count_df.article_id==article]['article_count'].iloc[0])\n            next_best_price.append(count_df[count_df.article_id==article]['article_min_price'].iloc[0])\n        else:\n            next_best_count.append(0)\n            next_best_price.append(0)\n\n    count_df['count_next_best'] = next_best_count\n    count_df['next_best_min_price'] = next_best_price\n        \n    more_popular_alternatives = count_df[(count_df.article_min_price >= count_df.next_best_min_price) & \n                                         (count_df.count_next_best > factor *count_df.article_count)].copy().reset_index(drop=True)\n    more_popular_alt_list = more_popular_alternatives.article_id.unique().tolist()\n    \n    if return_orig:\n        return more_popular_alt_list, more_popular_alternatives, count_df\n    else:\n        return more_popular_alt_list, more_popular_alternatives","metadata":{"papermill":{"duration":0.052909,"end_time":"2022-02-27T16:57:28.238839","exception":false,"start_time":"2022-02-27T16:57:28.18593","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alt_list_1v, alt_df_1v = get_alternate_most_popular(train2, 2, return_orig=False)\nalt_list_2v, alt_df_2v = get_alternate_most_popular(train3, 2, return_orig=False)\nalt_list_3v, alt_df_3v = get_alternate_most_popular(train4, 2, return_orig=False)\nalt_list_4v, alt_df_4v = get_alternate_most_popular(train5, 2, return_orig=False)","metadata":{"papermill":{"duration":2041.309454,"end_time":"2022-02-27T17:31:29.5798","exception":false,"start_time":"2022-02-27T16:57:28.270346","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alt_df_1v.shape, alt_df_2v.shape, alt_df_3v.shape, alt_df_4v.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Items which an user has bought in our train set time.","metadata":{"papermill":{"duration":4.461421,"end_time":"2022-02-27T17:31:38.652447","exception":false,"start_time":"2022-02-27T17:31:34.191026","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# List of all purchases per user (has repetitions)\npositive_items_per_user1 = train1.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user2 = train2.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user3 = train3.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user4 = train4.groupby(['customer_id'])['article_id'].apply(list)","metadata":{"papermill":{"duration":13.132548,"end_time":"2022-02-27T17:31:56.55243","exception":false,"start_time":"2022-02-27T17:31:43.419882","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next we do exponential weighting based popularity for items. This leads to items bought more recently having more weight in the popularity list. In simple words, item A bought 5 times on the first day of the train period is inferior than item B bought 4 times on the last day of the train period.","metadata":{"papermill":{"duration":4.47381,"end_time":"2022-02-27T17:32:05.615576","exception":false,"start_time":"2022-02-27T17:32:01.141766","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.concat([train1, train2], axis=0)\ntrain['pop_factor'] = train['t_dat'].apply(lambda x: 1/(datetime.datetime(2020,9,16) - x).days)\npopular_items_group = train.groupby(['article_id'])['pop_factor'].sum()\n\n_, popular_items = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1])\n\ntrain['pop_factor'].describe()","metadata":{"papermill":{"duration":16.191189,"end_time":"2022-02-27T17:32:26.341524","exception":false,"start_time":"2022-02-27T17:32:10.150335","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Moving on to Validation ...","metadata":{"papermill":{"duration":4.642484,"end_time":"2022-02-27T17:32:35.479874","exception":false,"start_time":"2022-02-27T17:32:30.83739","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def apk(actual, predicted, k=12):\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"papermill":{"duration":4.491817,"end_time":"2022-02-27T17:32:44.588241","exception":false,"start_time":"2022-02-27T17:32:40.096424","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Items bought by users in the validation period. Similar as the one for train set.","metadata":{"papermill":{"duration":4.450168,"end_time":"2022-02-27T17:32:53.686644","exception":false,"start_time":"2022-02-27T17:32:49.236476","status":"completed"},"tags":[]}},{"cell_type":"code","source":"positive_items_val = val.groupby(['customer_id'])['article_id'].apply(list)","metadata":{"papermill":{"duration":6.07262,"end_time":"2022-02-27T17:33:04.341476","exception":false,"start_time":"2022-02-27T17:32:58.268856","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creating validation set for metrics use case\nval_users = positive_items_val.keys()\nval_items = []\n\nfor i,user in tqdm(enumerate(val_users)):\n    val_items.append(positive_items_val[user])\n    \nprint(\"Total users in validation:\", len(val_users))","metadata":{"papermill":{"duration":4.97067,"end_time":"2022-02-27T17:33:13.876462","exception":false,"start_time":"2022-02-27T17:33:08.905792","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll now validate our algo on the validation set.","metadata":{"papermill":{"duration":4.734947,"end_time":"2022-02-27T17:33:23.192151","exception":false,"start_time":"2022-02-27T17:33:18.457204","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from collections import Counter\noutputs = []\ncnt = 0\n\npopular_items = list(popular_items)\n\nfor user in tqdm(val_users):\n    user_output = []\n    if user in positive_items_per_user1.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user1[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_1v:\n                al.append(alt_df_1v[alt_df_1v.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n        \n    if user in positive_items_per_user2.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user2[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_2v:\n                al.append(alt_df_2v[alt_df_2v.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n        \n    if user in positive_items_per_user3.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user3[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_3v:\n                al.append(alt_df_3v[alt_df_3v.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n        \n    if user in positive_items_per_user4.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user4[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_4v:\n                al.append(alt_df_4v[alt_df_4v.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n    \n    user_output += list(popular_items[:12 - len(user_output)])    \n    outputs.append(user_output)\n    \nprint(\"mAP Score on Validation set:\", mapk(val_items, outputs))","metadata":{"papermill":{"duration":17.930094,"end_time":"2022-02-27T17:33:54.740502","exception":false,"start_time":"2022-02-27T17:33:36.810408","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction on Test Set","metadata":{"papermill":{"duration":4.579377,"end_time":"2022-02-27T17:34:23.76886","exception":false,"start_time":"2022-02-27T17:34:19.189483","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train1 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,16)) & (data['t_dat'] < datetime.datetime(2020,9,23))]\ntrain2 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,8)) & (data['t_dat'] < datetime.datetime(2020,9,16))]\ntrain3 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,31)) & (data['t_dat'] < datetime.datetime(2020,9,8))]\ntrain4 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,23)) & (data['t_dat'] < datetime.datetime(2020,8,31))]\ntrain5 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,15)) & (data['t_dat'] < datetime.datetime(2020,8,23))]","metadata":{"papermill":{"duration":6.202178,"end_time":"2022-02-27T17:34:34.804638","exception":false,"start_time":"2022-02-27T17:34:28.60246","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alt_list_1, alt_df_1 = get_alternate_most_popular(train2, 2, return_orig=False)\nalt_list_2, alt_df_2 = alt_list_1v, alt_df_1v\nalt_list_3, alt_df_3 = alt_list_2v, alt_df_2v\nalt_list_4, alt_df_4 = alt_list_3v, alt_df_3v","metadata":{"papermill":{"duration":1997.774249,"end_time":"2022-02-27T18:07:57.31906","exception":false,"start_time":"2022-02-27T17:34:39.544811","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alt_df_1.shape, alt_df_2.shape, alt_df_3.shape, alt_df_4.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_items_per_user1 = train1.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user2 = train2.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user3 = train3.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user4 = train4.groupby(['customer_id'])['article_id'].apply(list)\n\ntrain = pd.concat([train1, train2], axis=0)\ntrain['pop_factor'] = train['t_dat'].apply(lambda x: 1/(datetime.datetime(2020,9,23) - x).days)\npopular_items_group = train.groupby(['article_id'])['pop_factor'].sum()\n\n_, popular_items = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1])\n\nuser_group = pd.concat([train1, train2, train3, train4], axis=0).groupby(['customer_id'])['article_id'].apply(list)","metadata":{"papermill":{"duration":41.081592,"end_time":"2022-02-27T18:08:47.633281","exception":false,"start_time":"2022-02-27T18:08:06.551689","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\")\nsubmission.head()","metadata":{"papermill":{"duration":15.478103,"end_time":"2022-02-27T18:09:12.61848","exception":false,"start_time":"2022-02-27T18:08:57.140377","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\noutputs = []\ncnt = 0\n\nfor user in tqdm(submission['customer_id']):\n    user_output = []\n    if user in positive_items_per_user1.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user1[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_1:\n                al.append(alt_df_1[alt_df_1.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n        \n    if user in positive_items_per_user2.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user2[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_2:\n                al.append(alt_df_2[alt_df_2.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n        \n    if user in positive_items_per_user3.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user3[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_3:\n                al.append(alt_df_3[alt_df_3.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al\n        user_output += l[:12]\n        \n    if user in positive_items_per_user4.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user4[user]).most_common()}\n        l = list(most_common_items_of_user.keys())\n        al = []\n        for j in range(0, len(l)):\n            if l[j] in alt_list_4:\n                al.append(alt_df_4[alt_df_4.article_id==l[j]]['next_best_article'].iloc[0])\n        l = l + al        \n        user_output += l[:12]\n    \n    user_output += list(popular_items[:12 - len(user_output)])\n    outputs.append(user_output)\n    \nstr_outputs = []\nfor output in outputs:\n    str_outputs.append(\" \".join([str(x) for x in output]))","metadata":{"papermill":{"duration":167.609166,"end_time":"2022-02-27T18:12:09.323078","exception":false,"start_time":"2022-02-27T18:09:21.713912","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['prediction'] = str_outputs\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":107.353021,"end_time":"2022-02-27T18:14:06.076056","exception":false,"start_time":"2022-02-27T18:12:18.723035","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"papermill":{"duration":9.557492,"end_time":"2022-02-27T18:14:24.977576","exception":false,"start_time":"2022-02-27T18:14:15.420084","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":9.296421,"end_time":"2022-02-27T18:14:43.98322","exception":false,"start_time":"2022-02-27T18:14:34.686799","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}