{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# H&M EDA & Customer Clustering by Kmeans (modified)","metadata":{}},{"cell_type":"markdown","source":"## information","metadata":{}},{"cell_type":"markdown","source":"- This note is based on \"[H&M EDA & Customer Clustering by Kmeans](https://www.kaggle.com/code/hirotakanogami/h-m-eda-customer-clustering-by-kmeans)\" \n- CPU only (without GPU)","metadata":{}},{"cell_type":"code","source":"import sys, warnings, time, os, copy, gc, re, random, pickle#, cudf\nwarnings.filterwarnings('ignore')\nfrom IPython.display import display\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n# pd.set_option('display.max_rows', 50)\n# pd.set_option('display.max_columns', None)\n# pd.set_option(\"display.max_colwidth\", 10000)\nimport seaborn as sns\nsns.set()\nfrom pandas.io.json import json_normalize\nfrom pprint import pprint\nfrom pathlib import Path\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom collections import Counter\nfrom datetime import datetime, timedelta\n#import cudf\n\nfrom sklearn.cluster import KMeans\nfrom sklearn import preprocessing","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:19.367915Z","iopub.execute_input":"2022-05-09T06:56:19.368469Z","iopub.status.idle":"2022-05-09T06:56:21.073476Z","shell.execute_reply.started":"2022-05-09T06:56:19.368336Z","shell.execute_reply":"2022-05-09T06:56:21.072299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Customer Clustering","metadata":{}},{"cell_type":"code","source":"DEBUG = False\nPATH_INPUT = r'../input/h-and-m-personalized-fashion-recommendations/'\n#PATH_INPUT = r'./'","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:24.232864Z","iopub.execute_input":"2022-05-09T06:56:24.233198Z","iopub.status.idle":"2022-05-09T06:56:24.237550Z","shell.execute_reply.started":"2022-05-09T06:56:24.233165Z","shell.execute_reply":"2022-05-09T06:56:24.236803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers = pd.read_csv(PATH_INPUT + 'customers.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:28.501455Z","iopub.execute_input":"2022-05-09T06:56:28.501896Z","iopub.status.idle":"2022-05-09T06:56:34.862929Z","shell.execute_reply.started":"2022-05-09T06:56:28.501859Z","shell.execute_reply":"2022-05-09T06:56:34.861899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Clustering_HandM():\n    def customers_preprocessing(self, customers, dropcol=['postal_code'] , **kwargs):\n        customers = customers.drop(dropcol, axis=1)\n        customers_col = list(customers.columns)\n        \n        if 'fashion_news_frequency' in customers_col :\n            customers['fashion_news_frequency'] = customers['fashion_news_frequency'].replace('NONE','None')\n            customers['fashion_news_frequency'] = customers['fashion_news_frequency'].replace({np.nan :0, 'None':0, 'Monthly':1, 'Regularly':2})\n            \n        if 'club_member_status' in customers_col:\n            customers['club_member_status'] = customers['club_member_status'].replace({np.nan :0, 'PRE-CREATE':1, 'ACTIVE':2, 'LEFT CLUB':-1})\n            \n        if 'age' in customers_col:\n            customers['age'] = customers['age'].fillna(-1)\n            \n        if 'FN' in customers_col:\n            customers['FN'] = customers['FN'].fillna(0)\n\n        if 'Active' in customers_col:\n            customers['Active'] = customers['Active'].fillna(0)\n            \n            print(f'###NULL DESCRIPTION###\\n{customers.isnull().sum()}')\n            \n        return customers\n    \n    def clustering(self, df, predcol, usecol, normmethod='StandardScaler', clusters=12, DEBUG=False):\n        \n        X = np.array(df[usecol])\n        \n        if normmethod == 'StandardScaler':\n            nm = preprocessing.StandardScaler()\n            X = nm.fit_transform(X)\n        elif normmethod == 'minMax':\n            nm = preprocessing.MinMaxScaler()\n            X = nm.fit_transform(X)\n        print(f'NormarlizationMethod:{normmethod}')\n        \n        #km = KMeans(n_clusters=clusters, random_state=2022)\n        km = KMeans(n_clusters=clusters, \n                    init='k-means++', \n                    random_state=2022)\n        km.fit(X)\n        distortion = km.inertia_\n        print('Distortion: %.2f'% km.inertia_)\n\n        pred = km.labels_\n        df_pred = pd.DataFrame(pred, columns=['pred'])\n        df_pred = pd.concat([df, df_pred], axis=1)\n        \n        df_norm = pd.DataFrame(X, columns=usecol)\n        #print(df_norm.describe())\n\n\n        if DEBUG:\n            df_norm = pd.concat([df[predcol], df_norm], axis=1)\n            return df_pred, distortion, df_norm\n        else:\n            return df_pred, distortion\n        ","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:34.864594Z","iopub.execute_input":"2022-05-09T06:56:34.864850Z","iopub.status.idle":"2022-05-09T06:56:34.881708Z","shell.execute_reply.started":"2022-05-09T06:56:34.864819Z","shell.execute_reply":"2022-05-09T06:56:34.881019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dealing with missing values and replacing some words with integer","metadata":{}},{"cell_type":"code","source":"print(\"FN:\", customers['FN'].unique())\nprint(\"Active:\", customers['Active'].unique())\nprint(\"club_member_status:\", customers['club_member_status'].unique())\nprint(\"fashion_news_frequency:\", customers['fashion_news_frequency'].unique())\nprint(\"age:\", customers['age'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:44.129333Z","iopub.execute_input":"2022-05-09T06:56:44.130041Z","iopub.status.idle":"2022-05-09T06:56:44.506413Z","shell.execute_reply.started":"2022-05-09T06:56:44.129999Z","shell.execute_reply":"2022-05-09T06:56:44.505374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clst = Clustering_HandM()\ncustomers = clst.customers_preprocessing(customers)\nusecol = ['club_member_status', 'fashion_news_frequency', 'age', 'FN', 'Active']\npredcol = ['customer_id']","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:48.934464Z","iopub.execute_input":"2022-05-09T06:56:48.934779Z","iopub.status.idle":"2022-05-09T06:56:51.338296Z","shell.execute_reply.started":"2022-05-09T06:56:48.934744Z","shell.execute_reply":"2022-05-09T06:56:51.337329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://qiita.com/deaikei/items/11a10fde5bb47a2cf2c2","metadata":{}},{"cell_type":"markdown","source":"### Determining K, number of clustering by Elbow method","metadata":{}},{"cell_type":"code","source":"distortions = []\nkx = []\nfor k in range(1,12):\n    print(f'---- K = {k} ----')\n    dfCustomers, dist = clst.clustering(customers, predcol=predcol, usecol=usecol, clusters=k)\n    distortions.append(dist)\n    kx.append(k)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:56:53.568122Z","iopub.execute_input":"2022-05-09T06:56:53.568394Z","iopub.status.idle":"2022-05-09T06:58:29.345436Z","shell.execute_reply.started":"2022-05-09T06:56:53.568365Z","shell.execute_reply":"2022-05-09T06:58:29.344129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Elbow method\nplt.plot(kx, distortions)\nplt.scatter(kx, distortions)\nplt.xlabel(\"k: number of clusters\")\nplt.ylabel(\"Distortion\")\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:29.347500Z","iopub.execute_input":"2022-05-09T06:58:29.347737Z","iopub.status.idle":"2022-05-09T06:58:29.671276Z","shell.execute_reply.started":"2022-05-09T06:58:29.347706Z","shell.execute_reply":"2022-05-09T06:58:29.670228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Clustering after optimizing K","metadata":{}},{"cell_type":"code","source":"K_NUMBER = 5","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:29.672809Z","iopub.execute_input":"2022-05-09T06:58:29.673147Z","iopub.status.idle":"2022-05-09T06:58:29.678308Z","shell.execute_reply.started":"2022-05-09T06:58:29.673104Z","shell.execute_reply":"2022-05-09T06:58:29.677164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfCustomers, dist = clst.clustering(customers, predcol=predcol, usecol=usecol, clusters=K_NUMBER)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:29.680144Z","iopub.execute_input":"2022-05-09T06:58:29.680468Z","iopub.status.idle":"2022-05-09T06:58:37.157374Z","shell.execute_reply.started":"2022-05-09T06:58:29.680422Z","shell.execute_reply":"2022-05-09T06:58:37.156320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listBin = [-1, 19, 29, 39, 49, 59, 69, 119]\ndfCustomers['age_bins'] = pd.cut(dfCustomers['age'], listBin)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:37.159170Z","iopub.execute_input":"2022-05-09T06:58:37.159869Z","iopub.status.idle":"2022-05-09T06:58:37.222491Z","shell.execute_reply.started":"2022-05-09T06:58:37.159830Z","shell.execute_reply":"2022-05-09T06:58:37.221041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.crosstab(dfCustomers['pred'], dfCustomers['age_bins'])","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:42.642797Z","iopub.execute_input":"2022-05-09T06:58:42.643111Z","iopub.status.idle":"2022-05-09T06:58:42.834903Z","shell.execute_reply.started":"2022-05-09T06:58:42.643079Z","shell.execute_reply":"2022-05-09T06:58:42.833739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfCustomers = dfCustomers.drop(['age_bins'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:47.085031Z","iopub.execute_input":"2022-05-09T06:58:47.085321Z","iopub.status.idle":"2022-05-09T06:58:47.181305Z","shell.execute_reply.started":"2022-05-09T06:58:47.085292Z","shell.execute_reply":"2022-05-09T06:58:47.180264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfTransactions = pd.read_csv(PATH_INPUT + 'transactions_train.csv',  \n                               usecols=['t_dat', 'customer_id', 'article_id'],\n                               dtype={'article_id': 'int32', 't_dat': 'string', 'customer_id': 'string'})\ndfTransactions['t_dat'] = pd.to_datetime(dfTransactions['t_dat'])\ndfTransactions.set_index('t_dat', inplace=True)\ndfTransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-09T06:58:48.981130Z","iopub.execute_input":"2022-05-09T06:58:48.981414Z","iopub.status.idle":"2022-05-09T07:00:01.390577Z","shell.execute_reply.started":"2022-05-09T06:58:48.981384Z","shell.execute_reply":"2022-05-09T07:00:01.389219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfRecent = dfTransactions.loc['2020-09-01' : '2020-09-21']\ndfRecent","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:01.392709Z","iopub.execute_input":"2022-05-09T07:00:01.394437Z","iopub.status.idle":"2022-05-09T07:00:02.021436Z","shell.execute_reply.started":"2022-05-09T07:00:01.394368Z","shell.execute_reply":"2022-05-09T07:00:02.020113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dfRecent = dfRecent.to_pandas()\n# dfRecent = dfRecent.merge(dfCustomers[['customer_id', 'age_bins']], on='customer_id', how='inner')\ndfRecent = dfRecent.merge(dfCustomers[['customer_id', 'pred']], on='customer_id', how='inner')\ndfRecent","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:16.046727Z","iopub.execute_input":"2022-05-09T07:00:16.047066Z","iopub.status.idle":"2022-05-09T07:00:17.337532Z","shell.execute_reply.started":"2022-05-09T07:00:16.047031Z","shell.execute_reply":"2022-05-09T07:00:17.336521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfRecent = dfRecent.groupby(['pred', 'article_id']).count().reset_index().rename(columns={'customer_id': 'counts'})\nlistUniBins = dfRecent['pred'].unique().tolist()\ndisplay(dfRecent, listUniBins)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:20.295828Z","iopub.execute_input":"2022-05-09T07:00:20.296160Z","iopub.status.idle":"2022-05-09T07:00:20.526365Z","shell.execute_reply.started":"2022-05-09T07:00:20.296128Z","shell.execute_reply":"2022-05-09T07:00:20.525694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict100 = {}\nfor uniBin in listUniBins:\n    # dfTemp = dfRecent[dfRecent['age_bins'] == uniBin]\n    dfTemp = dfRecent[dfRecent['pred'] == uniBin]\n    dfTemp = dfTemp.sort_values(by='counts', ascending=False)\n    dict100[uniBin] = dfTemp.head(100)['article_id'].values.tolist()\n\ndf100 = pd.DataFrame([dict100]).T.rename(columns={0:'top100'})","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:23.580036Z","iopub.execute_input":"2022-05-09T07:00:23.580540Z","iopub.status.idle":"2022-05-09T07:00:23.600639Z","shell.execute_reply.started":"2022-05-09T07:00:23.580488Z","shell.execute_reply":"2022-05-09T07:00:23.599505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df100","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:27.717769Z","iopub.execute_input":"2022-05-09T07:00:27.718076Z","iopub.status.idle":"2022-05-09T07:00:27.734545Z","shell.execute_reply.started":"2022-05-09T07:00:27.718039Z","shell.execute_reply":"2022-05-09T07:00:27.733369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in df100.index:\n    df100[index] = [len(set(df100.at[index, 'top100']) & set(df100.at[x, 'top100']))/100 for x in df100.index]\n\ndf100 = df100.drop(columns='top100')\nplt.figure(figsize=(10, 6))\nsns.heatmap(df100, annot=True, cbar=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:30.515381Z","iopub.execute_input":"2022-05-09T07:00:30.515898Z","iopub.status.idle":"2022-05-09T07:00:30.780633Z","shell.execute_reply.started":"2022-05-09T07:00:30.515847Z","shell.execute_reply":"2022-05-09T07:00:30.779792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfCustomers","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:34.130056Z","iopub.execute_input":"2022-05-09T07:00:34.130685Z","iopub.status.idle":"2022-05-09T07:00:34.159898Z","shell.execute_reply.started":"2022-05-09T07:00:34.130633Z","shell.execute_reply":"2022-05-09T07:00:34.157664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listUniBins = dfCustomers['pred'].sort_values().unique().tolist()\nprint(listUniBins)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:00:37.415051Z","iopub.execute_input":"2022-05-09T07:00:37.415686Z","iopub.status.idle":"2022-05-09T07:00:37.527401Z","shell.execute_reply.started":"2022-05-09T07:00:37.415643Z","shell.execute_reply":"2022-05-09T07:00:37.526367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_org  = pd.read_csv(PATH_INPUT + 'transactions_train.csv',\n                        usecols= ['t_dat', 'customer_id', 'article_id'], \n                        dtype={'article_id': 'int32', 't_dat': 'string', 'customer_id': 'string'})","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:03:28.252555Z","iopub.execute_input":"2022-05-09T07:03:28.252967Z","iopub.status.idle":"2022-05-09T07:04:20.481076Z","shell.execute_reply.started":"2022-05-09T07:03:28.252905Z","shell.execute_reply":"2022-05-09T07:04:20.479598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_org","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:20.538034Z","iopub.execute_input":"2022-05-09T07:04:20.538317Z","iopub.status.idle":"2022-05-09T07:04:20.566230Z","shell.execute_reply.started":"2022-05-09T07:04:20.538283Z","shell.execute_reply":"2022-05-09T07:04:20.565451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_ts = df_org['t_dat'].max()\nlast_ts = (datetime.fromisoformat(last_ts) - timedelta(days=7)).strftime(\"%Y-%m-%d\")\nprint(f'last day for train: {last_ts}')","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:20.567511Z","iopub.execute_input":"2022-05-09T07:04:20.567758Z","iopub.status.idle":"2022-05-09T07:04:25.425253Z","shell.execute_reply.started":"2022-05-09T07:04:20.567729Z","shell.execute_reply":"2022-05-09T07:04:25.424204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_org[df_org['t_dat'] <= last_ts]\ndf_val = df_org[df_org['t_dat'] > last_ts]\nlast_ts = df_train['t_dat'].max()\nprint(last_ts)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:25.426551Z","iopub.execute_input":"2022-05-09T07:04:25.426823Z","iopub.status.idle":"2022-05-09T07:04:44.829925Z","shell.execute_reply.started":"2022-05-09T07:04:25.426792Z","shell.execute_reply":"2022-05-09T07:04:44.829025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_train, df_val)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:44.831253Z","iopub.execute_input":"2022-05-09T07:04:44.831499Z","iopub.status.idle":"2022-05-09T07:04:44.859768Z","shell.execute_reply.started":"2022-05-09T07:04:44.831468Z","shell.execute_reply":"2022-05-09T07:04:44.858780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val[\"article_id\"] = df_val[\"article_id\"].astype(str)\ndf_val[\"article_id\"] = df_val[\"article_id\"].str.zfill(10)\ndf_val2 = df_val.groupby(['customer_id'])[\"article_id\"].apply(list)\ndf_val2","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:44.862165Z","iopub.execute_input":"2022-05-09T07:04:44.862453Z","iopub.status.idle":"2022-05-09T07:04:47.252461Z","shell.execute_reply.started":"2022-05-09T07:04:44.862424Z","shell.execute_reply":"2022-05-09T07:04:47.251036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define Weight Parameters","metadata":{}},{"cell_type":"code","source":"# 重み付け関数\na, b, c, d = 2.5e4, 1.5e6, 3e-1, 1e3\n#a, b, c, d = 2.5e4, 1.5e5, 2e-1, 1e3\nx = np.array(range(0, 100))\ny = a / np.sqrt(x) + b * np.exp(-c * x) - d\nplt.plot(x, y)\nplt.xlabel(\"delta [days]\")\nplt.ylabel(\"Weight\")\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:53.601528Z","iopub.execute_input":"2022-05-09T07:04:53.601852Z","iopub.status.idle":"2022-05-09T07:04:53.888682Z","shell.execute_reply.started":"2022-05-09T07:04:53.601816Z","shell.execute_reply":"2022-05-09T07:04:53.887697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction for validation","metadata":{"tags":[]}},{"cell_type":"markdown","source":"### mAP function","metadata":{}},{"cell_type":"code","source":"def apk(actual, predicted, k=12):\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    # 12回ループが回る\n    for i, p in enumerate(predicted):\n        # p ∈ actual, p ∉ predicted[:i] (このループ内で始めてでてくる、繰り返しでないという意味)\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:56.744345Z","iopub.execute_input":"2022-05-09T07:04:56.744652Z","iopub.status.idle":"2022-05-09T07:04:56.753803Z","shell.execute_reply.started":"2022-05-09T07:04:56.744619Z","shell.execute_reply":"2022-05-09T07:04:56.752622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### prediction for validation data","metadata":{}},{"cell_type":"code","source":"score_k = []\nnum_customers = []\nN = 12 # 一般的なおすすめとして点数の大きいものからトップから12番までランキング\nfor uniBin in listUniBins:\n    df = df_train.copy()\n    #df  = pd.read_csv('transactions_train.csv',\n    #                        usecols= ['t_dat', 'customer_id', 'article_id'], \n    #                        dtype={'article_id': 'int32', 't_dat': 'string', 'customer_id': 'string'})\n    \n    # segmantation data by kmeans \n    if str(uniBin) == 'nan':\n        dfCustomersTemp = dfCustomers[dfCustomers['pred'].isnull()]\n    else:\n        dfCustomersTemp = dfCustomers[dfCustomers['pred'] == uniBin]\n    dfCustomersTemp = dfCustomersTemp.drop(['pred'], axis=1)\n    #dfCustomersTemp = pd.from_pandas(dfCustomersTemp)\n    \n    #\n    # dfのデータにageの情報を加えている\n    #\n    df = df.merge(dfCustomersTemp[['customer_id', 'age']], on='customer_id', how='inner')\n    print(f'The shape of scope transaction for {uniBin} is {df.shape}. \\n')\n\n    #\n    # メモリを減らす工夫\n    # https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/308635\n    #\n    df['customer_id'] = df['customer_id'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    #df['customer_id'] = df ['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n    df['t_dat'] = pd.to_datetime(df['t_dat'])\n    last_ts = df['t_dat'].max()\n\n    #\n    # 売上を計算する火曜のdateの列を作る\n    #\n    #tmp = df[['t_dat']].copy().to_pandas()\n    tmp = df[['t_dat']].copy()\n    tmp['dow'] = tmp['t_dat'].dt.dayofweek\n    # Previous Tuesday を計算する\n    tmp['ldbw'] = tmp['t_dat'] - pd.TimedeltaIndex(tmp['dow'] - 1, unit='D')\n    # np.ones(len(tmp.loc[tmp['dow'] >=2])) 水, 木, 金, 土, 日 のデータ数分、中身が1の配列を作る => [1. 1. 1. ... 1. 1. 1.]\n    # * 7で 全体に7をかける: [1. 1. 1. ... 1. 1. 1.] -> [7. 7. 7. ... 7. 7. 7.]\n    # pd.TimedeltaIndex(np.ones(len(tmp.loc[tmp['dow'] >=2])) * 7, unit='D'))により['7 days', '7 days', '7 days', ...\n    # 先程計算したldbwに7日のデータが加算される 2018-09-18 (火曜) => 2018-09-25 (火曜)\n    tmp.loc[tmp['dow'] >=2 , 'ldbw'] = tmp.loc[tmp['dow'] >=2 , 'ldbw'] + pd.TimedeltaIndex(np.ones(len(tmp.loc[tmp['dow'] >=2])) * 7, unit='D')\n    \n    df['ldbw'] = tmp['ldbw'].values\n    \n    # カスタマー情報を抜いて、人気商品ランキングを作りたい\n    # 次の売上集計日: ldbw, 商品ID: article_id でgroupbyする. 売上は個数をカウントする\n    weekly_sales = df.drop('customer_id', axis=1).groupby(['ldbw', 'article_id']).count().reset_index()\n    weekly_sales = weekly_sales.rename(columns={'t_dat': 'count'})\n    \n    # 直近の販売個数情報がすべてのデータに付与される\n    df = df.merge(weekly_sales, on=['ldbw', 'article_id'], how = 'left')\n    \n    weekly_sales = weekly_sales.reset_index().set_index('article_id')\n\n    # \"count_targ\" の列が付与される。付与されるデータは最終日の販売個数\n    df = df.merge(\n        weekly_sales.loc[weekly_sales['ldbw']==last_ts, ['count']],\n        on='article_id', suffixes=(\"\", \"_targ\"))\n\n    # 最終日のみに売れたデータは、mergeしたときnanになる。それらのデータはゼロ埋め\n    df['count_targ'].fillna(0, inplace=True)\n    del weekly_sales\n    \n    # 最終日を基準として販売個数の割合を計算する。quotientの列を作成。\n    df['quotient'] = df['count_targ'] / df['count']\n    \n    # quotientをarticle_idごとにsumしたい\n    target_sales = df.drop('customer_id', axis=1).groupby('article_id')['quotient'].sum()\n    # N はBin数で N = 12と設定されている。\n    # quotientが大きい順に並べて、上から12個抜き出す。\n    #general_pred = target_sales.nlargest(N).index.to_pandas().tolist()\n    general_pred = target_sales.nlargest(N).index.tolist()\n    general_pred = ['0' + str(article_id) for article_id in general_pred]\n    general_pred_str =  ' '.join(general_pred)\n    del target_sales\n    \n    purchase_dict = {}\n\n    #tmp = df.copy().to_pandas()\n    tmp = df.copy()\n    # 9月22日からの差分、9月20日なら2という値がint型で tmp['x'] に格納される\n    tmp['x'] = ((last_ts - tmp['t_dat']) / np.timedelta64(1, 'D')).astype(int)\n    # 最終日の \"0\" が1に変換される\n    tmp['dummy_1'] = 1 \n    tmp['x'] = tmp[[\"x\", \"dummy_1\"]].max(axis=1)\n\n    #a, b, c, d = 2.5e4, 1.5e5, 2e-1, 1e3\n    tmp['y'] = a / np.sqrt(tmp['x']) + b * np.exp(-c*tmp['x']) - d\n\n    tmp['dummy_0'] = 0 \n    tmp['y'] = tmp[[\"y\", \"dummy_0\"]].max(axis=1)\n    tmp['value'] = tmp['quotient'] * tmp['y'] \n\n    # 'customer_id', 'article_id' はgroupbyの基準列で残り、あとはvalueの列のみsumをとって列に残している\n    tmp = tmp.groupby(['customer_id', 'article_id']).agg({'value': 'sum'})\n    tmp = tmp.reset_index()\n\n    tmp = tmp.loc[tmp['value'] > 0]\n    tmp['rank'] = tmp.groupby(\"customer_id\")[\"value\"].rank(\"dense\", ascending=False)\n    tmp = tmp.loc[tmp['rank'] <= 12]\n\n    purchase_df = tmp.sort_values(['customer_id', 'value'], ascending = False).reset_index(drop = True)\n    purchase_df['prediction'] = '0' + purchase_df['article_id'].astype(str) + ' '\n    purchase_df = purchase_df.groupby('customer_id').agg({'prediction': sum}).reset_index()\n    purchase_df['prediction'] = purchase_df['prediction'].str.strip()\n    purchase_df = pd.DataFrame(purchase_df)\n    \n    #sub  = pd.read_csv('sample_submission.csv',\n    #                        usecols= ['customer_id'], \n    #                        dtype={'customer_id': 'string'})\n    sub = df_val.groupby([\"customer_id\"]).count()\n    sub.reset_index()\n    \n    numCustomers = sub.shape[0]\n    \n    sub = sub.merge(dfCustomersTemp[['customer_id', 'age']], on='customer_id', how='inner')\n\n    sub['customer_id2'] = sub['customer_id'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    #sub['customer_id2'] = sub['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n\n    sub = sub.merge(purchase_df, left_on = 'customer_id2', right_on = 'customer_id', how = 'left',\n                   suffixes = ('', '_ignored'))\n\n    #sub = sub.to_pandas()\n    sub['prediction'] = sub['prediction'].fillna(general_pred_str)\n    sub['prediction'] = sub['prediction'] + ' ' +  general_pred_str\n    sub['prediction'] = sub['prediction'].str.strip()\n    sub['prediction'] = sub['prediction'].str[:131]\n    sub = sub[['customer_id', 'prediction']]\n    sub['prediction2'] = sub['prediction'].apply(lambda x: x.split())\n    sub2 = sub.merge(df_val2, on = 'customer_id', how = 'left', suffixes = ('article_id', 'val'))\n    val_items = sub2[\"article_id\"].tolist()\n    outputs = sub2[\"prediction2\"].tolist()\n    score = mapk(val_items, outputs)\n    print(\"mAP Score on Validation set:\", score)\n    print(f'prediction for {uniBin}. The shape is {sub.shape}. \\n')\n    \n    score_k.append(score)\n    num_customers.append(sub.shape[0])\n\n    print('-'*50)\nprint('Finished.\\n')\nprint('='*50)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:04:59.134174Z","iopub.execute_input":"2022-05-09T07:04:59.135192Z","iopub.status.idle":"2022-05-09T07:08:48.952954Z","shell.execute_reply.started":"2022-05-09T07:04:59.135137Z","shell.execute_reply":"2022-05-09T07:08:48.951472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Caliculate mAP Score for valid data","metadata":{}},{"cell_type":"code","source":"score_list = np.array(score_k)\nnum_list = np.array(num_customers)\n\ntotal = (score_list * num_list).sum()\ncount = num_list.sum()\nprint(\"mAP Score on Validation set:\", total/count)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:08:48.954764Z","iopub.execute_input":"2022-05-09T07:08:48.955055Z","iopub.status.idle":"2022-05-09T07:08:48.961662Z","shell.execute_reply.started":"2022-05-09T07:08:48.955020Z","shell.execute_reply":"2022-05-09T07:08:48.960612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ndel sub","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:08:54.142278Z","iopub.execute_input":"2022-05-09T07:08:54.142572Z","iopub.status.idle":"2022-05-09T07:08:54.150790Z","shell.execute_reply.started":"2022-05-09T07:08:54.142543Z","shell.execute_reply":"2022-05-09T07:08:54.149419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submittion data","metadata":{}},{"cell_type":"code","source":"for uniBin in listUniBins:\n    df = df_org.copy()\n    \n    if str(uniBin) == 'nan':\n        dfCustomersTemp = dfCustomers[dfCustomers['pred'].isnull()]\n    else:\n        dfCustomersTemp = dfCustomers[dfCustomers['pred'] == uniBin]\n    \n    dfCustomersTemp = dfCustomersTemp.drop(['pred'], axis=1)\n    #dfCustomersTemp = pd.from_pandas(dfCustomersTemp)\n    \n    #\n    # dfのデータにageの情報を加えている\n    #\n    df = df.merge(dfCustomersTemp[['customer_id', 'age']], on='customer_id', how='inner')\n    print(f'The shape of scope transaction for {uniBin} is {df.shape}. \\n')\n\n    #\n    # メモリを減らす工夫\n    # https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/308635\n    #\n    df['customer_id'] = df['customer_id'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    #df['customer_id'] = df ['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n    df['t_dat'] = pd.to_datetime(df['t_dat'])\n    last_ts = df['t_dat'].max()\n\n    #\n    # 売上を計算する火曜のdateの列を作る\n    #\n    #tmp = df[['t_dat']].copy().to_pandas()\n    tmp = df[['t_dat']].copy()\n    tmp['dow'] = tmp['t_dat'].dt.dayofweek\n    # Previous Tuesday を計算する\n    tmp['ldbw'] = tmp['t_dat'] - pd.TimedeltaIndex(tmp['dow'] - 1, unit='D')\n    # np.ones(len(tmp.loc[tmp['dow'] >=2])) 水, 木, 金, 土, 日 のデータ数分、中身が1の配列を作る => [1. 1. 1. ... 1. 1. 1.]\n    # * 7で 全体に7をかける: [1. 1. 1. ... 1. 1. 1.] -> [7. 7. 7. ... 7. 7. 7.]\n    # pd.TimedeltaIndex(np.ones(len(tmp.loc[tmp['dow'] >=2])) * 7, unit='D'))により['7 days', '7 days', '7 days', ...\n    # 先程計算したldbwに7日のデータが加算される 2018-09-18 (火曜) => 2018-09-25 (火曜)\n    tmp.loc[tmp['dow'] >=2 , 'ldbw'] = tmp.loc[tmp['dow'] >=2 , 'ldbw'] + pd.TimedeltaIndex(np.ones(len(tmp.loc[tmp['dow'] >=2])) * 7, unit='D')\n    \n    df['ldbw'] = tmp['ldbw'].values\n    \n    # カスタマー情報を抜いて、人気商品ランキングを作りたい\n    # 次の売上集計日: ldbw, 商品ID: article_id でgroupbyする. 売上は個数をカウントする\n    weekly_sales = df.drop('customer_id', axis=1).groupby(['ldbw', 'article_id']).count().reset_index()\n    weekly_sales = weekly_sales.rename(columns={'t_dat': 'count'})\n    \n    # 直近の販売個数情報がすべてのデータに付与される\n    df = df.merge(weekly_sales, on=['ldbw', 'article_id'], how = 'left')\n    \n    weekly_sales = weekly_sales.reset_index().set_index('article_id')\n\n    # \"count_targ\" の列が付与される。付与されるデータは最終日の販売個数\n    df = df.merge(\n        weekly_sales.loc[weekly_sales['ldbw']==last_ts, ['count']],\n        on='article_id', suffixes=(\"\", \"_targ\"))\n\n    # 最終日のみに売れたデータは、mergeしたときnanになる。それらのデータはゼロ埋め\n    df['count_targ'].fillna(0, inplace=True)\n    del weekly_sales\n    \n    # 最終日を基準として販売個数の割合を計算する。quotientの列を作成。\n    df['quotient'] = df['count_targ'] / df['count']\n    \n    # quotientをarticle_idごとにsumしたい\n    target_sales = df.drop('customer_id', axis=1).groupby('article_id')['quotient'].sum()\n    # N はBin数で N = 12と設定されている。\n    # quotientが大きい順に並べて、上から12個抜き出す。\n    #general_pred = target_sales.nlargest(N).index.to_pandas().tolist()\n    general_pred = target_sales.nlargest(N).index.tolist()\n    general_pred = ['0' + str(article_id) for article_id in general_pred]\n    general_pred_str =  ' '.join(general_pred)\n    del target_sales\n    \n    purchase_dict = {}\n\n    #tmp = df.copy().to_pandas()\n    tmp = df.copy()\n    # 9月22日からの差分、9月20日なら2という値がint型で tmp['x'] に格納される\n    tmp['x'] = ((last_ts - tmp['t_dat']) / np.timedelta64(1, 'D')).astype(int)\n    # 最終日の \"0\" が1に変換される\n    tmp['dummy_1'] = 1 \n    tmp['x'] = tmp[[\"x\", \"dummy_1\"]].max(axis=1)\n\n    #a, b, c, d = 2.5e4, 1.5e5, 2e-1, 1e3\n    tmp['y'] = a / np.sqrt(tmp['x']) + b * np.exp(-c*tmp['x']) - d\n\n    tmp['dummy_0'] = 0 \n    tmp['y'] = tmp[[\"y\", \"dummy_0\"]].max(axis=1)\n    tmp['value'] = tmp['quotient'] * tmp['y'] \n\n    # 'customer_id', 'article_id' はgroupbyの基準列で残り、あとはvalueの列のみsumをとって列に残している\n    tmp = tmp.groupby(['customer_id', 'article_id']).agg({'value': 'sum'})\n    tmp = tmp.reset_index()\n\n    tmp = tmp.loc[tmp['value'] > 0]\n    tmp['rank'] = tmp.groupby(\"customer_id\")[\"value\"].rank(\"dense\", ascending=False)\n    tmp = tmp.loc[tmp['rank'] <= 12]\n\n    purchase_df = tmp.sort_values(['customer_id', 'value'], ascending = False).reset_index(drop = True)\n    purchase_df['prediction'] = '0' + purchase_df['article_id'].astype(str) + ' '\n    purchase_df = purchase_df.groupby('customer_id').agg({'prediction': sum}).reset_index()\n    purchase_df['prediction'] = purchase_df['prediction'].str.strip()\n    purchase_df = pd.DataFrame(purchase_df)\n    \n    sub  = pd.read_csv(PATH_INPUT + 'sample_submission.csv',\n                            usecols= ['customer_id'], \n                            dtype={'customer_id': 'string'})\n    \n    numCustomers = sub.shape[0]\n    \n    sub = sub.merge(dfCustomersTemp[['customer_id', 'age']], on='customer_id', how='inner')\n\n    sub['customer_id2'] = sub['customer_id'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    #sub['customer_id2'] = sub['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n\n    sub = sub.merge(purchase_df, left_on = 'customer_id2', right_on = 'customer_id', how = 'left',\n                   suffixes = ('', '_ignored'))\n\n    #sub = sub.to_pandas()\n    sub['prediction'] = sub['prediction'].fillna(general_pred_str)\n    sub['prediction'] = sub['prediction'] + ' ' +  general_pred_str\n    sub['prediction'] = sub['prediction'].str.strip()\n    sub['prediction'] = sub['prediction'].str[:131]\n    sub = sub[['customer_id', 'prediction']]\n    sub.to_csv(f'submission_' + str(uniBin) + '.csv',index=False)\n    print(f'Saved prediction for {uniBin}. The shape is {sub.shape}. \\n')\n    print('-'*50)\nprint('Finished.\\n')\nprint('='*50)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:10:53.435173Z","iopub.execute_input":"2022-05-09T07:10:53.435530Z","iopub.status.idle":"2022-05-09T07:15:20.768203Z","shell.execute_reply.started":"2022-05-09T07:10:53.435498Z","shell.execute_reply":"2022-05-09T07:15:20.767064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df, purchase_df, sub)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:15:20.769985Z","iopub.execute_input":"2022-05-09T07:15:20.770315Z","iopub.status.idle":"2022-05-09T07:15:20.812841Z","shell.execute_reply.started":"2022-05-09T07:15:20.770285Z","shell.execute_reply":"2022-05-09T07:15:20.811978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction data for submittion ","metadata":{}},{"cell_type":"code","source":"for i, uniBin in enumerate(listUniBins):\n    dfTemp  = pd.read_csv(f'submission_' + str(uniBin) + '.csv')\n    if i == 0:\n        dfSub = dfTemp\n    else:\n        dfSub = pd.concat([dfSub, dfTemp], axis=0)\n\nassert dfSub.shape[0] == numCustomers, f'The number of dfSub rows is not correct. {dfSub.shape[0]} vs {numCustomers}.'\n\ndfSub.to_csv(f'submission.csv', index=False)\nprint(f'Saved submission.csv.')","metadata":{"execution":{"iopub.status.busy":"2022-05-09T07:15:20.814130Z","iopub.execute_input":"2022-05-09T07:15:20.814341Z","iopub.status.idle":"2022-05-09T07:15:37.899509Z","shell.execute_reply.started":"2022-05-09T07:15:20.814315Z","shell.execute_reply":"2022-05-09T07:15:37.898627Z"},"trusted":true},"execution_count":null,"outputs":[]}]}