{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create action df!!!","metadata":{}},{"cell_type":"markdown","source":"テストデータに含まれているユーザーの過去の行動からどのような商品に興味を持っているのかを調べてみます。","metadata":{}},{"cell_type":"markdown","source":"# 1.ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport time\nimport gc\nimport copy\nimport glob","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:42.298924Z","iopub.execute_input":"2022-12-30T15:02:42.299380Z","iopub.status.idle":"2022-12-30T15:02:42.304119Z","shell.execute_reply.started":"2022-12-30T15:02:42.299340Z","shell.execute_reply":"2022-12-30T15:02:42.303305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.データの作成","metadata":{}},{"cell_type":"code","source":"%%time\n# 与えられたパスのファイルを読み込む関数\ndef read_file(f):\n    return pd.DataFrame(data_cache[f])\n\n\n# ファイルのの容量をできるだけ小さくする関数。\ndef read_file_to_cache(f):\n    df = pd.read_parquet(f)\n    df.ts = ((df.ts // 1000)).astype('int32')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:42.323097Z","iopub.execute_input":"2022-12-30T15:02:42.323870Z","iopub.status.idle":"2022-12-30T15:02:42.330763Z","shell.execute_reply.started":"2022-12-30T15:02:42.323830Z","shell.execute_reply":"2022-12-30T15:02:42.329770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_cache = {}\nfiles = glob.glob('../input/otto-chunk-data-inparquet-format/test_parquet/*')\n\nfor f in files:\n    data_cache[f] = read_file_to_cache(f)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:42.345670Z","iopub.execute_input":"2022-12-30T15:02:42.346675Z","iopub.status.idle":"2022-12-30T15:02:45.056850Z","shell.execute_reply.started":"2022-12-30T15:02:42.346631Z","shell.execute_reply":"2022-12-30T15:02:45.055704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nDISK_PIECES = 1\n\nfor PART in range(DISK_PIECES):\n    print()\n    print('### DISK PART',PART+1)\n    \n    for f in range(len(files)):\n        df = read_file(files[f])\n        if f == 0:\n            tmp = df\n        else:\n            tmp = pd.concat([tmp, df], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:45.058678Z","iopub.execute_input":"2022-12-30T15:02:45.059033Z","iopub.status.idle":"2022-12-30T15:02:46.948689Z","shell.execute_reply.started":"2022-12-30T15:02:45.059004Z","shell.execute_reply":"2022-12-30T15:02:46.947426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_action_df = tmp.copy()\n\n# aidを文字列に変換し、前にスペースを入れます。\n# これをユーザー別、行動別にグループ分けし、足し合わせていきます。\n# ここで、文字列にしておいたことで、12345 12346のようにaidのリストのようなものを作ることができます。\ntest_action_df.aid = ' ' + tmp.aid.astype(str)\ntest_action_df = test_action_df.groupby(['session', 'type'])['aid'].sum().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:46.950149Z","iopub.execute_input":"2022-12-30T15:02:46.950530Z","iopub.status.idle":"2022-12-30T15:02:55.049509Z","shell.execute_reply.started":"2022-12-30T15:02:46.950495Z","shell.execute_reply":"2022-12-30T15:02:55.048506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ある商品をクリックした人にはその人が過去にクリックした商品をお勧めします。\nfor_clicks = test_action_df[test_action_df['type'] == 'clicks']\n\n# 同様にある商品をカートに入れた人にもその人が過去にクリックした商品をお勧めします。\nfor_carts = for_clicks.copy()\nfor_carts['type'] = 'carts'","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:55.051528Z","iopub.execute_input":"2022-12-30T15:02:55.052261Z","iopub.status.idle":"2022-12-30T15:02:55.478717Z","shell.execute_reply.started":"2022-12-30T15:02:55.052187Z","shell.execute_reply":"2022-12-30T15:02:55.477557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ある商品を注文した人にはその人が過去にクリックした商品と、カートに入れた商品の両方をお勧めします。\nfor_orders = test_action_df[test_action_df['type'] == 'carts']\nfor_orders = pd.merge(for_orders, for_clicks[['session', 'aid']], on='session', how='left')\nfor_orders['type'] = 'orders'\nfor_orders['aid'] = for_orders['aid_x'] + for_orders['aid_y']\nfor_orders.drop(['aid_x', 'aid_y'], axis=1, inplace=True)\nfor_orders","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:55.479932Z","iopub.execute_input":"2022-12-30T15:02:55.480265Z","iopub.status.idle":"2022-12-30T15:02:56.499408Z","shell.execute_reply.started":"2022-12-30T15:02:55.480236Z","shell.execute_reply":"2022-12-30T15:02:56.498116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for_orders","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:56.501025Z","iopub.execute_input":"2022-12-30T15:02:56.501436Z","iopub.status.idle":"2022-12-30T15:02:56.517219Z","shell.execute_reply.started":"2022-12-30T15:02:56.501398Z","shell.execute_reply":"2022-12-30T15:02:56.516060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for_clicks.to_parquet('for_clicks.parquet')\nfor_carts.to_parquet('for_carts.parquet')\nfor_orders.to_parquet('for_orders.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:56.519174Z","iopub.execute_input":"2022-12-30T15:02:56.519583Z","iopub.status.idle":"2022-12-30T15:02:58.489230Z","shell.execute_reply.started":"2022-12-30T15:02:56.519549Z","shell.execute_reply":"2022-12-30T15:02:58.488292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_df = tmp.groupby(['aid', 'type'])['ts'].count().reset_index()\naid_df","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:02:58.490757Z","iopub.execute_input":"2022-12-30T15:02:58.491445Z","iopub.status.idle":"2022-12-30T15:03:00.560317Z","shell.execute_reply.started":"2022-12-30T15:02:58.491408Z","shell.execute_reply":"2022-12-30T15:03:00.558949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_df_pivot = aid_df.pivot_table('ts', index='aid', columns='type').add_suffix('_count')\naid_df_pivot = aid_df_pivot.reindex(columns=['clicks_count', 'carts_count', 'orders_count']).fillna(0.0)\naid_df_pivot","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:00.562434Z","iopub.execute_input":"2022-12-30T15:03:00.562793Z","iopub.status.idle":"2022-12-30T15:03:01.510334Z","shell.execute_reply.started":"2022-12-30T15:03:00.562762Z","shell.execute_reply":"2022-12-30T15:03:01.508949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_df_pivot['all_count'] = aid_df_pivot['clicks_count'] + aid_df_pivot['carts_count'] + aid_df_pivot['orders_count']\n\naid_df_pivot['clicks_rate'] = aid_df_pivot['clicks_count'] / (aid_df_pivot['all_count'] + 1e-10)\naid_df_pivot['carts_rate'] = aid_df_pivot['carts_count'] / (aid_df_pivot['all_count'] + 1e-10)\naid_df_pivot['orders_rate'] = aid_df_pivot['orders_count'] / (aid_df_pivot['all_count'] + 1e-10)\n\naid_df_pivot['clicks_carts_rate'] = aid_df_pivot['carts_count'] / (aid_df_pivot['clicks_count'] + 1e-10)\naid_df_pivot['carts_orders_rate'] = aid_df_pivot['orders_count'] / (aid_df_pivot['carts_count'] + 1e-10)\naid_df_pivot['clicks_orders_rate'] = aid_df_pivot['orders_count'] / (aid_df_pivot['clicks_count'] + 1e-10)\n\naid_df_pivot","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:01.514560Z","iopub.execute_input":"2022-12-30T15:03:01.514954Z","iopub.status.idle":"2022-12-30T15:03:01.579442Z","shell.execute_reply.started":"2022-12-30T15:03:01.514918Z","shell.execute_reply":"2022-12-30T15:03:01.578493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# click時には、clickされていなくても購入される確率が高い商品をレコメンドする\naid_df_pivot.sort_values(by=['orders_rate', 'clicks_rate'], ascending=[False, True])\n# aid_df_pivot.sort_values(['orders_count', 'clicks_count'], ascending={'orders_count:False', 'clicks_count:True'})","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:01.580853Z","iopub.execute_input":"2022-12-30T15:03:01.581933Z","iopub.status.idle":"2022-12-30T15:03:01.760677Z","shell.execute_reply.started":"2022-12-30T15:03:01.581885Z","shell.execute_reply":"2022-12-30T15:03:01.759817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for_clicks_aid_list = aid_df_pivot.sort_values(by=['orders_rate', 'clicks_rate'], ascending=[False, True]).head(10).index.to_list()\nfor_clicks_aid = ' '\nfor aid in for_clicks_aid_list:\n    for_clicks_aid += ' ' + str(aid)\nfor_clicks_aid","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:01.761875Z","iopub.execute_input":"2022-12-30T15:03:01.762992Z","iopub.status.idle":"2022-12-30T15:03:01.930629Z","shell.execute_reply.started":"2022-12-30T15:03:01.762955Z","shell.execute_reply":"2022-12-30T15:03:01.929315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cart時には、cartに入れられていなくても、購入される確率が高い商品をレコメンドする\naid_df_pivot.sort_values(by=['orders_rate', 'carts_rate'], ascending=[False, True])","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:01.931867Z","iopub.execute_input":"2022-12-30T15:03:01.932246Z","iopub.status.idle":"2022-12-30T15:03:02.115445Z","shell.execute_reply.started":"2022-12-30T15:03:01.932207Z","shell.execute_reply":"2022-12-30T15:03:02.114259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for_carts_aid_list = aid_df_pivot.sort_values(by=['orders_rate', 'carts_rate'], ascending=[False, True]).head(10).index.to_list()\nfor_carts_aid = ' '\nfor aid in for_carts_aid_list:\n    for_carts_aid += ' ' + str(aid)\nfor_carts_aid","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:02.117144Z","iopub.execute_input":"2022-12-30T15:03:02.117891Z","iopub.status.idle":"2022-12-30T15:03:02.290814Z","shell.execute_reply.started":"2022-12-30T15:03:02.117846Z","shell.execute_reply":"2022-12-30T15:03:02.289533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cart時には、cartに入れられた後、購入される確率が高い商品をレコメンドする\naid_df_pivot.query('carts_count >= orders_count').sort_values('carts_orders_rate', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:02.292346Z","iopub.execute_input":"2022-12-30T15:03:02.292704Z","iopub.status.idle":"2022-12-30T15:03:02.521852Z","shell.execute_reply.started":"2022-12-30T15:03:02.292674Z","shell.execute_reply":"2022-12-30T15:03:02.520373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for_orders_aid_list = aid_df_pivot.query('carts_count >= orders_count').sort_values('carts_orders_rate', ascending=False).head(10).index.to_list()\nfor_orders_aid = ' '\nfor aid in for_orders_aid_list:\n    for_orders_aid += ' ' + str(aid)\nfor_orders_aid","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:02.523438Z","iopub.execute_input":"2022-12-30T15:03:02.523911Z","iopub.status.idle":"2022-12-30T15:03:02.686663Z","shell.execute_reply.started":"2022-12-30T15:03:02.523863Z","shell.execute_reply":"2022-12-30T15:03:02.685507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submitデータを作成","metadata":{}},{"cell_type":"code","source":"sample_sub = pd.read_csv('/kaggle/input/otto-recommender-system/sample_submission.csv')\nsample_sub","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:02.688346Z","iopub.execute_input":"2022-12-30T15:03:02.689056Z","iopub.status.idle":"2022-12-30T15:03:08.909677Z","shell.execute_reply.started":"2022-12-30T15:03:02.689020Z","shell.execute_reply":"2022-12-30T15:03:08.908347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = sample_sub.copy()\nsubmit_df['session'] = submit_df['session_type'].apply(lambda x: x.split('_')[0])\nsubmit_df['type'] = submit_df['session_type'].apply(lambda x: x.split('_')[1])\nsubmit_df","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:08.911335Z","iopub.execute_input":"2022-12-30T15:03:08.912113Z","iopub.status.idle":"2022-12-30T15:03:13.908976Z","shell.execute_reply.started":"2022-12-30T15:03:08.912066Z","shell.execute_reply":"2022-12-30T15:03:13.907844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_recommend = pd.concat([for_clicks, for_carts])\ndf_recommend = pd.concat([df_recommend, for_orders])\ndf_recommend","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:13.910493Z","iopub.execute_input":"2022-12-30T15:03:13.910948Z","iopub.status.idle":"2022-12-30T15:03:14.346316Z","shell.execute_reply.started":"2022-12-30T15:03:13.910907Z","shell.execute_reply":"2022-12-30T15:03:14.345182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 過去の履歴からのレコメンド商品に、新しく作ったレコメンド商品を連結\ndf_recommend.loc[df_recommend['type'] == \"clicks\", 'aid'] += for_clicks_aid\ndf_recommend.loc[df_recommend['type'] == \"carts\", 'aid'] += for_carts_aid\ndf_recommend.loc[df_recommend['type'] == \"orders\", 'aid'] += for_orders_aid\ndf_recommend","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:14.347662Z","iopub.execute_input":"2022-12-30T15:03:14.347992Z","iopub.status.idle":"2022-12-30T15:03:17.330582Z","shell.execute_reply.started":"2022-12-30T15:03:14.347962Z","shell.execute_reply":"2022-12-30T15:03:17.329203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_recommend['session'] = df_recommend['session'].astype('str')\nsubmit_df = pd.merge(submit_df, df_recommend, on=['session', 'type'], how='left')\nsubmit_df","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:17.332352Z","iopub.execute_input":"2022-12-30T15:03:17.332752Z","iopub.status.idle":"2022-12-30T15:03:26.292742Z","shell.execute_reply.started":"2022-12-30T15:03:17.332718Z","shell.execute_reply":"2022-12-30T15:03:26.291421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# レコメンドする商品がないところには、人気商品をレコメンドしておく\naid_df_pivot.sort_values(by=['orders_count', 'orders_rate'], ascending=[False, False])","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:26.294395Z","iopub.execute_input":"2022-12-30T15:03:26.295517Z","iopub.status.idle":"2022-12-30T15:03:26.427090Z","shell.execute_reply.started":"2022-12-30T15:03:26.295476Z","shell.execute_reply":"2022-12-30T15:03:26.425706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for_nans_aid_list = aid_df_pivot.sort_values(by=['orders_count', 'orders_rate'], ascending=[False, False]).head(20).index.to_list()\n\nfor_nans_aid = ''\nfor aid in for_nans_aid_list:\n    for_nans_aid += ' ' + str(aid)\nfor_nans_aid","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:26.428567Z","iopub.execute_input":"2022-12-30T15:03:26.429567Z","iopub.status.idle":"2022-12-30T15:03:26.541683Z","shell.execute_reply.started":"2022-12-30T15:03:26.429526Z","shell.execute_reply":"2022-12-30T15:03:26.540494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df['aid'] = submit_df['aid'].fillna(for_nans_aid)\nsubmit_df","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:03:26.543092Z","iopub.execute_input":"2022-12-30T15:03:26.543488Z","iopub.status.idle":"2022-12-30T15:03:27.218334Z","shell.execute_reply.started":"2022-12-30T15:03:26.543453Z","shell.execute_reply":"2022-12-30T15:03:27.217168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df2 = submit_df.drop(columns=['labels', 'session', 'type'])\nsubmit_df2 = submit_df2.rename(columns={'aid':'labels'})\nsubmit_df2","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:05:02.986629Z","iopub.execute_input":"2022-12-30T15:05:02.987077Z","iopub.status.idle":"2022-12-30T15:05:03.505519Z","shell.execute_reply.started":"2022-12-30T15:05:02.987039Z","shell.execute_reply":"2022-12-30T15:05:03.504077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df2.to_csv('/kaggle/working/nakano_model4_submission1.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T15:05:12.530747Z","iopub.execute_input":"2022-12-30T15:05:12.531205Z","iopub.status.idle":"2022-12-30T15:05:30.530558Z","shell.execute_reply.started":"2022-12-30T15:05:12.531152Z","shell.execute_reply":"2022-12-30T15:05:30.529091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}