{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Setup","metadata":{}},{"cell_type":"code","source":"import pathlib\nfrom tqdm import tqdm\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-08T12:01:43.905903Z","iopub.execute_input":"2022-11-08T12:01:43.906575Z","iopub.status.idle":"2022-11-08T12:01:43.917428Z","shell.execute_reply.started":"2022-11-08T12:01:43.906399Z","shell.execute_reply":"2022-11-08T12:01:43.916408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_dataset_directory = pathlib.Path('../input/otto-recommender-system')\npickled_dataset_directory = pathlib.Path('../input/otto-multi-objective-recommender-system-pickle')\n\ndf_train = pd.read_pickle(pickled_dataset_directory / 'train.pkl')\ndf_test = pd.read_pickle(pickled_dataset_directory / 'test.pkl')\n\nprint(f'Training Shape: {df_train.shape} - Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')\nprint(f'Test Shape: {df_test.shape} - Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB')","metadata":{"execution":{"iopub.status.busy":"2022-11-08T12:01:44.143125Z","iopub.execute_input":"2022-11-08T12:01:44.143960Z","iopub.status.idle":"2022-11-08T12:02:20.504546Z","shell.execute_reply.started":"2022-11-08T12:01:44.143916Z","shell.execute_reply":"2022-11-08T12:02:20.502381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Dataset Statistics\n\nTop 20 most frequent aids are extracted from training, test and training + test sets for every event at once and separately. Extracted aids will be used for filling predictions with less than 20 unique aids. ","metadata":{}},{"cell_type":"code","source":"df_train_aid_counts = df_train.groupby('aid')[['aid']].count()\ndf_test_aid_counts = df_test.groupby('aid')[['aid']].count()\n\ndf_train_aid_counts = df_train_aid_counts.rename(columns={'aid': 'count'}).reset_index()\ndf_test_aid_counts = df_test_aid_counts.rename(columns={'aid': 'count'}).reset_index()\ndf_all_aid_counts = pd.concat((df_train_aid_counts, df_test_aid_counts), axis=0, ignore_index=True).reset_index(drop=True)\ndf_all_aid_counts = df_all_aid_counts.groupby('aid')['count'].sum().reset_index()\n\ndf_train_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_test_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_all_aid_counts.sort_values(by='count', ascending=False, inplace=True)\n\ntrain_20_most_frequent_aids = df_train_aid_counts.set_index('aid').head(20).to_dict()['count']\ntest_20_most_frequent_aids = df_test_aid_counts.set_index('aid').head(20).to_dict()['count']\nall_20_most_frequent_aids = df_all_aid_counts.set_index('aid').head(20).to_dict()['count']\ndel df_train_aid_counts, df_test_aid_counts, df_all_aid_counts\n\ndf_train_aids_by_types = df_train.groupby(['type', 'aid'])[['aid']].count()\ndf_test_aids_by_types = df_test.groupby(['type', 'aid'])[['aid']].count()\ndf_train_click_aid_counts = df_train_aids_by_types.loc[0].rename(columns={'aid': 'count'}).reset_index()\ndf_train_cart_aid_counts = df_train_aids_by_types.loc[1].rename(columns={'aid': 'count'}).reset_index()\ndf_train_order_aid_counts = df_train_aids_by_types.loc[2].rename(columns={'aid': 'count'}).reset_index()\ndf_test_click_aid_counts = df_test_aids_by_types.loc[0].rename(columns={'aid': 'count'}).reset_index()\ndf_test_cart_aid_counts = df_test_aids_by_types.loc[1].rename(columns={'aid': 'count'}).reset_index()\ndf_test_order_aid_counts = df_test_aids_by_types.loc[2].rename(columns={'aid': 'count'}).reset_index()\ndel df_train_aids_by_types, df_test_aids_by_types\ndf_all_click_aid_counts = pd.concat((df_train_click_aid_counts, df_test_click_aid_counts), axis=0, ignore_index=True).reset_index(drop=True)\ndf_all_click_aid_counts = df_all_click_aid_counts.groupby('aid')['count'].sum().reset_index()\ndf_all_cart_aid_counts = pd.concat((df_train_cart_aid_counts, df_test_cart_aid_counts), axis=0, ignore_index=True).reset_index(drop=True)\ndf_all_cart_aid_counts = df_all_cart_aid_counts.groupby('aid')['count'].sum().reset_index()\ndf_all_order_aid_counts = pd.concat((df_train_order_aid_counts, df_test_order_aid_counts), axis=0, ignore_index=True).reset_index(drop=True)\ndf_all_order_aid_counts = df_all_order_aid_counts.groupby('aid')['count'].sum().reset_index()\n\ndf_train_click_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_test_click_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_all_click_aid_counts.sort_values(by='count', ascending=False, inplace=True)\n\ndf_train_cart_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_test_cart_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_all_cart_aid_counts.sort_values(by='count', ascending=False, inplace=True)\n\ndf_train_order_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_test_order_aid_counts.sort_values(by='count', ascending=False, inplace=True)\ndf_all_order_aid_counts.sort_values(by='count', ascending=False, inplace=True)\n\ntrain_20_most_frequent_click_aids = df_train_click_aid_counts.set_index('aid').head(20).to_dict()['count']\ntest_20_most_frequent_click_aids = df_test_click_aid_counts.set_index('aid').head(20).to_dict()['count']\nall_20_most_frequent_click_aids = df_all_click_aid_counts.set_index('aid').head(20).to_dict()['count']\ndel df_train_click_aid_counts, df_test_click_aid_counts, df_all_click_aid_counts\n\ntrain_20_most_frequent_cart_aids = df_train_cart_aid_counts.set_index('aid').head(20).to_dict()['count']\ntest_20_most_frequent_cart_aids = df_test_cart_aid_counts.set_index('aid').head(20).to_dict()['count']\nall_20_most_frequent_cart_aids = df_all_cart_aid_counts.set_index('aid').head(20).to_dict()['count']\ndel df_train_cart_aid_counts, df_test_cart_aid_counts, df_all_cart_aid_counts\n\ntrain_20_most_frequent_order_aids = df_train_order_aid_counts.set_index('aid').head(20).to_dict()['count']\ntest_20_most_frequent_order_aids = df_test_order_aid_counts.set_index('aid').head(20).to_dict()['count']\nall_20_most_frequent_order_aids = df_all_order_aid_counts.set_index('aid').head(20).to_dict()['count']\ndel df_train_order_aid_counts, df_test_order_aid_counts, df_all_order_aid_counts","metadata":{"execution":{"iopub.status.busy":"2022-11-08T12:09:52.555795Z","iopub.execute_input":"2022-11-08T12:09:52.556986Z","iopub.status.idle":"2022-11-08T12:10:50.087794Z","shell.execute_reply.started":"2022-11-08T12:09:52.556940Z","shell.execute_reply":"2022-11-08T12:10:50.085860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. aid Frequency Baseline\n\n[@Tawara](https://www.kaggle.com/ttahara) shared aid frequency baseline in [this](https://www.kaggle.com/code/ttahara/otto-mors-aid-frequency-baseline) notebook. However, his implementation doesn't predict 20 aids if number of unique aids is less than 20 in a session.\n\nAim of this notebook is comparing different ways of filling empty predictions and improve frequency baseline little bit further.\n\n* top 20 most frequent aids per session **-> 0.482** (Tawara's baseline)\n* top 20 most frequent aids per session concatenated with top x (number of aids left to predict) most frequent aids in training set **-> 0.484**\n* top 20 most frequent aids per session concatenated with top x (number of aids left to predict) most frequent aids in training + test set **-> 0.484**\n* top 20 most frequent aids per session concatenated with top x (number of aids left to predict) most frequent aids per event type in training set **-> 0.484**\n* top 20 most frequent aids per session concatenated with top x (number of aids left to predict) most frequent aids per event type in training + test set **-> 0.484**\n\nThere wasn't much improvement after concatenation. All of the concatenation types scored 0.484 on public leaderboard however \"most frequent aids per event type in training + test set\" yields the largest boost compared to others.","metadata":{}},{"cell_type":"code","source":"submission = []\n\ndf_test_session_aid_frequencies = df_test.groupby(['session', 'aid'])['aid'].count()\n# Sort values inside groups\ndf_test_session_aid_frequencies = df_test_session_aid_frequencies.sort_values(ascending=False).sort_index(level='session', sort_remaining=False)\ndf_test_session_aid_frequencies = df_test_session_aid_frequencies.rename('count').reset_index()\n# Create a dictionary of session id keys and list of top 20 most frequent aid values\ndf_test_session_aid_frequencies = df_test_session_aid_frequencies.groupby('session')['aid'].agg(lambda x: list(x)[:20]).to_dict()\n\nfor session_id, aids in tqdm(df_test_session_aid_frequencies.items()):\n\n    for event_type in ['click', 'cart', 'order']:\n        \n        predictions = aids.copy()\n        \n        if event_type == 'click':\n            predictions += list(all_20_most_frequent_click_aids.keys())[:20 - len(aids)]\n        elif event_type == 'cart':\n            predictions += list(all_20_most_frequent_cart_aids.keys())[:20 - len(aids)]\n        elif event_type == 'order':\n            predictions += list(all_20_most_frequent_order_aids.keys())[:20 - len(aids)]\n        \n        predictions = ' '.join([str(aid) for aid in predictions])\n        submission.append({\n            'session_type': f'{session_id}_{event_type}s',\n            'labels': predictions\n        })\n","metadata":{"execution":{"iopub.status.busy":"2022-11-08T12:20:04.428699Z","iopub.execute_input":"2022-11-08T12:20:04.429154Z","iopub.status.idle":"2022-11-08T12:20:43.875627Z","shell.execute_reply.started":"2022-11-08T12:20:04.429116Z","shell.execute_reply":"2022-11-08T12:20:43.874160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.DataFrame(submission)\ndf_submission","metadata":{"execution":{"iopub.status.busy":"2022-11-08T12:18:52.219342Z","iopub.execute_input":"2022-11-08T12:18:52.219799Z","iopub.status.idle":"2022-11-08T12:18:52.232417Z","shell.execute_reply.started":"2022-11-08T12:18:52.219761Z","shell.execute_reply":"2022-11-08T12:18:52.231212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}