{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport pathlib\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport datetime","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Original data","metadata":{}},{"cell_type":"code","source":"### Paths ###\n\nDATA_PATH = pathlib.Path('../input/otto-recommender-system')\nTRAIN_PATH = DATA_PATH/'train.jsonl'\nTEST_PATH = DATA_PATH/'test.jsonl'\nSAMPLE_SUB_PATH = pathlib.Path('../input/otto-recommender-system/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets check how many lines the training data has!\n# with open(TRAIN_PATH, 'r') as f:\n#     print(f\"We have {len(f.readlines()):,} lines in the training data\")\n\n# Load in a sample to a pandas df\nsample_size = 100\nchunks = pd.read_json(TRAIN_PATH, lines=True, chunksize = sample_size)\nfor c in chunks:\n    sample_train_df = c\n    break    \nsample_train_df.set_index('session', drop=True, inplace=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_colwidth = 100\nsample_train_df['no_events'] = sample_train_df[\"events\"].apply(lambda x: len(x))\nsample_train_df.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open(TEST_PATH, 'r') as f:\n#     print(f\"We have {len(f.readlines()):,} lines in the test data\")\n\n# Load in a sample to a pandas df\nsample_size = 150\nchunks = pd.read_json(TEST_PATH, lines=True, chunksize = sample_size)\nfor c in chunks:\n    sample_test_df = c\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df[\"no_events\"] = sample_test_df[\"events\"].apply(lambda x: len(x))\n\npd.options.display.max_colwidth = 120\nsample_test_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df.iloc[1].events","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(SAMPLE_SUB_PATH)\nsample_submission[\"count\"] = sample_submission[\"labels\"].apply(lambda x: len(x.split(\" \")))\n\npd.options.display.max_colwidth = 100\nsample_submission.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sample_submission)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pickled Data","metadata":{}},{"cell_type":"code","source":"### Paths ###\n\nDATA_PATH = pathlib.Path('../input/otto-recommender-system')\nPICKLED_DATA_DIR = pathlib.Path('../input/otto-multi-objective-recommender-system-pickle')\n\ndf_train = pd.read_pickle(PICKLED_DATA_DIR/'train.pkl')\ndf_test = pd.read_pickle(PICKLED_DATA_DIR/'test.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_train = list(df_train[\"aid\"].unique())\nproduct_test = list(df_test[\"aid\"].unique())\noverlap = list(set(product_train) & set(product_test))\nprint(f\"len_train: {len(product_train)} | len_test: {len(product_test)} | overlap: {len(overlap)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train['seconds'] = df_train['ts'].apply(lambda x: datetime.datetime.timestamp(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore features","metadata":{}},{"cell_type":"markdown","source":"## General","metadata":{}},{"cell_type":"code","source":"print(f'Training Shape: {df_train.shape} - Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')\nprint(f'Test Shape: {df_test.shape} - Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB')\n\nprint(\"List of columns in df_train\", list(df_train.columns))\nprint(\"List of columns in df_test\", list(df_test.columns))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = df_train.shape[0]\ntest_events = df_test.shape[0]\nprint(f'Number of Events - Training: {train_events} | Test: {test_events}')\n\ntrain_unique_sessions = df_train['session'].unique()\ntest_unique_sessions = df_test['session'].unique()\nprint(f'Number of Unique Sessions - Training: {len(train_unique_sessions)} | Test: {len(test_unique_sessions)}')\ndel train_unique_sessions, test_unique_sessions\n\ntrain_unique_aids = df_train['aid'].unique()\ntest_unique_aids = df_test['aid'].unique()\noverlapping_aids = set(train_unique_aids).intersection(set(test_unique_aids))\nprint(f'Number of Unique Products - Training: {len(train_unique_aids)} | Test: {len(test_unique_aids)} - ({len(overlapping_aids)} Overlapping Products)')\ndel train_unique_aids, test_unique_aids, overlapping_aids\n\ntrain_clicks = df_train[df_train['type'] == 0].shape[0]\ntest_clicks = df_test[df_test['type'] == 0].shape[0]\nprint(f'Number of Clicks - Training: {train_clicks} | Test: {test_clicks}')\n\ntrain_carts = df_train[df_train['type'] == 1].shape[0]\ntest_carts = df_test[df_test['type'] == 1].shape[0]\nprint(f'Number of Carts - Training: {train_carts} | Test: {test_carts}')\n\ntrain_orders = df_train[df_train['type'] == 2].shape[0]\ntest_orders = df_test[df_test['type'] == 2].shape[0]\nprint(f'Number of Orders - Training: {train_orders} | Test: {test_orders}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_start = df_train['ts'].min().strftime('%Y.%m.%d %X')\ntrain_end = df_train['ts'].max().strftime('%Y.%m.%d %X')\ntest_start = df_test['ts'].min().strftime('%Y.%m.%d %X')\ntest_end = df_test['ts'].max().strftime('%Y.%m.%d %X')\nprint(f'Events Time Range\\nTraining: {train_start} - {train_end}\\nTest: {test_start} - {test_end}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Product","metadata":{}},{"cell_type":"code","source":"# def stats_feature(df_train, df_test, feature, plot=False):\n#     df_f1 = df_train.groupby(feature)[[feature]].count()\n#     df_f2 = df_test.groupby(feature)[[feature]].count()\n    \n    \n#     if plot:\n#         plt.figure(figsize=(24, 6), dpi=100)\n\n#         plt.subplot(1,2,1)\n#         ax = sns.kdeplot(df_f1[feature], label='train', fill=True, log_scale = True)\n#         ax.tick_params(axis='x', labelsize=12.5)\n#         ax.tick_params(axis='y', labelsize=12.5)\n#         ax.set_xlabel('')\n#         ax.set_ylabel('')\n#         ax.legend(prop={'size': 15})\n\n#         plt.subplot(1,2,2)\n#         ax = sns.kdeplot(df_f2[feature], label='test', fill=True, log_scale = False)\n#         ax.tick_params(axis='x', labelsize=12.5)\n#         ax.tick_params(axis='y', labelsize=12.5)\n#         ax.set_xlabel('')\n#         ax.set_ylabel('')\n#         ax.legend(prop={'size': 15})\n\n#     title = (f'''\n#     Frequency of {\"event\" if feature is \"session\" else feature}\n#     Mean - Train: {df_f1[feature].mean():.2f} |  Test: {df_f2[feature].mean():.2f}\n#     Median - Train: {df_f1[feature].median():.2f} |  Test: {df_f2[feature].median():.2f}\n#     Std - Train: {df_f1[feature].std():.2f} |  Test: {df_f2[feature].std():.2f}\n#     Min - Train: {df_f1[feature].min():.2f} |  Test: {df_f2[feature].min():.2f}\n#     Max - Train: {df_f1[feature].max():.2f} |  Test: {df_f2[feature].max():.2f}\n#     ''')\n    \n#     if plot:\n#         plt.suptitle(title, size=20)\n#         plt.show()\n#     else:\n#         print(title)\n        \n# stats_feature(df_train, df_test, \"aid\")","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Stats feature aid in whole set\nfeature = \"aid\"\ngrouped_train_df = df_train.groupby(feature)[[feature]].count()\ngrouped_test_df = df_test.groupby(feature)[[feature]].count()\n\nstats_train = pd.DataFrame(grouped_train_df.describe()).reset_index().rename(columns={\"aid\": \"train\"})\nstats_test = pd.DataFrame(grouped_test_df.describe()).reset_index().rename(columns={\"aid\": \"test\"})\n\nstats_all = stats_train.merge(stats_test)\nstats_all.iloc[1:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 frequency aid (total type action)\nfeature = \"aid\"\nstats_train = df_train.groupby(feature)[[feature]].count()\nstats_train = stats_train.rename(columns={\"aid\":\"frequency\"}).reset_index().sort_values(by=[\"frequency\"], ascending=False)\nstats_train.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Events - Actions","metadata":{}},{"cell_type":"code","source":"# stats feature type in whole set\n\nfeature = \"type\"\ndf_tmp1 = pd.DataFrame(df_train.groupby([feature])[\"aid\"].count()).reset_index()\ndf_tmp2 = pd.DataFrame(df_test.groupby([feature])[\"aid\"].count()).reset_index()\n\n# df_tmp1[\"set\"] = \"train\"\n# df_tmp2[\"set\"] = \"test\"\n\n# df_tmp = pd.concat([df_tmp1, df_tmp2])\n\nax = sns.barplot(data=df_tmp1, x=feature, y= \"aid\")\nax.bar_label(ax.containers[0])\nplt.ylabel(\"Number of aids\")\nplt.title(f\"{feature} | Train\")\nplt.show()\n\nax = sns.barplot(data=df_tmp2, x=feature, y= \"aid\")\nax.bar_label(ax.containers[0])\nplt.ylabel(\"Number of aids\")\nplt.title(f\"{feature} | Test\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stats action in whole set\n\nfeature = \"session\"\ngrouped_train_df = df_train.groupby(feature)[[feature]].count()\ngrouped_test_df = df_test.groupby(feature)[[feature]].count()\n\nstats_train = pd.DataFrame(grouped_train_df.describe()).reset_index().rename(columns={feature: \"train\"})\nstats_test = pd.DataFrame(grouped_test_df.describe()).reset_index().rename(columns={feature: \"test\"})\n\nstats_all = stats_train.merge(stats_test)\nstats_all[1:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of session have only 2 actions in training set:\", len(grouped_train_df.session[grouped_train_df.session == grouped_train_df.session.min()]))\nprint(\"=\"*20)\nprint(grouped_train_df.session[grouped_train_df.session == grouped_train_df.session.min()])\nprint(\"=\"*20)\ndf_train[df_train.session == 10157]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of type per session\ndf_tmp = df_train.groupby([\"session\", \"type\"])[\"aid\"].count()\n\ndf_tmp = pd.DataFrame(df_tmp).reset_index()\n\n# stats each type in each session\ntype_name = [\"click\", \"cart\", \"order\"]\n\nfor i, name in enumerate(type_name):\n#     plt.hist(df_tmp[df_tmp.type == i][\"aid\"])\n#     plt.title(f'Number of {name} per session | Mean: {df_tmp[df_tmp.type == i][\"aid\"].mean():0.2f}')\n#     plt.show()\n    if i == 0:\n        type_per_session = pd.DataFrame(df_tmp[df_tmp.type == i][\"aid\"].describe()).reset_index().rename(columns={\"aid\": name})\n    else:\n        _df = pd.DataFrame(df_tmp[df_tmp.type == i][\"aid\"].describe()).reset_index().rename(columns={\"aid\": name})\n        type_per_session = type_per_session.merge(_df)\n        \ntype_per_session","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# i = 0\n# type_train_df = pd.DataFrame(df_train.groupby([\"session\"])[\"session\"])\n# type_train_df\n# # train_df[train_df.type == i].groupby([\"session\"])[\"type\"].count()","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_session_type_groupby = df_train.groupby(['session', 'type'])\ndf_train_sessions_types = train_session_type_groupby[['session']].count().rename(columns={'session': 'count'}).reset_index()\ndf_train_sessions_types['total'] = df_train_sessions_types.groupby('session')['count'].transform('sum')\ndf_train_sessions_types.sort_values(by=['total', 'session', 'type'], ascending=False, inplace=True)\ndf_train_sessions_types['rate'] = df_train_sessions_types['count'] / df_train_sessions_types['total']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_sessions_types","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Timestamp","metadata":{}},{"cell_type":"code","source":"train_parquet = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest_parquet = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_min = train_parquet.groupby([\"session\"])[\"ts\"].min()\nts_max = train_parquet.groupby([\"session\"])[\"ts\"].max()\n\nlength_session = (ts_max - ts_min).apply(lambda x: x / (24*3600))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"length_session.describe()[1:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_parquet[train_parquet.session == 10157]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_df = train_parquet[train_parquet.session == 747]\ntmp_df[\"date\"] = tmp_df[\"ts\"].apply(lambda x: datetime.datetime.fromtimestamp(x))\ntmp_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_min = df_train.groupby([\"session\"])[\"ts\"].min()\nts_max = df_train.groupby([\"session\"])[\"ts\"].max()\n\nlength_session = (ts_max - ts_min).apply(lambda x: x / (24*3600))","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train.session == 10157]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}