{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"---\n# [Predict Student Performance from Game Play][1]\n\nThe goal of this competition is to predict student performance during game-based learning in real-time.\n\n---\n#### **The aim of this notebook is to...**\n- **1. devide the data of game play logs on 'level_group' level or 'session_id' level by using clustering methods for categorical features. (Chapter4)**\n- **2. aggregate the game log data and make features for the purpose of clustering (Chapter3)**\n\n**Please note that the aim of this notebook is not just as same as the competition's one. Thus, I will not make predictions for submission in this notebook.**\n\n---\n#### **Results**\n- **1. Data points created by aggregations on 'level_group' level are clearly devided into clusters. (Chapter4.1)**\n- **2. A large part of data points created by aggregations on 'session_id' level are still not clearly devided into clusters, but there might be a specific small group. (Chapter4.3)**\n\n\n---\n**References:** Thanks to previous great codes, blogs and notebooks.\n\n- Japanese tech-blogs.\n - [質的（カテゴリカルな）な特徴量があるときのクラスタリング手法とPythonでの実装について][2]\n - [読了：Ahmad & Khan (2019) 量質混在データのクラスタリング手法レビュー ][3]\n - [Oisixのお客様をクラスタリングしてみた][4]\n\n---\n**If you find this notebook useful, or when you copy&edit this notebook, please give me an upvote. It helps me keep up my motivation.**\n\n---\n[1]: https://www.kaggle.com/competitions/predict-student-performance-from-game-play\n[2]: https://qiita.com/shinji_komine/items/b634146bd873d8d876e5\n[3]: https://elsur.jpn.org/mt/2019/12/002808.html\n[4]: https://creators.oisix.co.jp/entry/2020/06/25/194031#%E3%81%9D%E3%81%AE%E4%BB%96%E6%89%8B%E6%B3%95K-modesK-prototype","metadata":{}},{"cell_type":"markdown","source":"0. TOC
\n\n","metadata":{}},{"cell_type":"markdown","source":"1. Settings
","metadata":{}},{"cell_type":"code","source":"## Parameters\ndata_config = {\n 'train_csv_path': '/kaggle/input/predict-student-performance-from-game-play/train.csv',\n 'train_labels_csv_path': '/kaggle/input/predict-student-performance-from-game-play/train_labels.csv',\n 'test_csv_path': '/kaggle/input/predict-student-performance-from-game-play/test.csv',\n 'sample_submission_path': '/kaggle/input/predict-student-performance-from-game-play/sample_submission.csv',\n}\n\nexp_config = {\n 'sample_limits': 100000, \n 'data_loading_iter': 25,\n 'level_group_n_clusters': 3,\n 'session_id_n_clusters': 5,\n}\n\nprint('Parameters setted!')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:38:34.877256Z","iopub.execute_input":"2023-04-23T08:38:34.877672Z","iopub.status.idle":"2023-04-23T08:38:34.913661Z","shell.execute_reply.started":"2023-04-23T08:38:34.877636Z","shell.execute_reply":"2023-04-23T08:38:34.912603Z"},"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Import dependencies \nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport matplotlib.pyplot as plt \n%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.ticker as ticker\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport os, sys, pathlib, gc\nimport re, math, random, time\nimport datetime as dt\nfrom tqdm import tqdm\nfrom typing import Optional, Union, Tuple\nfrom collections import OrderedDict\n\nimport sklearn\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import OrdinalEncoder\n\nfrom kmodes.kprototypes import KPrototypes\n!pip install gower -q\nimport gower\nfrom sklearn.cluster import AgglomerativeClustering\nfrom scipy.cluster.hierarchy import dendrogram\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nimport umap\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nimport torch \nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint('import done!')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-23T08:43:00.381171Z","iopub.execute_input":"2023-04-23T08:43:00.381572Z","iopub.status.idle":"2023-04-23T08:43:44.975911Z","shell.execute_reply.started":"2023-04-23T08:43:00.381536Z","shell.execute_reply":"2023-04-23T08:43:44.974793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## For reproducible results \ndef seed_all(s):\n random.seed(s)\n np.random.seed(s)\n tf.random.set_seed(s)\n torch.manual_seed(s)\n torch.cuda.manual_seed(s)\n torch.backends.cudnn.deterministic = True\n torch.use_deterministic_algorithms = True\n os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\n os.environ['PYTHONHASHSEED'] = str(s) \n print('Seeds setted!')\nglobal_seed = 42\nseed_all(global_seed)\n\n## Limit GPU Memory in TensorFlow\n## Because TensorFlow, by default, allocates the full amount of available GPU memory when it is launched. \nphysical_devices = tf.config.list_physical_devices('GPU')\nif len(physical_devices) > 0:\n for device in physical_devices:\n tf.config.experimental.set_memory_growth(device, True)\n print('{} memory growth: {}'.format(device, tf.config.experimental.get_memory_growth(device)))\nelse:\n print(\"Not enough GPU hardware devices available\")\n \n## For Seaborn Setting\ncustom_params = {\n \"axes.spines.right\": False,\n \"axes.spines.top\": False,\n 'grid.alpha': 0.3,\n 'figure.figsize': (16, 6),\n 'axes.titlesize': 'Large',\n 'axes.labelsize': 'Large',\n 'figure.facecolor': '#fdfcf6',\n 'axes.facecolor': '#fdfcf6',\n}\ncluster_colors = ['#b4d2b1', '#568f8b', '#1d4a60', '#cd7e59', '#ddb247', '#d15252']\nsns.set_theme(\n style='whitegrid',\n #palette=sns.color_palette(cluster_colors),\n rc=custom_params,)","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-23T08:43:44.977871Z","iopub.execute_input":"2023-04-23T08:43:44.978912Z","iopub.status.idle":"2023-04-23T08:43:45.002298Z","shell.execute_reply.started":"2023-04-23T08:43:44.978873Z","shell.execute_reply":"2023-04-23T08:43:45.001421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. Data Loading
","metadata":{}},{"cell_type":"markdown","source":"---\n### [File and Data Field Descriptions](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/data)\n\n- **train.csv** - the training set\n - `session_id` - the ID of the session the event took place in\n - `index` - the index of the event for the session\n - `elapsed_time` - how much time has passed (in milliseconds) between the start of the session and when the event was recorded\n - `event_name` - the name of the event type\n - `name` - the event name (e.g. identifies whether a notebook_click is is opening or closing the notebook)\n - `level` - what level of the game the event occurred in (0 to 22)\n - `page` - the page number of the event (only for notebook-related events)\n - `room_coor_x` - the coordinates of the click in reference to the in-game room (only for click events)\n - `room_coor_y` - the coordinates of the click in reference to the in-game room (only for click events)\n - `screen_coor_x` - the coordinates of the click in reference to the player’s screen (only for click events)\n - `screen_coor_y` - the coordinates of the click in reference to the player’s screen (only for click events)\n - `hover_duration` - how long (in milliseconds) the hover happened for (only for hover events)\n - `text` - the text the player sees during this event\n - `fqid` - the fully qualified ID of the event\n - `room_fqid` - the fully qualified ID of the room the event took place in\n - `text_fqid` - the fully qualified ID of the\n - `fullscreen` - whether the player is in fullscreen mode\n - `hq` - whether the game is in high-quality\n - `music` - whether the game music is on or off\n - `level_group` - which group of levels - and group of questions - this row belongs to (0-4, 5-12, 13-22)\n\n\n- **test.csv** - the test set\n\n- **sample_submission.csv** - a sample submission file in the correct format\n\n- **train_labels.csv** - `correct` value for all 18 questions for each session in the training set\n\n---\n### [Submission & Evaluation](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/overview/evaluation)\n\n- Submissions will be evaluated based on their F1 score.\n\n---","metadata":{}},{"cell_type":"markdown","source":"---\n**Note**\n- train.csv is too large to load fully on VM. (In this competision's constraint, VMs will have only 2 CPUs, 8GB of RAM, and no GPU available.)\n- Thus, I will load the first 100,000 records for the present. (I will load additional data in later.) \n- P.S. Thanks for [this discussion](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/384359), I added the code for full loading of train.csv. However, the full record of train.csv is too huge to aggregate or compute clustering. Thus, I limited the loading data to a part of it. \n\n---","metadata":{}},{"cell_type":"code","source":"## Data Loading\n\nif exp_config['sample_limits'] is None:\n # Load the whole train.csv\n dtypes={\n 'session_id':'category', \n 'elapsed_time':np.int32,\n 'event_name':'category',\n 'name':'category',\n 'level':np.uint8,\n 'page':'category',\n 'room_coor_x':np.float32,\n 'room_coor_y':np.float32,\n 'screen_coor_x':np.float32,\n 'screen_coor_y':np.float32,\n 'hover_duration':np.float32,\n 'text':'category',\n 'fqid':'category',\n 'room_fqid':'category',\n 'text_fqid':'category',\n 'fullscreen':'category',\n 'hq':'category',\n 'music':'category',\n 'level_group':'category'\n }\n train_df = pd.read_csv(data_config['train_csv_path'], dtype=dtypes)\nelse:\n train_df = pd.read_csv(data_config['train_csv_path'], nrows=exp_config['sample_limits'])\n\ntrain_labels_df = pd.read_csv(data_config['train_labels_csv_path'], nrows=exp_config['sample_limits'])\ntest_df = pd.read_csv(data_config['test_csv_path'])\nsubmission_df = pd.read_csv(data_config['sample_submission_path'])\n\nprint(f'train_length: {len(train_df)}')\nprint(f'train_labels_length: {len(train_labels_df)}')\nprint(f'test_lenght: {len(test_df)}')\nprint(f'submission_length: {len(submission_df)}')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:43:45.003806Z","iopub.execute_input":"2023-04-23T08:43:45.004353Z","iopub.status.idle":"2023-04-23T08:43:45.674009Z","shell.execute_reply.started":"2023-04-23T08:43:45.004318Z","shell.execute_reply":"2023-04-23T08:43:45.672519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Null Value Check\nprint('train_df.info()'); print(train_df.info(), '\\n')\nprint('train_labels_df.info()'); print(train_labels_df.info(), '\\n')\nprint('test_df.info()'); print(test_df.info(), '\\n')\n\n## train_df Check\ntrain_df.head()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-23T08:43:45.675872Z","iopub.execute_input":"2023-04-23T08:43:45.676725Z","iopub.status.idle":"2023-04-23T08:43:45.785336Z","shell.execute_reply.started":"2023-04-23T08:43:45.676686Z","shell.execute_reply":"2023-04-23T08:43:45.784518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unique number of categorical features\nfor feature in train_df.columns:\n if train_df[feature].dtype=='object':\n print(f\"unique_#_of_{feature}: {train_df[feature].nunique()}\")\n\nprint()\n \nfor feature in ['session_id', 'level', 'fullscreen', 'hq', 'music']:\n print(f\"unique_#_of_{feature}: {train_df[feature].nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:43:45.786577Z","iopub.execute_input":"2023-04-23T08:43:45.787100Z","iopub.status.idle":"2023-04-23T08:43:45.845404Z","shell.execute_reply.started":"2023-04-23T08:43:45.787066Z","shell.execute_reply":"2023-04-23T08:43:45.844517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basic statistics of tain_df\ntrain_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:44:50.972204Z","iopub.execute_input":"2023-04-23T08:44:50.972651Z","iopub.status.idle":"2023-04-23T08:44:51.077950Z","shell.execute_reply.started":"2023-04-23T08:44:50.972614Z","shell.execute_reply":"2023-04-23T08:44:51.076193Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unique values of categorical features\nsession_ids = train_df['session_id'].unique()\nlevels = train_df['level'].unique()\nevent_names = train_df['event_name'].unique()\nnames = train_df['name'].unique()\nfqids = train_df['fqid'].unique()\nroom_fqids = train_df['room_fqid'].unique()\ntext_fqids = train_df['text_fqid'].unique()\nlevel_groups = train_df['level_group'].unique() # array(['0-4', '5-12', '13-22'], dtype=object)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:45:42.371067Z","iopub.execute_input":"2023-04-23T08:45:42.371525Z","iopub.status.idle":"2023-04-23T08:45:42.422879Z","shell.execute_reply.started":"2023-04-23T08:45:42.371487Z","shell.execute_reply":"2023-04-23T08:45:42.421808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. Feature Engineering
","metadata":{}},{"cell_type":"markdown","source":"3.1 Aggregate on 'level' levevl
","metadata":{}},{"cell_type":"code","source":"# Features\nlevel_features = [\n 'session_id', \n 'level',\n 'level_elapsed_time', \n 'max_room_coor_x',\n 'min_room_coor_x',\n 'mean_room_coor_x',\n 'std_room_coor_x',\n 'max_room_coor_y',\n 'min_room_coor_y',\n 'mean_room_coor_y',\n 'std_room_coor_y',\n 'fullscreen',\n 'hq',\n 'music',\n 'level_group',\n ]\nevent_flg_features = [f\"{event_name}_flg\" for event_name in event_names]\nfqid_flg_features = [f\"{fqid}_flg\" for fqid in fqids]\nlevel_features.extend(event_flg_features)\nlevel_features.extend(fqid_flg_features)\nprint(len(level_features))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:46:11.706808Z","iopub.execute_input":"2023-04-23T08:46:11.707317Z","iopub.status.idle":"2023-04-23T08:46:11.715844Z","shell.execute_reply.started":"2023-04-23T08:46:11.707270Z","shell.execute_reply":"2023-04-23T08:46:11.714736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This code took some minutes to compute\n\n# Aggregation\ndef aggregate_level_behavior(train_df, level_features):\n agg_level_df = pd.DataFrame(columns=level_features)\n session_ids = train_df['session_id'].unique()\n for session_id in session_ids:\n ex_df = train_df[train_df['session_id']==session_id]\n session_df = pd.DataFrame(columns=level_features)\n\n for i, level in enumerate(levels):\n value_dict = {}\n value_dict['session_id'] = session_id\n value_dict['level'] = level\n ex_level_df = ex_df[ex_df['level']==level]\n if len(ex_level_df) != 0:\n value_dict['level_elapsed_time'] = ex_level_df['elapsed_time'].iloc[-1] - ex_level_df['elapsed_time'].iloc[0]\n value_dict['fullscreen'] = ex_level_df['fullscreen'].mode()[0]\n value_dict['hq'] = ex_level_df['hq'].mode()[0]\n value_dict['music'] = ex_level_df['music'].mode()[0]\n value_dict['level_group'] = ex_level_df['level_group'].iloc[0]\n\n value_dict['max_room_coor_x'] = ex_level_df['room_coor_x'].max()\n value_dict['min_room_coor_x'] = ex_level_df['room_coor_x'].min()\n value_dict['mean_room_coor_x'] = ex_level_df['room_coor_x'].mean()\n value_dict['std_room_coor_x'] = ex_level_df['room_coor_x'].std()\n\n value_dict['max_room_coor_y'] = ex_level_df['room_coor_y'].max()\n value_dict['min_room_coor_y'] = ex_level_df['room_coor_y'].min()\n value_dict['mean_room_coor_y'] = ex_level_df['room_coor_y'].mean()\n value_dict['std_room_coor_y'] = ex_level_df['room_coor_y'].std()\n\n for event_name in event_names:\n tmp_count = ex_level_df[ex_level_df['event_name']==event_name]['event_name'].count()\n if tmp_count > 0:\n value_dict[f\"{event_name}_flg\"] = 1\n else:\n value_dict[f\"{event_name}_flg\"] = 0\n\n for fqid in fqids:\n tmp_count = ex_level_df[ex_level_df['fqid']==fqid]['fqid'].count()\n if tmp_count > 0:\n value_dict[f\"{fqid}_flg\"] = 1\n else:\n value_dict[f\"{fqid}_flg\"] = 0\n\n session_df.loc[i] = value_dict\n agg_level_df = pd.concat([agg_level_df, session_df], axis=0).reset_index(drop=True)\n return agg_level_df\n\n# Operation check\nagg_level_df = aggregate_level_behavior(train_df, level_features)\nprint(agg_level_df.shape)\ndisplay(agg_level_df.describe())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:46:24.586213Z","iopub.execute_input":"2023-04-23T08:46:24.586630Z","iopub.status.idle":"2023-04-23T08:49:20.085234Z","shell.execute_reply.started":"2023-04-23T08:46:24.586587Z","shell.execute_reply":"2023-04-23T08:49:20.084127Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3.2 Aggregate on 'level_group' levevl
","metadata":{}},{"cell_type":"code","source":"# Features\nlevel_group_features = [\n 'session_id', \n 'mean_level_elapsed_time', \n 'max_room_coor_x',\n 'min_room_coor_x',\n 'mean_mean_room_coor_x',\n 'mean_std_room_coor_x',\n 'max_room_coor_y',\n 'min_room_coor_y',\n 'mean_mean_room_coor_y',\n 'mean_std_room_coor_y',\n 'fullscreen',\n 'hq',\n 'music',\n 'level_group',\n ]\nevent_flg_features = [f\"{event_name}_flg\" for event_name in event_names]\nfqid_flg_features = [f\"{fqid}_flg\" for fqid in fqids]\nlevel_group_features.extend(event_flg_features)\nlevel_group_features.extend(fqid_flg_features)\nprint(len(level_group_features))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:49:34.694195Z","iopub.execute_input":"2023-04-23T08:49:34.694649Z","iopub.status.idle":"2023-04-23T08:49:34.702721Z","shell.execute_reply.started":"2023-04-23T08:49:34.694610Z","shell.execute_reply":"2023-04-23T08:49:34.701514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aggregation\ndef aggregate_level_group_behavior(agg_level_df, level_group_features):\n agg_group_df = pd.DataFrame(columns=level_group_features)\n session_ids = agg_level_df['session_id'].unique()\n for session_id in session_ids:\n ex_df = agg_level_df[agg_level_df['session_id']==session_id]\n session_df = pd.DataFrame(columns=level_group_features)\n\n for i, level_group in enumerate(level_groups):\n value_dict = {}\n value_dict['session_id'] = session_id\n value_dict['level_group'] = level_group\n ex_group_df = ex_df[ex_df['level_group']==level_group]\n if len(ex_group_df) != 0:\n value_dict['mean_level_elapsed_time'] = ex_group_df['level_elapsed_time'].mean()\n value_dict['fullscreen'] = ex_group_df['fullscreen'].mode()[0]\n value_dict['hq'] = ex_group_df['hq'].mode()[0]\n value_dict['music'] = ex_group_df['music'].mode()[0]\n\n value_dict['max_room_coor_x'] = ex_group_df['max_room_coor_x'].max()\n value_dict['min_room_coor_x'] = ex_group_df['min_room_coor_x'].min()\n value_dict['mean_mean_room_coor_x'] = ex_group_df['mean_room_coor_x'].mean()\n value_dict['mean_std_room_coor_x'] = ex_group_df['std_room_coor_x'].mean()\n\n value_dict['max_room_coor_y'] = ex_group_df['max_room_coor_y'].max()\n value_dict['min_room_coor_y'] = ex_group_df['min_room_coor_y'].min()\n value_dict['mean_mean_room_coor_y'] = ex_group_df['mean_room_coor_y'].mean()\n value_dict['mean_std_room_coor_y'] = ex_group_df['std_room_coor_y'].mean()\n\n for event_name in event_names:\n tmp_count = ex_group_df[f'{event_name}_flg'].sum()\n if tmp_count > 0:\n value_dict[f\"{event_name}_flg\"] = 1\n else:\n value_dict[f\"{event_name}_flg\"] = 0\n\n for fqid in fqids:\n tmp_count = ex_group_df[f'{fqid}_flg'].sum()\n if tmp_count > 0:\n value_dict[f\"{fqid}_flg\"] = 1\n else:\n value_dict[f\"{fqid}_flg\"] = 0\n\n else:\n empty_group_features = [\n #'session_id', \n 'mean_level_elapsed_time', \n 'max_room_coor_x',\n 'min_room_coor_x',\n 'mean_mean_room_coor_x',\n 'mean_std_room_coor_x',\n 'max_room_coor_y',\n 'min_room_coor_y',\n 'mean_mean_room_coor_y',\n 'mean_std_room_coor_y',\n 'fullscreen',\n 'hq',\n 'music',\n #'level_group',\n ]\n event_flg_features = [f\"{event_name}_flg\" for event_name in event_names]\n fqid_flg_features = [f\"{fqid}_flg\" for fqid in fqids]\n empty_group_features.extend(event_flg_features)\n empty_group_features.extend(fqid_flg_features)\n for feature in empty_group_features:\n value_dict[feature] = np.nan\n\n session_df.loc[i] = value_dict\n agg_group_df = pd.concat([agg_group_df, session_df], axis=0).reset_index(drop=True)\n return agg_group_df\n \n# Operation check\nagg_group_df = aggregate_level_group_behavior(agg_level_df, level_group_features)\nprint(agg_group_df.shape)\ndisplay(agg_group_df.describe())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:49:37.362483Z","iopub.execute_input":"2023-04-23T08:49:37.362889Z","iopub.status.idle":"2023-04-23T08:49:47.659698Z","shell.execute_reply.started":"2023-04-23T08:49:37.362853Z","shell.execute_reply":"2023-04-23T08:49:47.658516Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3.3 Load and aggregate additional data
","metadata":{}},{"cell_type":"code","source":"sample_limits = exp_config['sample_limits']\ndata_loading_iter = exp_config['data_loading_iter']\ntrain_df_columns = train_df.columns\n\nall_agg_group_df = pd.DataFrame(columns=level_group_features)\n\n# The data on the edge is excluded from the aggregation.\ngap_session_ids = []\n\nif exp_config['sample_limits'] is not None:\n if data_loading_iter > 1:\n for i in range(data_loading_iter): \n train_df = pd.read_csv(data_config['train_csv_path'],\n skiprows=sample_limits*i,\n nrows=sample_limits,\n header=None,\n )\n train_df.columns = train_df_columns\n\n agg_level_df = aggregate_level_behavior(train_df, level_features)\n agg_df = aggregate_level_group_behavior(agg_level_df, level_group_features)\n\n gap_session_ids.append(agg_df['session_id'].iloc[-1])\n all_agg_group_df = pd.concat([all_agg_group_df, agg_df], axis=0).reset_index(drop=True)\n\n # The data on the edge is excluded from the aggregation.\n all_agg_group_df = all_agg_group_df[~all_agg_group_df['session_id'].isin(gap_session_ids)]\n all_agg_group_df = all_agg_group_df.reset_index(drop=True)\n\n else:\n all_agg_group_df = agg_group_df\nelse:\n all_agg_group_df = agg_group_df\n \nprint(all_agg_group_df.shape)\ndisplay(all_agg_group_df.describe())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:50:32.080565Z","iopub.execute_input":"2023-04-23T08:50:32.081060Z","iopub.status.idle":"2023-04-23T08:50:32.139524Z","shell.execute_reply.started":"2023-04-23T08:50:32.081019Z","shell.execute_reply":"2023-04-23T08:50:32.138535Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3.4 Aggregate on 'session_id' levevl
","metadata":{}},{"cell_type":"code","source":"# Features\nsession_id_features = [\n 'session_id', \n ]\nquality_features = [\n 'fullscreen',\n 'hq',\n 'music',\n ]\nlevel_group_quality_features = \\\n [f\"{level_group}_{quality}\" for quality in quality_features for level_group in level_groups]\ncoor_x_features = [\n 'max_room_coor_x',\n 'min_room_coor_x',\n 'mean_mean_room_coor_x',\n 'mean_std_room_coor_x',\n ]\nlevel_group_coor_x_features = \\\n [f\"{level_group}_{coor_x}\" for coor_x in coor_x_features for level_group in level_groups]\ncoor_y_features = [\n 'max_room_coor_y',\n 'min_room_coor_y',\n 'mean_mean_room_coor_y',\n 'mean_std_room_coor_y',\n ]\nlevel_group_coor_y_features = \\\n [f\"{level_group}_{coor_y}\" for coor_y in coor_y_features for level_group in level_groups]\nlevel_group_elapsed_time_features = \\\n [f\"{level_group}_mean_level_elapsed_time\" for level_group in level_groups]\nlevel_group_event_flg_features = \\\n [f\"{level_group}_{event_name}_flg\" for event_name in event_names for level_group in level_groups]\nlevel_group_fqid_flg_features = \\\n [f\"{level_group}_{fqid}_flg\" for fqid in fqids for level_group in level_groups]\nsession_id_features.extend(level_group_quality_features)\nsession_id_features.extend(level_group_coor_x_features)\nsession_id_features.extend(level_group_coor_y_features)\nsession_id_features.extend(level_group_elapsed_time_features)\nsession_id_features.extend(level_group_event_flg_features)\nsession_id_features.extend(level_group_fqid_flg_features)\nprint(len(session_id_features))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:50:58.245406Z","iopub.execute_input":"2023-04-23T08:50:58.245831Z","iopub.status.idle":"2023-04-23T08:50:58.257089Z","shell.execute_reply.started":"2023-04-23T08:50:58.245794Z","shell.execute_reply":"2023-04-23T08:50:58.255734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aggregation\ndef aggregate_session_behavior(\n agg_level_group_df,\n session_id_features,\n level_group_features,\n level_groups=level_groups,\n ):\n agg_session_df = pd.DataFrame(columns=session_id_features)\n level_group_features_copy = level_group_features.copy()\n level_group_features_copy.remove('session_id')\n session_ids = agg_level_group_df['session_id'].unique()\n for i, session_id in enumerate(session_ids):\n ex_df = agg_level_group_df[agg_level_group_df['session_id']==session_id].reset_index(drop=True)\n value_dict = {}\n value_dict['session_id'] = session_id\n if len(ex_df) > 0:\n for level_group in level_groups:\n level_group_df = ex_df[ex_df['level_group']==level_group].reset_index(drop=True)\n if len(level_group_df) > 0:\n for level_group_feature in level_group_features_copy:\n value_dict[f\"{level_group}_{level_group_feature}\"] = \\\n level_group_df[level_group_feature].iloc[0]\n else:\n for level_group in level_groups:\n for level_group_feature in level_group_features_copy:\n value_dict[f\"{level_group}_{level_group_feature}\"] = np.nan\n\n agg_session_df.loc[i] = value_dict\n agg_session_df = agg_session_df.reset_index(drop=True)\n return agg_session_df\n\n# Operation check\nagg_session_df = aggregate_session_behavior(\n all_agg_group_df, session_id_features, level_group_features)\nprint(agg_session_df.shape)\ndisplay(agg_session_df.describe())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:51:07.691660Z","iopub.execute_input":"2023-04-23T08:51:07.692070Z","iopub.status.idle":"2023-04-23T08:51:13.213982Z","shell.execute_reply.started":"2023-04-23T08:51:07.692036Z","shell.execute_reply":"2023-04-23T08:51:13.212996Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. Clustering
","metadata":{}},{"cell_type":"markdown","source":"---\n## How to clustering the data points including both numerical and categorical features.\n\n---\n#### 【Gower’s Distance】\n**Calculation** \nGower’s Distance is calculated based on Gower's similarity score. \nGower's similarity score represents the similarity between two data points, $i$ and $j$. \nPlease note that the larger gower's similarity score means that the two samples are more similar. \nGower's similarity $S_{\\text{Gower}}(x_i, x_j)$ is caluculated as follows: \n\n- for each feature $k$, \n\n $S_{ij} = \\frac{\\sum_k s_{ijk}}{\\sum_k \\delta_{ijk}}$\n\n How to calculate the $s_{ijk}$ and $\\delta_{ijk}$ is as follows:\n \n - $s_{ijk}$ : \n - for numerical features -> $s_{ijk} = 1 - \\frac{\\|x_i-x_j\\|}{R_k}$ \n - for binary features -> 1 if both are 1, else 0 \n - for categorical features -> 1 if $x_{ij} = x_{jk}$, 0 if $x_{ij} \\neq x_{jk}$ \n - note: $R_k$ represents the range of the values of feature $k$ (max - min)\n \n - $\\delta_{ijk}$ : \n - for numerical features -> 1 \n - for binary features -> 0 if both are 0, else 1 \n - for categorical features -> 1 \n \n
\n \n - note1: for binary features -> It means that 0-0 is not treated as 'match' case \n - note2: for categorical features -> It is similar to Hamming distance or Sørensen–Dice coefficient \n - note3: When the valuable has hierarchical structure, you can (somehow) allocate the weights $w_k$ and calculate as follows: \n $S_{ij} = \\frac{\\sum_k s_{ijk} w_k}{\\sum_k \\delta_{ijk} w_k}$ \n \nNow, Gower's distance $S_{\\text{Gower}}(x_i, x_j)$ is caluculated as follows: \n\n$ d_{\\text{Gower}} = \\sqrt{1-S_{\\text{Gower}}} $\n \n
\n\n**Library** : gower\n ```python\n pip install gower\n ```\n
\n\n---\n#### 【K-modes】\n**Calculation** \nK-modes is the application of K-means to categorical features. \nThe distance of two data points $d_{ij}$ are calculated as follows:\n\n- When $x$ contains $p$-dimentional numerical features and $m$-dimentional categorical features, \n\n $d_{ij} = \\sum^p_{k=1}(x_{ik} - x_{jk})^2 + \\gamma \\sum^m_{k=p+1} \\delta_{ijk}$ \n $\\delta_{ijk} = 0 \\, \\text{if} \\, x_{ik} = x_{jk} \\, , \\text{else} \\, 1$ \n\n - note: $\\gamma$ is the parameter which controls the weight balances between numerical and categorical features. \n \n\n
\n\n**Library** : kmodes\n ```python\n pip install kmodes\n ``` \n- note1: The parameter $\\gamma$ is automatically computed based on the variances of the categorical features by default. \n- note2: We can use K-prototype algorithm which combines K-means and K-modes for the clustering based on both numerical and categorical features.\n\n
\n\n---","metadata":{}},{"cell_type":"markdown","source":"4.1 Clustering on 'level_group' features
","metadata":{}},{"cell_type":"code","source":"# Features\ncategorical_features = [ \n 'fullscreen',\n 'hq',\n 'music',\n 'level_group'\n ]\ncategorical_features.extend(event_flg_features)\ncategorical_features.extend(fqid_flg_features)\n# Of course, I will not use 'level_group' as a feature for clustering.\n\nnumerical_features = [\n 'mean_level_elapsed_time', \n 'max_room_coor_x',\n 'min_room_coor_x',\n 'mean_mean_room_coor_x',\n 'mean_std_room_coor_x',\n 'max_room_coor_y',\n 'min_room_coor_y',\n 'mean_mean_room_coor_y',\n 'mean_std_room_coor_y',\n ]\n \nprint(len(numerical_features), len(categorical_features))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:52:07.935680Z","iopub.execute_input":"2023-04-23T08:52:07.936116Z","iopub.status.idle":"2023-04-23T08:52:07.944305Z","shell.execute_reply.started":"2023-04-23T08:52:07.936052Z","shell.execute_reply":"2023-04-23T08:52:07.942995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The number of clusters\nn_clusters = exp_config['level_group_n_clusters']\n\n# Data preprocessing\ndef data_preprocessing(\n df, numerical_features, categorical_features):\n\n X_df = df.copy()\n\n # Drop the records which contain null values in numerical features\n X_df[numerical_features] = X_df[numerical_features].astype('float64')\n X_df.dropna(subset=numerical_features, axis=0, inplace=True)\n X_df = X_df.reset_index(drop=True)\n\n # Numerical and categorical features for clustering\n X_numerical = X_df[numerical_features]\n X_categorical = X_df[categorical_features]\n\n # Standardization of numerical features\n stdscl = StandardScaler()\n X_numerical_scl = pd.DataFrame(stdscl.fit_transform(X_numerical) , columns=X_numerical.columns)\n\n # Standardization of categorical features (convert to str type)\n X_categorical_scl = X_categorical.astype(str)\n\n # Preprocessed features\n X_scl = pd.concat([X_numerical_scl, X_categorical_scl], axis=1, join='inner')\n \n return X_scl\n \n# Operation check\nX_scl = data_preprocessing(\n all_agg_group_df, \n numerical_features=numerical_features, \n categorical_features=categorical_features)\nprint(X_scl.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:52:19.834389Z","iopub.execute_input":"2023-04-23T08:52:19.835151Z","iopub.status.idle":"2023-04-23T08:52:19.865615Z","shell.execute_reply.started":"2023-04-23T08:52:19.835110Z","shell.execute_reply":"2023-04-23T08:52:19.864701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\nClustering by K-Prototype method\n\n---","metadata":{}},{"cell_type":"code","source":"# Positions of categorical columns (this is nedded as a parameter of K-Prototype) \ncatColumnsPos = [X_scl.columns.get_loc(col) for col in list(X_scl.select_dtypes('object').columns)]\n\n# Execution of clustering\nkprototype = KPrototypes(n_jobs=-1, n_clusters=n_clusters, init='Huang')\nkprototype.fit_predict(X_scl, categorical=catColumnsPos)\nall_agg_group_df['k_proto_label'] = kprototype.labels_\ndisplay(all_agg_group_df.groupby('k_proto_label')['session_id'].count())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:52:41.423679Z","iopub.execute_input":"2023-04-23T08:52:41.424098Z","iopub.status.idle":"2023-04-23T08:52:48.729221Z","shell.execute_reply.started":"2023-04-23T08:52:41.424049Z","shell.execute_reply":"2023-04-23T08:52:48.728038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Elbow method by K-Prototype\ndef plot_elbow(X_scl, n_max=8):\n catColumnsPos = [ X_scl.columns.get_loc(col) for col in list(X_scl.select_dtypes('object').columns) ]\n n_clusters_list = []\n cost_list = []\n for n in tqdm(range(1, n_max+1)):\n try:\n kprototype = KPrototypes(n_jobs=-1, n_clusters=n, init='Huang')\n kprototype.fit_predict(X_scl, categorical=catColumnsPos)\n n_clusters_list.append(n)\n cost_list.append(kprototype.cost_)\n except:\n break\n plt.plot(n_clusters_list, cost_list)\n plt.xlabel('n_clusters')\n plt.ylabel('cost')\n return \n\n# Operation check\nplot_elbow(X_scl, n_max=6)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:55:51.965687Z","iopub.execute_input":"2023-04-23T08:55:51.966103Z","iopub.status.idle":"2023-04-23T08:56:57.909946Z","shell.execute_reply.started":"2023-04-23T08:55:51.966057Z","shell.execute_reply":"2023-04-23T08:56:57.908824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n**Results:**\n- n_clusters should be 3.\n\n---","metadata":{}},{"cell_type":"markdown","source":"---\nClustering based on gower's distance\n\n---","metadata":{}},{"cell_type":"code","source":"# Computation of gower's distance\nX_distance = gower.gower_matrix(X_scl)\nprint(X_distance.shape)\n\n# Hierarchical clustering\nAggCluster = AgglomerativeClustering(\n n_clusters=n_clusters, linkage='complete',\n affinity='precomputed', compute_distances=True)\nAggCluster.fit(X_distance)\nall_agg_group_df['gower_label'] = AggCluster.fit_predict(X_distance) \ndisplay(all_agg_group_df.groupby('gower_label')['session_id'].count())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:57:17.948578Z","iopub.execute_input":"2023-04-23T08:57:17.948962Z","iopub.status.idle":"2023-04-23T08:57:18.893186Z","shell.execute_reply.started":"2023-04-23T08:57:17.948929Z","shell.execute_reply":"2023-04-23T08:57:18.892001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The function for the execution of clustering on both numerical and categorical features\n# This fuction executes the clustering based on gower's distance by default\ndef make_clustering_labels(df,\n numerical_features,\n categorical_features,\n n_clusters=5,\n K_proto_flg=False, \n gower_flg=True, \n verbose=True):\n X_df = df.copy()\n X_df[numerical_features] = X_df[numerical_features].astype('float')\n X_df[categorical_features] =X_df[categorical_features].astype('str')\n \n if verbose:\n print(f'------ before dropna for numerical features ------: \\n{X_df.shape}')\n X_df.dropna(subset=numerical_features, inplace=True)\n X_df = X_df.reset_index(drop=True)\n if verbose:\n print(f'------ after dropna for numerical features ------: \\n{X_df.shape}')\n\n X_numerical = X_df[numerical_features]\n X_categorical = X_df[categorical_features]\n \n stdscl = StandardScaler()\n X_numerical_scl = pd.DataFrame(stdscl.fit_transform(X_numerical) , columns=X_numerical.columns)\n X_categorical_scl = X_categorical.astype(str)\n\n if verbose:\n for categorical_feature in categorical_features:\n print('------ unique numbers for categorical features ------')\n print(f\"{categorical_feature} unique_number: {X_df[categorical_feature].nunique()}\")\n\n X_scl =pd.concat([X_numerical_scl, X_categorical_scl], axis=1, join='inner')\n\n if K_proto_flg: # Clustering by K-Prototype method\n catColumnsPos = [X_scl.columns.get_loc(col) for col in list(X_scl.select_dtypes('object').columns)]\n kprototype = KPrototypes(n_jobs=-1, n_clusters=n_clusters, init='Huang', random_state=0)\n kprototype.fit_predict(X_scl, categorical=catColumnsPos)\n X_df['k_proto_label'] = kprototype.labels_\n\n if gower_flg: # Clustering based on gower's distance\n X_distance = gower.gower_matrix(X_scl)\n AggCluster = AgglomerativeClustering(n_clusters=n_clusters, linkage='complete',\n affinity='precomputed', compute_distances=True)\n AggCluster.fit(X_distance)\n X_df['gower_label'] = AggCluster.fit_predict(X_distance)\n return X_df, X_distance\n \n return X_df\n\n# Operation check\nall_agg_group_df, X_distance = make_clustering_labels(\n all_agg_group_df,\n numerical_features=numerical_features, \n categorical_features=categorical_features, \n n_clusters=n_clusters,\n verbose=False,\n )\ndisplay(all_agg_group_df.groupby('gower_label')['session_id'].count())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:57:49.839568Z","iopub.execute_input":"2023-04-23T08:57:49.839971Z","iopub.status.idle":"2023-04-23T08:57:50.787609Z","shell.execute_reply.started":"2023-04-23T08:57:49.839937Z","shell.execute_reply":"2023-04-23T08:57:50.786718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\nDimensionality reduction with PCA, t-SNE and UMAP\n\n---","metadata":{}},{"cell_type":"code","source":"# Dimensionality reduction with PCA\npca = PCA()\npca.fit(X_distance)\nX_distance_pca = pca.transform(X_distance)\n#print(X_distance.shape)\n\n# Visualization of the variances\n#var = pd.DataFrame(X_distance_pca).var()\n#plt.plot(np.arange(0,len(X_distance_pca)), var)\n#plt.show()\n#print(pca.explained_variance_ratio_)\n\n# Plot in a 2D graph\nplt.figure(figsize=(4,4))\nplt.scatter(X_distance_pca[:,0],X_distance_pca[:,1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:58:35.352627Z","iopub.execute_input":"2023-04-23T08:58:35.353021Z","iopub.status.idle":"2023-04-23T08:58:35.548759Z","shell.execute_reply.started":"2023-04-23T08:58:35.352987Z","shell.execute_reply":"2023-04-23T08:58:35.547817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with t-SNE\ntsne = TSNE(random_state=0)\nX_distance_tsne = tsne.fit_transform(X_distance)\n\n# Plot in a 2D graph\nplt.figure(figsize=(4,4))\nplt.scatter(X_distance_tsne[:,0],X_distance_tsne[:,1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:58:52.039730Z","iopub.execute_input":"2023-04-23T08:58:52.040122Z","iopub.status.idle":"2023-04-23T08:58:53.449733Z","shell.execute_reply.started":"2023-04-23T08:58:52.040087Z","shell.execute_reply":"2023-04-23T08:58:53.448488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with UMAP\nmapper = umap.UMAP(random_state=0)\nX_distance_umap = mapper.fit_transform(X_distance)\n\n# Plot in a 2D graph\nplt.figure(figsize=(4,4))\nplt.scatter(X_distance_umap[:,0],X_distance_umap[:,1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:59:00.929170Z","iopub.execute_input":"2023-04-23T08:59:00.929597Z","iopub.status.idle":"2023-04-23T08:59:09.929180Z","shell.execute_reply.started":"2023-04-23T08:59:00.929560Z","shell.execute_reply":"2023-04-23T08:59:09.926800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Color-coded display according to a categorical feature.\ndef plot_category(X, X_distance, label_feature, figsize=(4,4), mapper='umap'):\n if mapper == 'tsne':\n mapper = TSNE(random_state=0)\n elif mapper == 'pca':\n mapper = PCA()\n else:\n mapper = umap.UMAP(random_state=0)\n X_distance_emb = mapper.fit_transform(X_distance)\n cmap_keyword = \"jet\"\n cmap = plt.get_cmap(cmap_keyword)\n plt.figure(figsize=figsize)\n labels = list(X[label_feature].unique())\n labels = sorted(labels)\n for idx, label in enumerate(labels):\n condition = X[label_feature] == label\n c = cmap(idx/(X[label_feature].nunique()-1))\n plt.scatter(X_distance_emb[condition,0],X_distance_emb[condition,1], color=c, label=label)\n plt.title(label_feature)\n plt.legend(loc='upper left', bbox_to_anchor=(1, 1))\n plt.show()\n \n# Operation check\nplot_category(all_agg_group_df, X_distance,\n 'gower_label', mapper='umap')\nplot_category(all_agg_group_df, X_distance,\n 'k_proto_label', mapper='umap')\nplot_category(all_agg_group_df, X_distance,\n 'level_group', mapper='umap')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:59:12.438239Z","iopub.execute_input":"2023-04-23T08:59:12.438672Z","iopub.status.idle":"2023-04-23T08:59:18.012987Z","shell.execute_reply.started":"2023-04-23T08:59:12.438635Z","shell.execute_reply":"2023-04-23T08:59:18.011728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n**Results:**\n- Data points created by aggregations on 'level_group' are clearly devided by level_groups. \n- Please note that I didn't use 'level_group' as a feature for clustering. \n- It means that there are distinct differences in user experiences by level_groups.\n\n---","metadata":{}},{"cell_type":"code","source":"# Display basic statistics on a numerical feature\ndef make_numerical_table(\n cluster_df, \n numerical_feature,\n label_feature='gower_label',\n label_groups=None):\n \n value_list = []\n if label_groups is None:\n label_groups = list(cluster_df.groupby(label_feature)[numerical_feature].groups.keys())\n label_groups = sorted(label_groups)\n\n for group in label_groups:\n value_list.append(cluster_df.groupby(label_feature)[numerical_feature] \\\n .get_group(group).describe())\n value_df = pd.concat(value_list, axis=1, join='inner')\n value_df.columns = label_groups\n return value_df \n# Operation check\n#value_df = make_numerical_table(all_agg_group_df, 'mean_level_elapsed_time', )\n#display(value_df)\n\n# Display specified statistics on numerical features in clusters\ndef make_stats_numerical(\n df,\n numerical_features,\n stats='mean',\n label_feature='gower_label'):\n \n label_groups = list(df.groupby(label_feature)[numerical_features[0]].groups.keys())\n label_groups = sorted(label_groups)\n stats_df = pd.DataFrame(columns=label_groups)\n\n for numerical_feature in numerical_features:\n value_df = make_numerical_table(df, \n numerical_feature,\n label_feature,\n label_groups=label_groups)\n stats_df.loc[numerical_feature] = value_df.loc[stats]\n stats_df.columns = label_groups\n return stats_df\n\n# Operation check\nmean_df = make_stats_numerical(\n all_agg_group_df,\n numerical_features=numerical_features,\n stats='mean',\n)\nprint('mean: ')\ndisplay(mean_df)\n\nstd_df = make_stats_numerical(\n all_agg_group_df,\n numerical_features=numerical_features,\n stats='std',\n)\nprint('std: ')\ndisplay(std_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:00:39.811019Z","iopub.execute_input":"2023-04-23T09:00:39.811561Z","iopub.status.idle":"2023-04-23T09:00:40.049244Z","shell.execute_reply.started":"2023-04-23T09:00:39.811515Z","shell.execute_reply":"2023-04-23T09:00:40.048043Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aggregation of common items in a categorical feature\ndef make_categorical_table(\n cluster_df, \n categorical_feature,\n label_feature='gower_label', \n join='inner'):\n value_list = []\n label_groups = list(cluster_df.groupby(label_feature)[categorical_feature].groups.keys())\n label_groups = sorted(label_groups)\n for group in label_groups:\n value_list.append(cluster_df.groupby(label_feature)[categorical_feature] \\\n .get_group(group).value_counts(sort=False, normalize=True))\n value_df = pd.concat(value_list, axis=1, join=join)\n value_df.columns = label_groups\n return value_df \n# Operation check\n#value_df = make_categorical_table(all_agg_group_df, 'fullscreen', join='inner')\n#display(value_df)\n\n# Display ratios of common items in each categorical feature as a bar plot\ndef plot_bar_category(n_rows, n_cols,\n df, categorical_cols,\n label_feature='gower_label', color_set=False,\n figsize=(4, 4)):\n figsize = tuple([l*x for l, x in zip(figsize, [n_cols, n_rows])])\n fig, axes = plt.subplots(n_rows, n_cols,\n figsize=figsize,\n tight_layout=True,\n sharey=True)\n if color_set:\n cmap_keyword = \"jet\"\n cmap = plt.get_cmap(cmap_keyword)\n\n for i in range(n_rows):\n for j in range(n_cols):\n idx = (i * n_cols) + j\n if idx >= len(categorical_cols):\n break\n feature = categorical_cols[idx]\n # Aggregation of categorical features\n value_list = []\n label_groups = list(df.groupby(label_feature)[feature].groups.keys())\n label_groups = sorted(label_groups)\n for group in label_groups:\n value_list.append(df.groupby(label_feature)[feature] \\\n .get_group(group).value_counts(sort=False, normalize=True))\n value_df = pd.concat(value_list, axis=1, join='inner')\n value_df.columns = label_groups\n \n # display\n x = np.arange(len(value_df))\n labels = value_df.index\n margin = 0.2 #0 4.2 Clustering on 'session_id' features
","metadata":{}},{"cell_type":"code","source":"# Features\nnumerical_features = []\nnumerical_features.extend(level_group_coor_x_features)\nnumerical_features.extend(level_group_coor_y_features)\nnumerical_features.extend(level_group_elapsed_time_features)\n\ncategorical_features = []\ncategorical_features.extend(level_group_quality_features)\ncategorical_features.extend(level_group_event_flg_features)\ncategorical_features.extend(level_group_fqid_flg_features)\n\nprint(len(numerical_features), len(categorical_features))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:01:22.872869Z","iopub.execute_input":"2023-04-23T09:01:22.873301Z","iopub.status.idle":"2023-04-23T09:01:22.880841Z","shell.execute_reply.started":"2023-04-23T09:01:22.873265Z","shell.execute_reply":"2023-04-23T09:01:22.879681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The number of clusters\nn_clusters = exp_config['session_id_n_clusters']\n\n# Data preprocessing\nX_scl = data_preprocessing(\n agg_session_df, \n numerical_features=numerical_features, \n categorical_features=categorical_features)\nprint(X_scl.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:01:30.822019Z","iopub.execute_input":"2023-04-23T09:01:30.822420Z","iopub.status.idle":"2023-04-23T09:01:30.855414Z","shell.execute_reply.started":"2023-04-23T09:01:30.822384Z","shell.execute_reply":"2023-04-23T09:01:30.854400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clustering by K-Prototype method\nagg_session_df = make_clustering_labels(\n agg_session_df,\n numerical_features=numerical_features, \n categorical_features=categorical_features, \n n_clusters=n_clusters,\n K_proto_flg=True, \n gower_flg=False,\n verbose=False,\n )\nprint('Clustering by K-Prototype method')\ndisplay(agg_session_df.groupby('k_proto_label')['session_id'].count())\n\nprint()\n\n# Clustering based on gower's distance\nagg_session_df, X_distance = make_clustering_labels(\n agg_session_df,\n numerical_features=numerical_features, \n categorical_features=categorical_features, \n n_clusters=n_clusters,\n verbose=False,\n )\nprint('Clustering by K-Prototype method')\ndisplay(agg_session_df.groupby('gower_label')['session_id'].count())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:01:40.559384Z","iopub.execute_input":"2023-04-23T09:01:40.559797Z","iopub.status.idle":"2023-04-23T09:01:53.522038Z","shell.execute_reply.started":"2023-04-23T09:01:40.559759Z","shell.execute_reply":"2023-04-23T09:01:53.521002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display means and stds on numerical features in clusters\nmean_df = make_stats_numerical(\n agg_session_df,\n numerical_features=numerical_features,\n stats='mean',\n)\nprint('mean: ')\ndisplay(mean_df)\n\nstd_df = make_stats_numerical(\n agg_session_df,\n numerical_features=numerical_features,\n stats='std',\n)\nprint('std: ')\ndisplay(std_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:02:34.688600Z","iopub.execute_input":"2023-04-23T09:02:34.689918Z","iopub.status.idle":"2023-04-23T09:02:35.258251Z","shell.execute_reply.started":"2023-04-23T09:02:34.689867Z","shell.execute_reply":"2023-04-23T09:02:35.257183Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with PCA\nplot_category(agg_session_df, X_distance,\n 'gower_label', mapper='pca')\n\nplot_category(agg_session_df, X_distance,\n 'k_proto_label', mapper='pca')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:04:29.378175Z","iopub.execute_input":"2023-04-23T09:04:29.378644Z","iopub.status.idle":"2023-04-23T09:04:29.962612Z","shell.execute_reply.started":"2023-04-23T09:04:29.378606Z","shell.execute_reply":"2023-04-23T09:04:29.961647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with t-SNE\nplot_category(agg_session_df, X_distance,\n 'gower_label', mapper='tsne')\n\nplot_category(agg_session_df, X_distance,\n 'k_proto_label', mapper='tsne')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:04:39.683178Z","iopub.execute_input":"2023-04-23T09:04:39.683614Z","iopub.status.idle":"2023-04-23T09:04:41.292211Z","shell.execute_reply.started":"2023-04-23T09:04:39.683577Z","shell.execute_reply":"2023-04-23T09:04:41.291121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with UMAP\nplot_category(agg_session_df, X_distance,\n 'gower_label', mapper='umap')\nplot_category(agg_session_df, X_distance,\n 'k_proto_label', mapper='umap')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:04:48.502697Z","iopub.execute_input":"2023-04-23T09:04:48.503120Z","iopub.status.idle":"2023-04-23T09:04:51.371128Z","shell.execute_reply.started":"2023-04-23T09:04:48.503083Z","shell.execute_reply":"2023-04-23T09:04:51.369940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n**Results:** \n- Data points created by aggregations on 'session_id' level are not clearly devided into clusters.\n\n---","metadata":{}},{"cell_type":"markdown","source":"4.3 Convert categorical features from sparse matrix to dense matrix
","metadata":{}},{"cell_type":"code","source":"# Dimensionality reduction on categorical features with SVD\nsvd = sklearn.decomposition.TruncatedSVD(n_components=50)\ndecomposed_features = svd.fit_transform(agg_session_df[categorical_features])\nX_num_cat_decompose = np.concatenate([X_scl[numerical_features], decomposed_features], axis=1)\n\n# Data preprocessing\nstdscl = StandardScaler()\nX_num_cat_decompose_scl = stdscl.fit_transform(X_num_cat_decompose)\n\n# Clustering by K-means\nkmeans = sklearn.cluster.KMeans(n_clusters=n_clusters)\npred = kmeans.fit_predict(X_num_cat_decompose_scl)\nagg_session_df['decomposed_kmeans_label'] = pred\n\ndisplay(agg_session_df.groupby('decomposed_kmeans_label')['session_id'].count())","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:05:51.374193Z","iopub.execute_input":"2023-04-23T09:05:51.374741Z","iopub.status.idle":"2023-04-23T09:05:51.458152Z","shell.execute_reply.started":"2023-04-23T09:05:51.374698Z","shell.execute_reply":"2023-04-23T09:05:51.457120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with PCA\nplot_category(agg_session_df, X_num_cat_decompose_scl,\n 'decomposed_kmeans_label', mapper='pca')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:06:04.422739Z","iopub.execute_input":"2023-04-23T09:06:04.423135Z","iopub.status.idle":"2023-04-23T09:06:04.675727Z","shell.execute_reply.started":"2023-04-23T09:06:04.423097Z","shell.execute_reply":"2023-04-23T09:06:04.674688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with t-SNE\nplot_category(agg_session_df, X_num_cat_decompose_scl,\n 'decomposed_kmeans_label', mapper='tsne')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:06:13.404746Z","iopub.execute_input":"2023-04-23T09:06:13.405175Z","iopub.status.idle":"2023-04-23T09:06:14.336501Z","shell.execute_reply.started":"2023-04-23T09:06:13.405137Z","shell.execute_reply":"2023-04-23T09:06:14.335405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensionality reduction with UMAP\nplot_category(agg_session_df, X_num_cat_decompose_scl,\n 'decomposed_kmeans_label', mapper='umap')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T09:06:23.392275Z","iopub.execute_input":"2023-04-23T09:06:23.393407Z","iopub.status.idle":"2023-04-23T09:06:24.843960Z","shell.execute_reply.started":"2023-04-23T09:06:23.393359Z","shell.execute_reply":"2023-04-23T09:06:24.842899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n**Results:** \n- A large part of data points created by aggregations on 'session_id' level are still not clearly devided into clusters, but there might be a specific small group.\n\n---","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}