{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np, pandas as pd, matplotlib.pyplot as plt, matplotlib.image as mpimg, seaborn as sns, gc, warnings\nfrom wordcloud import WordCloud\nsns.set(style=\"darkgrid\", color_codes=True)\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = {'session_id': 'category',\n          'elapsed_time': np.int32,\n          'event_name': 'category',\n          'name': 'category',\n          'level': np.uint8,\n          'page': 'category',\n          'room_coor_x': np.float32,\n          'room_coor_y': np.float32,\n          'screen_coor_x': np.float32,\n          'screen_coor_y': np.float32,\n          'hover_duration': np.float32,\n          'text': 'category',\n          'fqid': 'category',\n          'room_fqid': 'category',\n          'text_fqid': 'category',\n          'fullscreen': np.int8,\n          'hq': np.int8,\n          'music': np.int8,\n          'level_group': 'category'}\n\ntrain = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)\ntest = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv', dtype=dtypes)\nlabels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\nprint('Train data shape:', train.shape)\nprint('Test data shape:', test.shape)\nprint('Labels data shape:', labels.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Sample of train data:\")\ntrain.head()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Sample of test data:\")\ntest.head()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train data statistics:\")\ntrain.describe().apply(lambda x: x.apply('{0:.1f}'.format))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Test data statistics:\")\ntest.describe().apply(lambda x: x.apply('{0:.1f}'.format))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat = pd.DataFrame([train.nunique(), test.nunique()]).T.fillna(0)\nstat.columns = ['Number of unique values in train', 'Number of unique values in test']\nstat.head(30).style.format(\"{:,.0f}\").background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_missing = train.isna().sum() / len(train) *100\nplt.figure(figsize=(20, 10))\npal = sns.color_palette(\"flare\", len(train_missing))\nrank = train_missing.argsort().argsort()\ng = sns.barplot(x=train_missing.index, y=train_missing, palette=np.array(pal[::])[rank])\ng.axes.set_title(\"Share of missing values in train data\", fontsize=18)\ng.bar_label(g.containers[0], fmt=\"%.1f%%\")\ng.xaxis.set_tick_params(rotation=35)\ndel train_missing, g, pal, rank","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_missing = test.isna().sum() / len(test) *100\nplt.figure(figsize=(20, 10))\npal = sns.color_palette(\"flare\", len(test_missing))\nrank = test_missing.argsort().argsort()\ng = sns.barplot(x=test_missing.index, y=test_missing, palette=np.array(pal[::])[rank])\ng.axes.set_title(\"Share of missing values in test data\", fontsize=18)\ng.bar_label(g.containers[0], fmt=\"%.1f%%\")\ng.xaxis.set_tick_params(rotation=35)\ndel test_missing, g, pal, rank","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(columns=['session_level'], inplace=True)\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\ng = sns.heatmap(train.corr(), annot=True, square=True, cmap='coolwarm', annot_kws={'size': 15},fmt='.2f')\ng.tick_params(axis='x', labelsize=15)\ng.tick_params(axis='y', labelsize=15)\ng.set_title('Correlations in train data', size=20, pad=15)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\nplt.figure(figsize=(20, 10))\ng = sns.heatmap(test.corr(), annot=True, square=True, cmap='coolwarm', annot_kws={'size': 15},fmt='.2f')\ng.tick_params(axis='x', labelsize=15)\ng.tick_params(axis='y', labelsize=15)\ng.set_title('Correlations in test data', size=20, pad=15)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Sample of labels data:\")\nlabels.head()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.describe().T","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[['session_id_id', 'question_number']] = labels['session_id'].str.split('_', 1, expand=True)\nlabels['session_id_id'] = labels['session_id_id'].astype(np.int64)\nmean_correct = (labels['correct'].mean()*100).astype(np.float64)\n\nlabels_perc = labels.groupby('question_number')['correct'].value_counts(normalize=True).mul(100).rename('Percent').reset_index()\nlabels_perc['number'] = labels_perc['question_number'].apply(lambda x: int(x[1:])).astype(np.int64)\nlabels_perc = labels_perc[labels_perc['correct'] == 1]\npal = sns.color_palette(\"RdYlGn\", len(labels_perc))\nrank = labels_perc.sort_values('number')['Percent'].argsort().argsort()\nplt.figure(figsize=(20, 10))\ng = sns.barplot(data=labels_perc, x='number', y=\"Percent\", palette=np.array(pal[::])[rank])\ng.axhline(mean_correct, color=\"coral\")\nplt.text(9, mean_correct+1, f'Average ={round(mean_correct, 1)}%')\ng.axes.set_title(\"Share of correct answers by questions\", fontsize=18)\ng.set(xlabel='Question number', ylabel='Percent of correct answers')\ng.bar_label(g.containers[0], fmt=\"%.1f%%\")\ndel g, labels_perc, pal, rank, mean_correct","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct_answers_per_session = labels.groupby('session_id_id')['correct'].sum()\nmean_correct_answers_per_session = correct_answers_per_session.mean()\nmedian_correct_answers_per_session = correct_answers_per_session.median()\ncorrect_answers_per_session = correct_answers_per_session.value_counts()\nplt.figure(figsize=(20, 8))\ng = sns.barplot(x=correct_answers_per_session.index, y=correct_answers_per_session.values, color='rosybrown')\nplt.title('Distribution of games by number of correct answers', fontsize=18)\ng.set_xticklabels(['{}'.format(int(num + 1)) for num in g.get_xticks()])\ng.set(xlabel='Number of correct answers', ylabel='Count of sessions')\ng.axvline(x=mean_correct_answers_per_session-1, color=\"coral\")\ng.text(mean_correct_answers_per_session-1, 1500, f'Average ={round(mean_correct_answers_per_session, 1)}', rotation=90)\ng.axvline(x=median_correct_answers_per_session-1, color=\"peru\")\ng.text(median_correct_answers_per_session-1, 1500, f'Median ={round(median_correct_answers_per_session, 1)}', rotation=90)\nfor i, v in enumerate(correct_answers_per_session.sort_index().values):\n    plt.text(i-0.2, v, str(int(v)))\ndel correct_answers_per_session, mean_correct_answers_per_session, median_correct_answers_per_session, i, v\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del labels\ngc.collect()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_counts = train['session_id'].value_counts()\ntest_events_counts = test['session_id'].value_counts()\n\nstat = pd.DataFrame([train_events_counts.describe().index,\n                    np.round(train_events_counts.describe(), 2).values,\n                    np.round(test_events_counts.describe(), 2).values]).T\nstat.columns = [' ', 'train', 'test']\nstat[['train', 'test']] = stat[['train', 'test']].astype(np.float64)\nprint(f'Unique sessions in train: {len(train_events_counts)}')\nprint(f'Unique sessions in test: {len(test_events_counts)}')\nprint('\\n\\nStatistics of events per session:')\nstat[1:].style.hide_index().format(precision=1).background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_train_events_counts = train_events_counts.mean()\nmedian_train_events_counts = train_events_counts.median()\nplt.figure(figsize=(20, 10))\ng = sns.histplot(data=train_events_counts)\nplt.title('Number of events per session for train (outliers with > 4000 excluded)', fontsize=18)\ng.set(xlabel='Number of events per session', ylabel='Count of sessions')\nplt.axvline(x=mean_train_events_counts, color=\"coral\")\nplt.text(mean_train_events_counts, 500, f'Average ={round(mean_train_events_counts, 1)}', rotation=90)\nplt.axvline(x=median_train_events_counts, color=\"peru\")\nplt.text(median_train_events_counts, 500, f'Median ={round(median_train_events_counts, 1)}', rotation=90)\nplt.xlim(600, 4000)\nplt.show()\ndel train_events_counts, mean_train_events_counts, median_train_events_counts, test_events_counts, stat, g","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_session_id_features(data):\n    session_ids = pd.DataFrame()\n    session_ids['session_id'] = data['session_id'].unique()\n    session_ids['year'] = session_ids['session_id'].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    session_ids['month'] = session_ids['session_id'].apply(lambda x: int(str(x)[2:4]) + 1).astype(np.uint8)\n    session_ids['weekday'] = session_ids['session_id'].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    session_ids['hour'] = session_ids['session_id'].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    session_ids['minute'] = session_ids['session_id'].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    session_ids['second'] = session_ids['session_id'].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    session_ids['ms'] = session_ids['session_id'].apply(lambda x: int(str(x)[12:15])).astype(np.uint16)\n    session_ids['unknown_part'] = session_ids['session_id'].apply(lambda x: int(str(x)[15:17])).astype(np.uint8)\n    return session_ids\n\ntrain_session_ids = get_session_id_features(train)\ntest_session_ids = get_session_id_features(test)\ntrain_session_ids.head()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_session_ids['year'] = train_session_ids['year'] + 2000\ntrain_session_ids['date'] = (train_session_ids['year']).astype(str) + '-' + (train_session_ids['month']).astype(str)\nplt.figure(figsize=(20, 8))\ng = sns.countplot(x='date', data=train_session_ids)\nplt.title('Number of sessions for each month in train', fontsize=18)\ng.set_yticklabels(['{}'.format(int(num)) for num in g.get_yticks()])\ng.bar_label(g.containers[0])\ng.xaxis.set_tick_params(rotation=45)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 5))\ng = sns.countplot(x='weekday', data=train_session_ids)\nplt.title('Number of sessions for each weekday in train', fontsize=18)\ng.set_yticklabels(['{}'.format(int(num)) for num in g.get_yticks()])\ng.bar_label(g.containers[0])\ng.xaxis.set_tick_params()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 8))\ng = sns.countplot(x='hour', data=train_session_ids)\nplt.title('Number of sessions for each hour in train', fontsize=18)\ng.set_yticklabels(['{}'.format(int(num)) for num in g.get_yticks()])\ng.bar_label(g.containers[0])\ng.xaxis.set_tick_params()\nplt.show()\ndel train_session_ids, g","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = train[['session_id', 'index']]\nidx_dupl = idx[idx.duplicated()]\nprint(f\"\\nNumber of duplicated combinations session_id/index: {len(idx_dupl)}\")\nprint(f\"\\nNumber of sessions with duplicates: {len(idx_dupl['session_id'].unique())}\")\nprint(\"\\nExamples of sessions with duplicates:\", idx_dupl['session_id'].unique()[:4])","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del idx, idx_dupl\ntrain.loc[(train['session_id'] == '20110507081078290') & (train['index'].isin([624, 625]))].sort_values('index')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_zero_index = train.groupby('session_id').filter(lambda x: 0 not in x[\"index\"].values)['session_id'].unique()\nprint(f\"Number session_id without index==0: {len(non_zero_index)}\")\nprint(\"\\nExamples of sessions without index==0:\", non_zero_index[:4])","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"elapsed_time_train = np.round((train['elapsed_time']).astype(np.float64)/60000.0, 1)\nelapsed_time_test = np.round((test['elapsed_time']).astype(np.float64)/60000.0, 1)\n\nstat = pd.DataFrame([elapsed_time_train.describe().index,\n                    np.round(elapsed_time_train.describe(), 2).values,\n                    np.round(elapsed_time_test.describe(), 2).values]).T\nstat.columns = [' ', 'train', 'test']\nstat[['train', 'test']] = stat[['train', 'test']].astype(np.float64)\nprint('Elapsed time statistics for events:')\nstat[1:].style.hide_index().format(precision=1).background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_elapsed_time_train = elapsed_time_train.mean()\nmedian_elapsed_time_train = elapsed_time_train.median()\nelapsed_time_train_counts = np.round(elapsed_time_train, 0).astype(np.int64).value_counts()\n\nplt.figure(figsize=(20, 10))\ng = sns.barplot(x=elapsed_time_train_counts.index, y=elapsed_time_train_counts.values, color='limegreen')\nplt.title('Elapsed time for events in train (outliers not shown)', fontsize=18)\ng.set_xticklabels(['{}'.format(int(num)) if i % 10 == 0 else '' for i, num in enumerate(g.get_xticks())])\ng.set(xlabel='elapsed_time, minutes', ylabel='Events')\ng.axvline(x=mean_elapsed_time_train, color=\"coral\")\ng.text(mean_elapsed_time_train, 600000, f'Average ={round(mean_elapsed_time_train, 1)}', rotation=90)\ng.axvline(x=median_elapsed_time_train, color=\"peru\")\ng.text(median_elapsed_time_train, 600000, f'Median ={round(median_elapsed_time_train, 1)}', rotation=90)\ndel mean_elapsed_time_train, median_elapsed_time_train, elapsed_time_train_counts\nplt.xlim(0, 180)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_elapsed_time_test = elapsed_time_test.mean()\nmedian_elapsed_time_test = elapsed_time_test.median()\nelapsed_time_test_counts  = np.round(elapsed_time_test, 0).astype(np.int64).value_counts()\n\nplt.figure(figsize=(20, 10))\ng = sns.barplot(x=elapsed_time_test_counts.index, y=elapsed_time_test_counts.values, color='lightgreen')\nplt.title('Elapsed time for events in test', fontsize=18)\ng.set_xticklabels(['{}'.format(int(num)) if i % 10 == 0 else '' for i, num in enumerate(g.get_xticks())])\ng.set(xlabel='elapsed_time, minutes', ylabel='Events')\ng.axvline(x=mean_elapsed_time_test, color=\"coral\")\ng.text(mean_elapsed_time_test, 80, f'Average ={round(mean_elapsed_time_test, 1)}', rotation=90)\ng.axvline(x=median_elapsed_time_test, color=\"peru\")\ng.text(median_elapsed_time_test, 80, f'Median ={round(median_elapsed_time_test, 1)}', rotation=90)\ndel mean_elapsed_time_test, median_elapsed_time_test, elapsed_time_test_counts\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_elapsed_time = train.groupby('session_id')['elapsed_time'].max()\nmax_elapsed_time = np.round(max_elapsed_time / 60000.0, 1)\n\nmean_elapsed_time = max_elapsed_time.mean()\nmedian_elapsed_time = max_elapsed_time.median()\nplt.figure(figsize=(20, 10))\ng = sns.histplot(max_elapsed_time, kde=False, color='mistyrose')\nplt.title('Maximum elapsed time for sessions in train (outliers not shown)', fontsize=18)\nplt.ticklabel_format(style='plain', axis='x')\ng.set(xlabel='Elapsed time, minutes', ylabel='Sessions')\nplt.axvline(x=mean_elapsed_time, color=\"coral\")\nplt.text(mean_elapsed_time, 600, f'Average ={round(mean_elapsed_time, 1)}', rotation=90)\nplt.axvline(x=median_elapsed_time, color=\"peru\")\nplt.text(median_elapsed_time, 600, f'Median ={round(median_elapsed_time, 1)}', rotation=90)\nplt.xlim(0, 360)\ndel max_elapsed_time, mean_elapsed_time, median_elapsed_time\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['elapsed_time_change'] = (train.groupby('session_id')['elapsed_time'].diff()).astype(np.float32)\nprint(f\"Number of events with negative elapsed time change: {len(train[train['elapsed_time_change'] < 0])}\")\nprint(f\"Percent of events with negative elapsed time change: {round(len(train[train['elapsed_time_change'] < 0])/len(train)*100.0, 2)}\\n\")\nprint(f\"Number of sessions with at least one negative elapsed time change: {len(train.loc[train['elapsed_time_change'] < 0, 'session_id'].unique())}\")\nprint(f\"Percent of sessions with at least one negative elapsed time change: {round(len(train.loc[train['elapsed_time_change'] < 0, 'session_id'].unique())*100.0/len(train['session_id'].unique()), 2)}\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"elapsed_time_change = np.round(train[train['elapsed_time_change'].notna()]['elapsed_time_change']/1000.0, 1)\nmean_elapsed_time_change = elapsed_time_change.mean()\nmedian_elapsed_time_change = elapsed_time_change.median()\nelapsed_time_change_counts = (np.round(elapsed_time_change, 0).astype(np.int64).value_counts()/1000).sort_index()\nplt.figure(figsize=(20, 10))\ng = sns.barplot(x=elapsed_time_change_counts.index, y=elapsed_time_change_counts.values, color='thistle')\nplt.title('elapsed_time change for events in train (outliers not shown)', fontsize=18)\ng.set(xlabel='elapsed_time_change, seconds', ylabel='Count, thousands')\ng.axvline(x=mean_elapsed_time_change+14, color=\"coral\")\ng.text(mean_elapsed_time_change+14, 5000, f'Average ={round(mean_elapsed_time_change, 1)}', rotation=90)\ng.axvline(x=median_elapsed_time_change+14, color=\"peru\")\ng.text(median_elapsed_time_change+14, 5000, f'Median ={round(median_elapsed_time_change, 1)}', rotation=90)\ndel mean_elapsed_time_change, median_elapsed_time_change, elapsed_time_change_counts, elapsed_time_change\ngc.collect()\nplt.xlim(5, 30)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_names_counts_train = train['event_name'].value_counts()/1000\nnames_counts_train = train['name'].value_counts()/1000\nfqid_counts_train = train['fqid'].value_counts()\nevent_names_counts_test = test['event_name'].value_counts()\nnames_counts_test = test['name'].value_counts()\nfqid_counts_test = test['fqid'].value_counts()\n\nprint(\"Number of unique event names:\", len(event_names_counts_train))\nprint(\"Number of unique names:\", len(names_counts_train))\nprint(\"Number of unique fqid:\", len(fqid_counts_train))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 9))\n\nplt.subplot(1, 2, 1)\nplt.bar(event_names_counts_train.index, event_names_counts_train.values, color='beige')\nplt.ylabel(\"Thousands of events\", fontsize=12)\nplt.title(\"Frequency of 'event_name' (=type) in train\", fontsize=18)\nfor i, v in enumerate(event_names_counts_train.values):\n    plt.text(i-0.4, v, str(round(v, 1)))\nplt.xticks(rotation=45)\n\nplt.subplot(1, 2, 2)\nplt.bar(event_names_counts_test.index, event_names_counts_test.values, color='cornsilk')\nplt.ylabel(\"Events\", fontsize=12)\nplt.title(\"Frequency of 'event_name' (=type) in test\", fontsize=18)\nfor i, v in enumerate(event_names_counts_test.values):\n    plt.text(i-0.4, v, str(int(v)))\nplt.xticks(rotation=45)\n\ndel i, v\nplt.show()\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_name_session_counts = pd.DataFrame(train.groupby('event_name')['session_id'].value_counts())\nevent_name_session_counts.columns = ['count']\nevent_name_session_counts = event_name_session_counts.reset_index(drop=False)\nprint(\"'event_name' counts per session in train:\")\nevent_name_session_counts.groupby('event_name')['count'].describe().drop(columns=['count']).style.format(precision=2).background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del event_name_session_counts\nplt.figure(figsize=(20, 7))\n\nplt.subplot(1, 2, 1)\nplt.bar(names_counts_train.index, names_counts_train.values, color='honeydew')\nplt.ylabel(\"Thousands of events\", fontsize=12)\nplt.title(\"Frequency of 'name' in train\", fontsize=18)\nfor i, v in enumerate(names_counts_train.values):\n    plt.text(i-0.3, v, str(round(v, 1)))\nplt.xticks(rotation=45)\n\nplt.subplot(1, 2, 2)\nplt.bar(names_counts_test.index, names_counts_test.values, color='aliceblue')\nplt.ylabel(\"Events\", fontsize=12)\nplt.title(\"Frequency of 'name' in test\", fontsize=18)\nfor i, v in enumerate(names_counts_test.values):\n    plt.text(i-0.2, v, str(int(v)))\nplt.xticks(rotation=45)\n\ndel i, v\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_name_session_counts = pd.DataFrame(train.groupby(['event_name', 'name'])['session_id'].value_counts())\nevent_name_session_counts.columns = ['count']\nevent_name_session_counts = event_name_session_counts.reset_index(drop=False)\nprint(\"'event_name'/'name' counts per session in train:\")\nevent_name_session_counts.groupby(['event_name', 'name'])['count'].describe().drop(columns=['count']).style.format(precision=2).background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del event_name_session_counts\npivot = train.pivot_table(index='event_name', columns='name', aggfunc='size')\npivot = (pivot.fillna(0) / 1000).round(decimals = 1)\nplt.figure(figsize=(20, 8))\nannotations = pivot.astype(str)\nannotations[pivot == 0] = \"\"\ng = sns.heatmap(pivot, annot=annotations, fmt='', cmap='YlGn')\nplt.title(\"Frequency of 'event_name' vs 'name' in train, thousands\", fontsize=16)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot = test.pivot_table(index='event_name', columns='name', aggfunc='size')\npivot = pivot.fillna(0).astype(int)\nplt.figure(figsize=(20, 8))\nannotations = pivot.astype(str)\nannotations[pivot == 0] = \"\"\ng = sns.heatmap(pivot, annot=annotations, fmt='', cmap='YlGn')\nplt.title(\"Frequency of 'event_name' vs 'name' in train data\", fontsize=16)\nplt.show()\ndel g, pivot, annotations","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_freq_train = dict(zip(fqid_counts_train.index.tolist(), fqid_counts_train.values.tolist()))\nwordcloud_train = WordCloud(width=2000, height=1000, background_color=\"white\").generate_from_frequencies(word_freq_train)\nplt.figure(figsize=(20, 10))\nplt.imshow(wordcloud_train, interpolation='bilinear')\nplt.title(\"Most frequent fqid values in train\", fontsize=16)\nplt.axis(\"off\")\ndel word_freq_train, wordcloud_train\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_freq_test = dict(zip(fqid_counts_test.index.tolist(), fqid_counts_test.values.tolist()))\nwordcloud_test = WordCloud(width=2000, height=1000, background_color=\"white\").generate_from_frequencies(word_freq_test)\nplt.figure(figsize=(20, 10))\nplt.imshow(wordcloud_test, interpolation='bilinear')\nplt.title(\"Most frequent fqid values in test\", fontsize=16)\nplt.axis(\"off\")\ndel word_freq_test, wordcloud_test\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('TOP-20 fqid for events:')\nstat = pd.DataFrame([fqid_counts_train, fqid_counts_test]).T\nstat.columns = ['train', 'test']\nstat = stat.sort_values('train', ascending=False)\ndel event_names_counts_train, names_counts_train, fqid_counts_train, event_names_counts_test, names_counts_test, fqid_counts_test\nstat.head(20).style.format(precision=0)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"room_counts_train = train['room_fqid'].value_counts()/1000\nroom_counts_test = test['room_fqid'].value_counts()\n\nprint(\"Number of unique rooms:\", len(room_counts_train))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 9))\nplt.bar(room_counts_train.index, room_counts_train.values, color='thistle')\nplt.ylabel(\"Thousands of events\", fontsize=12)\nplt.title(\"Frequency of rooms in train\", fontsize=18)\nfor i, v in enumerate(room_counts_train.values):\n    plt.text(i-0.35, v, str(round(v, 1)))\nplt.xticks(rotation=90)\nplt.show()\n\nplt.figure(figsize=(20, 9))\nplt.bar(room_counts_test.index, room_counts_test.values, color='pink')\nplt.ylabel(\"Events\", fontsize=12)\nplt.title(\"Frequency of rooms in test\", fontsize=18)\nfor i, v in enumerate(room_counts_test.values):\n    plt.text(i-0.2, v, str(int(v)))\nplt.xticks(rotation=90)\n\ndel i, v\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_levels_per_session = train[['session_id', 'level']].drop_duplicates()\nmissing_sessions = unique_levels_per_session.groupby('session_id').count().reset_index(drop=False)\nmissing_sessions = missing_sessions[missing_sessions['level'] != 23].reset_index(drop=True)['session_id']\nprint(f\"Number of sessions in which one of the 22 levels is not present: {len(missing_sessions)}\")\nfor lvl in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]:\n    x = unique_levels_per_session.loc[(unique_levels_per_session['session_id'].isin(missing_sessions)) & (unique_levels_per_session['level'].eq(lvl))]\n    if len(x) != len(missing_sessions):\n        print(f\"\\nLevel {lvl} is missing in {len(x)} sessions\")\n        print(f\"Examples of sessions with missing {lvl} level:\", x['session_id'].to_list()[:5])\ndel lvl, x, unique_levels_per_session, missing_sessions\n\nlevel_counts_train = train['level'].value_counts()/1000\nlevel_group_counts_train = train['level_group'].value_counts()/1000\nlevel_counts_test = test['level'].value_counts()\nlevel_group_counts_test = test['level_group'].value_counts()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.subplot(1, 3, (1, 2))\nplt.bar(level_counts_train.index, level_counts_train.values, color='lightsalmon')\nplt.ylabel(\"Thousands of events\", fontsize=12)\nplt.title(\"Frequency of level in train\", fontsize=18)\nfor i, v in enumerate(level_counts_train.sort_index().values):\n    plt.text(i-0.6, v, str(round(v, 1)))\nplt.xticks(level_counts_train.index)\n\nplt.subplot(1, 3, 3)\nplt.bar(level_group_counts_train.index, level_group_counts_train.values, color='peachpuff')\nplt.ylabel(\"Thousands of events\", fontsize=12)\nplt.title(\"Frequency of level_group in train\", fontsize=18)\nfor i, v in enumerate(level_group_counts_train):\n    plt.text(i-0.15, v, str(round(v, 1)))\nplt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.subplot(1, 3, (1, 2))\nplt.bar(level_counts_test.index, level_counts_test.values, color='violet')\nplt.ylabel(\"Events\", fontsize=12)\nplt.title(\"Frequency of level in test\", fontsize=18)\nfor i, v in enumerate(level_counts_test.sort_index().values):\n    plt.text(i-0.35, v, str(int(v)))\nplt.xticks(level_counts_train.index)\n\nplt.subplot(1, 3, 3)\nplt.bar(level_group_counts_test.index, level_group_counts_test.values, color='thistle')\nplt.ylabel(\"Events\", fontsize=12)\nplt.title(\"Frequency of level_group in test\", fontsize=18)\nfor i, v in enumerate(level_group_counts_test):\n    plt.text(i-0.15, v, str(int(v)))\nplt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot = train.pivot_table(index='room_fqid', columns='level', aggfunc='size')\npivot = (pivot.fillna(0) / 1000).round(decimals = 1)\nplt.figure(figsize=(20, 12))\nannotations = pivot.astype(str)\nannotations[pivot == 0] = \"\"\ng = sns.heatmap(pivot, annot=annotations, fmt='', cmap='YlGn')\nplt.title(\"Frequency of 'level' vs 'room_fqid' in train, thousands events\", fontsize=16)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot = train.pivot_table(index='event_name', columns='level', aggfunc='size')\npivot = (pivot.fillna(0) / 1000).round(decimals = 1)\nplt.figure(figsize=(20, 9))\nannotations = pivot.astype(str)\nannotations[pivot == 0] = \"\"\ng = sns.heatmap(pivot, annot=annotations, fmt='', cmap='YlGn')\nplt.title(\"Frequency of 'level' vs 'event_name' in train, thousands events\", fontsize=16)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot = train.pivot_table(index='name', columns='level', aggfunc='size')\npivot = (pivot.fillna(0) / 1000).round(decimals = 1)\nplt.figure(figsize=(20, 6))\nannotations = pivot.astype(str)\nannotations[pivot == 0] = \"\"\ng = sns.heatmap(pivot, annot=annotations, fmt='', cmap='YlGn')\nplt.title(\"Frequency of 'level' vs 'name' in train, thousands events\", fontsize=16)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Share of events with geo-location in train: {round(train['room_coor_x'].count() / len(train)*100, 2)}\")\nprint(f\"Share of events with geo-location in test: {round(test['room_coor_x'].count() / len(test)*100, 2)}\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat = pd.DataFrame([train['room_coor_x'].describe().index.to_list(),\n                    train['room_coor_x'].astype('float64').describe().values,\n                    train['room_coor_y'].astype('float64').describe().values,\n                    train['screen_coor_x'].astype('float64').describe().values,\n                    train['screen_coor_y'].astype('float64').describe().values,\n                    test['room_coor_x'].astype('float64').describe().values,\n                    test['room_coor_y'].astype('float64').describe().values,\n                    test['screen_coor_x'].astype('float64').describe().values,\n                    test['screen_coor_y'].astype('float64').describe().values]).T\nstat.columns = [' ', 'room_coor_x in train', 'room_coor_y in train', 'screen_coor_x in train', 'screen_coor_y in train',\n                     'room_coor_x in test', 'room_coor_y in test', 'screen_coor_x in test', 'screen_coor_y in test']\nstat[stat.columns.to_list()[1:]] = stat[stat.columns.to_list()[1:]].apply(pd.to_numeric, errors='coerce' )\nprint('Geo-location statistics for events:')\nstat[1:].style.hide_index().format(precision=1).background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rooms = train['room_fqid'].unique().to_list()\nfor room in rooms:\n    data_room = train[train['room_fqid'] == room]\n    data_room = data_room[['room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y']].dropna().reset_index(drop=True)\n    data_room = data_room.apply(pd.to_numeric, errors='coerce')\n    g = sns.pairplot(\n        data=data_room,\n        x_vars=[\"room_coor_x\"],\n        y_vars=[\"room_coor_y\"],\n        kind='hist',\n        height=6\n    )\n    g.fig.suptitle(f\"{room}: game room coordinates\")\n    g.savefig('g0.png', dpi=300)\n    plt.close(g.fig)\n    \n    g = sns.pairplot(\n        data=data_room,\n        x_vars=[\"screen_coor_x\"],\n        y_vars=[\"screen_coor_y\"],\n        kind='hist',\n        height=6\n    )\n    g.fig.suptitle(f\"{room}: player's screen coordinates\")\n    g.savefig('g1.png', dpi=300)\n    plt.close(g.fig)\n    \n    f, ax = plt.subplots(1, 2, figsize=(20, 20))\n    ax[0].imshow(mpimg.imread('g0.png'))\n    ax[1].imshow(mpimg.imread('g1.png'))\n    [axarr.set_axis_off() for axarr in ax.ravel()]\n    plt.tight_layout()\n    plt.show()\n\ndel room, rooms, data_room, g, f, ax","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_geo_location(df, session_id):\n    session_df = df[df['session_id'] == session_id]\n    rooms = session_df['room_fqid'].unique().to_list()\n    for room in rooms:\n        session_df_room = session_df[session_df['room_fqid'] == room]\n        session_df_room = session_df_room[['room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y']].dropna().reset_index(drop=True)        \n        plt.figure(figsize=(15, 6))        \n        \n        # room\n        x = session_df_room['room_coor_x']\n        y = session_df_room['room_coor_y']\n        plt.subplot(1, 5, (1, 3))\n        plt.plot(x, y, zorder=0, lw=0.5, color='steelblue')\n        plt.scatter(x, y, s=5, color='grey')\n        plt.scatter(x[0], y[0], s=200, lw=5, color='gold', marker='*')\n        plt.scatter(x[-1:], y[-1:], s=200, lw=5, color='crimson', marker='*')\n        plt.title(f\"{session_id}: {room} (room)\")\n        plt.legend(['Cursor path', 'Click position', 'Start', 'End'])\n        plt.gca().set_aspect('equal', adjustable='box')\n        plt.xlim(-2000, 1300)\n        plt.ylim(-920, 550)\n        plt.xlabel(\"room_coor_x\")\n        plt.ylabel(\"room_coor_y\")      \n    \n        # screen\n        x = session_df_room['screen_coor_x']\n        y = session_df_room['screen_coor_y']\n        plt.subplot(1, 5, (4, 5))\n        plt.plot(x, y, zorder=0, lw=0.5, color='lightcoral')\n        plt.scatter(x, y, s=5, color='grey')\n        plt.scatter(x[0], y[0], s=200, lw=5, color='gold', marker='*')\n        plt.scatter(x[-1:], y[-1:], s=200, lw=5, color='crimson', marker='*')\n        plt.title(f\"session {session_id}: {room} (screen)\")\n        plt.legend(['Cursor path', 'Click position', 'Start', 'End'])\n        plt.gca().set_aspect('equal', adjustable='box')\n        plt.xlabel(\"screen_coor_x\")\n        plt.ylabel(\"screen_coor_y\")\n        plt.xlim(0, 2000)\n        plt.ylim(0, 1500)\n        plt.tight_layout()\n        plt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_dupl = train[['session_id', 'index']]\nidx_dupl = idx_dupl[idx_dupl.duplicated()]\nsession_ids = train['session_id'].unique()\nsession_ids = session_ids[np.isin(session_ids, idx_dupl['session_id'].unique(), invert=True)]\ndel idx_dupl\ngc.collect()\nplot_geo_location(train, np.random.choice(session_ids))\ndel session_ids","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of unique text messages in train: {len(train['text'].unique())}\")\nprint(f\"Number of unique text messages in test: {len(test['text'].unique())}\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_counts_train = test['text'].value_counts()\nword_freq_train = dict(zip(text_counts_train.index.tolist(), text_counts_train.values.tolist()))\nwordcloud_train = WordCloud(width=2000, height=1000, background_color=\"white\").generate_from_frequencies(word_freq_train)\nplt.figure(figsize=(20, 10))\nplt.imshow(wordcloud_train, interpolation='bilinear')\nplt.title(\"Top text messages in train\", fontsize=16)\nplt.axis(\"off\")\ndel word_freq_train, wordcloud_train, text_counts_train\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"TOP-20 text messages:\")\nstat = pd.DataFrame([train['text'].value_counts(), test['text'].value_counts()]).T\nstat.columns = ['train', 'test']\nstat = stat.sort_values('train', ascending=False)\nstat.head(20).style.format(precision=0)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_counts_train = test['text_fqid'].value_counts()\nword_freq_train = dict(zip(text_counts_train.index.tolist(), text_counts_train.values.tolist()))\nwordcloud_train = WordCloud(width=2000, height=1000, background_color=\"white\").generate_from_frequencies(word_freq_train)\nplt.figure(figsize=(20, 10))\nplt.imshow(wordcloud_train, interpolation='bilinear')\nplt.title(\"Top text_fqid values in train\", fontsize=16)\nplt.axis(\"off\")\ndel word_freq_train, wordcloud_train, text_counts_train\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"TOP-20 text_fqid messages:\")\nstat = pd.DataFrame([train['text_fqid'].value_counts(), test['text_fqid'].value_counts()]).T\nstat.columns = ['train', 'test']\nstat = stat.sort_values('train', ascending=False)\nstat.head(20).style.format(precision=0)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"TOP-30 text_fqid and text combinations:\")\nstat = pd.DataFrame([train[['text_fqid', 'text']].value_counts(), test[['text_fqid', 'text']].value_counts()]).T\nstat.columns = ['train', 'test']\nstat = stat.sort_values('train', ascending=False)\nstat.head(30).style.format(precision=0)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['room_fqid', 'fqid', 'text_fqid']].dropna().drop_duplicates().head().style.hide_index()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Share of events with page in train: {round(train['page'].count() / len(train)*100, 2)}%\")\nprint(f\"Share of events with page in test: {round(test['page'].count() / len(test)*100, 2)}%\")\n\npage_counts_train = train['page'].value_counts()/1000\npage_counts_test = test['page'].value_counts()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 8))\nplt.subplot(1, 2, 1)\nplt.bar(page_counts_train.index, page_counts_train.values, color='skyblue')\nplt.ylabel(\"Thousands of events\", fontsize=12)\nplt.title(\"Frequency of page in train\", fontsize=18)\nfor i, v in enumerate(page_counts_train.values):\n    plt.text(i-0.25, v, str(round(v, 1)))\nplt.xticks()\n\nplt.subplot(1, 2, 2)\nplt.bar(page_counts_test.index, page_counts_test.values, color='lightblue')\nplt.ylabel(\"Events\", fontsize=12)\nplt.title(\"Frequency of page in test\", fontsize=18)\nfor i, v in enumerate(page_counts_test.values):\n    plt.text(i-0.15, v, str(int(v)))\ndel page_counts_train, page_counts_test\nplt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Share of events with hover_duration in train: {round(train['hover_duration'].count() / len(train)*100, 2)}%\")\nprint(f\"Share of events with hover_duration in test: {round(test['hover_duration'].count() / len(test)*100, 2)}%\")\n\nhover_duration_train = np.round(train[train['hover_duration'].notna()]['hover_duration'].astype(np.int32)/1000.0, 1)\nhover_duration_test = np.round(test[test['hover_duration'].notna()]['hover_duration'].astype(np.int32)/1000.0, 1)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat = pd.DataFrame([hover_duration_train.describe().index.to_list(),\n                    hover_duration_train.describe().values,\n                    hover_duration_test.describe().values]).T\nstat.columns = [' ', 'hover_duration in train, seconds', 'hover_duration in test, seconds']\nstat[stat.columns.to_list()[1:]] = stat[stat.columns.to_list()[1:]].apply(pd.to_numeric, errors='coerce' )\nprint('hover_duration statistics:')\nstat[1:].style.hide_index().format(precision=2).background_gradient(cmap='YlGn')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_hover_duration_train = hover_duration_train.mean()\nmedian_hover_duration_train = hover_duration_train.median()\nhover_duration_train_counts = np.round(hover_duration_train*10, 0).astype(np.int64).value_counts()\n\nplt.figure(figsize=(20, 9))\ng = sns.barplot(x=hover_duration_train_counts.index, y=np.round(hover_duration_train_counts.values, 0), color='wheat')\nplt.title('hover_duration for events in train (outliers not shown)', fontsize=18)\ng.set_xticklabels(['{}'.format(int(num/10)) if i % 10 == 0 else '' for i, num in enumerate(g.get_xticks())])\ng.set(xlabel='hover_duration, seconds', ylabel='Count')\ng.axvline(x=mean_hover_duration_train*10, color=\"peachpuff\")\ng.text(mean_hover_duration_train*10, 150000, f'Average ={round(mean_hover_duration_train, 1)}', rotation=90)\nplt.axvline(x=median_hover_duration_train*10, color=\"peru\")\nplt.text(median_hover_duration_train*10, 150000, f'Median ={round(median_hover_duration_train, 1)}', rotation=90)\nplt.xlim(-1, 8*10)\ndel mean_hover_duration_train, median_hover_duration_train, hover_duration_train_counts\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_hover_duration_test = hover_duration_test.mean()\nmedian_hover_duration_test = hover_duration_test.median()\nhover_duration_test_counts = np.round(hover_duration_test*10, 0).astype(np.int64).value_counts()\n\nplt.figure(figsize=(20, 7))\ng = sns.barplot(x=hover_duration_test_counts.index, y=hover_duration_test_counts.values, color='wheat')\nplt.title('hover_duration for events in test', fontsize=18)\ng.set_xticklabels(['{}'.format(int(num/10)) if i % 10 == 0 else '' for i, num in enumerate(g.get_xticks())])\ng.set(xlabel='hover_duration, seconds', ylabel='Count')\ng.axvline(x=mean_hover_duration_test*10, color=\"coral\")\ng.text(mean_hover_duration_test*10, 30, f'Average ={round(mean_hover_duration_test, 1)}', rotation=90)\ng.axvline(x=median_hover_duration_test*10, color=\"peru\")\ng.text(median_hover_duration_test*10, 30, f'Median ={round(median_hover_duration_test, 1)}', rotation=90)\ndel mean_hover_duration_test, median_hover_duration_test, hover_duration_test_counts, hover_duration_train, hover_duration_test\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 14))\nplt.subplot(1, 3, 1)\nplt.pie(train['fullscreen'].value_counts().sort_index(), labels = ['No (0)', 'Yes (1)'], colors = ['pink', 'lightgreen'], autopct='%.1f%%')\nplt.title('fullscreen, % of events', fontsize=18)\n\nplt.subplot(1, 3, 2)\nplt.pie(train['hq'].value_counts().sort_index(), labels = ['No (0)', 'Yes (1)'], colors = ['pink', 'lightgreen'], autopct='%.1f%%')\nplt.title('high-quality resolution (hq), % of events', fontsize=18)\n\nplt.subplot(1, 3, 3)\nplt.pie(train['music'].value_counts().sort_index(), labels = ['No (0)', 'Yes (1)'], colors = ['pink', 'lightgreen'], autopct='%.1f%%')\nplt.title('music, % of events', fontsize=18)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Percent of sessions where fullscreen is the same for all events: {train.groupby('session_id')['fullscreen'].nunique().eq(1).sum()/train['session_id'].nunique()*100}%\")\nprint(f\"Percent of sessions where hq is the same for all events: {train.groupby('session_id')['hq'].nunique().eq(1).sum()/train['session_id'].nunique()*100}%\")\nprint(f\"Percent of sessions where music is the same for all events: {train.groupby('session_id')['music'].nunique().eq(1).sum()/train['session_id'].nunique()*100}%\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\nplt.figure(figsize=(20, 14))\nplt.subplot(1, 3, 1)\nplt.pie(train.groupby('session_id')['fullscreen'].median().value_counts().sort_index(), \n        labels = ['No (0)', 'Yes (1)'], colors = ['pink', 'lightgreen'], autopct='%.1f%%')\nplt.title('fullscreen, % of sessions', fontsize=18)\n\nplt.subplot(1, 3, 2)\nplt.pie(train.groupby('session_id')['hq'].median().value_counts().sort_index(), \n        labels = ['No (0)', 'Yes (1)'], colors = ['pink', 'lightgreen'], autopct='%.1f%%')\nplt.title('high-quality resolution (hq), % of sessions', fontsize=18)\n\nplt.subplot(1, 3, 3)\nplt.pie(train.groupby('session_id')['music'].median().value_counts().sort_index(), \n        labels = ['No (0)', 'Yes (1)'], colors = ['pink', 'lightgreen'], autopct='%.1f%%')\nplt.title('music, % of sessions', fontsize=18)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}