{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Loading the DataSet 😎\n\n\n\n\n","metadata":{"id":"mXE-0_8oQN-j"}},{"cell_type":"markdown","source":"Importando bibliotecas","metadata":{"id":"eveLBPXKQc_U"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \n\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nimport seaborn as sns","metadata":{"id":"Ot2VPUzCQSkb","execution":{"iopub.status.busy":"2023-05-25T00:21:04.026839Z","iopub.execute_input":"2023-05-25T00:21:04.027769Z","iopub.status.idle":"2023-05-25T00:21:05.750034Z","shell.execute_reply.started":"2023-05-25T00:21:04.027677Z","shell.execute_reply":"2023-05-25T00:21:05.748854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Definindo tipos e colunas usadas","metadata":{"id":"JfL97hD3RGnc"}},{"cell_type":"code","source":"PATH = '/kaggle/input/predict-student-performance-from-game-play'\n\ndtypes = {\"session_id\": 'int64',\n          \"index\": np.int16,\n          \"elapsed_time\": np.int32,\n          \"event_name\": 'category',\n          \"name\": 'category',\n          \"level\": np.int8,\n          \"page\": np.float16,\n          \"room_coor_x\": np.float16,\n          \"room_coor_y\": np.float16,\n          \"screen_coor_x\": np.float16,\n          \"screen_coor_y\": np.float16,\n          \"hover_duration\": np.float32,\n          \"text\": 'category',\n          \"fqid\": 'category',\n          \"room_fqid\": 'category',\n          \"text_fqid\": 'category',\n          \"fullscreen\": np.int8,\n          \"hq\": np.int8,\n          \"music\": np.int8,\n          \"level_group\": 'category'\n          }\n# Specify the list of columns you are using\nuse_col = ['session_id', 'index', 'elapsed_time', 'event_name', 'name', 'level', 'page', 'room_coor_x', 'room_coor_y', \n           'screen_coor_x', 'screen_coor_y', 'hover_duration', 'text', 'fqid', 'room_fqid', 'text_fqid', 'fullscreen', 'hq', 'music', 'level_group']\n","metadata":{"id":"z2MKtrdeQpME","execution":{"iopub.status.busy":"2023-05-25T00:21:05.752387Z","iopub.execute_input":"2023-05-25T00:21:05.753017Z","iopub.status.idle":"2023-05-25T00:21:05.764017Z","shell.execute_reply.started":"2023-05-25T00:21:05.752974Z","shell.execute_reply":"2023-05-25T00:21:05.762355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Importando train.csv sozinho devido ao tamanho","metadata":{"id":"gGxZHoXtR1tk"}},{"cell_type":"code","source":"train_original = pd.read_csv(f'{PATH}/train.csv', dtype=dtypes, usecols=use_col)","metadata":{"id":"9DgNqOXzRuGM","execution":{"iopub.status.busy":"2023-05-25T00:21:05.766411Z","iopub.execute_input":"2023-05-25T00:21:05.766887Z","iopub.status.idle":"2023-05-25T00:23:23.020124Z","shell.execute_reply.started":"2023-05-25T00:21:05.766836Z","shell.execute_reply":"2023-05-25T00:23:23.018685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Importando arquivos restantes","metadata":{"id":"jKiphw94R5XP"}},{"cell_type":"code","source":"test_original = pd.read_csv(f'{PATH}/test.csv')\ntrain_labels = pd.read_csv(f'{PATH}/train_labels.csv')\nsample_submission = pd.read_csv(f'{PATH}/sample_submission.csv')","metadata":{"id":"pYzHsaqxRVae","execution":{"iopub.status.busy":"2023-05-25T00:23:23.024510Z","iopub.execute_input":"2023-05-25T00:23:23.025314Z","iopub.status.idle":"2023-05-25T00:23:23.492135Z","shell.execute_reply.started":"2023-05-25T00:23:23.025264Z","shell.execute_reply":"2023-05-25T00:23:23.490966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ____________________________\n\n---\n\n","metadata":{"id":"LwTwW6K57sGm"}},{"cell_type":"markdown","source":"#***Feature Engineering***\n\n\n","metadata":{"id":"j0e4ONTc5k3_"}},{"cell_type":"code","source":"sessions = train_original['session_id'].unique().tolist()","metadata":{"id":"KHmFI-e2-BqM","execution":{"iopub.status.busy":"2023-05-25T00:23:23.493825Z","iopub.execute_input":"2023-05-25T00:23:23.494476Z","iopub.status.idle":"2023-05-25T00:23:23.654631Z","shell.execute_reply.started":"2023-05-25T00:23:23.494434Z","shell.execute_reply":"2023-05-25T00:23:23.653208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fixing Elapsed Time 😎","metadata":{"id":"V4Tl-p9CHgiu"}},{"cell_type":"code","source":"def fix_elapsed(df):\n    elapsed_list = df['elapsed_time'].values.tolist()\n\n    for i in range(1, len(elapsed_list)):\n      if elapsed_list[i] < elapsed_list[i-1] and elapsed_list[i+1] != 0:\n        for ii in range(i,len(elapsed_list)):\n          if elapsed_list[ii] == 0: break\n          if elapsed_list[ii] >= elapsed_list[i-1]:\n            elapsed_list[i] = int(round((elapsed_list[ii] + elapsed_list[i-1])/2))\n            break\n    return elapsed_list","metadata":{"id":"EKQxUZ2nOV3a","execution":{"iopub.status.busy":"2023-05-25T00:23:23.656340Z","iopub.execute_input":"2023-05-25T00:23:23.657015Z","iopub.status.idle":"2023-05-25T00:23:23.666720Z","shell.execute_reply.started":"2023-05-25T00:23:23.656971Z","shell.execute_reply":"2023-05-25T00:23:23.664635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting Time and  Date of start of dataset per session 😎","metadata":{"id":"CsQT22JCzjq3"}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/pdnartreb/session-id-reverse-engineering\n\nhttps://www.kaggle.com/code/janmpia/session-start-time-eda","metadata":{"id":"80VRDQMH7w62"}},{"cell_type":"code","source":"def get_time_info(df):\n    year, month, weekday, hour, minute, second, ms = [], [], [], [], [], [], []\n\n    for c in df['session_id'].unique().tolist():\n      year.append(int(str(c)[:2]))\n      month.append(int(str(c)[2:4]))\n      weekday.append(int(str(c)[4:6]))\n      hour.append(int(str(c)[6:8]))\n      minute.append(int(str(c)[8:10]))\n      second.append(int(str(c)[10:12]))\n      ms.append(int(str(c)[12:15]))\n\n    date_df = pd.DataFrame(df['session_id'].unique().tolist(), columns =['session_id'])\n    date_df['year'], date_df['month'], date_df['weekday'], date_df['hour'], date_df['minute'], date_df['second'], date_df['ms'] = year, month, weekday, hour, minute, second, ms\n    return date_df","metadata":{"id":"4OdkwqcLznxX","outputId":"3c139f9d-bbda-47f8-c46a-58df2f1d1946","execution":{"iopub.status.busy":"2023-05-25T00:23:23.670925Z","iopub.execute_input":"2023-05-25T00:23:23.671896Z","iopub.status.idle":"2023-05-25T00:23:23.685927Z","shell.execute_reply.started":"2023-05-25T00:23:23.671844Z","shell.execute_reply":"2023-05-25T00:23:23.684296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_date_info(df):\n  date_df = get_time_info(df).copy()  \n  sessions_initial_elapsed_time = df.groupby('session_id')['elapsed_time2'].agg('min').tolist()\n\n  year = date_df['year'].tolist()\n  month = date_df['month'].tolist()\n  day = date_df['weekday'].tolist()\n  hour = date_df['hour'].tolist()\n  minute = date_df['minute'].tolist()\n\n  session_weekday = []\n  session_hour = []\n  session_month = []\n  session_year = []\n\n  for i in range(len(sessions_initial_elapsed_time)): #shape : 23562, elements: number of events of each session\n    ms = sessions_initial_elapsed_time[i]\n\n    hours = ms/3.6e+6\n    days = hours//24\n    hours = hours - days*24\n    hours = round(hours + hour[i])\n\n    weekday = day[i] + days - (((day[i] + days)//6)*6)\n    session_weekday.append(weekday)\n    a = 0 #chech if minute is over 30; if so, round to one hour above\n    if minute[i] > 30: a=1\n    session_hour.append(hours+a)\n    session_month.append(month[i])\n    session_year.append(year[i])\n\n  s1 = pd.Series(session_weekday, index = sessions)\n  s1.name = 'session_weekday'\n  s1 = s1.rename_axis(\"session_id\")\n  s2 = pd.Series(session_hour, index = sessions)\n  s2.name = 'session_hour'\n  s2 = s2.rename_axis(\"session_id\")\n  s3 = pd.Series(session_month, index = sessions)\n  s3.name = 'session_month'\n  s3 = s3.rename_axis(\"session_id\")\n  s4 = pd.Series(session_year, index = sessions)\n  s4.name = 'session_year'\n  s4 = s4.rename_axis(\"session_id\")\n  return [s1, s2, s3, s4]","metadata":{"id":"N_NXDUEhP9Rf","execution":{"iopub.status.busy":"2023-05-25T00:23:23.687860Z","iopub.execute_input":"2023-05-25T00:23:23.688296Z","iopub.status.idle":"2023-05-25T00:23:23.712710Z","shell.execute_reply.started":"2023-05-25T00:23:23.688252Z","shell.execute_reply":"2023-05-25T00:23:23.711018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Total time and Number of events per session 😎","metadata":{"id":"bXKdI3tChV10"}},{"cell_type":"code","source":"def get_time_and_events_for_each_session(df): \n\n  s1 = df.groupby('session_id')['session_id'].agg('count')\n  s1.name = 'number_of_events'\n\n  s2 = df.groupby('session_id')['elapsed_time2'].agg('max') - df.groupby('session_id')['elapsed_time2'].agg('min')\n  s2.name = 'total_session_time'\n\n  return [s1,s2]","metadata":{"id":"bOPa80ZUYSzJ","execution":{"iopub.status.busy":"2023-05-25T00:23:23.715465Z","iopub.execute_input":"2023-05-25T00:23:23.715947Z","iopub.status.idle":"2023-05-25T00:23:23.737698Z","shell.execute_reply.started":"2023-05-25T00:23:23.715904Z","shell.execute_reply":"2023-05-25T00:23:23.736073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Counting unique events of text and text_fqid 😎","metadata":{"id":"gUvgxjw0qKSx"}},{"cell_type":"code","source":"def get_unique_texts(df):\n  dfs = []\n  for c in ['text','text_fqid']:\n    tmp = df.groupby('session_id')[c].agg('nunique')\n    tmp.name = tmp.name + '_nunique'\n    dfs.append(tmp)\n  return dfs","metadata":{"id":"dlaQiyU4qJvZ","execution":{"iopub.status.busy":"2023-05-25T00:23:23.743798Z","iopub.execute_input":"2023-05-25T00:23:23.744237Z","iopub.status.idle":"2023-05-25T00:23:23.753906Z","shell.execute_reply.started":"2023-05-25T00:23:23.744190Z","shell.execute_reply":"2023-05-25T00:23:23.752589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting the data from the x and y coordinates (room) 😎","metadata":{"id":"EZrN3LQArvdh"}},{"cell_type":"code","source":"def get_room_coord_data(df):\n  dfs = []\n  for c in ['room_coor_x', 'room_coor_y']:\n    tmp = df.groupby('session_id')[c].agg('mean')\n    tmp.name = tmp.name + '_mean'\n    dfs.append(tmp)  \n  for c in ['room_coor_x', 'room_coor_y']:\n    tmp = df.groupby('session_id')[c].agg('std')\n    tmp.name = tmp.name + '_std'\n    dfs.append(tmp)    \n  return dfs","metadata":{"id":"tQc3DrqKrutQ","execution":{"iopub.status.busy":"2023-05-25T00:23:23.755845Z","iopub.execute_input":"2023-05-25T00:23:23.756321Z","iopub.status.idle":"2023-05-25T00:23:23.768918Z","shell.execute_reply.started":"2023-05-25T00:23:23.756277Z","shell.execute_reply":"2023-05-25T00:23:23.767515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting data from Elapsed Time 😎","metadata":{"id":"bHdAVCo8uyXY"}},{"cell_type":"code","source":"def get_elapsed_time_dt(df):\n  dfs = []\n\n  tmp = df.groupby('session_id')['elapsed_time2'].agg('mean')\n  tmp.name = tmp.name + '_mean'\n  dfs.append(tmp)\n\n  tmp = df.groupby('session_id')['elapsed_time2'].agg('std')\n  tmp.name = tmp.name + '_std'\n  dfs.append(tmp)  \n\n  return dfs","metadata":{"id":"h7aYAA_-u0kn","execution":{"iopub.status.busy":"2023-05-25T00:23:23.770846Z","iopub.execute_input":"2023-05-25T00:23:23.771294Z","iopub.status.idle":"2023-05-25T00:23:23.793456Z","shell.execute_reply.started":"2023-05-25T00:23:23.771248Z","shell.execute_reply":"2023-05-25T00:23:23.791835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting data from Page 😎","metadata":{"id":"SS6gxQ1PtTz4"}},{"cell_type":"code","source":"def get_data_page(df):\n  dfs = []\n\n  tmp = df.groupby('session_id')['page'].agg('mean')\n  tmp.name = tmp.name + '_mean'\n  dfs.append(tmp)\n\n  tmp = df.groupby('session_id')['page'].agg('std')\n  tmp.name = tmp.name + '_std'\n  dfs.append(tmp)  \n\n  return dfs","metadata":{"id":"GRZ5314UtTSQ","execution":{"iopub.status.busy":"2023-05-25T00:23:23.795155Z","iopub.execute_input":"2023-05-25T00:23:23.796065Z","iopub.status.idle":"2023-05-25T00:23:23.809380Z","shell.execute_reply.started":"2023-05-25T00:23:23.796019Z","shell.execute_reply":"2023-05-25T00:23:23.808122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting data from Level 😎","metadata":{"id":"_m-aOI-1uVAQ"}},{"cell_type":"code","source":"def get_data_level(df):\n  dfs = []\n\n  tmp = df.groupby('session_id')['level'].agg('mean')\n  tmp.name = tmp.name + '_mean'\n  dfs.append(tmp)\n\n  tmp = df.groupby('session_id')['level'].agg('std')\n  tmp.name = tmp.name + '_std'\n  dfs.append(tmp)  \n\n  return dfs","metadata":{"id":"JnJYc8FPuUWo","execution":{"iopub.status.busy":"2023-05-25T00:23:23.810753Z","iopub.execute_input":"2023-05-25T00:23:23.812340Z","iopub.status.idle":"2023-05-25T00:23:23.824486Z","shell.execute_reply.started":"2023-05-25T00:23:23.812289Z","shell.execute_reply":"2023-05-25T00:23:23.823335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Counting the number of events for each event name 🧠","metadata":{"id":"dh5fGC5_GHWZ"}},{"cell_type":"markdown","source":"Confuso? Sim porém ao ver o dataset final tudo fica mais simples","metadata":{"id":"BBueNFwaGce1"}},{"cell_type":"code","source":"# This function counts the number of events for each event name in a DataFrame\n\ndef get_number_of_events_for_each_event_name(df):\n    # Initialize an empty list to store the counts for each event name\n    event_counts_list = []\n    \n    # Group the DataFrame by session ID and event name, and count the number of events for each group\n    grouped_counts = df.groupby(by=[\"session_id\", \"event_name\"])[\"index\"].count()\n    \n    # Get a list of unique event names in the DataFrame\n    event_names = grouped_counts.index.get_level_values(1).unique()\n    \n    # Iterate through each event name and extract the counts for that name\n    for event_name in event_names:\n        counts_for_event_name = grouped_counts.loc[:, event_name]\n        \n        # Rename the column to include the event name\n        counts_for_event_name = counts_for_event_name.rename(f\"{event_name}_count\")\n        \n        # Add the counts to the list of event counts\n        event_counts_list.append(counts_for_event_name)\n    \n    # Return the list of event counts\n    return event_counts_list","metadata":{"id":"p1UTD2k6GQmV","execution":{"iopub.status.busy":"2023-05-25T00:23:23.825970Z","iopub.execute_input":"2023-05-25T00:23:23.827194Z","iopub.status.idle":"2023-05-25T00:23:23.962393Z","shell.execute_reply.started":"2023-05-25T00:23:23.827144Z","shell.execute_reply":"2023-05-25T00:23:23.961229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Counting the number of events for each level 🧠","metadata":{"id":"_K-Bk7YVHM41"}},{"cell_type":"code","source":"# This function counts the number of events for each level in a DataFrame\n\ndef get_number_of_events_for_each_level(df):\n    # Initialize an empty list to store the counts for each level\n    level_counts_list = []\n    \n    # Group the DataFrame by session ID and level, and count the number of events for each group\n    grouped_counts = df.groupby(by=[\"session_id\", \"level\"])[\"index\"].count()\n    \n    # Get a list of unique levels in the DataFrame\n    levels = grouped_counts.index.get_level_values(1).unique()\n    \n    # Iterate through each level and extract the counts for that level\n    for level in levels:\n        counts_for_level = grouped_counts.loc[:, level]\n        \n        # Rename the column to include the level number\n        counts_for_level = counts_for_level.rename(f\"level{level}_count\")\n        \n        # Add the counts to the list of level counts\n        level_counts_list.append(counts_for_level)\n    \n    # Return the list of level counts\n    return level_counts_list","metadata":{"id":"BJoUWWWrHTTC","execution":{"iopub.status.busy":"2023-05-25T00:23:23.964611Z","iopub.execute_input":"2023-05-25T00:23:23.965061Z","iopub.status.idle":"2023-05-25T00:23:23.989921Z","shell.execute_reply.started":"2023-05-25T00:23:23.965022Z","shell.execute_reply":"2023-05-25T00:23:23.988358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bringing together the data of the \"hover\" variable 🧠","metadata":{"id":"exFBeVBIHnjP"}},{"cell_type":"code","source":"# This function calculates aggregate statistics for the \"hover_duration\" column in a DataFrame\n\ndef get_hover_duration_aggregate_data(df):\n    # Initialize an empty list to store the results for each aggregate statistic\n    results_list = []\n    \n    # Calculate the sum of hover durations for each session\n    hover_duration_sum = df.groupby(by=[\"session_id\"])[\"hover_duration\"].sum()\n    hover_duration_sum = hover_duration_sum.rename(\"hover_duration_sum\")\n    results_list.append(hover_duration_sum)\n    \n    # Calculate the mean of hover durations for each session\n    hover_duration_mean = df.groupby(by=[\"session_id\"])[\"hover_duration\"].mean()\n    hover_duration_mean = hover_duration_mean.rename(\"hover_duration_mean\")\n    results_list.append(hover_duration_mean)\n    \n    # Calculate the standard deviation of hover durations for each session\n    hover_duration_std = df.groupby(by=[\"session_id\"])[\"hover_duration\"].std()\n    hover_duration_std = hover_duration_std.rename(\"hover_duration_std\")\n    results_list.append(hover_duration_std)\n    \n    # Calculate the count of hover durations for each session\n    hover_duration_count = df.groupby(by=[\"session_id\"])[\"hover_duration\"].count()\n    hover_duration_count = hover_duration_count.rename(\"hover_duration_count\")\n    results_list.append(hover_duration_count)\n    \n    # Return the list of results\n    return results_list\n","metadata":{"id":"bbhfMHuiHyky","execution":{"iopub.status.busy":"2023-05-25T00:23:23.991917Z","iopub.execute_input":"2023-05-25T00:23:23.992415Z","iopub.status.idle":"2023-05-25T00:23:24.015403Z","shell.execute_reply.started":"2023-05-25T00:23:23.992375Z","shell.execute_reply":"2023-05-25T00:23:24.013820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bringing together all the game configurations 🧠\n\n\n\n","metadata":{"id":"7jaNfKj9IawN"}},{"cell_type":"code","source":"# Define a function to extract game configuration data from a DataFrame\ndef get_game_config_data(df):\n    feature_dfs = []  # Create an empty list to hold DataFrames for each feature\n    # Loop over each game configuration option of interest\n    for config_option in [\"fullscreen\", \"hq\", \"music\"]:\n        # Group the DataFrame by session ID and extract the first value of the configuration option for each session\n        config_values = df.groupby(by=[\"session_id\"])[config_option].first()\n        config_values = config_values.rename(config_option)  # Rename the resulting Series to match the configuration option\n        feature_dfs.append(config_values)  # Append the Series to the list of feature DataFrames\n    return feature_dfs  # Return the list of feature DataFrames","metadata":{"id":"dC0GhvJLI01W","execution":{"iopub.status.busy":"2023-05-25T00:23:24.016946Z","iopub.execute_input":"2023-05-25T00:23:24.017334Z","iopub.status.idle":"2023-05-25T00:23:24.036064Z","shell.execute_reply.started":"2023-05-25T00:23:24.017296Z","shell.execute_reply":"2023-05-25T00:23:24.034977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bringing together all the new features 🧠","metadata":{"id":"yOJiVT27I6o6"}},{"cell_type":"code","source":"def concat_features(df):\n    dfs1 = get_number_of_events_for_each_event_name(df) #For each event name (Ex: checkpoint, cutscene, mapclick), get number of appearences for session\n    dfs2 = get_number_of_events_for_each_level(df) #For each level, get number of events/actions that occured in it\n    dfs3 = get_hover_duration_aggregate_data(df) #Calculates some statistical data about the column 'hover', like its sum, mean, std, for each session\n    dfs4 = get_game_config_data(df) #Simple yes or no column about the configurations fullscreen, hq and music\n\n    #dfs5 = get_unique_texts(df) #Getting number of unique texts that appeared, both for text and text_fqid, per session\n    #dfs6 = get_room_coord_data(df) #Getting statistical data both from the x and y room coordinates, like before (mean, std), for each session\n    #dfs7 = get_elapsed_time_dt(df) #Getting statistical data from the (new and fixed) elapsed_time column for each session\n    #dfs8 = get_data_page(df) #Getting statistical data from the page column (mean, std)\n    #dfs9 = get_data_level(df) #Getting statistical data from the level column (mean, std)\n        \n    dfs10 = get_time_and_events_for_each_session(df) #Getting the number of events and total time for each session\n    #dfs11 = get_date_info(df) # Getting info about the date and hour of when the first event of each session in the dataframe started (hour, day of the week, month, year)\n\n    dfs_total = (dfs1 + dfs2 + dfs3 + dfs4 + dfs10)\n\n    independent_variables = pd.concat(dfs_total, axis = 1)\n    #independent_variables = independent_variables.fillna(-1) #fill NaN with -1. Done specially for data regarding room coordinates and page, since there are a lot missing\n    independent_variables = independent_variables.replace(np.nan, 0)\n    independent_variables = independent_variables.reset_index()\n    independent_variables = independent_variables.set_index('session_id')\n\n    return independent_variables","metadata":{"id":"1WalN0xNx9tX","execution":{"iopub.status.busy":"2023-05-25T00:23:24.037979Z","iopub.execute_input":"2023-05-25T00:23:24.038917Z","iopub.status.idle":"2023-05-25T00:23:24.059591Z","shell.execute_reply.started":"2023-05-25T00:23:24.038870Z","shell.execute_reply":"2023-05-25T00:23:24.058474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we can split the data by group of questions since the submission API gives us level segments 'level_group'.","metadata":{"id":"LOJqEYabW4rv"}},{"cell_type":"code","source":"def feature_engineer(df):\n    my_df = df.copy()\n    my_df['elapsed_time2'] = fix_elapsed(my_df)\n    \n    early_game_questions = my_df[my_df[\"level_group\"] == \"0-4\"]\n    EARLY_FEATURES = concat_features(early_game_questions)\n    del early_game_questions\n    \n    mid_game_questions = my_df[my_df[\"level_group\"] == \"5-12\"]\n    MID_FEATURES = concat_features(mid_game_questions)\n    del mid_game_questions\n    \n    late_game_questions = my_df[my_df[\"level_group\"] == \"13-22\"]\n    LATE_FEATURES = concat_features(late_game_questions)\n    del late_game_questions\n    \n    del my_df\n    return [EARLY_FEATURES,MID_FEATURES,LATE_FEATURES]","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:23:24.061015Z","iopub.execute_input":"2023-05-25T00:23:24.061618Z","iopub.status.idle":"2023-05-25T00:23:24.080818Z","shell.execute_reply.started":"2023-05-25T00:23:24.061575Z","shell.execute_reply":"2023-05-25T00:23:24.079728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer2(df):\n    my_df = df.copy()\n    my_df['elapsed_time2'] = fix_elapsed(my_df)\n    \n    my_df = concat_features(my_df)\n    \n    return my_df","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:23:24.082272Z","iopub.execute_input":"2023-05-25T00:23:24.083007Z","iopub.status.idle":"2023-05-25T00:23:24.102726Z","shell.execute_reply.started":"2023-05-25T00:23:24.082967Z","shell.execute_reply":"2023-05-25T00:23:24.101043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training the model 🧠","metadata":{"id":"XoQB6z9OP7Pd"}},{"cell_type":"markdown","source":"\n\n*   [Notebook](https://www.kaggle.com/code/cdeotte/xgboost-baseline-0-680#XGBoost-Baseline---LB-0.678) for the baseline XGB model.\n\n\n","metadata":{"id":"qbZHzvvlZU6y"}},{"cell_type":"code","source":"def get_dependent_variable_matrix(df):\n    df[\"question\"] = df[\"session_id\"].str.split(\"_\").str[1].str[1:].astype(int)\n    df[\"session\"] = df[\"session_id\"].str.split(\"_\").str[0]\n    return df\n    \nDEPENDENT_VARIABLES = get_dependent_variable_matrix(train_labels)\nDEPENDENT_VARIABLES","metadata":{"id":"NeWw-x8UKhAH","outputId":"c5da3414-de91-4e4b-d419-89e20bb489ce","execution":{"iopub.status.busy":"2023-05-25T00:23:24.104512Z","iopub.execute_input":"2023-05-25T00:23:24.105303Z","iopub.status.idle":"2023-05-25T00:23:27.452137Z","shell.execute_reply.started":"2023-05-25T00:23:24.105244Z","shell.execute_reply":"2023-05-25T00:23:27.450837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fe = feature_engineer(train_original)\nEARLY_FEATURES = fe[0]\nMID_FEATURES = fe[1]\nLATE_FEATURES = fe[2]","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:23:27.453859Z","iopub.execute_input":"2023-05-25T00:23:27.454329Z","iopub.status.idle":"2023-05-25T00:24:04.240029Z","shell.execute_reply.started":"2023-05-25T00:23:27.454292Z","shell.execute_reply":"2023-05-25T00:24:04.238719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make = error + to + stop + code","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:24:04.241737Z","iopub.execute_input":"2023-05-25T00:24:04.242111Z","iopub.status.idle":"2023-05-25T00:24:04.247677Z","shell.execute_reply.started":"2023-05-25T00:24:04.242073Z","shell.execute_reply":"2023-05-25T00:24:04.246220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import necessary libraries\nfrom sklearn.model_selection import GridSearchCV\nimport xgboost as xgb\n\n# Set the model parameters for grid search\nparameters = {\n    'max_depth': [3, 4], \n    'n_estimators': [20, 50, 100],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'max_depth': [1, 2]\n}\n\n# Initialize dictionaries to store models and f1 scores\nMODELS = {}\nF1 = {}\n\n# Loop through each question and train a model\nfor question in range(1, 19):\n    print(f'\\nTRAIN QUESTION {question} MODEL')\n    \n    # Set X features based on the current question\n    if question <= 3:\n        X = EARLY_FEATURES\n    elif question <= 13:\n        X = MID_FEATURES\n    elif question <= 18:\n        X = LATE_FEATURES\n    \n    # Set y target based on the current question\n    y = DEPENDENT_VARIABLES[DEPENDENT_VARIABLES[\"question\"] == question][\"correct\"]\n    \n    # Initialize a new XGBoost classifier and perform grid search to find the best hyperparameters\n    model_xgb = xgb.XGBClassifier(random_state = 1)\n    model_xgb = GridSearchCV(\n        model_xgb, \n        parameters, \n        cv=3,\n        scoring='f1')\n    model_xgb.fit(X, y)\n    \n    # Store the model and its f1 score in their respective dictionaries\n    MODELS[f\"question {question} model\"] = model_xgb\n    F1[f\"question {question} f1\"] = model_xgb.best_score_\n    \n    # Print the best f1 score and hyperparameters for the current question\n    print(f\"\\tf1 score is {model_xgb.best_score_:.3f}\")\n    print(f\"\\tbest params are {model_xgb.best_params_}\")\n    print(f'QUESTION {question} MODEL COMPLETE')\n","metadata":{"id":"hAa0zY1mLzej","outputId":"327a1c1f-ba8e-4fd5-cdd5-f6a5e1e033e2","execution":{"iopub.status.busy":"2023-05-25T00:24:04.249266Z","iopub.execute_input":"2023-05-25T00:24:04.249919Z","iopub.status.idle":"2023-05-25T00:34:10.829020Z","shell.execute_reply.started":"2023-05-25T00:24:04.249877Z","shell.execute_reply":"2023-05-25T00:34:10.827724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submeter\n","metadata":{"id":"RtPyz8mIlA2W"}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:34:10.834404Z","iopub.execute_input":"2023-05-25T00:34:10.836831Z","iopub.status.idle":"2023-05-25T00:34:10.849703Z","shell.execute_reply.started":"2023-05-25T00:34:10.836768Z","shell.execute_reply":"2023-05-25T00:34:10.848529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test: \n    sessions = test['session_id'].unique().tolist()\n    \n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    models = list(MODELS.values())\n    predictions = []\n    \n    my_test = test[test[\"level_group\"] == grp]\n    df = feature_engineer2(my_test)\n    \n    for t in range(a,b):\n        p = models[t-1].predict(df)\n        p = p[0]\n        predictions.append(p)\n    sample_submission['correct'] = predictions\n    \n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:34:10.851830Z","iopub.execute_input":"2023-05-25T00:34:10.852654Z","iopub.status.idle":"2023-05-25T00:34:11.458209Z","shell.execute_reply.started":"2023-05-25T00:34:10.852608Z","shell.execute_reply":"2023-05-25T00:34:11.457054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('submission.csv')\nprint( df.shape )\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:34:11.464932Z","iopub.execute_input":"2023-05-25T00:34:11.465788Z","iopub.status.idle":"2023-05-25T00:34:11.483258Z","shell.execute_reply.started":"2023-05-25T00:34:11.465743Z","shell.execute_reply":"2023-05-25T00:34:11.482160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.correct.mean())","metadata":{"execution":{"iopub.status.busy":"2023-05-25T00:34:11.484738Z","iopub.execute_input":"2023-05-25T00:34:11.485458Z","iopub.status.idle":"2023-05-25T00:34:11.492514Z","shell.execute_reply.started":"2023-05-25T00:34:11.485410Z","shell.execute_reply":"2023-05-25T00:34:11.491358Z"},"trusted":true},"execution_count":null,"outputs":[]}]}