{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn import tree\nfrom sklearn.model_selection import train_test_split # Import train_test_split function\nfrom sklearn import metrics\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-22T09:04:28.710397Z","iopub.execute_input":"2023-05-22T09:04:28.710830Z","iopub.status.idle":"2023-05-22T09:04:28.725947Z","shell.execute_reply.started":"2023-05-22T09:04:28.710798Z","shell.execute_reply":"2023-05-22T09:04:28.724866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"def read_data(instead_read_test_data = False):\n\n    import pandas as pd\n    \n    # specifying datatypes and reducing some datatypes to avoid \n    # memory error\n    dtypes={\n    'elapsed_time':np.int32,\n    'event_name':'category',\n    'name':'category',\n    'level':np.uint8,\n    'room_coor_x':np.float32,\n    'room_coor_y':np.float32,\n    'screen_coor_x':np.float32,\n    'screen_coor_y':np.float32,\n    'hover_duration':np.float32,\n    'text':'category',\n    'fqid':'category',\n    'room_fqid':'category',\n    'text_fqid':'category',\n    'fullscreen':np.int32,\n    'hq':np.int32,\n    'music':np.int32,\n    'level_group':'category'}\n\n    if instead_read_test_data == True:\n\n        print(\"reading_test_Data file\")\n\n        df = pd.read_csv(\"/kaggle/input/student-performance-and-game-play/test.csv\"\n                        , dtype = dtypes)\n\n    else:\n\n        print(\"Loading train data\")\n\n        df = pd.read_csv(\"/kaggle/input/student-performance-and-game-play/train.csv\",\n                         dtype = dtypes)\n\n        print(\"Train data Loaded\")\n\n    return df\n\ndef read_data_labels():\n\n    import pandas as pd\n\n    print(\"Loading train labels\")\n\n    df = pd.read_csv(\"/kaggle/input/student-performance-and-game-play/train_labels.csv\")\n\n    print(\"Training Labels Loaded\")\n\n    return df\n\n\n\ndef preparing_training_labels_dataset(df_train_labels):\n\n    \"\"\"\"\"\"\n    print(\"Perparing the training labels dataset\")\n    #extracting session id and question number\n    df_train_labels[\"question_number\"] = df_train_labels[\"session_id\"].apply(lambda x: x.split(\"_\")[1][1:])\n\n    df_train_labels[\"session_id\"] = df_train_labels[\"session_id\"].apply(lambda x: x.split(\"_\")[0])\n\n    # Conversting the question number column to integer format from object\n    df_train_labels[\"question_number\"] = df_train_labels[\"question_number\"].apply(int)\n\n    # assigning appropriate data types\n    df_train_labels[\"session_id\"] = df_train_labels[\"session_id\"].apply(int)\n    \n    df_train_labels.sort_values(\"session_id\", inplace = True)\n    \n    df_train_labels.set_index(\"session_id\", inplace= True)\n    \n    return df_train_labels[[\"question_number\", \"correct\"]]\n\n\n\ndef creating_complete_dataset_with_labels_features():\n\n    import pandas as pd\n    import calendar\n    import time\n\n    current_GMT = time.gmtime()\n\n    time_stamp = calendar.timegm(current_GMT)\n\n\n    df_train_labels = preparing_training_labels_dataset()\n    merged = aggregating_merging_numeric_and_cat_train_data()\n\n    print(\"Merging the training labels dataset and the features dataset\")\n\n    final_df = pd.merge(df_train_labels, merged, \n             how = \"left\", left_on= [\"session_id\", \"level_group\"], right_on= [\"session_id\", \"level_group\"])\n    \n\n#     print(\"Saving data with time stamp\")\n#     final_df.to_csv(\"data/processed_data/\" + str(time_stamp)+\"_final_df_with_features.csv\")\n    \n    return final_df\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:04:28.766226Z","iopub.execute_input":"2023-05-22T08:04:28.766857Z","iopub.status.idle":"2023-05-22T08:04:28.785098Z","shell.execute_reply.started":"2023-05-22T08:04:28.766814Z","shell.execute_reply":"2023-05-22T08:04:28.784221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_train = read_data()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:04:30.648627Z","iopub.execute_input":"2023-05-22T08:04:30.649431Z","iopub.status.idle":"2023-05-22T08:06:45.367651Z","shell.execute_reply.started":"2023-05-22T08:04:30.649392Z","shell.execute_reply":"2023-05-22T08:06:45.366532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading labels\ndf_labels = read_data_labels()\n# Getting question number from\n# for each session is in the labels dataset\n\ndf_labels = preparing_training_labels_dataset(df_labels)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:45.369629Z","iopub.execute_input":"2023-05-22T08:06:45.370240Z","iopub.status.idle":"2023-05-22T08:06:47.014908Z","shell.execute_reply.started":"2023-05-22T08:06:45.370203Z","shell.execute_reply":"2023-05-22T08:06:47.013419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# First look\n\nFirst look at the raw datset to understand what it contains.\nThis will give us an idea on how to prepare feature for training data.","metadata":{}},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:47.016708Z","iopub.execute_input":"2023-05-22T08:06:47.017119Z","iopub.status.idle":"2023-05-22T08:06:47.047253Z","shell.execute_reply.started":"2023-05-22T08:06:47.017085Z","shell.execute_reply":"2023-05-22T08:06:47.046026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:47.050277Z","iopub.execute_input":"2023-05-22T08:06:47.050680Z","iopub.status.idle":"2023-05-22T08:06:47.084844Z","shell.execute_reply.started":"2023-05-22T08:06:47.050644Z","shell.execute_reply":"2023-05-22T08:06:47.083911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- There a re numerical and categorical features that will contribute to the question being right or wrong.\n- Each session id has multiple rows worth of data -- hence chronology of event occuring might play a role or not.","metadata":{}},{"cell_type":"code","source":"df_labels.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:47.086277Z","iopub.execute_input":"2023-05-22T08:06:47.086844Z","iopub.status.idle":"2023-05-22T08:06:47.097946Z","shell.execute_reply.started":"2023-05-22T08:06:47.086809Z","shell.execute_reply":"2023-05-22T08:06:47.096816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_labels.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:47.099566Z","iopub.execute_input":"2023-05-22T08:06:47.100256Z","iopub.status.idle":"2023-05-22T08:06:47.121412Z","shell.execute_reply.started":"2023-05-22T08:06:47.100213Z","shell.execute_reply":"2023-05-22T08:06:47.120025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\n\n_ developing features that can be used for training a model","metadata":{}},{"cell_type":"code","source":"def feature_engineer(train):\n    \"\"\"\n    Developing features by aggregating data\n    \"\"\"\n    # Defininig variable in categories\n    \n    CATS = ['event_name', 'fqid', 'room_fqid', 'text']\n    NUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']\n\n# https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data\n    EVENTS = ['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'checkpoint']\n    \n    dfs = [] # empty dataset \n    \n    #getting number of unique events\n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    \n\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    for c in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    train = train.drop(EVENTS,axis=1)\n        \n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:47.123025Z","iopub.execute_input":"2023-05-22T08:06:47.123416Z","iopub.status.idle":"2023-05-22T08:06:47.139062Z","shell.execute_reply.started":"2023-05-22T08:06:47.123384Z","shell.execute_reply":"2023-05-22T08:06:47.138032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndf_f = feature_engineer(df_train)\ndf_f.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:06:47.140470Z","iopub.execute_input":"2023-05-22T08:06:47.141050Z","iopub.status.idle":"2023-05-22T08:07:55.036277Z","shell.execute_reply.started":"2023-05-22T08:06:47.141001Z","shell.execute_reply":"2023-05-22T08:07:55.034867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_f.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:07:55.038015Z","iopub.execute_input":"2023-05-22T08:07:55.039190Z","iopub.status.idle":"2023-05-22T08:07:55.063007Z","shell.execute_reply.started":"2023-05-22T08:07:55.039148Z","shell.execute_reply":"2023-05-22T08:07:55.061867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"%%time\ndf_ana = []\n\nfor t in range(1,19): #each question number\n        \n    # USE THIS TRAIN DATA WITH THESE QUESTIONS\n    if t<=3: grp = '0-4'\n    elif t<=13: grp = '5-12'\n    elif t<=22: grp = '13-22'\n\n    df_ana.append(pd.merge(df_f[df_f[\"level_group\"]== grp], \n             df_labels[df_labels[\"question_number\"]== t],\n             on = \"session_id\", how = \"left\"))\ndf_analysis = pd.concat(df_ana, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:25:50.095270Z","iopub.execute_input":"2023-05-22T08:25:50.095698Z","iopub.status.idle":"2023-05-22T08:25:50.640508Z","shell.execute_reply.started":"2023-05-22T08:25:50.095665Z","shell.execute_reply":"2023-05-22T08:25:50.637787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Viewing correlation between labels and features\n\n    \ndf_analysis.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:26:27.617701Z","iopub.execute_input":"2023-05-22T08:26:27.618136Z","iopub.status.idle":"2023-05-22T08:26:27.656859Z","shell.execute_reply.started":"2023-05-22T08:26:27.618097Z","shell.execute_reply":"2023-05-22T08:26:27.655899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_analysis.corr()\n# Increase the size of the heatmap.\nplt.figure(figsize=(20, 20))\n\nmask = np.triu(np.ones_like(df_analysis.corr(), dtype=np.bool))\n\n# Store heatmap object in a variable to easily access it when you want to include more features (such as title).\n# Set the range of values to be displayed on the colormap from 0 to 1, and set the annotation to True to display the correlation values on the heatmap.\nheatmap = sns.heatmap(abs(df_analysis.corr()), mask = mask, vmin=0, vmax=1, annot=True, cmap= \"BrBG\")\n# Give a title to the heatmap. Pad defines the distance of the title from the top of the heatmap.\nheatmap.set_title('Correlation Heatmap', fontdict={'fontsize':12}, pad=12);","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:36:38.107109Z","iopub.execute_input":"2023-05-22T08:36:38.107524Z","iopub.status.idle":"2023-05-22T08:36:44.923167Z","shell.execute_reply.started":"2023-05-22T08:36:38.107491Z","shell.execute_reply":"2023-05-22T08:36:44.921959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"for grp in df_train[\"level_group\"].unique().tolist():\n    df_plot = df_analysis[df_analysis[\"level_group\"] == grp].corr()\n    plt.figure(figsize=(20, 20))\n\n    mask = np.triu(np.ones_like(df_plot.corr(), dtype=np.bool))\n\n    # Store heatmap object in a variable to easily access it when you want to include more features (such as title).\n    # Set the range of values to be displayed on the colormap from 0 to 1, and set the annotation to True to display the correlation values on the heatmap.\n    heatmap = sns.heatmap(abs(df_plot.corr()), mask = mask, vmin=0, vmax=1, annot=True, cmap= \"BrBG\")\n    # Give a title to the heatmap. Pad defines the distance of the title from the top of the heatmap.\n    heatmap.set_title('Correlation Heatmap: {}'.format(grp), fontdict={'fontsize':12}, pad=12);\n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:45:49.703407Z","iopub.execute_input":"2023-05-22T08:45:49.703845Z","iopub.status.idle":"2023-05-22T08:46:00.145989Z","shell.execute_reply.started":"2023-05-22T08:45:49.703810Z","shell.execute_reply":"2023-05-22T08:46:00.144656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training\n\n Training a logistic regression model for each question, total 18 models will be trained, Group_level would be used to determine which data to be used for training which model.","metadata":{}},{"cell_type":"code","source":"FEATURES = [c for c in df_f.columns if c != 'level_group']\nprint('We will train with', len(FEATURES) ,'features')\nALL_USERS = df_f.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"execution":{"iopub.status.busy":"2023-05-22T08:47:54.666228Z","iopub.execute_input":"2023-05-22T08:47:54.666639Z","iopub.status.idle":"2023-05-22T08:47:54.677572Z","shell.execute_reply.started":"2023-05-22T08:47:54.666605Z","shell.execute_reply":"2023-05-22T08:47:54.676321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\n\nfor t in range(1,19): #each question number\n        \n    # USE THIS TRAIN DATA WITH THESE QUESTIONS\n    if t<=3: grp = '0-4'\n    elif t<=13: grp = '5-12'\n    elif t<=22: grp = '13-22'\n        \n    X = df_f[df_f[\"level_group\"] == grp][FEATURES].sort_index()\n    y = df_labels[df_labels['question_number'] == t][\"correct\"]\n\n    # train and validation set split\n    X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.20, random_state=42)\n\n    clf = make_pipeline(StandardScaler(),\n                        LogisticRegression(random_state=0, max_iter=10000, verbose = 0))\n    \n    clf.fit(X_train, y_train)\n    \n    print(\"For question number {}\".format(t))\n    print(\"The accuracy of the model is {}\".format(metrics.accuracy_score(clf.predict(X_val), y_val)))\n    print(\"The f1 score of the model is {}\".format(metrics.f1_score(clf.predict(X_val), y_val)))\n    # SAVE MODEL, PREDICT VALID OOF\n    models[f'{grp}_{t}'] = clf\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-22T09:14:54.069891Z","iopub.execute_input":"2023-05-22T09:14:54.070330Z","iopub.status.idle":"2023-05-22T09:14:56.931561Z","shell.execute_reply.started":"2023-05-22T09:14:54.070296Z","shell.execute_reply":"2023-05-22T09:14:56.930492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models","metadata":{"execution":{"iopub.status.busy":"2023-05-22T09:15:00.148919Z","iopub.execute_input":"2023-05-22T09:15:00.149358Z","iopub.status.idle":"2023-05-22T09:15:00.211473Z","shell.execute_reply.started":"2023-05-22T09:15:00.149324Z","shell.execute_reply":"2023-05-22T09:15:00.210402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring Models\n\n- Finding out which features are important and which are not based on logistic regression","metadata":{}},{"cell_type":"code","source":"for t in range(1,19): #each question number\n        \n    # USE THIS TRAIN DATA WITH THESE QUESTIONS\n    if t<=3: grp = '0-4'\n    elif t<=13: grp = '5-12'\n    elif t<=22: grp = '13-22'\n    plt.figure(figsize=(20,4))\n    plt.bar(models[f'{grp}_{t}'].feature_names_in_.tolist(),\n            models[f'{grp}_{t}'].named_steps['logisticregression'].coef_.tolist()[0]\n            )\n    plt.title(f\"For Question number {t}\")\n    plt.xticks(rotation = 45)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T09:17:22.331426Z","iopub.execute_input":"2023-05-22T09:17:22.331944Z","iopub.status.idle":"2023-05-22T09:17:30.091079Z","shell.execute_reply.started":"2023-05-22T09:17:22.331903Z","shell.execute_reply":"2023-05-22T09:17:30.089668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Pipeline","metadata":{}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    \n    # FEATURE ENGINEER TEST DATA\n    df = feature_engineer(test)\n    \n    # INFER TEST DATA\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict(df[FEATURES])\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int(p)\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-05-22T09:06:06.930270Z","iopub.execute_input":"2023-05-22T09:06:06.930725Z","iopub.status.idle":"2023-05-22T09:06:07.777634Z","shell.execute_reply.started":"2023-05-22T09:06:06.930690Z","shell.execute_reply":"2023-05-22T09:06:07.776332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-05-22T09:06:12.547922Z","iopub.execute_input":"2023-05-22T09:06:12.548359Z","iopub.status.idle":"2023-05-22T09:06:13.720140Z","shell.execute_reply.started":"2023-05-22T09:06:12.548325Z","shell.execute_reply":"2023-05-22T09:06:13.718701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Archived Functions\n\n- These functions were developed in the early process but now serve no or litte purpose.","metadata":{}},{"cell_type":"code","source":"def aggregating_merging_numeric_and_cat_train_data(get_all_features = True, \n                                                   instead_read_test_data = False ):\n\n    \"\"\"\n    Train data contains data in different formats some are numeric while some \n    are objects and text fields. The session_id is unique to an individual and the \n    group_level, however many rows have same session_ids as a single user session can span over \n    some time creating lots of data. For performing analyses we must create features pertaining\n    each unique session_id. \n    \"\"\"\n    df = read_train_data(instead_read_test_data = instead_read_test_data)\n\n    # extracting columns pertaining to numeric data\n    numeric_columns = df.select_dtypes(include= [\"int64\", \"float64\"]).columns\n    \n    #grouping by session id and level group as questions are asked at the end of certain levels\n    # aggregating to create new features per session id\n\n    print(\"Extracting features from numerical data\")\n\n    if get_all_features == False:\n\n        grouped = df.groupby([\"session_id\", \"level_group\"])[numeric_columns[2:]].agg(\n            \n            # getting the count of entries in each group\n            count = (\"elapsed_time\", \"count\"),\n\n            # Get max of the elapsed time column for each group\n            max_elapsed_time = (\"elapsed_time\", \"max\"),\n\n            # Get min of the elapsed time column for each group\n            min_elapsed_time = (\"elapsed_time\", \"min\"),\n\n            # Get min of the elapsed time column for each group\n            mean_elapsed_time = (\"elapsed_time\", \"mean\"),\n\n            # Get min of the level column for each group\n            min_level = (\"level\", \"min\"),\n\n            # Get min of the level column for each group\n            max_level = (\"level\", \"max\"),\n\n            # Get min of the level column for each group\n            mean_level = (\"level\", \"mean\")\n        \n        )\n\n    else:\n        grouped_num = df.groupby([\"session_id\", \"level_group\"])[numeric_columns[2:]].agg(\n            ['min', \"max\",\"sum\", \"mean\", \"var\"])\n        \n        grouped_num.columns = [\"_\".join(x) for x in grouped_num.columns.tolist()]\n\n    print(\"Extracted features from numerical data\")\n    \n#     aggregating categorical features\n\n    print(\"Extracting features from Categorical data\")\n\n    non_numeric_columns = df.select_dtypes(include= [\"category\"]).columns\n\n    grouped_cat = df.groupby([\"session_id\", \"level_group\"])[non_numeric_columns].agg(['unique', \"nunique\"])\n\n    grouped_cat.columns = [\"_\".join(x) for x in grouped_cat.columns.tolist()]\n\n    print(\"Extracted features from categorical data\")\n\n    print(\"merging data frames\")\n\n    merged = grouped_cat.join(grouped_num).sort_values(\n                                                   by = [\"session_id\", \"level_group\"])\n    \n    print(\"dataframes merged\")\n    \n    return merged.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-05-21T10:55:05.180042Z","iopub.status.idle":"2023-05-21T10:55:05.181078Z","shell.execute_reply.started":"2023-05-21T10:55:05.180732Z","shell.execute_reply":"2023-05-21T10:55:05.180761Z"},"trusted":true},"execution_count":null,"outputs":[]}]}