{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-14T17:06:25.676735Z","iopub.execute_input":"2023-06-14T17:06:25.677161Z","iopub.status.idle":"2023-06-14T17:06:25.690159Z","shell.execute_reply.started":"2023-06-14T17:06:25.677127Z","shell.execute_reply":"2023-06-14T17:06:25.688736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data preparation**","metadata":{}},{"cell_type":"code","source":"!pip install fastparquet\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:06:25.692582Z","iopub.execute_input":"2023-06-14T17:06:25.693271Z","iopub.status.idle":"2023-06-14T17:06:35.134635Z","shell.execute_reply.started":"2023-06-14T17:06:25.693210Z","shell.execute_reply":"2023-06-14T17:06:35.133346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import csv\n\n#def count_rows(file_path):\n#    with open(file_path, 'r', newline='') as csvfile:\n#        reader = csv.reader(csvfile)\n#        row_count = sum(1 for _ in reader)  # Use generator to count rows\n#\n#    return row_count\n\n## Example usage\n#csv_file_path = '/kaggle/input/predict-student-performance-from-game-play/train.csv'\n#row_count = count_rows(csv_file_path)\n#print(f\"Row count: {row_count}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:06:35.135773Z","iopub.execute_input":"2023-06-14T17:06:35.136078Z","iopub.status.idle":"2023-06-14T17:06:35.148131Z","shell.execute_reply.started":"2023-06-14T17:06:35.136049Z","shell.execute_reply":"2023-06-14T17:06:35.145962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\n\n# Specify the chunk size and total number of rows\nchunksize = 1000000\ntotal_rows = 10000000\n\n# Initialize an empty DataFrame to store the chunks\ndf_chunks = []\n\n# Load the dataset in chunks with progress bar\nwith tqdm(total=total_rows, desc='Loading dataset') as pbar:\n    for chunk in pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', chunksize=chunksize, nrows=total_rows):\n        df_chunks.append(chunk)\n        pbar.update(chunksize)\n        del chunk  # Release memory\n\n# Concatenate all the chunks into a single DataFrame\ndf_train = pd.concat(df_chunks)\n\n# Reset the index of the DataFrame\ndf_train.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:06:35.151982Z","iopub.execute_input":"2023-06-14T17:06:35.152580Z","iopub.status.idle":"2023-06-14T17:07:10.118175Z","shell.execute_reply.started":"2023-06-14T17:06:35.152404Z","shell.execute_reply":"2023-06-14T17:07:10.116425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:10.120132Z","iopub.execute_input":"2023-06-14T17:07:10.121101Z","iopub.status.idle":"2023-06-14T17:07:10.149508Z","shell.execute_reply.started":"2023-06-14T17:07:10.121062Z","shell.execute_reply":"2023-06-14T17:07:10.148370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\noutput_dir = '/kaggle/working/data'\nos.makedirs(output_dir, exist_ok=True)\ndf_train.to_parquet(\"/kaggle/working/data/train.parquet\", compression=None)\ndf_test.to_parquet(\"/kaggle/working/data/test.parquet\", compression=None)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:10.151024Z","iopub.execute_input":"2023-06-14T17:07:10.151576Z","iopub.status.idle":"2023-06-14T17:07:17.912361Z","shell.execute_reply.started":"2023-06-14T17:07:10.151546Z","shell.execute_reply":"2023-06-14T17:07:17.910392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastparquet import write, ParquetFile\ndf_train = pd.read_parquet(\"/kaggle/working/data/train.parquet\", engine=\"fastparquet\")\ndf_test = pd.read_parquet(\"/kaggle/working/data/test.parquet\", engine=\"fastparquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:17.913851Z","iopub.execute_input":"2023-06-14T17:07:17.914190Z","iopub.status.idle":"2023-06-14T17:07:20.763880Z","shell.execute_reply.started":"2023-06-14T17:07:17.914160Z","shell.execute_reply":"2023-06-14T17:07:20.762310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.info())","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:20.765423Z","iopub.execute_input":"2023-06-14T17:07:20.765777Z","iopub.status.idle":"2023-06-14T17:07:20.782291Z","shell.execute_reply.started":"2023-06-14T17:07:20.765747Z","shell.execute_reply":"2023-06-14T17:07:20.780751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Describe missing values\nmissing_values_count = df_train.isnull().sum()  # Count missing values for each column\nmissing_values_percentage = (missing_values_count / len(df_train)) * 100  # Calculate missing values percentage\n\nmissing_data = pd.DataFrame({\n    'Missing Values Count': missing_values_count,\n    'Missing Values Percentage': missing_values_percentage\n})\nprint(missing_data)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:20.787078Z","iopub.execute_input":"2023-06-14T17:07:20.788476Z","iopub.status.idle":"2023-06-14T17:07:29.686432Z","shell.execute_reply.started":"2023-06-14T17:07:20.788394Z","shell.execute_reply":"2023-06-14T17:07:29.684865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling missing values\n# Option 1: Drop rows with missing values\n#df_train_droped = df_train.dropna()\n\n# Option 2: Fill missing values with appropriate methods (e.g., mean, median, mode)\n#df_train_mean = df_train.fillna(df_train.mean())","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:29.687797Z","iopub.execute_input":"2023-06-14T17:07:29.688105Z","iopub.status.idle":"2023-06-14T17:07:29.693445Z","shell.execute_reply.started":"2023-06-14T17:07:29.688079Z","shell.execute_reply":"2023-06-14T17:07:29.692131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train['session_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:29.695006Z","iopub.execute_input":"2023-06-14T17:07:29.695640Z","iopub.status.idle":"2023-06-14T17:07:29.747776Z","shell.execute_reply.started":"2023-06-14T17:07:29.695603Z","shell.execute_reply":"2023-06-14T17:07:29.746548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select the first 1350 unique session IDs\nunique_session_ids = df_train['session_id'].unique()[:3350]\n\n# Filter the dataframe based on the selected session IDs\ndf_train_1350 = df_train[df_train['session_id'].isin(unique_session_ids)].copy()\n\n# Reset the index of the new dataframe\ndf_train_1350.reset_index(drop=True, inplace=True)\n\n# Print the number of rows in the new dataframe\nprint(f\"Number of rows in df_train_1350: {len(df_train_1350)}\")\n\n# Delete the original df_train DataFrame to free up memory\ndel df_train\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:29.749304Z","iopub.execute_input":"2023-06-14T17:07:29.749852Z","iopub.status.idle":"2023-06-14T17:07:30.887497Z","shell.execute_reply.started":"2023-06-14T17:07:29.749821Z","shell.execute_reply":"2023-06-14T17:07:30.885447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1350.to_parquet(\"/kaggle/working/data/train_1350.parquet\", compression=None)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:30.889033Z","iopub.execute_input":"2023-06-14T17:07:30.889389Z","iopub.status.idle":"2023-06-14T17:07:33.722204Z","shell.execute_reply.started":"2023-06-14T17:07:30.889360Z","shell.execute_reply":"2023-06-14T17:07:33.721314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **1350**","metadata":{}},{"cell_type":"code","source":"from fastparquet import write, ParquetFile\ndf_train_1350 = pd.read_parquet(\"/kaggle/working/data/train_1350.parquet\", engine=\"fastparquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:33.723974Z","iopub.execute_input":"2023-06-14T17:07:33.724722Z","iopub.status.idle":"2023-06-14T17:07:34.735981Z","shell.execute_reply.started":"2023-06-14T17:07:33.724688Z","shell.execute_reply":"2023-06-14T17:07:34.734003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1350.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:34.737992Z","iopub.execute_input":"2023-06-14T17:07:34.739457Z","iopub.status.idle":"2023-06-14T17:07:34.756794Z","shell.execute_reply.started":"2023-06-14T17:07:34.739409Z","shell.execute_reply":"2023-06-14T17:07:34.756002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select only numeric columns\nnumeric_columns = df_train_1350.select_dtypes(include=np.number).columns\n\n# Order the dataframe by session_id, elapsed_time, and index\ndf_train_1350_sorted = df_train_1350.sort_values(by=['session_id', 'elapsed_time', 'index'])\ndf_test_sorted = df_test.sort_values(by=['session_id', 'elapsed_time', 'index'])\n\n# Calculate differences between consecutive rows for numeric columns within each session ID\ndf_train_1350_diff = df_train_1350_sorted.groupby('session_id')[numeric_columns].diff()\ndf_test_diff = df_test_sorted.groupby('session_id')[numeric_columns].diff()\n\n# Add the difference columns to the df_train_1350 dataframe\ndf_train_1350[numeric_columns + '_diff'] = df_train_1350_diff\ndf_test[numeric_columns + '_diff'] = df_test_diff\n\n# Order the dataframe by session_id, elapsed_time, and index\ndf_train_1350 = df_train_1350.sort_values(by=['session_id', 'elapsed_time', 'index'])\ndf_test = df_test.sort_values(by=['session_id', 'elapsed_time', 'index'])\n\n# Print the first few rows of the resulting dataframe\nprint(df_train_1350.head())\n\n# Delete the intermediate datasets to free up memory\ndel df_train_1350_sorted, df_test_sorted, df_train_1350_diff, df_test_diff\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:34.757852Z","iopub.execute_input":"2023-06-14T17:07:34.758266Z","iopub.status.idle":"2023-06-14T17:07:42.144136Z","shell.execute_reply.started":"2023-06-14T17:07:34.758243Z","shell.execute_reply":"2023-06-14T17:07:42.142665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the number of unique values in each column\nunique_counts = df_train_1350.nunique()\n\n# Filter out columns with only one unique value\ncolumns_to_drop = unique_counts[unique_counts == 1].index\n\n# Drop the columns from the DataFrame\ndf_train_1350 = df_train_1350.drop(columns=columns_to_drop)\ndf_test = df_test.drop(columns=columns_to_drop)\n\n# Print the updated DataFrame\nprint(df_train_1350.head())\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:42.145828Z","iopub.execute_input":"2023-06-14T17:07:42.146180Z","iopub.status.idle":"2023-06-14T17:07:46.191360Z","shell.execute_reply.started":"2023-06-14T17:07:42.146150Z","shell.execute_reply":"2023-06-14T17:07:46.190146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1350.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:46.193199Z","iopub.execute_input":"2023-06-14T17:07:46.193626Z","iopub.status.idle":"2023-06-14T17:07:49.551256Z","shell.execute_reply.started":"2023-06-14T17:07:46.193603Z","shell.execute_reply":"2023-06-14T17:07:49.549759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoding","metadata":{}},{"cell_type":"code","source":"# # Identify columns for one-hot encoding\n# columns_to_encode = []\n# for col in df_train_1350.columns:\n#     if df_train_1350[col].nunique() < 20 or df_train_1350[col].dtype == 'object':\n#         columns_to_encode.append(col)\n        \n# # Names to remove\n# names_to_remove = ['page', 'level_diff', 'page_diff']\n\n\n# # Remove elements by name\n# columns_to_encode = [item for item in columns_to_encode if item not in names_to_remove]\n# columns_to_encode","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.552764Z","iopub.execute_input":"2023-06-14T17:07:49.553135Z","iopub.status.idle":"2023-06-14T17:07:49.559060Z","shell.execute_reply.started":"2023-06-14T17:07:49.553076Z","shell.execute_reply":"2023-06-14T17:07:49.557340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Perform one-hot encoding\n# df_train_1350_encoded = pd.get_dummies(df_train_1350, columns=columns_to_encode)\n\n# # Print the first few rows of the encoded dataframe\n# df_train_1350_encoded.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.561030Z","iopub.execute_input":"2023-06-14T17:07:49.561403Z","iopub.status.idle":"2023-06-14T17:07:49.577034Z","shell.execute_reply.started":"2023-06-14T17:07:49.561375Z","shell.execute_reply":"2023-06-14T17:07:49.575570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_1350_encoded.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.579113Z","iopub.execute_input":"2023-06-14T17:07:49.580011Z","iopub.status.idle":"2023-06-14T17:07:49.591126Z","shell.execute_reply.started":"2023-06-14T17:07:49.579963Z","shell.execute_reply":"2023-06-14T17:07:49.589447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_1350_encoded.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.592685Z","iopub.execute_input":"2023-06-14T17:07:49.593033Z","iopub.status.idle":"2023-06-14T17:07:49.605842Z","shell.execute_reply.started":"2023-06-14T17:07:49.593004Z","shell.execute_reply":"2023-06-14T17:07:49.604502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_1350_encoded.to_parquet(\"/kaggle/working/data/train_1350_encoded.parquet\", compression=None)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.607269Z","iopub.execute_input":"2023-06-14T17:07:49.608021Z","iopub.status.idle":"2023-06-14T17:07:49.618688Z","shell.execute_reply.started":"2023-06-14T17:07:49.607989Z","shell.execute_reply":"2023-06-14T17:07:49.617874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **1350_encoded loading**","metadata":{}},{"cell_type":"code","source":"# from fastparquet import write, ParquetFile\n# df_train_1350_encoded = pd.read_parquet(\"/kaggle/working/data/train_1350_encoded.parquet\", engine=\"fastparquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.622092Z","iopub.execute_input":"2023-06-14T17:07:49.623011Z","iopub.status.idle":"2023-06-14T17:07:49.632511Z","shell.execute_reply.started":"2023-06-14T17:07:49.622966Z","shell.execute_reply":"2023-06-14T17:07:49.630810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_1350_encoded.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.634487Z","iopub.execute_input":"2023-06-14T17:07:49.634960Z","iopub.status.idle":"2023-06-14T17:07:49.645993Z","shell.execute_reply.started":"2023-06-14T17:07:49.634931Z","shell.execute_reply":"2023-06-14T17:07:49.644662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature enginering","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df):\n    # Create a new DataFrame to store the grouped features\n    df_grouped = pd.DataFrame()\n\n    # Add 'session_id' and 'level_group' to the df_grouped DataFrame\n    df_grouped['session_id'] = df['session_id']\n    df_grouped['level_group'] = df['level_group']\n\n    # Feature 5: Maximum room coordinate X\n    df_grouped['max_room_coor_x'] = df.groupby(['session_id', 'level_group'])['room_coor_x'].transform('max')\n\n    # Feature 6: Minimum room coordinate Y\n    df_grouped['min_room_coor_y'] = df.groupby(['session_id', 'level_group'])['room_coor_y'].transform('min')\n\n    # Feature 7: Sum of screen coordinate X\n    df_grouped['sum_screen_coor_x'] = df.groupby(['session_id', 'level_group'])['screen_coor_x'].transform('sum')\n\n    # Feature 8: Average screen coordinate Y\n    df_grouped['average_screen_coor_y'] = df.groupby(['session_id', 'level_group'])['screen_coor_y'].transform('mean')\n\n    # Feature 9: Total hover duration\n    df_grouped['total_hover_duration'] = df.groupby(['session_id', 'level_group'])['hover_duration'].transform('sum')\n\n    # Feature 10: Average hover duration\n    df_grouped['average_hover_duration'] = df.groupby(['session_id', 'level_group'])['hover_duration'].transform('mean')\n\n    # Feature 11: Total time spent with fullscreen mode\n    df_grouped['total_time_fullscreen'] = df.groupby(['session_id', 'level_group'])['fullscreen'].transform('sum')\n\n    # Feature 13: Maximum level achieved\n    df_grouped['max_level'] = df.groupby(['session_id', 'level_group'])['level'].transform('max')\n\n    # Feature 14: Minimum elapsed time\n    df_grouped['min_elapsed_time'] = df.groupby(['session_id', 'level_group'])['elapsed_time'].transform('min')\n\n    # Feature 15: Mean fullscreen usage\n    df_grouped['mean_fullscreen_usage'] = df.groupby(['session_id', 'level_group'])['fullscreen'].transform('mean')\n\n    # Feature 16: Sum of hq usage\n    df_grouped['sum_hq_usage'] = df.groupby(['session_id', 'level_group'])['hq'].transform('sum')\n\n    # Feature 17: Maximum music usage\n    df_grouped['max_music_usage'] = df.groupby(['session_id', 'level_group'])['music'].transform('max')\n\n    # Feature 18: Total number of unique pages visited\n    df_grouped['unique_page_count'] = df.groupby(['session_id', 'level_group'])['page'].transform('nunique')\n\n    # Feature 19: Maximum room coordinate Y\n    df_grouped['max_room_coor_y'] = df.groupby(['session_id', 'level_group'])['room_coor_y'].transform('max')\n\n    # Feature 20: Minimum screen coordinate X\n    df_grouped['min_screen_coor_x'] = df.groupby(['session_id', 'level_group'])['screen_coor_x'].transform('min')\n\n    # Feature 21: Average screen coordinate X\n    df_grouped['average_screen_coor_x'] = df.groupby(['session_id', 'level_group'])['screen_coor_x'].transform('mean')\n\n    # ... (continue creating more features)\n\n    # Feature 41: Mean of elapsed time\n    df_grouped['mean_elapsed_time'] = df.groupby(['session_id', 'level_group'])['elapsed_time'].transform('mean')\n\n    # Drop duplicates\n    df_grouped.drop_duplicates(inplace=True)\n\n    # Reset the index\n    df_grouped = df_grouped.reset_index()\n\n    return df_grouped\n\n# Replace the original DataFrame name with the appropriate name in your code\ndf_train_1350_grouped = feature_engineering(df_train_1350)\n\n# Delete the original df_train_1350 dataset\ndel df_train_1350\n\n# Apply feature engineering to the test set\ndf_test_grouped = feature_engineering(df_test)\n\n# Delete the original df_test dataset\ndel df_test\n\n# Print the first few rows of the grouped DataFrame for the test set\nprint(df_train_1350_grouped.head())\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:49.647496Z","iopub.execute_input":"2023-06-14T17:07:49.647932Z","iopub.status.idle":"2023-06-14T17:07:57.068339Z","shell.execute_reply.started":"2023-06-14T17:07:49.647894Z","shell.execute_reply":"2023-06-14T17:07:57.067112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1350_grouped","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:57.069665Z","iopub.execute_input":"2023-06-14T17:07:57.070149Z","iopub.status.idle":"2023-06-14T17:07:57.095322Z","shell.execute_reply.started":"2023-06-14T17:07:57.070117Z","shell.execute_reply":"2023-06-14T17:07:57.094196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_grouped.to_parquet(\"/kaggle/working/data/test_features.parquet\", compression=None)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:57.096564Z","iopub.execute_input":"2023-06-14T17:07:57.097004Z","iopub.status.idle":"2023-06-14T17:07:57.107749Z","shell.execute_reply.started":"2023-06-14T17:07:57.096977Z","shell.execute_reply":"2023-06-14T17:07:57.106529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1350_grouped.to_parquet(\"/kaggle/working/data/train_1350_features.parquet\", compression=None)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:07:57.109395Z","iopub.execute_input":"2023-06-14T17:07:57.109901Z","iopub.status.idle":"2023-06-14T17:07:57.140440Z","shell.execute_reply.started":"2023-06-14T17:07:57.109869Z","shell.execute_reply":"2023-06-14T17:07:57.138660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Lables**","metadata":{}},{"cell_type":"code","source":"# Load the dataset\ndf_train_labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\ndf_train_labels = df_train_labels.rename(columns={'session_id': 'orig_session_id'})","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:11.674970Z","iopub.execute_input":"2023-06-14T17:08:11.675322Z","iopub.status.idle":"2023-06-14T17:08:12.029232Z","shell.execute_reply.started":"2023-06-14T17:08:11.675299Z","shell.execute_reply":"2023-06-14T17:08:12.027544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separate session_id column\ndf_train_labels[['session_id', 'question']] = df_train_labels['orig_session_id'].str.split('_', expand=True)\ndf_train_labels[['test', 'q_number']] = df_train_labels['question'].str.split('q', expand=True)\n\n# Rename columns\ndf_train_labels = df_train_labels.rename(columns={'session_id': 'session_id', 'q_number': 'q_number'})\n\n\n# Drop the original test column\ndf_train_labels.drop('test', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:22.888857Z","iopub.execute_input":"2023-06-14T17:08:22.889274Z","iopub.status.idle":"2023-06-14T17:08:25.230031Z","shell.execute_reply.started":"2023-06-14T17:08:22.889240Z","shell.execute_reply":"2023-06-14T17:08:25.228552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_labels","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:30.017074Z","iopub.execute_input":"2023-06-14T17:08:30.017493Z","iopub.status.idle":"2023-06-14T17:08:30.033550Z","shell.execute_reply.started":"2023-06-14T17:08:30.017456Z","shell.execute_reply":"2023-06-14T17:08:30.032199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert 'session_id' column to int64 in df_train_labels\ndf_train_labels['session_id'] = df_train_labels['session_id'].astype('int64')\ndf_train_labels['q_number'] = df_train_labels['q_number'].astype('int64')\n\ndf_merged = pd.merge(df_train_labels, df_train_1350_grouped, how='left', on=['session_id'])\n\ndf_merged = df_merged[df_merged['index'].notna()]","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:34.269248Z","iopub.execute_input":"2023-06-14T17:08:34.269741Z","iopub.status.idle":"2023-06-14T17:08:34.587189Z","shell.execute_reply.started":"2023-06-14T17:08:34.269708Z","shell.execute_reply":"2023-06-14T17:08:34.585948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:40.454740Z","iopub.execute_input":"2023-06-14T17:08:40.455163Z","iopub.status.idle":"2023-06-14T17:08:40.526233Z","shell.execute_reply.started":"2023-06-14T17:08:40.455131Z","shell.execute_reply":"2023-06-14T17:08:40.524530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a list to store the selected dataframes\nselected_dfs = []\n\nSUB_LEVELS = {'0-4': [1, 2, 3, 4],\n              '5-12': [5, 6, 7, 8, 9, 10, 11, 12],\n              '13-22': [13, 14, 15, 16, 17, 18, 19, 20, 21, 22]}\n\n# Iterate over each key in SUB_LEVELS\nfor sub_level, q_numbers in SUB_LEVELS.items():\n    # Select rows where level_group matches the key and q_number is in the q_numbers list\n    rows = df_merged[(df_merged['level_group'] == sub_level) & (df_merged['q_number'].isin(q_numbers))]\n    \n    # Append the selected rows to the list\n    selected_dfs.append(rows)\n\n# Concatenate the selected dataframes into a single dataframe\nselected_rows = pd.concat(selected_dfs)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:47.321458Z","iopub.execute_input":"2023-06-14T17:08:47.321827Z","iopub.status.idle":"2023-06-14T17:08:47.384404Z","shell.execute_reply.started":"2023-06-14T17:08:47.321803Z","shell.execute_reply":"2023-06-14T17:08:47.382722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_rows","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:51.834988Z","iopub.execute_input":"2023-06-14T17:08:51.835357Z","iopub.status.idle":"2023-06-14T17:08:51.884336Z","shell.execute_reply.started":"2023-06-14T17:08:51.835330Z","shell.execute_reply":"2023-06-14T17:08:51.882917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract the features and target variables\nfeatures = selected_rows.drop(['orig_session_id', 'correct', 'session_id', 'question', 'level_group'], axis=1)\ntargets = selected_rows['correct']","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:08:57.465422Z","iopub.execute_input":"2023-06-14T17:08:57.465860Z","iopub.status.idle":"2023-06-14T17:08:57.476573Z","shell.execute_reply.started":"2023-06-14T17:08:57.465820Z","shell.execute_reply":"2023-06-14T17:08:57.474735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:09:00.970367Z","iopub.execute_input":"2023-06-14T17:09:00.970750Z","iopub.status.idle":"2023-06-14T17:09:00.985675Z","shell.execute_reply.started":"2023-06-14T17:09:00.970725Z","shell.execute_reply":"2023-06-14T17:09:00.984174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Moddel","metadata":{}},{"cell_type":"code","source":"#Describe missing values\nmissing_values_count = features.isnull().sum()  # Count missing values for each column\nmissing_values_percentage = (missing_values_count / len(features)) * 100  # Calculate missing values percentage\n\nmissing_data = pd.DataFrame({\n    'Missing Values Count': missing_values_count,\n    'Missing Values Percentage': missing_values_percentage\n})\nprint(missing_data.sort_values('Missing Values Count'))","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:09:07.309717Z","iopub.execute_input":"2023-06-14T17:09:07.310123Z","iopub.status.idle":"2023-06-14T17:09:07.324037Z","shell.execute_reply.started":"2023-06-14T17:09:07.310089Z","shell.execute_reply":"2023-06-14T17:09:07.322485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.preprocessing import StandardScaler\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport numpy as np\n\n# Split the data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(features, targets, test_size=0.2, random_state=42)\n\n# Normalize the training data\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_train_tensor = torch.tensor(X_train_scaled).float()\ny_train_tensor = torch.tensor(y_train.values).unsqueeze(1).float()  # Reshape the target tensor\n\n# Normalize the test data\nX_test_scaled = scaler.transform(X_test)\nX_test_tensor = torch.tensor(X_test_scaled).float()\ny_test_tensor = torch.tensor(y_test.values).unsqueeze(1).float()  # Reshape the target tensor\n\n# Replace NaN values with 0\nX_train_tensor = torch.nan_to_num(X_train_tensor, nan=0.0)\nX_test_tensor = torch.nan_to_num(X_test_tensor, nan=0.0)\n\n# Define the neural network model\nclass NeuralNet(nn.Module):\n    def __init__(self, input_size, hidden_size):\n        super(NeuralNet, self).__init__()\n        self.fc1 = nn.Linear(input_size, hidden_size)\n        self.relu = nn.ReLU()\n        self.fc2 = nn.Linear(hidden_size, 1)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)  # Apply sigmoid activation function\n        return x\n\ninput_size = X_train_tensor.shape[1]\nhidden_size = 64\nmodel = NeuralNet(input_size, hidden_size)\n\n# Define the loss function and optimizer\ncriterion = nn.BCELoss()  # Use binary cross-entropy loss for binary classification\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Train the model\nnum_epochs = 200\nbatch_size = 10\n\nfor epoch in range(num_epochs):\n    for i in range(0, len(X_train_tensor), batch_size):\n        batch_X = X_train_tensor[i:i+batch_size]\n        batch_y = y_train_tensor[i:i+batch_size]\n\n        outputs = model(batch_X)\n        loss = criterion(outputs, batch_y)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n    print(f'Epoch {epoch+1}/{num_epochs}, Loss: {loss.item():.4f}')\n\n# Test the model on the testing data\nwith torch.no_grad():\n    outputs = model(X_test_tensor)\n    predicted = torch.round(outputs)\n\n    f1 = f1_score(y_test_tensor.numpy(), predicted.numpy())\n    print(f'Test F1 Score: {f1:.4f}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n# Split the data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(features, targets, test_size=0.2, random_state=42)\n\n# Convert the data to PyTorch tensors\nX_train_tensor = torch.tensor(X_train.values).float()\ny_train_tensor = torch.tensor(y_train.values).long()\n\nX_test_tensor = torch.tensor(X_test.values).float()\ny_test_tensor = torch.tensor(y_test.values).long()\n\n# Normalize the training data\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_train_tensor = torch.tensor(X_train_scaled).float()\n\n# Normalize the test data\nX_test_scaled = scaler.transform(X_test)\nX_test_tensor = torch.tensor(X_test_scaled).float()\n\n# Replace NaN values with 0\nX_train_tensor = torch.nan_to_num(X_train_tensor, nan=0.0)\nX_test_tensor = torch.nan_to_num(X_test_tensor, nan=0.0)\n\nclass NeuralNet(nn.Module):\n    def __init__(self, input_size, hidden_size, output_size, dropout_rate):\n        super(NeuralNet, self).__init__()\n        self.fc1 = nn.Linear(input_size, hidden_size)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(dropout_rate)\n        self.fc2 = nn.Linear(hidden_size, output_size)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.dropout(x)\n        x = self.fc2(x)\n        return x\n\ndef train_model(model, X_train, y_train, X_test, y_test, criterion, optimizer, num_epochs, batch_size):\n    train_losses = []\n    train_scores = []\n    test_losses = []\n    test_scores = []\n\n    for epoch in range(num_epochs):\n        batch_losses = []\n        batch_scores = []\n\n        for i in range(0, len(X_train), batch_size):\n            batch_X = X_train[i:i+batch_size]\n            batch_y = y_train[i:i+batch_size]\n\n            optimizer.zero_grad()\n            outputs = model(batch_X)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            optimizer.step()\n\n            batch_losses.append(loss.item())\n            _, predicted = torch.max(outputs.data, 1)\n            batch_scores.append(f1_score(batch_y.numpy(), predicted.numpy(), average='micro'))\n\n        train_loss = np.mean(batch_losses)\n        train_score = np.mean(batch_scores)\n        train_losses.append(train_loss)\n        train_scores.append(train_score)\n\n        with torch.no_grad():\n            outputs = model(X_test)\n            test_loss = criterion(outputs, y_test)\n            _, predicted = torch.max(outputs.data, 1)\n            test_score = f1_score(y_test.numpy(), predicted.numpy(), average='micro')\n\n            test_losses.append(test_loss.item())\n            test_scores.append(test_score)\n\n        print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_loss:.4f}, Train F1 Score: {train_score:.4f}, Test Loss: {test_loss.item():.4f}, Test F1 Score: {test_score:.4f}')\n\n    return train_losses, train_scores, test_losses, test_scores\n\n# Define the neural network model\ninput_size = X_train_tensor.shape[1]\nhidden_size = 64\noutput_size = len(targets.unique())  # Number of unique question labels\ndropout_rate = 0.5\nmodel = NeuralNet(input_size, hidden_size, output_size, dropout_rate)\n\n# Define the loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adadelta(model.parameters(), lr=1.0)\n\n# Train the model\nnum_epochs = 100\nbatch_size = 5\n\ntrain_losses, train_scores, test_losses, test_scores = train_model(model, X_train_tensor, y_train_tensor, X_test_tensor, y_test_tensor, criterion, optimizer, num_epochs, batch_size)\n\n# Plot the learning curve\nplt.figure(figsize=(10, 4))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(test_losses, label='Test Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(train_scores, label='Train F1 Score')\nplt.plot(test_scores, label='Test F1 Score')\nplt.xlabel('Epoch')\nplt.ylabel('F1 Score')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:11:12.938207Z","iopub.execute_input":"2023-06-14T17:11:12.938622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install torchsummary if not already installed\n!pip install torchsummary\n\n# Import torchsummary\nfrom torchsummary import summary\n\n# Define the neural network model\n# ...\n\n# Print the model summary\nsummary(model, input_size=(input_size,))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the trained model\ntorch.save(model.state_dict(), '/kaggle/working/trained_model.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an instance of the neural network model\nmodel = NeuralNet(input_size, hidden_size, output_size, dropout_rate)\n\n# Load the saved model state\nmodel.load_state_dict(torch.load('/kaggle/working/trained_model.pth'))\n","metadata":{},"execution_count":null,"outputs":[]}]}