{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler # encode\n# SVM\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import classification_report\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-20T18:50:48.327385Z","iopub.execute_input":"2023-04-20T18:50:48.328359Z","iopub.status.idle":"2023-04-20T18:50:48.341152Z","shell.execute_reply.started":"2023-04-20T18:50:48.328158Z","shell.execute_reply":"2023-04-20T18:50:48.339848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n\ndf = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\nprint(df.head())\nprint(df.columns)\nprint(df.shape)\n\n'''","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:48.343892Z","iopub.execute_input":"2023-04-20T18:50:48.347146Z","iopub.status.idle":"2023-04-20T18:50:48.371670Z","shell.execute_reply.started":"2023-04-20T18:50:48.346825Z","shell.execute_reply":"2023-04-20T18:50:48.370386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Useful Notebook: https://www.kaggle.com/code/chiaenchen/eda-predict-student-performance-from-game-play#Conclusion\n\n\n","metadata":{}},{"cell_type":"markdown","source":"1. Drop the feature which 'null' ratio > 50%\n2. Encode string value, event_name and name\n3. Fill missing item in coordinates by Forward fill\n4. Remove features that are not relevant to predicting game performance, hover_duration, fullscreen, hq,music","metadata":{}},{"cell_type":"code","source":"dtypes = {\"session_id\": 'int64',\n          \"index\": np.int16,\n          \"elapsed_time\": np.int32,\n          \"event_name\": 'category',\n          \"name\": 'category',\n          \"level\": np.int8,\n          \"page\": np.float16,\n          \"room_coor_x\": np.float64,\n          \"room_coor_y\": np.float64,\n          \"screen_coor_x\": np.float64,\n          \"screen_coor_y\": np.float64,\n          \"hover_duration\": np.float32,\n          \"text\": 'category',\n          \"fqid\": 'category',\n          \"room_fqid\": 'category',\n          \"text_fqid\": 'category',\n          \"fullscreen\": np.int8,\n          \"hq\": np.int8,\n          \"music\": np.int8,\n          \"level_group\": 'category'\n         }","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:48.373869Z","iopub.execute_input":"2023-04-20T18:50:48.375612Z","iopub.status.idle":"2023-04-20T18:50:48.386151Z","shell.execute_reply.started":"2023-04-20T18:50:48.375513Z","shell.execute_reply":"2023-04-20T18:50:48.384820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 'page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n   'hover_duration', 'text_fqid', 'fullscreen', 'hq',\n   'music', 'level_group'","metadata":{}},{"cell_type":"code","source":"#avoid memory error, use chunk\ndef process(chunk,data):\n    # process data here\n    #print(chunk.shape)\n    \n    # Remove the 'page','text_fqid,'fqid','text' column\n    chunk.drop('page', axis=1, inplace=True)\n    #chunk.drop('text_fqid', axis=1, inplace=True)\n    #chunk.drop('fqid', axis=1, inplace=True)\n    #chunk.drop('text', axis=1, inplace=True)\n    \n    \n    # Remove Unrelate value\n    chunk.drop('hover_duration', axis=1, inplace=True)\n    chunk.drop('fullscreen', axis=1, inplace=True)\n    chunk.drop('hq', axis=1, inplace=True)\n    chunk.drop('music', axis=1, inplace=True)\n    \n    '''\n    # Encode the 'even_name',  variable\n    encoder = LabelEncoder()\n    chunk['event_name'] = encoder.fit_transform(chunk['event_name'])\n    chunk['name'] = encoder.fit_transform(chunk['name'])\n    chunk['room_fqid'] = encoder.fit_transform(chunk['room_fqid'])\n    '''\n    \n    # fill missing item by previous value, \n    chunk[['room_coor_x', 'room_coor_y']].fillna(method='ffill', inplace=True)\n    chunk[['screen_coor_x', 'screen_coor_y']].fillna(method='ffill', inplace=True)\n    \n    # scale the data\n    ...\n\n    # Add the preprocessed chunk to the list\n    data.append(chunk)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:48.391498Z","iopub.execute_input":"2023-04-20T18:50:48.392762Z","iopub.status.idle":"2023-04-20T18:50:48.406758Z","shell.execute_reply.started":"2023-04-20T18:50:48.392705Z","shell.execute_reply":"2023-04-20T18:50:48.405407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load in data\n\n#avoid memory error\nchunk_size = 10000\ndata = []\ndata_generator = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', chunksize=chunk_size, dtype=dtypes,nrows=10000)\nfor chunk in data_generator:\n    process(chunk,data)\n# Concatenate all preprocessed chunks into a single DataFrame\ndata = pd.concat(data, axis=0)\n\n# Load target label\ntarget = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:48.408466Z","iopub.execute_input":"2023-04-20T18:50:48.409200Z","iopub.status.idle":"2023-04-20T18:50:49.066497Z","shell.execute_reply.started":"2023-04-20T18:50:48.409158Z","shell.execute_reply":"2023-04-20T18:50:49.065137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:49.068220Z","iopub.execute_input":"2023-04-20T18:50:49.068575Z","iopub.status.idle":"2023-04-20T18:50:49.106328Z","shell.execute_reply.started":"2023-04-20T18:50:49.068540Z","shell.execute_reply":"2023-04-20T18:50:49.104648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The session_id of target contains the number of topics (q1-q8), which cannot be matched with the session_id of data. Because, add a new user_id which in target.(this idea refer to chiaenchen)\nUse regular expressions to achieve this.","metadata":{}},{"cell_type":"code","source":"import re #regular expression\n\n# split session_id by \"-\". -> 20090312431273200_q1: 20090312431273200 and 1\ntarget[\"session_id\"],target[\"q_id\"] = target.session_id.str.split(\"_\", expand = True)[0],target.session_id.str.split(\"_\", expand = True)[1] # session_id is 20090312431273200\ntarget[\"q_id\"] = target[\"q_id\"].apply(lambda x : re.sub(\"\\D\", \"\",x)) # remove all non-digit characters from the \"question\" column \n\n# to numeric\ntarget[\"q_id\"] = pd.to_numeric(target[\"q_id\"])\ntarget[\"session_id\"] = pd.to_numeric(target[\"session_id\"])\ntarget.head(3)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:49.108482Z","iopub.execute_input":"2023-04-20T18:50:49.108897Z","iopub.status.idle":"2023-04-20T18:50:53.377033Z","shell.execute_reply.started":"2023-04-20T18:50:49.108844Z","shell.execute_reply":"2023-04-20T18:50:53.375867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"just_dummies = pd.get_dummies(data['event_name'])\ndata = pd.concat([data, just_dummies], axis=1)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:53.378805Z","iopub.execute_input":"2023-04-20T18:50:53.379186Z","iopub.status.idle":"2023-04-20T18:50:53.409539Z","shell.execute_reply.started":"2023-04-20T18:50:53.379151Z","shell.execute_reply":"2023-04-20T18:50:53.408221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"list of question; \nnumber of user in data; \nnumber of user in target; \n\n\n","metadata":{}},{"cell_type":"code","source":"data['event_name'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:53.411125Z","iopub.execute_input":"2023-04-20T18:50:53.411499Z","iopub.status.idle":"2023-04-20T18:50:53.426566Z","shell.execute_reply.started":"2023-04-20T18:50:53.411442Z","shell.execute_reply":"2023-04-20T18:50:53.425308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.q_id.unique()\ndata.session_id.nunique()\ntarget.session_id.nunique()\ndata.shape,target.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:53.430671Z","iopub.execute_input":"2023-04-20T18:50:53.431320Z","iopub.status.idle":"2023-04-20T18:50:53.460776Z","shell.execute_reply.started":"2023-04-20T18:50:53.431272Z","shell.execute_reply":"2023-04-20T18:50:53.459731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.to_csv(\"/kaggle/working/extended_target\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:53.462260Z","iopub.execute_input":"2023-04-20T18:50:53.463450Z","iopub.status.idle":"2023-04-20T18:50:54.059296Z","shell.execute_reply.started":"2023-04-20T18:50:53.463406Z","shell.execute_reply":"2023-04-20T18:50:54.058057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data analysis","metadata":{}},{"cell_type":"code","source":"# data info\ndata.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.060729Z","iopub.execute_input":"2023-04-20T18:50:54.061168Z","iopub.status.idle":"2023-04-20T18:50:54.097502Z","shell.execute_reply.started":"2023-04-20T18:50:54.061136Z","shell.execute_reply":"2023-04-20T18:50:54.096159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user1's data\nid1 = data.session_id[0]\ndata1 = data.loc[data.session_id == id1]\ndata1\n# user1's answer\ntarget[target.session_id == id1]\n# group by session_id\ngrouped_data = data.groupby('session_id')\ngrouped_data.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.099041Z","iopub.execute_input":"2023-04-20T18:50:54.099351Z","iopub.status.idle":"2023-04-20T18:50:54.156786Z","shell.execute_reply.started":"2023-04-20T18:50:54.099320Z","shell.execute_reply":"2023-04-20T18:50:54.155489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Process data","metadata":{}},{"cell_type":"code","source":"def convert_time(df_sessions):\n    df_final = pd.DataFrame()\n    df_final['session_id'] = df_sessions['session_id'].unique()\n    df_final['year'] = df_final['session_id'].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    df_final['month'] = df_final['session_id'].apply(lambda x: int(str(x)[2:4]) + 1).astype(np.uint8)\n    df_final['weekday'] = df_final['session_id'].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    df_final['hour'] = df_final['session_id'].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    df_final['minute'] = df_final['session_id'].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    df_final['second'] = df_final['session_id'].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    df_final['ms'] = df_final['session_id'].apply(lambda x: int(str(x)[12:15])).astype(np.uint16)\n    df_final['noise'] = df_final['session_id'].apply(lambda x: int(str(x)[15:17])).astype(np.uint8)\n    return df_final","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.159913Z","iopub.execute_input":"2023-04-20T18:50:54.161281Z","iopub.status.idle":"2023-04-20T18:50:54.172200Z","shell.execute_reply.started":"2023-04-20T18:50:54.161235Z","shell.execute_reply":"2023-04-20T18:50:54.170722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merge = pd.merge(data, target, on = 'session_id')\ndf_final = convert_time(df_merge)\ndf_final = df_final.set_index(['session_id'])\ndf_final.index.get_level_values('session_id')\nday_map = {0: 'monday', 1: 'tuesday', 2:'wednesday', 3:'thursday', 4:'friday', 5:'saturday', 6:'sunday'}\ndf_final = pd.merge(data, df_final, on= 'session_id')\ndf_final.weekday = df_final.weekday.map(day_map)\ndf_final.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_target = target.groupby('session_id')\ntarget.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.298424Z","iopub.execute_input":"2023-04-20T18:50:54.298764Z","iopub.status.idle":"2023-04-20T18:50:54.311947Z","shell.execute_reply.started":"2023-04-20T18:50:54.298731Z","shell.execute_reply":"2023-04-20T18:50:54.310829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_var = ['event_name', 'fqid','room_fqid', 'text']\nmean_var = ['elapsed_time','level']\nevent_var = ['navigate_click','person_click','cutscene_click','object_click','map_hover','notification_click',\n            'map_click','observation_click','checkpoint','elapsed_time']","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.315679Z","iopub.execute_input":"2023-04-20T18:50:54.316081Z","iopub.status.idle":"2023-04-20T18:50:54.321700Z","shell.execute_reply.started":"2023-04-20T18:50:54.316045Z","shell.execute_reply":"2023-04-20T18:50:54.320667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### reference: https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook","metadata":{}},{"cell_type":"code","source":"def click_convert(train):\n    dfs = []\n    for c in count_var:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in mean_var:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        dfs.append(tmp)\n    for c in event_var:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.323487Z","iopub.execute_input":"2023-04-20T18:50:54.324201Z","iopub.status.idle":"2023-04-20T18:50:54.337745Z","shell.execute_reply.started":"2023-04-20T18:50:54.324161Z","shell.execute_reply":"2023-04-20T18:50:54.336149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final = click_convert(df_final)\nprint( df_final.shape )","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.339706Z","iopub.execute_input":"2023-04-20T18:50:54.340409Z","iopub.status.idle":"2023-04-20T18:50:54.432987Z","shell.execute_reply.started":"2023-04-20T18:50:54.340366Z","shell.execute_reply":"2023-04-20T18:50:54.431504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by session_id\ngrouped_data = data.groupby('session_id')\ngrouped_data.head(3)\ndf_final","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.435031Z","iopub.execute_input":"2023-04-20T18:50:54.435420Z","iopub.status.idle":"2023-04-20T18:50:54.473685Z","shell.execute_reply.started":"2023-04-20T18:50:54.435387Z","shell.execute_reply":"2023-04-20T18:50:54.472719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Split the data into train, validation, and test sets\nX_train, X_val, y_train, y_val = train_test_split(data, target, test_size=0.2, random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_val, y_val, test_size=0.5, random_state=42)\n\n# Create an SVM classifier with a linear kernel\nsvm = SVC(kernel='linear')\n\n# Train the model on the training set\nsvm.fit(X_train, y_train)\n\n# Predict on the validation set and evaluate the performance\ny_pred = svm.predict(X_val)\nprint(classification_report(y_val, y_pred))\n\n# Predict on the test set and evaluate the performance\ny_pred_test = svm.predict(X_test)\nprint(classification_report(y_test, y_pred_test))\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-04-20T18:50:54.475241Z","iopub.execute_input":"2023-04-20T18:50:54.475849Z","iopub.status.idle":"2023-04-20T18:50:54.488407Z","shell.execute_reply.started":"2023-04-20T18:50:54.475813Z","shell.execute_reply":"2023-04-20T18:50:54.486915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}