{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:22:43.496715Z","iopub.execute_input":"2023-06-05T17:22:43.498201Z","iopub.status.idle":"2023-06-05T17:22:43.518431Z","shell.execute_reply.started":"2023-06-05T17:22:43.498148Z","shell.execute_reply":"2023-06-05T17:22:43.516886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:22:48.753654Z","iopub.execute_input":"2023-06-05T17:22:48.754586Z","iopub.status.idle":"2023-06-05T17:22:48.854476Z","shell.execute_reply.started":"2023-06-05T17:22:48.754531Z","shell.execute_reply":"2023-06-05T17:22:48.853508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:22:56.754685Z","iopub.execute_input":"2023-06-05T17:22:56.755090Z","iopub.status.idle":"2023-06-05T17:24:51.565891Z","shell.execute_reply.started":"2023-06-05T17:22:56.755058Z","shell.execute_reply":"2023-06-05T17:24:51.564884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"we create this two functions to be sure about deleting the column text note that in the train file you have 26296947 rows ","metadata":{}},{"cell_type":"code","source":"def verify(data, col1, col2):\n    i = 0  \n    for index, row in data.iterrows():\n        if row[col1] != 'checkpoint' and pd.isnull(row[col2]):\n            i += 1\n\n    return i","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"verify(data2, 'event_name','text')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def verifyy(data, col1, col2):\n    i = 0  \n    for index, row in data.iterrows():\n        if row[col1] == 'checkpoint' and pd.isnull(row[col2]):\n            i += 1\n\n    return i","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"verifyy(data2, 'event_name','text')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_unique_values(data, column_name):\n    if column_name not in data.columns:\n        raise ValueError(f\"Column '{column_name}' does not exist in the data.\")\n\n    unique_values = data[column_name].unique().tolist()\n    return unique_values\ncolumn_name = 'page'  \nunique_values = get_unique_values(data2, column_name)\nprint(f\"Unique values in column '{column_name}':\")\nfor value in unique_values:\n    print(value)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:32:40.352658Z","iopub.execute_input":"2023-06-05T17:32:40.353512Z","iopub.status.idle":"2023-06-05T17:32:40.450613Z","shell.execute_reply.started":"2023-06-05T17:32:40.353476Z","shell.execute_reply":"2023-06-05T17:32:40.449532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:14:02.059065Z","iopub.execute_input":"2023-06-03T20:14:02.059438Z","iopub.status.idle":"2023-06-03T20:14:02.091391Z","shell.execute_reply.started":"2023-06-03T20:14:02.059405Z","shell.execute_reply":"2023-06-03T20:14:02.090259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def change_data(data, col_to_change, col_to_del, save_path):\n    col_not_found = [col for col in col_to_del if col not in data.columns]\n    if col_not_found : \n        print(\"col not found\")\n        return\n    \n    data.drop(col_to_del, axis=1, inplace= True)\n    data[col_to_change]=data[col_to_change]/1000.0\n    data.to_pickle(save_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:32:49.208266Z","iopub.execute_input":"2023-06-05T17:32:49.208628Z","iopub.status.idle":"2023-06-05T17:32:49.216634Z","shell.execute_reply.started":"2023-06-05T17:32:49.208597Z","shell.execute_reply":"2023-06-05T17:32:49.215820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"change_data(data,'elapsed_time',['hover_duration','fullscreen','hq','music', 'text'],'cleaneddata.pickle')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:32:54.537480Z","iopub.execute_input":"2023-06-05T17:32:54.538173Z","iopub.status.idle":"2023-06-05T17:33:10.849467Z","shell.execute_reply.started":"2023-06-05T17:32:54.538138Z","shell.execute_reply":"2023-06-05T17:33:10.848503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:34:52.463742Z","iopub.execute_input":"2023-06-05T17:34:52.464744Z","iopub.status.idle":"2023-06-05T17:34:53.064561Z","shell.execute_reply.started":"2023-06-05T17:34:52.464707Z","shell.execute_reply":"2023-06-05T17:34:53.063584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:34:56.864043Z","iopub.execute_input":"2023-06-05T17:34:56.864392Z","iopub.status.idle":"2023-06-05T17:34:56.967049Z","shell.execute_reply.started":"2023-06-05T17:34:56.864364Z","shell.execute_reply":"2023-06-05T17:34:56.965861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_pickle(\"/kaggle/working/cleaneddata.pickle\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:35:04.744904Z","iopub.execute_input":"2023-06-05T17:35:04.745269Z","iopub.status.idle":"2023-06-05T17:35:14.786135Z","shell.execute_reply.started":"2023-06-05T17:35:04.745238Z","shell.execute_reply":"2023-06-05T17:35:14.785104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:35:17.167018Z","iopub.execute_input":"2023-06-05T17:35:17.167725Z","iopub.status.idle":"2023-06-05T17:35:17.263696Z","shell.execute_reply.started":"2023-06-05T17:35:17.167678Z","shell.execute_reply":"2023-06-05T17:35:17.262655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicated = data[data.duplicated()]\nif not duplicated.empty:\n    print(\"Duplicate rows found:\")\n    print(duplicated)\nelse:\n    print(\"No duplicate rows found.\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:40:12.892915Z","iopub.execute_input":"2023-06-05T17:40:12.893524Z","iopub.status.idle":"2023-06-05T17:41:22.025788Z","shell.execute_reply.started":"2023-06-05T17:40:12.893487Z","shell.execute_reply":"2023-06-05T17:41:22.024706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_empty_value = data[data.isnull().any(axis = 1)]\nif not row_empty_value.empty:\n    print(\"Rows with empty values:\")\n    print(row_empty_value)\nelse:\n    print(\"No rows with empty values found.\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:49:05.189517Z","iopub.execute_input":"2023-06-05T17:49:05.189915Z","iopub.status.idle":"2023-06-05T17:49:49.837098Z","shell.execute_reply.started":"2023-06-05T17:49:05.189877Z","shell.execute_reply":"2023-06-05T17:49:49.836089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_same_data_type (data) : \n    object_col = []\n    int_col = []\n    float_col =[]\n    for column in data.columns : \n        if data[column].dtype =='object':\n            object_col.append(column)\n        elif data[column].dtype == 'int64':\n            int_col.append(column)\n        elif data[column].dtype == 'float64' : \n            float_col.append(column)\n    return(object_col, int_col, float_col)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:55:36.493221Z","iopub.execute_input":"2023-06-05T17:55:36.493619Z","iopub.status.idle":"2023-06-05T17:55:36.499442Z","shell.execute_reply.started":"2023-06-05T17:55:36.493580Z","shell.execute_reply":"2023-06-05T17:55:36.498542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_col, int_col, float_col = split_same_data_type(data)\nobject_data = data[object_col]\nint_data = data[int_col]\nfloat_data = data[float_col]","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:55:45.243860Z","iopub.execute_input":"2023-06-05T17:55:45.244219Z","iopub.status.idle":"2023-06-05T17:55:47.476761Z","shell.execute_reply.started":"2023-06-05T17:55:45.244189Z","shell.execute_reply":"2023-06-05T17:55:47.475804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:55:52.577756Z","iopub.execute_input":"2023-06-05T17:55:52.578119Z","iopub.status.idle":"2023-06-05T17:55:52.682225Z","shell.execute_reply.started":"2023-06-05T17:55:52.578088Z","shell.execute_reply":"2023-06-05T17:55:52.681319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_data.to_csv('object_data.csv', index= False)\ngc.collect()\nint_data.to_csv('int_data.csv', index= False)\ngc.collect()\nfloat_data.to_csv('float_data.csv', index= False)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:56:00.773690Z","iopub.execute_input":"2023-06-05T17:56:00.774071Z","iopub.status.idle":"2023-06-05T18:02:57.571542Z","shell.execute_reply.started":"2023-06-05T17:56:00.774039Z","shell.execute_reply":"2023-06-05T18:02:57.570684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T18:51:42.980854Z","iopub.execute_input":"2023-06-05T18:51:42.981212Z","iopub.status.idle":"2023-06-05T18:51:43.086321Z","shell.execute_reply.started":"2023-06-05T18:51:42.981182Z","shell.execute_reply":"2023-06-05T18:51:43.085415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data","metadata":{"execution":{"iopub.status.busy":"2023-06-05T18:51:37.169682Z","iopub.execute_input":"2023-06-05T18:51:37.170380Z","iopub.status.idle":"2023-06-05T18:51:37.696166Z","shell.execute_reply.started":"2023-06-05T18:51:37.170345Z","shell.execute_reply":"2023-06-05T18:51:37.695069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object = pd.read_csv('/kaggle/working/object_data.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T18:52:11.003840Z","iopub.execute_input":"2023-06-05T18:52:11.004196Z","iopub.status.idle":"2023-06-05T18:52:41.044419Z","shell.execute_reply.started":"2023-06-05T18:52:11.004166Z","shell.execute_reply":"2023-06-05T18:52:41.043395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T21:39:40.377661Z","iopub.execute_input":"2023-06-03T21:39:40.378484Z","iopub.status.idle":"2023-06-03T21:39:40.395057Z","shell.execute_reply.started":"2023-06-03T21:39:40.378447Z","shell.execute_reply":"2023-06-03T21:39:40.393888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colmn= 'fqid'\ncol_empty = object[colmn].isnull().any()\nif col_empty :\n     print(f\"The column '{colmn}' contains empty values.\")\nelse:\n    print(f\"The column '{colmn}' does not contain empty values.\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T19:09:45.266205Z","iopub.execute_input":"2023-06-05T19:09:45.266561Z","iopub.status.idle":"2023-06-05T19:09:47.369797Z","shell.execute_reply.started":"2023-06-05T19:09:45.266530Z","shell.execute_reply":"2023-06-05T19:09:47.368825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colmn= 'fqid'\nobject[colmn].fillna('', inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T19:09:40.088615Z","iopub.execute_input":"2023-06-05T19:09:40.089252Z","iopub.status.idle":"2023-06-05T19:09:42.264662Z","shell.execute_reply.started":"2023-06-05T19:09:40.089217Z","shell.execute_reply":"2023-06-05T19:09:42.263550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T19:10:04.513066Z","iopub.execute_input":"2023-06-05T19:10:04.513427Z","iopub.status.idle":"2023-06-05T19:10:04.647437Z","shell.execute_reply.started":"2023-06-05T19:10:04.513395Z","shell.execute_reply":"2023-06-05T19:10:04.646501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del object\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T21:39:45.028774Z","iopub.execute_input":"2023-06-03T21:39:45.029803Z","iopub.status.idle":"2023-06-03T21:39:45.200571Z","shell.execute_reply.started":"2023-06-03T21:39:45.029753Z","shell.execute_reply":"2023-06-03T21:39:45.199374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float = pd.read_pickle('/kaggle/working/float_data.pickle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del float\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int = pd.read_csv('/kaggle/working/int_data.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-03T21:40:03.232688Z","iopub.execute_input":"2023-06-03T21:40:03.233058Z","iopub.status.idle":"2023-06-03T21:40:13.973753Z","shell.execute_reply.started":"2023-06-03T21:40:03.233026Z","shell.execute_reply":"2023-06-03T21:40:13.972719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T21:40:21.767367Z","iopub.execute_input":"2023-06-03T21:40:21.767765Z","iopub.status.idle":"2023-06-03T21:40:21.777175Z","shell.execute_reply.started":"2023-06-03T21:40:21.767731Z","shell.execute_reply":"2023-06-03T21:40:21.775974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del int \ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info(memory_usage=\"deep\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:46:32.494262Z","iopub.execute_input":"2023-06-03T20:46:32.494621Z","iopub.status.idle":"2023-06-03T20:46:32.601882Z","shell.execute_reply.started":"2023-06-03T20:46:32.494592Z","shell.execute_reply":"2023-06-03T20:46:32.600928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['session_id'] = data['session_id'].astype('int32')\ndata['index'] = data['index'].astype('int32')\ndata['elapsed_time']= data['elapsed_time'].astype('float32')\ndata['event_name']= data['event_name'].astype('category')\ndata['name']= data['name'].astype('category')\ndata['level']=data['level'].astype('int32')\ndata['page']=data['page'].astype('float32')\ndata['room_coor_x']= data['room_coor_x'].astype('float32')\ndata['room_coor_y']= data['room_coor_y'].astype('float32')\ndata['screen_coor_x'] = data['screen_coor_x'].astype('float32')\ndata['screen_coor_y'] = data['screen_coor_y'].astype('float32')\ndata['text'] = data['text'].astype('category')   \ndata['fqid'] = data['fqid'].astype('category')\ndata['room_fqid'] = data['room_fqid'].astype('category')\ndata['text_fqid'] = data['text_fqid'].astype('category')\ndata['level_group'] = data['level_group'].astype('category') ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.to_pickle('output_pickle_file.pickle')\ndata2 = pd.read_pickle('/kaggle/working/output_pickle_file.pickle')\ndel data","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}