{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-27T07:56:51.084732Z","iopub.execute_input":"2023-04-27T07:56:51.085591Z","iopub.status.idle":"2023-04-27T07:56:51.126870Z","shell.execute_reply.started":"2023-04-27T07:56:51.085545Z","shell.execute_reply":"2023-04-27T07:56:51.125625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndtypes={\n        'session_id':'category', \n        'elapsed_time':np.int32,\n        'event_name':'category',\n        'name':'category',\n        'level':np.uint8,\n        'page':'category',\n        'room_coor_x':np.float32,\n        'room_coor_y':np.float32,\n        'screen_coor_x':np.float32,\n        'screen_coor_y':np.float32,\n        'hover_duration':np.float32,\n         'text':'category',\n         'fqid':'category',\n         'room_fqid':'category',\n         'text_fqid':'category',\n         'fullscreen':'category',\n         'hq':'category',\n         'music':'category',\n         'level_group':'category'\n        }\ntrain = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype = dtypes)\ntest = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv', dtype = dtypes)\ntrain_label = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:56:51.129487Z","iopub.execute_input":"2023-04-27T07:56:51.130239Z","iopub.status.idle":"2023-04-27T07:58:46.656163Z","shell.execute_reply.started":"2023-04-27T07:56:51.130198Z","shell.execute_reply":"2023-04-27T07:58:46.654761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:58:46.662614Z","iopub.execute_input":"2023-04-27T07:58:46.663793Z","iopub.status.idle":"2023-04-27T07:58:46.846421Z","shell.execute_reply.started":"2023-04-27T07:58:46.663745Z","shell.execute_reply":"2023-04-27T07:58:46.845079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"portion_null = train.isnull().sum()/len(train) \nprint(\"Remove features: \", portion_null[portion_null > 0.5].index)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T07:25:43.303748Z","iopub.execute_input":"2023-04-26T07:25:43.304417Z","iopub.status.idle":"2023-04-26T07:25:50.038266Z","shell.execute_reply.started":"2023-04-26T07:25:43.304376Z","shell.execute_reply":"2023-04-26T07:25:50.036926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remove the feature if the portion of null value is over 50%.","metadata":{}},{"cell_type":"code","source":"cols = portion_null[portion_null <0.5].index.to_list()\nrm_null_train = train[cols]\nrm_null_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T07:30:18.898985Z","iopub.execute_input":"2023-04-26T07:30:18.899645Z","iopub.status.idle":"2023-04-26T07:30:20.932828Z","shell.execute_reply.started":"2023-04-26T07:30:18.899606Z","shell.execute_reply":"2023-04-26T07:30:20.931630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nimport re\nfrom nltk.tokenize import word_tokenize\n\ntext = train[train['text'] != 'undefined']['text'].dropna()\n\ntoken_sen = []\nfor i in text[:100000]:\n    if i:\n        tmp = re.sub(r'[^\\w\\s]','',i)\n        token_sen.append(nltk.word_tokenize(tmp))","metadata":{"execution":{"iopub.status.busy":"2023-04-27T09:15:13.364795Z","iopub.execute_input":"2023-04-27T09:15:13.365214Z","iopub.status.idle":"2023-04-27T09:15:25.642561Z","shell.execute_reply.started":"2023-04-27T09:15:13.365178Z","shell.execute_reply":"2023-04-27T09:15:25.641416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remove stop words","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nstop_list = stopwords.words(\"english\")\nrm_stop_sen = []\nfor sen in token_sen:\n    tmp = [w for w in sen if w.lower() not in stop_list]\n    rm_stop_sen.append(tmp)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T09:15:03.639680Z","iopub.execute_input":"2023-04-27T09:15:03.640073Z","iopub.status.idle":"2023-04-27T09:15:03.795963Z","shell.execute_reply.started":"2023-04-27T09:15:03.640037Z","shell.execute_reply":"2023-04-27T09:15:03.794600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label['q'] = train_label['session_id'].str.split('_q', expand = True)[1]\ntrain_label['session_id'] = train_label['session_id'].str.split('_q', expand = True)[0]","metadata":{"execution":{"iopub.status.busy":"2023-04-27T09:28:25.709900Z","iopub.execute_input":"2023-04-27T09:28:25.710854Z","iopub.status.idle":"2023-04-27T09:28:26.462366Z","shell.execute_reply.started":"2023-04-27T09:28:25.710816Z","shell.execute_reply":"2023-04-27T09:28:26.461156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}