{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\nfor dirname, _, filenames in os.walk('.'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename),os.path.join(os.path.abspath(dirname), filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-08T13:32:44.206666Z","iopub.execute_input":"2023-05-08T13:32:44.207090Z","iopub.status.idle":"2023-05-08T13:32:44.214401Z","shell.execute_reply.started":"2023-05-08T13:32:44.207054Z","shell.execute_reply":"2023-05-08T13:32:44.213586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport os.path as path\nimport random\n\nimport matplotlib.pyplot as plt\nimport numpy\nimport pandas\nimport seaborn as sns\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import GroupKFold\nfrom xgboost import XGBClassifier","metadata":{"execution":{"iopub.status.busy":"2023-05-08T13:32:47.470104Z","iopub.execute_input":"2023-05-08T13:32:47.470508Z","iopub.status.idle":"2023-05-08T13:32:47.476687Z","shell.execute_reply.started":"2023-05-08T13:32:47.470473Z","shell.execute_reply":"2023-05-08T13:32:47.475576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_df = pandas.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\nprint(target_df.shape)\ntarget_df['session_id'] = target_df['session_id'].astype(dtype=str)\ngb = target_df.groupby(['session_id'])\ntmp = gb['correct'].agg('sum')\ntarget_dict = tmp.to_dict()\nprint('0: ' + str(len(target_df[target_df['correct'] == 0])), '1: ' + str(len(target_df[target_df['correct'] == 1])))\ndel target_df\ngc.collect()\n\nv_list =[1]*5\nv_list.append(2)\nv_list.append(2)\nv_list.append(2)\nv_list.append(2)\n# v_list =[1]*999\n# v_list.append(2)\nlines = []\ntotal = 0\nif path.exists('/kaggle/working/train2.csv'):\n    os.remove('/kaggle/working/train2.csv')\nwith open('/kaggle/working/train2.csv', 'w', encoding='utf8') as writer:\n    with open('/kaggle/input/predict-student-performance-from-game-play/train.csv', 'r', encoding='utf8') as reader:\n        while True:\n            random.shuffle(v_list)\n            line = reader.readline()\n            total += 1\n            if (len(lines)>0 and v_list[0] == 1):\n                continue\n            if not line:\n                break\n            lines.append(line)\n            if len(lines) > 10000:\n                writer.writelines(lines)\n                lines = []\n    if len(lines) > 0:\n        writer.writelines(lines)\n\nprint('total = ' + str(total))\ntrain_df = pandas.read_csv('/kaggle/working/train2.csv')\nprint(len(train_df),train_df.columns)\ntest_df = pandas.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv')\nprint(len(test_df),test_df.columns)\nall_df = [train_df, test_df]\n","metadata":{"execution":{"iopub.status.busy":"2023-05-08T13:32:51.004147Z","iopub.execute_input":"2023-05-08T13:32:51.004561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_screen_coor_x(x: float):\n    # (-1.916, 638.667] < (638.667, 1277.333] < (1277.333, 1916.0]\n    x = x if x is not None else 0\n    if x <= 638.667:\n        return '0'\n    elif x <= 1277.333:\n        return '1'\n    else:\n        return '2'\n\n\ndef process_screen_coor_y(y: float):\n    # (-1.439, 479.667] < (479.667, 959.333] < (959.333, 1439.0]\n    y = y if y is not None else 0\n    if y <= 479.667:\n        return '0'\n    elif y <= 959.333:\n        return '1'\n    else:\n        return '2'\n\n\ndef process_room_coor_x(x: float):\n    # (-1995.254, -907.333] < (-907.333, 177.333] <  (177.333, 1262.0]\n    x = x if x is not None else 0\n    if x <= -907.333:\n        return '0'\n    elif x <= 177.333:\n        return '1'\n    else:\n        return '2'\n\n\ndef process_room_coor_y(y: float):\n    # (-919.462, -430.667] < (-430.667, 56.667] < (56.667, 544.0]\n    y = y if y is not None else 0\n    if y <= -430.667:\n        return '0'\n    elif y <= 56.667:\n        return '1'\n    else:\n        return '2'\n\n\ndef process_elapsed_time(t: float):\n    # (-1749293.395, 583097798.333] < (583097798.333, 1166195596.667] <  (1166195596.667, 1749293395.0]\n    t = t if t is not None else 0\n    if t <= 583097798.333:\n        return '0'\n    elif t <= 1166195596.667:\n        return '1'\n    else:\n        return '2'\n\n\ndef level_group_convert(level_group: str):\n    if '0-4' == level_group:\n        return 1, 4\n    elif '5-12' == level_group:\n        return 4, 14\n    else:\n        return 14, 19","metadata":{"execution":{"iopub.status.busy":"2023-05-07T14:13:41.073783Z","iopub.execute_input":"2023-05-07T14:13:41.075691Z","iopub.status.idle":"2023-05-07T14:13:41.090392Z","shell.execute_reply.started":"2023-05-07T14:13:41.075577Z","shell.execute_reply":"2023-05-07T14:13:41.089389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(df: pandas.DataFrame):\n    data_list = []\n    data_group = {}\n    for index in df.index:\n#         if index >= 1000000:\n#             break\n        row = df.iloc[index, :]\n#         print(row)\n        session_id = str(row['session_id'])\n        level_group = row['level_group']\n        key = '{}_{}'.format(session_id, level_group)\n        data_dict = data_group.get(key)\n        if not data_dict:\n            data_dict = {}\n            data_group.setdefault(key, data_dict)\n            data_list.append(data_dict)\n        t = 't_{}'.format(process_elapsed_time(row['elapsed_time']))\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = '{}_{}'.format(process_screen_coor_x(row['screen_coor_x']), process_screen_coor_y(row['screen_coor_y']))\n        t = 'screen_coor_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = '{}_{}'.format(process_room_coor_x(row['room_coor_x']), process_room_coor_y(row['room_coor_y']))\n        t = 'room_coor_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = str(row['event_name'])\n        t = 'event_name_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = str(row['level'])\n        t = 'level_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = str(row['fqid'])\n        t = 'fqid_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = str(row['room_fqid'])\n        t = 'room_fqid_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        t = str(row['name'])\n        t = 'name_{}'.format(t)\n        data_dict[t] = data_dict.get(t, 0) + 1\n        start, end = level_group_convert(level_group)\n        for i in range(start, end):\n            data_dict['q_' + str(i)] = target_dict.get(str(session_id) + '_q' + str(i), 0)\n        data_dict['session_id'] = session_id\n        data_dict['level_group'] = level_group\n    return pandas.DataFrame(data_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T14:13:45.467887Z","iopub.execute_input":"2023-05-07T14:13:45.468279Z","iopub.status.idle":"2023-05-07T14:13:45.483753Z","shell.execute_reply.started":"2023-05-07T14:13:45.468244Z","shell.execute_reply":"2023-05-07T14:13:45.482727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df2 = []\nall_columns = set()\nfor df in all_df:\n    print(df.shape)\n    df_new = preprocess_data(df)\n    del df\n    print(df_new.shape)\n    all_df2.append(df_new)\n    all_columns.update(set(df_new.columns))\nprint(all_columns)\nall_df.clear()\nfor df in all_df2:\n    col_list = df.columns\n    for col in all_columns:\n        if col not in col_list:\n            df = pandas.concat([df, pandas.DataFrame({col: [0] * len(df)})], axis=1)\n            continue\n        if col == 'session_id' or col == 'level_group' or col.startswith('q_'):\n            continue\n        df[col].fillna(0)\n    all_df.append(df)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T14:13:50.064684Z","iopub.execute_input":"2023-05-07T14:13:50.065104Z","iopub.status.idle":"2023-05-07T14:25:44.648287Z","shell.execute_reply.started":"2023-05-07T14:13:50.065066Z","shell.execute_reply":"2023-05-07T14:25:44.647115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\nFEATURES = []\nresult_list=[]\nfor col in all_columns:\n    if col.startswith('q_'):\n        continue\n    FEATURES.append(col)\nFEATURES.remove('session_id')\nFEATURES.remove('level_group')\ntrain_df, test_df = all_df\ngkf = GroupKFold(n_splits=5)\nmodels = {}\nfor i, (train_index, test_index) in enumerate(gkf.split(X=train_df, groups=train_df.index)):\n    print('*' * 100)\n    test_result = []\n    for t in range(1, 19):\n        if t <= 3:\n            grp = '0-4'\n        elif t <= 13:\n            grp = '5-12'\n        elif t <= 22:\n            grp = '13-22'\n\n        # TRAIN DATA\n        train_x = train_df.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_x2 = train_x.loc[:, :]\n        train_x2['correct'] = train_x2['q_' + str(t)]\n\n        # VALID DATA\n        valid_x = train_df.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_x2 = valid_x.loc[:, :]\n        valid_x2['correct'] = valid_x2['q_' + str(t)]\n\n        # TRAIN MODEL\n        clf = XGBClassifier()\n        clf.fit(train_x2[FEATURES].astype('float32'), train_x2['correct'])\n        score1 = clf.score(valid_x2[FEATURES].astype('float32'), valid_x2['correct'])\n        models[f'{grp}_{t}'] = clf\n\n        test_df2 = test_df.loc[test_df['level_group'] == grp]\n        test_df2_copy = test_df2.copy().reset_index()\n        test_df2_copy['correct'] = pandas.Series(clf.predict(test_df2_copy[FEATURES].astype('float32')))\n        test_df2_copy['correct'] = test_df2_copy['correct'].astype(dtype=numpy.int64)\n        test_df2_copy['q'] = pandas.Series([str(t)] * len(test_df2_copy))\n        test_df2_copy['session_level'] = test_df2_copy['session_id'].astype(dtype=str) + '_' + test_df2_copy[\n            'level_group']\n        test_df2_copy['session_id'] = test_df2_copy['session_id'].astype(dtype=str) + '_q' + test_df2_copy['q']\n        test_result.append(test_df2_copy[['session_id', 'correct', 'session_level']])\n\n        lc = LGBMClassifier()\n        lc.fit(train_x2[FEATURES].astype('float32'), train_x2['correct'])\n        score2 = lc.score(valid_x2[FEATURES].astype('float32'), valid_x2['correct'])\n        print(f'XGB => {grp}_{t}：{score1} ; lc => {grp}_{t}：{score2}')\n    result_df = pandas.concat(test_result, axis=0).reset_index()\n    result_list.append(result_df)\nresult_df=pandas.concat(result_list, axis=0).reset_index()\nresult_df = result_df.drop(columns=['level_0','index'], errors='ignore')\nresult_df.to_csv('result_submission.csv',index=False)\nprint('run end,size={}'.format(len(result_list)))","metadata":{"execution":{"iopub.status.busy":"2023-05-07T14:48:08.398294Z","iopub.execute_input":"2023-05-07T14:48:08.398742Z","iopub.status.idle":"2023-05-07T15:06:57.954864Z","shell.execute_reply.started":"2023-05-07T14:48:08.398702Z","shell.execute_reply":"2023-05-07T15:06:57.953669Z"},"trusted":true},"execution_count":null,"outputs":[]}]}