{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pickle\nimport glob\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-12T03:32:25.280844Z","iopub.execute_input":"2023-02-12T03:32:25.281313Z","iopub.status.idle":"2023-02-12T03:32:25.402822Z","shell.execute_reply.started":"2023-02-12T03:32:25.281278Z","shell.execute_reply":"2023-02-12T03:32:25.401854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = ['event_name', 'fqid', 'room_fqid', 'text']\nNUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']\n\n# https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data\nEVENTS = ['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'checkpoint']\n\nFEATURES = ['event_name_nunique','fqid_nunique','room_fqid_nunique','text_nunique','elapsed_time_mean',\n         'level_mean','page_mean','room_coor_x_mean','room_coor_y_mean','screen_coor_x_mean',\n         'screen_coor_y_mean','hover_duration_mean','elapsed_time_std',\n         'level_std','page_std','room_coor_x_std','room_coor_y_std','screen_coor_x_std','screen_coor_y_std',\n         'hover_duration_std','navigate_click_sum','person_click_sum','cutscene_click_sum','object_click_sum',\n         'map_hover_sum','notification_click_sum','map_click_sum','observation_click_sum','checkpoint_sum','elapsed_time_sum']","metadata":{"execution":{"iopub.status.busy":"2023-02-12T03:32:46.808459Z","iopub.execute_input":"2023-02-12T03:32:46.808945Z","iopub.status.idle":"2023-02-12T03:32:46.815188Z","shell.execute_reply.started":"2023-02-12T03:32:46.808875Z","shell.execute_reply":"2023-02-12T03:32:46.814024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    \n    dfs = []\n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    for c in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    train = train.drop(EVENTS,axis=1)\n        \n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-02-12T03:32:47.343240Z","iopub.execute_input":"2023-02-12T03:32:47.344554Z","iopub.status.idle":"2023-02-12T03:32:47.354287Z","shell.execute_reply.started":"2023-02-12T03:32:47.344506Z","shell.execute_reply":"2023-02-12T03:32:47.353335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## debug inference","metadata":{}},{"cell_type":"code","source":"valid_files = glob.glob('/kaggle/input/psp-validation-dataset/valid_parquet_v2/*')\nsubmit_files = glob.glob('/kaggle/input/psp-validation-dataset/submit_parquet_v2/*')\nvalid_files = sorted(valid_files)\nsubmit_files = sorted(submit_files)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T03:32:48.622746Z","iopub.execute_input":"2023-02-12T03:32:48.623573Z","iopub.status.idle":"2023-02-12T03:32:48.886380Z","shell.execute_reply.started":"2023-02-12T03:32:48.623532Z","shell.execute_reply":"2023-02-12T03:32:48.885416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/psp-public-models/models/xgb_models.pkl','rb') as f:\n    models = pickle.load(f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\nbest_threshold = 0.63\nfor sub_path,test_path in tqdm(zip(submit_files,valid_files)):\n    sample_submission = pd.read_parquet(sub_path)\n    test = pd.read_parquet(test_path)\n    \n    # FEATURE ENGINEER TEST DATA\n    df = feature_engineer(test)\n    \n    # INFER TEST DATA\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURES].astype('float32'))[:,1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int(p.item()>best_threshold)\n    \n#     env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T03:32:55.891566Z","iopub.execute_input":"2023-02-12T03:32:55.891996Z","iopub.status.idle":"2023-02-12T03:33:00.249389Z","shell.execute_reply.started":"2023-02-12T03:32:55.891963Z","shell.execute_reply":"2023-02-12T03:33:00.248306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}