{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\n\nsys.path.append('../input/predict-student-performance-from-game-play')\nimport jo_wilder_310","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-04T10:36:08.363697Z","iopub.execute_input":"2023-07-04T10:36:08.364652Z","iopub.status.idle":"2023-07-04T10:36:08.441574Z","shell.execute_reply.started":"2023-07-04T10:36:08.364608Z","shell.execute_reply":"2023-07-04T10:36:08.439640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from cffi import FFI\nimport json\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\n\nffi = FFI()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:08.446962Z","iopub.execute_input":"2023-07-04T10:36:08.447344Z","iopub.status.idle":"2023-07-04T10:36:11.499315Z","shell.execute_reply.started":"2023-07-04T10:36:08.447314Z","shell.execute_reply":"2023-07-04T10:36:11.498275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREPROCESSED_DIR = '/kaggle/input/jowilder-2nd-place-solution-0-preprocess-data/'\nFEATURE_GEN_CODE_DIR = '/kaggle/input/jowilder-2nd-place-solution-1-features-code/'\nMODEL_DIR = '/kaggle/input/jowilder-2nd-place-solution-3-train-model/'","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.501455Z","iopub.execute_input":"2023-07-04T10:36:11.501771Z","iopub.status.idle":"2023-07-04T10:36:11.508308Z","shell.execute_reply.started":"2023-07-04T10:36:11.501743Z","shell.execute_reply":"2023-07-04T10:36:11.507144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(FEATURE_GEN_CODE_DIR)\n\nimport JoWilder_C_features\nimport JoWilder_numba_features","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.510027Z","iopub.execute_input":"2023-07-04T10:36:11.510652Z","iopub.status.idle":"2023-07-04T10:36:11.548299Z","shell.execute_reply.started":"2023-07-04T10:36:11.510621Z","shell.execute_reply":"2023-07-04T10:36:11.546996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(PREPROCESSED_DIR)\n\nimport JoWilder_preprocess_functions","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.551579Z","iopub.execute_input":"2023-07-04T10:36:11.552288Z","iopub.status.idle":"2023-07-04T10:36:11.576498Z","shell.execute_reply.started":"2023-07-04T10:36:11.552253Z","shell.execute_reply":"2023-07-04T10:36:11.575442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(PREPROCESSED_DIR + \"/preprocess_info.json\", \"r\") as f:\n    preprocess_info = json.loads(f.read())","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.580111Z","iopub.execute_input":"2023-07-04T10:36:11.580789Z","iopub.status.idle":"2023-07-04T10:36:11.594132Z","shell.execute_reply.started":"2023-07-04T10:36:11.580746Z","shell.execute_reply":"2023-07-04T10:36:11.593033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level_group_map = preprocess_info['level_group_map']\n\nquestions_per_level = np.array([3, 10, 5])\nquestions_splits_per_level = np.array([0, 3, 13, 18])","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.599190Z","iopub.execute_input":"2023-07-04T10:36:11.600046Z","iopub.status.idle":"2023-07-04T10:36:11.608555Z","shell.execute_reply.started":"2023-07-04T10:36:11.600001Z","shell.execute_reply":"2023-07-04T10:36:11.607535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(FEATURE_GEN_CODE_DIR + 'FEATURES_GENERATION_INFO.json', 'r') as f:\n    FEATURES_GENERATION_INFO = json.loads(f.read())\n    \nNUM_FEATURES = FEATURES_GENERATION_INFO['NUM_FEATURES']\nHISTORY_LEN = FEATURES_GENERATION_INFO['HISTORY_LEN']","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.611883Z","iopub.execute_input":"2023-07-04T10:36:11.615097Z","iopub.status.idle":"2023-07-04T10:36:11.625828Z","shell.execute_reply.started":"2023-07-04T10:36:11.615057Z","shell.execute_reply":"2023-07-04T10:36:11.624658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(MODEL_DIR + 'MODEL_INFO.json', 'r') as f:\n    MODEL_INFO = json.loads(f.read())\n    \nTHRESHOLD = MODEL_INFO['THRESHOLD']","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.626954Z","iopub.execute_input":"2023-07-04T10:36:11.627257Z","iopub.status.idle":"2023-07-04T10:36:11.645675Z","shell.execute_reply.started":"2023-07-04T10:36:11.627231Z","shell.execute_reply":"2023-07-04T10:36:11.644348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_FEATURES, HISTORY_LEN, THRESHOLD","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.647247Z","iopub.execute_input":"2023-07-04T10:36:11.647576Z","iopub.status.idle":"2023-07-04T10:36:11.656248Z","shell.execute_reply.started":"2023-07-04T10:36:11.647549Z","shell.execute_reply":"2023-07-04T10:36:11.655237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = lgb.Booster(model_file=f'{MODEL_DIR}/model_0_0.txt')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:11.657531Z","iopub.execute_input":"2023-07-04T10:36:11.657872Z","iopub.status.idle":"2023-07-04T10:36:12.156664Z","shell.execute_reply.started":"2023-07-04T10:36:11.657842Z","shell.execute_reply":"2023-07-04T10:36:12.155638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_features(\n    session_id,\n    level_group_index,\n    elapsed_time,\n    event_name,\n    name,\n    level,\n    hover_duration,\n    session_weekday,\n    building,\n    room,\n    fqids,\n    session_event_index,\n    room_coor_x,\n    room_coor_y,\n    screen_coor_x,\n    screen_coor_y,\n    text_length,\n    text_numerical,\n    text_fqid_numerical,\n    room_fqid_numerical,\n    page,\n    session_hour,\n    hist,\n):\n    \n    hist_pointer = ffi.from_buffer('double[]', hist)\n    \n    num_questions_in_level = questions_per_level[level_group_index]\n    \n    questions_start_number = questions_splits_per_level[level_group_index]\n    questions_end_number = questions_splits_per_level[level_group_index + 1]\n    question_number = np.arange(questions_start_number, questions_end_number)\n    \n    features = np.full((num_questions_in_level, NUM_FEATURES), np.nan, dtype=np.float32)\n    \n    features_pointer = ffi.from_buffer('float[]', features)\n    \n    x_et_pointer = ffi.from_buffer('long[]', elapsed_time)\n    x_en_pointer = ffi.from_buffer('long[]', event_name)\n    x_n_pointer = ffi.from_buffer('long[]', name)\n    x_hover_duration_pointer = ffi.from_buffer('double[]', hover_duration)\n    x_session_weekday_pointer = ffi.from_buffer('long[]', session_weekday)\n    x_b_pointer = ffi.from_buffer('long[]', building)\n    x_r_pointer = ffi.from_buffer('long[]', room)\n    x_fqids_pointer = ffi.from_buffer('long[]', fqids)\n    x_l_pointer = ffi.from_buffer('long[]', level)\n    x_index_pointer = ffi.from_buffer('long[]', session_event_index)\n    x_rc_x_pointer = ffi.from_buffer('double[]', room_coor_x)\n    x_rc_y_pointer = ffi.from_buffer('double[]', room_coor_y)\n    x_sc_x_pointer = ffi.from_buffer('double[]', screen_coor_x)\n    x_sc_y_pointer = ffi.from_buffer('double[]', screen_coor_y)\n    x_tl_pointer = ffi.from_buffer('long[]', text_length)\n    x_text_numerical_pointer = ffi.from_buffer('long[]', text_numerical)\n    x_text_fqid_numerical_pointer = ffi.from_buffer('long[]', text_fqid_numerical)\n    x_room_fqid_numerical_pointer = ffi.from_buffer('long[]', room_fqid_numerical)\n    x_page_pointer = ffi.from_buffer('double[]', page)\n    x_hour_pointer = ffi.from_buffer('long[]', session_hour)\n    \n    x_et = elapsed_time\n    x_en = event_name\n    x_n = name\n    x_hover_duration = hover_duration\n    x_session_weekday = session_weekday\n    x_b = building\n    x_r = room\n    x_fqids = fqids\n    x_l = level\n    x_index = session_event_index\n    x_rc_x = room_coor_x\n    x_rc_y = room_coor_y\n    x_sc_x = screen_coor_x\n    x_sc_y = screen_coor_y\n    x_tl = text_length\n    x_text_numerical = text_numerical\n    x_text_fqid_numerical = text_fqid_numerical\n    x_room_fqid_numerical = room_fqid_numerical\n    x_page = page\n    x_hour = session_hour\n\n    JoWilder_numba_features.process_single(\n        level_group_index,\n        x_et,\n        x_en,\n        x_n,\n        x_hover_duration,\n        x_session_weekday,\n        x_b,\n        x_r,\n        x_fqids,\n        x_l,\n        x_index,\n        x_rc_x,\n        x_rc_y,\n        x_sc_x,\n        x_sc_y,\n        x_tl,\n        x_text_numerical,\n        x_text_fqid_numerical,\n        x_room_fqid_numerical,\n        x_page,\n        x_hour,\n        features,\n        hist,\n    )\n\n    number_of_events = x_et.shape[0]\n\n    JoWilder_C_features.lib.fill_history(\n        level_group_index,\n        x_l_pointer,\n        hist_pointer,\n        x_index_pointer,\n        x_text_numerical_pointer,\n        x_en_pointer,\n        x_n_pointer,\n        x_fqids_pointer,\n        x_et_pointer,\n        x_rc_x_pointer,\n        x_rc_y_pointer,\n        x_sc_x_pointer,\n        x_sc_y_pointer,\n        x_b_pointer,\n        x_r_pointer,\n        x_room_fqid_numerical_pointer,\n        x_text_fqid_numerical_pointer,\n        x_page_pointer,\n        x_hover_duration_pointer,\n        number_of_events,\n    )\n\n    JoWilder_C_features.lib.add_features_batch(\n        level_group_index,\n        features_pointer,\n        hist_pointer,\n        num_questions_in_level\n    )\n    \n    return features, question_number","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:12.158034Z","iopub.execute_input":"2023-07-04T10:36:12.158362Z","iopub.status.idle":"2023-07-04T10:36:12.188315Z","shell.execute_reply.started":"2023-07-04T10:36:12.158334Z","shell.execute_reply":"2023-07-04T10:36:12.187397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES_MASK = np.load(MODEL_DIR + f'/FEATURE_MASK.npy')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:12.193015Z","iopub.execute_input":"2023-07-04T10:36:12.195368Z","iopub.status.idle":"2023-07-04T10:36:12.224950Z","shell.execute_reply.started":"2023-07-04T10:36:12.195329Z","shell.execute_reply":"2023-07-04T10:36:12.224094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = jo_wilder_310.make_env()\niter_test = env.iter_test()\n\nhistory = {}\n\nfor (test_df, sample_submission) in iter_test:\n    test_df.sort_values(by=['elapsed_time', 'index'], inplace=True)\n    \n    original_session_order = sample_submission['session_id'].values.copy()\n    \n    sample_submission['questions'] = sample_submission['session_id'].str.split('_').str[1].str[1:].astype(int)\n    sample_submission = sample_submission.sort_values(by = 'questions')\n    sample_submission = sample_submission[['session_id', 'correct']]\n    \n    #############################\n    \n    level_group = test_df['level_group'].values[0]\n    session_id = test_df['session_id'].values[0]\n    \n    try:\n        hist = history[session_id]\n    except KeyError:\n        hist = np.full(HISTORY_LEN, np.nan, dtype=np.float64)\n        history[session_id] = hist\n        \n    level_group_index = level_group_map[level_group]\n    \n    elapsed_time = test_df['elapsed_time'].values\n    index = test_df['index'].values\n\n    n = elapsed_time.shape[0]\n\n    temp = test_df['event_name'].values.astype('<U18')\n    numerical_event_name = JoWilder_preprocess_functions.event_name_to_number(temp.view(np.uint32).reshape((temp.shape[0], -1)))\n\n    temp = test_df['name'].values.astype('<U9')\n    numerical_name = JoWilder_preprocess_functions.name_to_number(temp.view(np.uint32).reshape((temp.shape[0], -1)))\n\n    level = test_df['level'].values\n    hover_duration = test_df['hover_duration'].values\n\n    temp = test_df['fqid'].fillna(\"\").values.astype('<U30')\n    numerical_fqid = JoWilder_preprocess_functions.fqids_to_number(temp.view(np.uint32).reshape((temp.shape[0], -1)))\n\n    room_coor_x = test_df['room_coor_x'].values\n    room_coor_y = test_df['room_coor_y'].values\n    screen_coor_x = test_df['screen_coor_x'].values\n    screen_coor_y = test_df['screen_coor_y'].values\n\n    _, _, session_weekday, session_hour = JoWilder_preprocess_functions.session_id_parser(session_id)\n\n    session_weekday = np.full(n, session_weekday, dtype=np.uint8)\n    session_hour = np.full(n, session_hour, dtype=np.uint8)\n\n    temp = test_df['text'].fillna(\"\").values.astype('<U89')\n    temp = temp.view(np.uint32).reshape((temp.shape[0], -1))\n    numerical_text = JoWilder_preprocess_functions.text_to_number(temp)\n\n    text_length = JoWilder_preprocess_functions.calculate_text_length(temp)\n\n    temp = test_df['text_fqid'].fillna(\"\").values.astype('<U71')\n    numerical_text_fqid = JoWilder_preprocess_functions.text_fqid_to_number(temp.view(np.uint32).reshape((temp.shape[0], -1)))\n\n    temp = test_df['room_fqid'].fillna(\"\").values.astype('<U39')\n    temp = temp.view(np.uint32).reshape((temp.shape[0], -1))\n    numerical_room_fqid = JoWilder_preprocess_functions.room_fqids_to_number(temp)# .astype(np.int64)\n\n    numerical_room_fqid_building, numerical_room_fqid_room = JoWilder_preprocess_functions.get_building_and_room(temp)\n\n    page = test_df['page'].values\n\n    features, question_number = generate_features(\n        session_id,\n        level_group_index,\n        elapsed_time,\n        numerical_event_name,\n        numerical_name,\n        level,\n        hover_duration,\n        session_weekday,\n        numerical_room_fqid_building,\n        numerical_room_fqid_room,\n        numerical_fqid,\n        index,\n        room_coor_x,\n        room_coor_y,\n        screen_coor_x,\n        screen_coor_y,\n        text_length,\n        numerical_text,\n        numerical_text_fqid,\n        numerical_room_fqid,\n        page,\n        session_hour,\n        hist,\n    )\n\n    features = features[:, FEATURES_MASK]\n    \n    p = model.predict(features)\n    \n    sample_submission['correct'] = (p > THRESHOLD).astype(int)\n    \n    #############################\n    \n    sample_submission = sample_submission.set_index('session_id').reindex(original_session_order).reset_index()\n    \n    assert np.all(sample_submission['session_id'].values == original_session_order)\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T10:36:12.227790Z","iopub.execute_input":"2023-07-04T10:36:12.233710Z","iopub.status.idle":"2023-07-04T10:36:12.506401Z","shell.execute_reply.started":"2023-07-04T10:36:12.233666Z","shell.execute_reply":"2023-07-04T10:36:12.505387Z"},"trusted":true},"execution_count":null,"outputs":[]}]}