{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"MODELS_PATH = '/kaggle/input/jo-wilder-catboost-6k-final'","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:43:48.106173Z","iopub.execute_input":"2023-06-29T03:43:48.106657Z","iopub.status.idle":"2023-06-29T03:43:48.149928Z","shell.execute_reply.started":"2023-06-29T03:43:48.106615Z","shell.execute_reply":"2023-06-29T03:43:48.148444Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook uses Cython to extract 6k features from each session_id + level_group combination. It is highly optimized and can finish execution in 2 hours. The original feature extraction method based on DataFrame group by will not finish in 9 hours.","metadata":{}},{"cell_type":"code","source":"THRESHOLD = 0.59","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:43:48.151977Z","iopub.execute_input":"2023-06-29T03:43:48.152931Z","iopub.status.idle":"2023-06-29T03:43:48.159239Z","shell.execute_reply.started":"2023-06-29T03:43:48.152893Z","shell.execute_reply":"2023-06-29T03:43:48.157301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext Cython","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:43:48.161831Z","iopub.execute_input":"2023-06-29T03:43:48.162194Z","iopub.status.idle":"2023-06-29T03:43:49.458852Z","shell.execute_reply.started":"2023-06-29T03:43:48.162167Z","shell.execute_reply":"2023-06-29T03:43:49.457999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%cython -2\n# distutils: language = c++\n# cython: language_level=2\n\nfrom __future__ import print_function\n\nimport numpy as np\ncimport numpy as cnp\n\nfrom libc.math cimport isnan, sqrt, floor, fabs\nfrom cython.view cimport array as cvarray\nfrom libcpp.vector cimport vector\nfrom libcpp.set cimport set as cppset\nfrom libc.stdio cimport printf\n\nfrom libcpp.algorithm cimport sort\ncimport cython\n\nctypedef signed char INT8\nctypedef signed short INT16\nctypedef signed int INT32\nctypedef float FLOAT32\n\nctypedef fused DTYPE_INT:\n    INT8\n    INT16\n    INT32\n    # signed char  // int8\n    # signed short // int16\n    # signed int   // int32\n\nctypedef fused DTYPE_FLOAT:\n    FLOAT32\n    # float        // float32\n\nctypedef fused DTYPE_NUMBER:\n    DTYPE_INT\n    DTYPE_FLOAT\n\ncdef INT8 CAT_NAN = -1\n\ncdef FLOAT32 FILL_NAN = -999.0\n\ncdef INT32 DDOF = 1\n\ncdef INT32 NUM_OF_FEATURES = 6306\ncdef INT32 NUM_OF_NUMERICAL = 7\ncdef INT32 NUM_OF_CATEGORICAL = 8\ncdef INT32 NUM_OF_NUMERICAL_METRICS = 8\ncdef INT32 NUM_OF_CATEGORICAL_METRICS = 1\ncdef INT32 NUM_OF_NUMERICAL_QUANTILES = 6\ncdef FLOAT32 NUMERICAL_QUANTILES[6]\nNUMERICAL_QUANTILES[:] = [0.01, 0.2, 0.4, 0.6, 0.8, 0.99]\n\ncdef INT32 COMPOSITE_NUM_OF_NUMERICAL_METRICS = 3\ncdef INT32 COMPOSITE_NUM_OF_OVERALL_METRICS = 1\ncdef INT32 COMPOSITE_NUM_OF_NUMERICAL = 1\n\ncdef INT8 EVENT_NAME_CAT_SIZE = 11\ncdef INT8 NAME_CAT_SIZE = 6\ncdef INT8 LEVEL_CAT_SIZE = 23\ncdef INT8 PAGE_CAT_SIZE = 7\ncdef INT16 FQID_CAT_SIZE = 128\ncdef INT8 ROOM_FQID_CAT_SIZE = 19\ncdef INT8 TEXT_FQID_CAT_SIZE = 126\ncdef INT8 LEVEL_GROUP_CAT_SIZE = 3\n\ncdef INT32 STAGES_INDEX[12]\nSTAGES_INDEX[:] = [0, 56, 98, 106, 1394, 2360, 2544, 2981, 4292, 5356, 6154, 6306]\n\ncdef INT32 ROOM_COOR_X_IDX = 0\ncdef INT32 ROOM_COOR_Y_IDX = 1\ncdef INT32 SCREEN_COOR_X_IDX = 2\ncdef INT32 SCREEN_COOR_Y_IDX = 3\ncdef INT32 HOVER_DURATION_IDX = 4\ncdef INT32 ELAPSED_TIME_IDX = 5\ncdef INT32 DELTA_TIME_IDX = 6\n\ncdef INT32 TEXT_FQID_IDX = 0\ncdef INT32 FQID_IDX = 1\ncdef INT32 ROOM_FQID_IDX = 2\ncdef INT32 LEVEL_GROUP_IDX = 3\ncdef INT32 EVENT_NAME_IDX = 4\ncdef INT32 NAME_IDX = 5\ncdef INT32 PAGE_IDX = 6\ncdef INT32 LEVEL_IDX = 7\n\ncdef INT32 STD_IDX = 0\ncdef INT32 MEAN_IDX = 1\ncdef INT32 VAR_IDX = 2\ncdef INT32 SEM_IDX = 3\ncdef INT32 MEDIAN_IDX = 4\ncdef INT32 SUM_IDX = 5\ncdef INT32 MIN_IDX = 6\ncdef INT32 MAX_IDX = 7\n\ncdef INT32 NUNIQUE_IDX = 0\n\ncdef INT32 COMPOSITE_SUM_IDX = 0\ncdef INT32 COMPOSITE_MEAN_IDX = 1\ncdef INT32 COMPOSITE_MEDIAN_IDX = 2\n\n# for reference only, do not call these Python constants in code\n# NUMERICAL_METRICS = [\n#     'std', 'mean', 'var', 'sem', 'median', 'sum', 'min', 'max',\n# ]\n# CATEGORICAL_METRICS = ['nunique']\n# CONTINUOUS_NUMBER_COLUMNS = [\n#     \"room_coor_x\",\n#     \"room_coor_y\",\n#     \"screen_coor_x\",\n#     \"screen_coor_y\",\n#     \"hover_duration\",\n#     \"elapsed_time\",\n#     \"time_diff_with_prev_event\", # delta_time\n# ]\n# CATEGORY_COLUMNS = [\n#     \"text_fqid\",\n#     \"fqid\",\n#     \"room_fqid\",\n#     \"level_group\",\n#     \"event_name\",\n#     \"name\",\n#     \"page\",\n#     'level',\n# ]\n#\n# def compute_column_names() -> List[str]:\n#     BASE_GROUPING = ['session_id', 'level_group']\n#     NUMERICAL = CONTINUOUS_NUMBER_COLUMNS + [DELTA_TIME]\n#     CATEGORICAL = CATEGORY_COLUMNS + []\n#     CATEGORICAL.remove('text')\n#\n#     LEVELS = COLUMN_DTYPES['level'].categories\n#     ROOM_FQIDS = COLUMN_DTYPES['room_fqid'].categories\n#\n#     ret = []\n#\n#     print(len(ret))  # 0\n#\n#     for col in NUMERICAL:\n#         for m in NUMERICAL_METRICS:\n#             ret.append(f\"{col}_{m}\")\n#\n#     print(len(ret)) # 1\n#\n#     for col in NUMERICAL:\n#         for q in NUMERICAL_QUANTILES:\n#             ret.append(f\"{col}_{q:.04f}\")\n#\n#     print(len(ret)) # 2\n#\n#     for col in CATEGORICAL:\n#         for m in CATEGORICAL_METRICS:\n#             ret.append(f\"{col}_{m}\")\n#\n#     print(len(ret)) # 3\n#\n#     for col in NUMERICAL:\n#         for m in NUMERICAL_METRICS:\n#             for l in LEVELS:\n#                 ret.append(f\"{col}_{m}_{l}\")\n#\n#     print(len(ret)) # 4\n#\n#     for col in NUMERICAL:\n#         for q in NUMERICAL_QUANTILES:\n#             for l in LEVELS:\n#                 ret.append(f\"{col}_{q:.04f}_{l}\")\n#\n#     print(len(ret)) # 5\n#\n#     for col in CATEGORICAL:\n#         for m in CATEGORICAL_METRICS:\n#             for l in LEVELS:\n#                 ret.append(f\"{col}_{m}_{l}\")\n#\n#     print(len(ret)) # 6\n#\n#     # size\n#     for rf in ROOM_FQIDS:\n#         for l in LEVELS:\n#             ret.append(f\"{rf}_{l}\")\n#\n#     print(len(ret)) # 7\n#\n#     for col in [DELTA_TIME]:\n#         for m in ['sum', 'mean', 'median']:\n#             for rf in ROOM_FQIDS:\n#                 for l in LEVELS:\n#                     ret.append(f\"{col}_{m}_{rf}_{l}\")\n#\n#     print(len(ret)) # 8\n#\n#     for col in NUMERICAL:\n#         for m in NUMERICAL_METRICS:\n#             for rf in ROOM_FQIDS:\n#                 ret.append(f\"{col}_{m}_{rf}\")\n#\n#     print(len(ret)) # 9\n#\n#     for col in NUMERICAL:\n#         for q in NUMERICAL_QUANTILES:\n#             for rf in ROOM_FQIDS:\n#                 ret.append(f\"{col}_{q:.04f}_{rf}\")\n#\n#     print(len(ret)) # 10\n#\n#     for col in CATEGORICAL:\n#         for m in CATEGORICAL_METRICS:\n#             for rf in ROOM_FQIDS:\n#                 ret.append(f\"{col}_{m}_{rf}\")\n#\n#     print(len(ret)) # 11\n#\n#     return ret\n#\n# for reference end\n\ncdef struct NumericalStats:\n    FLOAT32 std, mean, var, sem, median, sum, min, max\n\ncdef struct NumericalQuantiles:\n    FLOAT32 p1, p20, p40, p60, p80, p99\n\ncdef struct CategoricalStats:\n    FLOAT32 nunique\n\ncdef NumericalStats compute_numerical_stats(vector[FLOAT32]& values) nogil:\n\n    cdef NumericalStats ret\n    cdef Py_ssize_t n = values.size()\n    if n == 0:\n        ret.std = FILL_NAN\n        ret.mean = FILL_NAN\n        ret.var = FILL_NAN\n        ret.sem = FILL_NAN\n        ret.median = FILL_NAN\n        ret.sum = 0.0\n        ret.min = FILL_NAN\n        ret.max = FILL_NAN\n        return ret\n    cdef Py_ssize_t i\n    ret.sum = 0.0\n    for i in range(n):\n        ret.sum += values[i]\n    ret.mean = ret.sum / n\n    ret.min = values[0]\n    ret.max = values[n-1]\n    if n % 2 == 1:\n        ret.median = values[(n-1)//2]\n    else:\n        ret.median = (values[(n-1)//2] + values[n//2]) / 2.0\n    cdef FLOAT32 sq_dev_sum = 0.0\n    for i in range(n):\n        sq_dev_sum += (values[i] - ret.mean) * (values[i] - ret.mean)\n    if n == 1:\n        ret.var = FILL_NAN\n        ret.std = FILL_NAN\n        ret.sem = FILL_NAN\n    else:\n        ret.var = sq_dev_sum / (n - DDOF)\n        ret.std = sqrt(ret.var)\n        ret.sem = ret.std / sqrt(<FLOAT32>n)\n    return ret\n\ncdef FLOAT32 find_quantile(vector[FLOAT32]& values, FLOAT32 q) nogil:\n    # assume that values.size() > 0 (size check must be done already)\n    # assume that 0.0 <= q <= 1.0\n    cdef FLOAT32 alpha = 1, beta = 1\n    cdef Py_ssize_t n = values.size()\n    cdef FLOAT32 t = q * (n-alpha-beta+1) + alpha\n    cdef INT32 i = <INT32>floor(t)\n    cdef FLOAT32 g = t - i\n    # Python uses 0-indexing\n    i -= 1\n    cdef INT32 j = i + 1\n    if i < 0 or j <= 0:\n        return values[0]\n    if i >= n - 1 or j > n - 1:\n        return values[n-1]\n    return values[i] * (1.0 - g) + values[j] * g\n\ncdef NumericalQuantiles compute_numerical_quantiles(vector[FLOAT32]& values) nogil:\n    cdef NumericalQuantiles ret\n    if values.size() <= 0:\n        ret.p1 = FILL_NAN\n        ret.p20 = FILL_NAN\n        ret.p40 = FILL_NAN\n        ret.p60 = FILL_NAN\n        ret.p80 = FILL_NAN\n        ret.p99 = FILL_NAN\n        return ret\n    ret.p1 = find_quantile(values, 0.01)\n    ret.p20 = find_quantile(values, 0.20)\n    ret.p40 = find_quantile(values, 0.40)\n    ret.p60 = find_quantile(values, 0.60)\n    ret.p80 = find_quantile(values, 0.80)\n    ret.p99 = find_quantile(values, 0.99)\n    return ret\n\ncdef CategoricalStats compute_categorical_stats(vector[INT16]& values) nogil:\n    cdef CategoricalStats ret\n    ret.nunique = 0.0\n    cdef Py_ssize_t i\n    cdef Py_ssize_t n = values.size()\n    cdef cppset[INT32] s\n    for i in range(n):\n        s.insert(values[i])\n    ret.nunique = <FLOAT32>s.size()\n    return ret\n\ncdef inline void put_numerical_stats(FLOAT32[:] result_arr, INT32 head_idx, NumericalStats& stats) nogil:\n    result_arr[head_idx + STD_IDX] = stats.std\n    result_arr[head_idx + MEAN_IDX] = stats.mean\n    result_arr[head_idx + VAR_IDX] = stats.var\n    result_arr[head_idx + SEM_IDX] = stats.sem\n    result_arr[head_idx + MEDIAN_IDX] = stats.median\n    result_arr[head_idx + SUM_IDX] = stats.sum\n    result_arr[head_idx + MIN_IDX] = stats.min\n    result_arr[head_idx + MAX_IDX] = stats.max\n\ncdef inline void put_numerical_quantiles(FLOAT32[:] result_arr, INT32 head_idx, NumericalQuantiles& quantiles) nogil:\n    result_arr[head_idx + 0] = quantiles.p1\n    result_arr[head_idx + 1] = quantiles.p20\n    result_arr[head_idx + 2] = quantiles.p40\n    result_arr[head_idx + 3] = quantiles.p60\n    result_arr[head_idx + 4] = quantiles.p80\n    result_arr[head_idx + 5] = quantiles.p99\n\ncdef inline void put_categorical_stats(FLOAT32[:] result_arr, INT32 head_idx, CategoricalStats& stats) nogil:\n    result_arr[head_idx + NUNIQUE_IDX] = stats.nunique\n\ncdef inline void add_to_array_0_f32(\n    vector[vector[FLOAT32]]& dest,\n    INT32 feat_idx,\n    FLOAT32[:] source,\n    INT32 row_idx,\n) nogil:\n    if not isnan(source[row_idx]):\n        dest[feat_idx].push_back(source[row_idx])\n\ncdef inline void add_to_array_0_i16(\n    vector[vector[INT16]]& dest,\n    INT32 feat_idx,\n    DTYPE_INT[:] source,\n    INT32 row_idx,\n) nogil:\n    if source[row_idx] != CAT_NAN:\n        dest[feat_idx].push_back(source[row_idx])\n\ncdef inline void add_to_array_1_f32(\n    vector[vector[vector[FLOAT32]]]& dest,\n    INT32 feat_idx,\n    INT32 l1_idx,\n    FLOAT32[:] source,\n    INT32 row_idx,\n) nogil:\n    if not isnan(source[row_idx]) and l1_idx >= 0:\n        dest[feat_idx][l1_idx].push_back(source[row_idx])\n\ncdef inline void add_to_array_1_i16(\n    vector[vector[vector[INT16]]]& dest,\n    INT32 feat_idx,\n    INT32 l1_idx,\n    DTYPE_INT[:] source,\n    INT32 row_idx,\n) nogil:\n    if source[row_idx] != CAT_NAN and l1_idx >= 0:\n        dest[feat_idx][l1_idx].push_back(source[row_idx])\n\ncdef inline void add_to_array_2_f32(\n    vector[vector[vector[vector[FLOAT32]]]]& dest,\n    INT32 feat_idx,\n    INT32 l1_idx,\n    INT32 l2_idx,\n    FLOAT32[:] source,\n    INT32 row_idx,\n) nogil:\n    if not isnan(source[row_idx]) and l1_idx >= 0 and l2_idx >= 0:\n        dest[feat_idx][l1_idx][l2_idx].push_back(source[row_idx])\n\ncdef inline void add_to_array_2_i16(\n    vector[vector[vector[vector[INT16]]]]& dest,\n    INT32 feat_idx,\n    INT32 l1_idx,\n    INT32 l2_idx,\n    DTYPE_INT[:] source,\n    INT32 row_idx,\n) nogil:\n    if source[row_idx] != CAT_NAN and l1_idx >= 0 and l2_idx >= 0:\n        dest[feat_idx][l1_idx][l2_idx].push_back(source[row_idx])\n\ncdef inline void swap(DTYPE_NUMBER[:] arr, INT32 i, INT32 j) nogil:\n    arr[i], arr[j] = arr[j], arr[i]\n\ncdef inline void swap_all(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 i,\n    INT32 j,\n) nogil:\n    swap(elapsed_time_arr, i, j)\n    swap(event_name_arr, i, j)\n    swap(name_arr, i, j)\n    swap(level_arr, i, j)\n    swap(page_arr, i, j)\n    swap(room_coor_x_arr, i, j)\n    swap(room_coor_y_arr, i, j)\n    swap(screen_coor_x_arr, i, j)\n    swap(screen_coor_y_arr, i, j)\n    swap(hover_duration_arr, i, j)\n    swap(fqid_arr, i, j)\n    swap(room_fqid_arr, i, j)\n    swap(text_fqid_arr, i, j)\n    swap(level_group_arr, i, j)\n\ncdef INT32 partition_by_elapsed_time(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 low,\n    INT32 high,\n) nogil:\n    cdef FLOAT32 pivot = elapsed_time_arr[high]\n    cdef INT32 i = low - 1\n    for j in range(low, high):\n        if elapsed_time_arr[j] <= pivot:\n            i += 1\n            swap_all(\n                elapsed_time_arr,\n                event_name_arr,\n                name_arr,\n                level_arr,\n                page_arr,\n                room_coor_x_arr,\n                room_coor_y_arr,\n                screen_coor_x_arr,\n                screen_coor_y_arr,\n                hover_duration_arr,\n                fqid_arr,\n                room_fqid_arr,\n                text_fqid_arr,\n                level_group_arr,\n                i,\n                j\n            )\n    swap_all(\n        elapsed_time_arr,\n        event_name_arr,\n        name_arr,\n        level_arr,\n        page_arr,\n        room_coor_x_arr,\n        room_coor_y_arr,\n        screen_coor_x_arr,\n        screen_coor_y_arr,\n        hover_duration_arr,\n        fqid_arr,\n        room_fqid_arr,\n        text_fqid_arr,\n        level_group_arr,\n        i+1,\n        high\n    )\n    return i + 1\n\ncdef void heapify_by_elapsed_time(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 N,\n    INT32 i,\n) nogil:\n    cdef INT32 largest = i\n    cdef INT32 left = 2 * i + 1\n    cdef INT32 right = 2 * i + 2\n    if left < N and elapsed_time_arr[left] > elapsed_time_arr[largest]:\n        largest = left\n    if right < N and elapsed_time_arr[right] > elapsed_time_arr[largest]:\n        largest = right\n    if largest != i:\n        swap_all(\n            elapsed_time_arr,\n            event_name_arr,\n            name_arr,\n            level_arr,\n            page_arr,\n            room_coor_x_arr,\n            room_coor_y_arr,\n            screen_coor_x_arr,\n            screen_coor_y_arr,\n            hover_duration_arr,\n            fqid_arr,\n            room_fqid_arr,\n            text_fqid_arr,\n            level_group_arr,\n            i,\n            largest\n        )\n        heapify_by_elapsed_time(\n            elapsed_time_arr,\n            event_name_arr,\n            name_arr,\n            level_arr,\n            page_arr,\n            room_coor_x_arr,\n            room_coor_y_arr,\n            screen_coor_x_arr,\n            screen_coor_y_arr,\n            hover_duration_arr,\n            fqid_arr,\n            room_fqid_arr,\n            text_fqid_arr,\n            level_group_arr,\n            N,\n            largest\n        )\n\n\ncdef void heapsort_by_elapsed_time(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 N,\n) nogil:\n    cdef INT32 i\n    for i in range(N/2-1, -1, -1):\n        heapify_by_elapsed_time(\n            elapsed_time_arr,\n            event_name_arr,\n            name_arr,\n            level_arr,\n            page_arr,\n            room_coor_x_arr,\n            room_coor_y_arr,\n            screen_coor_x_arr,\n            screen_coor_y_arr,\n            hover_duration_arr,\n            fqid_arr,\n            room_fqid_arr,\n            text_fqid_arr,\n            level_group_arr,\n            N,\n            i\n        )\n    for i in range(N-1, 0, -1):\n        swap_all(\n            elapsed_time_arr,\n            event_name_arr,\n            name_arr,\n            level_arr,\n            page_arr,\n            room_coor_x_arr,\n            room_coor_y_arr,\n            screen_coor_x_arr,\n            screen_coor_y_arr,\n            hover_duration_arr,\n            fqid_arr,\n            room_fqid_arr,\n            text_fqid_arr,\n            level_group_arr,\n            0,\n            i\n        )\n        heapify_by_elapsed_time(\n            elapsed_time_arr,\n            event_name_arr,\n            name_arr,\n            level_arr,\n            page_arr,\n            room_coor_x_arr,\n            room_coor_y_arr,\n            screen_coor_x_arr,\n            screen_coor_y_arr,\n            hover_duration_arr,\n            fqid_arr,\n            room_fqid_arr,\n            text_fqid_arr,\n            level_group_arr,\n            i,\n            0\n        )\n\n\ncdef void sort_by_elapsed_time(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 low,\n    INT32 high,\n) nogil:\n    if low >= high:\n        return\n    cdef INT32 p = partition_by_elapsed_time(\n        elapsed_time_arr,\n        event_name_arr,\n        name_arr,\n        level_arr,\n        page_arr,\n        room_coor_x_arr,\n        room_coor_y_arr,\n        screen_coor_x_arr,\n        screen_coor_y_arr,\n        hover_duration_arr,\n        fqid_arr,\n        room_fqid_arr,\n        text_fqid_arr,\n        level_group_arr,\n        low,\n        high\n    )\n    sort_by_elapsed_time(\n        elapsed_time_arr,\n        event_name_arr,\n        name_arr,\n        level_arr,\n        page_arr,\n        room_coor_x_arr,\n        room_coor_y_arr,\n        screen_coor_x_arr,\n        screen_coor_y_arr,\n        hover_duration_arr,\n        fqid_arr,\n        room_fqid_arr,\n        text_fqid_arr,\n        level_group_arr,\n        low,\n        p - 1\n    )\n    sort_by_elapsed_time(\n        elapsed_time_arr,\n        event_name_arr,\n        name_arr,\n        level_arr,\n        page_arr,\n        room_coor_x_arr,\n        room_coor_y_arr,\n        screen_coor_x_arr,\n        screen_coor_y_arr,\n        hover_duration_arr,\n        fqid_arr,\n        room_fqid_arr,\n        text_fqid_arr,\n        level_group_arr,\n        p + 1,\n        high\n    )\n\ncdef FLOAT32[:] compute_stat_inner(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 arr_len,\n):\n    heapsort_by_elapsed_time(\n        elapsed_time_arr,\n        event_name_arr,\n        name_arr,\n        level_arr,\n        page_arr,\n        room_coor_x_arr,\n        room_coor_y_arr,\n        screen_coor_x_arr,\n        screen_coor_y_arr,\n        hover_duration_arr,\n        fqid_arr,\n        room_fqid_arr,\n        text_fqid_arr,\n        level_group_arr,\n        arr_len\n    )\n    cdef FLOAT32[:] delta_time_arr = cvarray(shape=(arr_len,), itemsize=sizeof(FLOAT32), format=\"f\")\n    delta_time_arr[0] = 0.0\n    cdef Py_ssize_t idx\n    for idx in range(1, arr_len):\n        delta_time_arr[idx] = elapsed_time_arr[idx] - elapsed_time_arr[idx-1]\n\n    cdef FLOAT32[:] result_arr = cvarray(shape=(NUM_OF_FEATURES,), itemsize=sizeof(FLOAT32), format=\"f\")\n\n    # feat, values\n    cdef vector[vector[FLOAT32]] numerical_values = vector[vector[FLOAT32]](NUM_OF_NUMERICAL)\n\n    # feat, values\n    cdef vector[vector[INT16]] categorical_values = vector[vector[INT16]](NUM_OF_CATEGORICAL)\n\n    # feat, level, values\n    cdef vector[vector[vector[FLOAT32]]] numerical_level_values = vector[vector[vector[FLOAT32]]](NUM_OF_NUMERICAL, vector[vector[FLOAT32]](LEVEL_CAT_SIZE))\n\n    # feat, level, values\n    cdef vector[vector[vector[INT16]]] categorical_level_values = vector[vector[vector[INT16]]](NUM_OF_CATEGORICAL, vector[vector[INT16]](LEVEL_CAT_SIZE))\n\n    # feat, room_fqid, level, values\n    cdef vector[vector[vector[vector[FLOAT32]]]] numerical_room_fqid_level_values = vector[vector[vector[vector[FLOAT32]]]](NUM_OF_NUMERICAL, vector[vector[vector[FLOAT32]]](ROOM_FQID_CAT_SIZE, vector[vector[FLOAT32]](LEVEL_CAT_SIZE)))\n\n    # feat, room_fqid, level, values`\n    cdef vector[vector[vector[vector[INT16]]]] categorical_room_fqid_level_values = vector[vector[vector[vector[INT16]]]](NUM_OF_CATEGORICAL, vector[vector[vector[INT16]]](ROOM_FQID_CAT_SIZE, vector[vector[INT16]](LEVEL_CAT_SIZE)))\n\n    # feat, room_fqid, values\n    cdef vector[vector[vector[FLOAT32]]] numerical_room_fqid_values = vector[vector[vector[FLOAT32]]](NUM_OF_NUMERICAL, vector[vector[FLOAT32]](ROOM_FQID_CAT_SIZE))\n\n    # feat, room_fqid, values\n    cdef vector[vector[vector[INT16]]] categorical_room_fqid_values = vector[vector[vector[INT16]]](NUM_OF_CATEGORICAL, vector[vector[INT16]](ROOM_FQID_CAT_SIZE))\n\n    # slottinh all values in\n    for row_idx in range(arr_len):\n        add_to_array_0_f32(numerical_values, ROOM_COOR_X_IDX, room_coor_x_arr, row_idx)\n        add_to_array_0_f32(numerical_values, ROOM_COOR_Y_IDX, room_coor_y_arr, row_idx)\n        add_to_array_0_f32(numerical_values, SCREEN_COOR_X_IDX, screen_coor_x_arr, row_idx)\n        add_to_array_0_f32(numerical_values, SCREEN_COOR_Y_IDX, screen_coor_y_arr, row_idx)\n        add_to_array_0_f32(numerical_values, HOVER_DURATION_IDX, hover_duration_arr, row_idx)\n        add_to_array_0_f32(numerical_values, ELAPSED_TIME_IDX, elapsed_time_arr, row_idx)\n        add_to_array_0_f32(numerical_values, DELTA_TIME_IDX, delta_time_arr, row_idx)\n        add_to_array_0_i16(categorical_values, TEXT_FQID_IDX, text_fqid_arr, row_idx)\n        add_to_array_0_i16(categorical_values, FQID_IDX, fqid_arr, row_idx)\n        add_to_array_0_i16(categorical_values, ROOM_FQID_IDX, room_fqid_arr, row_idx)\n        add_to_array_0_i16(categorical_values, LEVEL_GROUP_IDX, level_group_arr, row_idx)\n        add_to_array_0_i16(categorical_values, EVENT_NAME_IDX, event_name_arr, row_idx)\n        add_to_array_0_i16(categorical_values, NAME_IDX, name_arr, row_idx)\n        add_to_array_0_i16(categorical_values, PAGE_IDX, page_arr, row_idx)\n        add_to_array_0_i16(categorical_values, LEVEL_IDX, level_arr, row_idx)\n\n        add_to_array_1_f32(numerical_level_values, ROOM_COOR_X_IDX, level_arr[row_idx], room_coor_x_arr, row_idx)\n        add_to_array_1_f32(numerical_level_values, ROOM_COOR_Y_IDX, level_arr[row_idx], room_coor_y_arr, row_idx)\n        add_to_array_1_f32(numerical_level_values, SCREEN_COOR_X_IDX, level_arr[row_idx], screen_coor_x_arr, row_idx)\n        add_to_array_1_f32(numerical_level_values, SCREEN_COOR_Y_IDX, level_arr[row_idx], screen_coor_y_arr, row_idx)\n        add_to_array_1_f32(numerical_level_values, HOVER_DURATION_IDX, level_arr[row_idx], hover_duration_arr, row_idx)\n        add_to_array_1_f32(numerical_level_values, ELAPSED_TIME_IDX, level_arr[row_idx], elapsed_time_arr, row_idx)\n        add_to_array_1_f32(numerical_level_values, DELTA_TIME_IDX, level_arr[row_idx], delta_time_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, TEXT_FQID_IDX, level_arr[row_idx], text_fqid_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, FQID_IDX, level_arr[row_idx], fqid_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, ROOM_FQID_IDX, level_arr[row_idx], room_fqid_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, LEVEL_GROUP_IDX, level_arr[row_idx], level_group_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, EVENT_NAME_IDX, level_arr[row_idx], event_name_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, NAME_IDX, level_arr[row_idx], name_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, PAGE_IDX, level_arr[row_idx], page_arr, row_idx)\n        add_to_array_1_i16(categorical_level_values, LEVEL_IDX, level_arr[row_idx], level_arr, row_idx)\n\n        add_to_array_2_f32(numerical_room_fqid_level_values, ROOM_COOR_X_IDX, room_fqid_arr[row_idx], level_arr[row_idx], room_coor_x_arr, row_idx)\n        add_to_array_2_f32(numerical_room_fqid_level_values, ROOM_COOR_Y_IDX, room_fqid_arr[row_idx], level_arr[row_idx], room_coor_y_arr, row_idx)\n        add_to_array_2_f32(numerical_room_fqid_level_values, SCREEN_COOR_X_IDX, room_fqid_arr[row_idx], level_arr[row_idx], screen_coor_x_arr, row_idx)\n        add_to_array_2_f32(numerical_room_fqid_level_values, SCREEN_COOR_Y_IDX, room_fqid_arr[row_idx], level_arr[row_idx], screen_coor_y_arr, row_idx)\n        add_to_array_2_f32(numerical_room_fqid_level_values, HOVER_DURATION_IDX, room_fqid_arr[row_idx], level_arr[row_idx], hover_duration_arr, row_idx)\n        add_to_array_2_f32(numerical_room_fqid_level_values, ELAPSED_TIME_IDX, room_fqid_arr[row_idx], level_arr[row_idx], elapsed_time_arr, row_idx)\n        add_to_array_2_f32(numerical_room_fqid_level_values, DELTA_TIME_IDX, room_fqid_arr[row_idx], level_arr[row_idx], delta_time_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, TEXT_FQID_IDX, room_fqid_arr[row_idx], level_arr[row_idx], text_fqid_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, FQID_IDX, room_fqid_arr[row_idx], level_arr[row_idx], fqid_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, ROOM_FQID_IDX, room_fqid_arr[row_idx], level_arr[row_idx], room_fqid_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, LEVEL_GROUP_IDX, room_fqid_arr[row_idx], level_arr[row_idx], level_group_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, EVENT_NAME_IDX, room_fqid_arr[row_idx], level_arr[row_idx], event_name_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, NAME_IDX, room_fqid_arr[row_idx], level_arr[row_idx], name_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, PAGE_IDX, room_fqid_arr[row_idx], level_arr[row_idx], page_arr, row_idx)\n        add_to_array_2_i16(categorical_room_fqid_level_values, LEVEL_IDX, room_fqid_arr[row_idx], level_arr[row_idx], level_arr, row_idx)\n\n        add_to_array_1_f32(numerical_room_fqid_values, ROOM_COOR_X_IDX, room_fqid_arr[row_idx], room_coor_x_arr, row_idx)\n        add_to_array_1_f32(numerical_room_fqid_values, ROOM_COOR_Y_IDX, room_fqid_arr[row_idx], room_coor_y_arr, row_idx)\n        add_to_array_1_f32(numerical_room_fqid_values, SCREEN_COOR_X_IDX, room_fqid_arr[row_idx], screen_coor_x_arr, row_idx)\n        add_to_array_1_f32(numerical_room_fqid_values, SCREEN_COOR_Y_IDX, room_fqid_arr[row_idx], screen_coor_y_arr, row_idx)\n        add_to_array_1_f32(numerical_room_fqid_values, HOVER_DURATION_IDX, room_fqid_arr[row_idx], hover_duration_arr, row_idx)\n        add_to_array_1_f32(numerical_room_fqid_values, ELAPSED_TIME_IDX, room_fqid_arr[row_idx], elapsed_time_arr, row_idx)\n        add_to_array_1_f32(numerical_room_fqid_values, DELTA_TIME_IDX, room_fqid_arr[row_idx], delta_time_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, TEXT_FQID_IDX, room_fqid_arr[row_idx], text_fqid_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, FQID_IDX, room_fqid_arr[row_idx], fqid_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, ROOM_FQID_IDX, room_fqid_arr[row_idx], room_fqid_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, LEVEL_GROUP_IDX, room_fqid_arr[row_idx], level_group_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, EVENT_NAME_IDX, room_fqid_arr[row_idx], event_name_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, NAME_IDX, room_fqid_arr[row_idx], name_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, PAGE_IDX, room_fqid_arr[row_idx], page_arr, row_idx)\n        add_to_array_1_i16(categorical_room_fqid_values, LEVEL_IDX, room_fqid_arr[row_idx], level_arr, row_idx)\n\n    cdef Py_ssize_t i, j, k, m\n\n    # sort all arrays\n    for i in range(NUM_OF_NUMERICAL):\n        sort(numerical_values[i].begin(), numerical_values[i].end())\n    for i in range(NUM_OF_CATEGORICAL):\n        sort(categorical_values[i].begin(), categorical_values[i].end())\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(LEVEL_CAT_SIZE):\n            sort(numerical_level_values[i][j].begin(), numerical_level_values[i][j].end())\n    for i in range(NUM_OF_CATEGORICAL):\n        for j in range(LEVEL_CAT_SIZE):\n            sort(categorical_level_values[i][j].begin(), categorical_level_values[i][j].end())\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            for k in range(LEVEL_CAT_SIZE):\n                sort(numerical_room_fqid_level_values[i][j][k].begin(), numerical_room_fqid_level_values[i][j][k].end())\n    for i in range(NUM_OF_CATEGORICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            for k in range(LEVEL_CAT_SIZE):\n                sort(categorical_room_fqid_level_values[i][j][k].begin(), categorical_room_fqid_level_values[i][j][k].end())\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            sort(numerical_room_fqid_values[i][j].begin(), numerical_room_fqid_values[i][j].end())\n    for i in range(NUM_OF_CATEGORICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            sort(categorical_room_fqid_values[i][j].begin(), categorical_room_fqid_values[i][j].end())\n\n    cdef NumericalStats n_stats\n    cdef NumericalQuantiles n_quantiles\n    cdef CategoricalStats c_stats\n    cdef INT32 head_idx\n\n    # perform computation and then put result into result_arr\n    for i in range(NUM_OF_NUMERICAL):\n        n_stats = compute_numerical_stats(numerical_values[i])\n        head_idx = STAGES_INDEX[0] + i * NUM_OF_NUMERICAL_METRICS\n        put_numerical_stats(result_arr, head_idx, n_stats)\n    for i in range(NUM_OF_NUMERICAL):\n        n_quantiles = compute_numerical_quantiles(numerical_values[i])\n        head_idx = STAGES_INDEX[1] + i * NUM_OF_NUMERICAL_QUANTILES\n        put_numerical_quantiles(result_arr, head_idx, n_quantiles)\n    for i in range(NUM_OF_CATEGORICAL):\n        c_stats = compute_categorical_stats(categorical_values[i])\n        head_idx = STAGES_INDEX[2] + i * NUM_OF_CATEGORICAL_METRICS\n        put_categorical_stats(result_arr, head_idx, c_stats)\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(LEVEL_CAT_SIZE):\n            n_stats = compute_numerical_stats(numerical_level_values[i][j])\n            head_idx = STAGES_INDEX[3]\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + STD_IDX * LEVEL_CAT_SIZE + j] = n_stats.std\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + MEAN_IDX * LEVEL_CAT_SIZE + j] = n_stats.mean\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + VAR_IDX * LEVEL_CAT_SIZE + j] = n_stats.var\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + SEM_IDX * LEVEL_CAT_SIZE + j] = n_stats.sem\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + MEDIAN_IDX * LEVEL_CAT_SIZE + j] = n_stats.median\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + SUM_IDX * LEVEL_CAT_SIZE + j] = n_stats.sum\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + MIN_IDX * LEVEL_CAT_SIZE + j] = n_stats.min\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * LEVEL_CAT_SIZE + MAX_IDX * LEVEL_CAT_SIZE + j] = n_stats.max\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(LEVEL_CAT_SIZE):\n            n_quantiles = compute_numerical_quantiles(numerical_level_values[i][j])\n            head_idx = STAGES_INDEX[4]\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * LEVEL_CAT_SIZE + 0 * LEVEL_CAT_SIZE + j] = n_quantiles.p1\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * LEVEL_CAT_SIZE + 1 * LEVEL_CAT_SIZE + j] = n_quantiles.p20\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * LEVEL_CAT_SIZE + 2 * LEVEL_CAT_SIZE + j] = n_quantiles.p40\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * LEVEL_CAT_SIZE + 3 * LEVEL_CAT_SIZE + j] = n_quantiles.p60\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * LEVEL_CAT_SIZE + 4 * LEVEL_CAT_SIZE + j] = n_quantiles.p80\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * LEVEL_CAT_SIZE + 5 * LEVEL_CAT_SIZE + j] = n_quantiles.p99\n    for i in range(NUM_OF_CATEGORICAL):\n        for j in range(LEVEL_CAT_SIZE):\n            c_stats = compute_categorical_stats(categorical_level_values[i][j])\n            head_idx = STAGES_INDEX[5]\n            result_arr[head_idx + i * NUM_OF_CATEGORICAL_METRICS * LEVEL_CAT_SIZE + NUNIQUE_IDX * LEVEL_CAT_SIZE + j] = c_stats.nunique\n    for i in range(ROOM_FQID_CAT_SIZE):\n        for j in range(LEVEL_CAT_SIZE):\n            head_idx = STAGES_INDEX[6]\n            # A quick hack based on the fact that elapsed_time is never nan\n            result_arr[head_idx + i * LEVEL_CAT_SIZE + j] = <FLOAT32>numerical_room_fqid_level_values[ELAPSED_TIME_IDX][i][j].size()\n    for i in range(COMPOSITE_NUM_OF_NUMERICAL): # DELTA_TIME_IDX only\n        for j in range(COMPOSITE_NUM_OF_NUMERICAL_METRICS):\n            for k in range(ROOM_FQID_CAT_SIZE):\n                for m in range(LEVEL_CAT_SIZE):\n                    n_stats = compute_numerical_stats(numerical_room_fqid_level_values[DELTA_TIME_IDX][k][m])\n                    head_idx = STAGES_INDEX[7]\n                    result_arr[\n                        head_idx +\n                        i * COMPOSITE_NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE * LEVEL_CAT_SIZE +\n                        COMPOSITE_SUM_IDX * ROOM_FQID_CAT_SIZE * LEVEL_CAT_SIZE +\n                        k * LEVEL_CAT_SIZE +\n                        m\n                    ] = n_stats.sum\n                    result_arr[\n                        head_idx +\n                        i * COMPOSITE_NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE * LEVEL_CAT_SIZE +\n                        COMPOSITE_MEAN_IDX * ROOM_FQID_CAT_SIZE * LEVEL_CAT_SIZE +\n                        k * LEVEL_CAT_SIZE +\n                        m\n                    ] = n_stats.mean\n                    result_arr[\n                        head_idx +\n                        i * COMPOSITE_NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE * LEVEL_CAT_SIZE +\n                        COMPOSITE_MEDIAN_IDX * ROOM_FQID_CAT_SIZE * LEVEL_CAT_SIZE +\n                        k * LEVEL_CAT_SIZE +\n                        m\n                    ] = n_stats.median\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            n_stats = compute_numerical_stats(numerical_room_fqid_values[i][j])\n            head_idx = STAGES_INDEX[8]\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + STD_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.std\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + MEAN_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.mean\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + VAR_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.var\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + SEM_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.sem\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + MEDIAN_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.median\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + SUM_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.sum\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + MIN_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.min\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_METRICS * ROOM_FQID_CAT_SIZE + MAX_IDX * ROOM_FQID_CAT_SIZE + j] = n_stats.max\n    for i in range(NUM_OF_NUMERICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            n_quantiles = compute_numerical_quantiles(numerical_room_fqid_values[i][j])\n            head_idx = STAGES_INDEX[9]\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * ROOM_FQID_CAT_SIZE + 0 * ROOM_FQID_CAT_SIZE + j] = n_quantiles.p1\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * ROOM_FQID_CAT_SIZE + 1 * ROOM_FQID_CAT_SIZE + j] = n_quantiles.p20\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * ROOM_FQID_CAT_SIZE + 2 * ROOM_FQID_CAT_SIZE + j] = n_quantiles.p40\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * ROOM_FQID_CAT_SIZE + 3 * ROOM_FQID_CAT_SIZE + j] = n_quantiles.p60\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * ROOM_FQID_CAT_SIZE + 4 * ROOM_FQID_CAT_SIZE + j] = n_quantiles.p80\n            result_arr[head_idx + i * NUM_OF_NUMERICAL_QUANTILES * ROOM_FQID_CAT_SIZE + 5 * ROOM_FQID_CAT_SIZE + j] = n_quantiles.p99\n    for i in range(NUM_OF_CATEGORICAL):\n        for j in range(ROOM_FQID_CAT_SIZE):\n            c_stats = compute_categorical_stats(categorical_room_fqid_values[i][j])\n            head_idx = STAGES_INDEX[10]\n            result_arr[head_idx + i * NUM_OF_CATEGORICAL_METRICS * ROOM_FQID_CAT_SIZE + NUNIQUE_IDX * ROOM_FQID_CAT_SIZE + j] = c_stats.nunique\n\n    return result_arr\n\n\ndef compute_stat(\n    FLOAT32[:] elapsed_time_arr,\n    INT8[:] event_name_arr,\n    INT8[:] name_arr,\n    INT8[:] level_arr,\n    INT8[:] page_arr,\n    FLOAT32[:] room_coor_x_arr,\n    FLOAT32[:] room_coor_y_arr,\n    FLOAT32[:] screen_coor_x_arr,\n    FLOAT32[:] screen_coor_y_arr,\n    FLOAT32[:] hover_duration_arr,\n    INT16[:] fqid_arr,\n    INT8[:] room_fqid_arr,\n    INT8[:] text_fqid_arr,\n    INT8[:] level_group_arr,\n    INT32 arr_len,\n):\n    # sort by elapsed_time first to get delta_time\n    assert arr_len >= 0\n    assert arr_len == elapsed_time_arr.shape[0]\n    assert arr_len == event_name_arr.shape[0]\n    assert arr_len == name_arr.shape[0]\n    assert arr_len == level_arr.shape[0]\n    assert arr_len == page_arr.shape[0]\n    assert arr_len == room_coor_x_arr.shape[0]\n    assert arr_len == room_coor_y_arr.shape[0]\n    assert arr_len == screen_coor_x_arr.shape[0]\n    assert arr_len == screen_coor_y_arr.shape[0]\n    assert arr_len == hover_duration_arr.shape[0]\n    assert arr_len == fqid_arr.shape[0]\n    assert arr_len == room_fqid_arr.shape[0]\n    assert arr_len == text_fqid_arr.shape[0]\n    assert arr_len == level_group_arr.shape[0]\n\n    return np.asarray(\n        compute_stat_inner(\n            elapsed_time_arr,\n            event_name_arr,\n            name_arr,\n            level_arr,\n            page_arr,\n            room_coor_x_arr,\n            room_coor_y_arr,\n            screen_coor_x_arr,\n            screen_coor_y_arr,\n            hover_duration_arr,\n            fqid_arr,\n            room_fqid_arr,\n            text_fqid_arr,\n            level_group_arr,\n            arr_len\n        )\n    )\n\ndef compute_stat_copy(\n    const FLOAT32[:] elapsed_time_arr,\n    const INT8[:] event_name_arr,\n    const INT8[:] name_arr,\n    const INT8[:] level_arr,\n    const INT8[:] page_arr,\n    const FLOAT32[:] room_coor_x_arr,\n    const FLOAT32[:] room_coor_y_arr,\n    const FLOAT32[:] screen_coor_x_arr,\n    const FLOAT32[:] screen_coor_y_arr,\n    const FLOAT32[:] hover_duration_arr,\n    const INT16[:] fqid_arr,\n    const INT8[:] room_fqid_arr,\n    const INT8[:] text_fqid_arr,\n    const INT8[:] level_group_arr,\n    INT32 arr_len,\n):\n    # sort by elapsed_time first to get delta_time\n    assert arr_len >= 0\n    assert arr_len == elapsed_time_arr.shape[0]\n    assert arr_len == event_name_arr.shape[0]\n    assert arr_len == name_arr.shape[0]\n    assert arr_len == level_arr.shape[0]\n    assert arr_len == page_arr.shape[0]\n    assert arr_len == room_coor_x_arr.shape[0]\n    assert arr_len == room_coor_y_arr.shape[0]\n    assert arr_len == screen_coor_x_arr.shape[0]\n    assert arr_len == screen_coor_y_arr.shape[0]\n    assert arr_len == hover_duration_arr.shape[0]\n    assert arr_len == fqid_arr.shape[0]\n    assert arr_len == room_fqid_arr.shape[0]\n    assert arr_len == text_fqid_arr.shape[0]\n    assert arr_len == level_group_arr.shape[0]\n\n\n    # array format spec https://docs.python.org/3/library/struct.html#module-struct\n    cdef FLOAT32[:] elapsed_time_arr_copy = cvarray(shape=(elapsed_time_arr.shape[0],), itemsize=sizeof(FLOAT32), format=\"f\")\n    cdef INT8[:] event_name_arr_copy = cvarray(shape=(event_name_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n    cdef INT8[:] name_arr_copy = cvarray(shape=(name_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n    cdef INT8[:] level_arr_copy = cvarray(shape=(level_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n    cdef INT8[:] page_arr_copy = cvarray(shape=(page_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n    cdef FLOAT32[:] room_coor_x_arr_copy = cvarray(shape=(room_coor_x_arr.shape[0],), itemsize=sizeof(FLOAT32), format=\"f\")\n    cdef FLOAT32[:] room_coor_y_arr_copy = cvarray(shape=(room_coor_y_arr.shape[0],), itemsize=sizeof(FLOAT32), format=\"f\")\n    cdef FLOAT32[:] screen_coor_x_arr_copy = cvarray(shape=(screen_coor_x_arr.shape[0],), itemsize=sizeof(FLOAT32), format=\"f\")\n    cdef FLOAT32[:] screen_coor_y_arr_copy = cvarray(shape=(screen_coor_y_arr.shape[0],), itemsize=sizeof(FLOAT32), format=\"f\")\n    cdef FLOAT32[:] hover_duration_arr_copy = cvarray(shape=(hover_duration_arr.shape[0],), itemsize=sizeof(FLOAT32), format=\"f\")\n    cdef INT16[:] fqid_arr_copy = cvarray(shape=(fqid_arr.shape[0],), itemsize=sizeof(INT16), format=\"h\")\n    cdef INT8[:] room_fqid_arr_copy = cvarray(shape=(room_fqid_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n    cdef INT8[:] text_fqid_arr_copy = cvarray(shape=(text_fqid_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n    cdef INT8[:] level_group_arr_copy = cvarray(shape=(level_group_arr.shape[0],), itemsize=sizeof(INT8), format=\"b\")\n\n    elapsed_time_arr_copy[...] = elapsed_time_arr\n    event_name_arr_copy[...] = event_name_arr\n    name_arr_copy[...] = name_arr\n    level_arr_copy[...] = level_arr\n    page_arr_copy[...] = page_arr\n    room_coor_x_arr_copy[...] = room_coor_x_arr\n    room_coor_y_arr_copy[...] = room_coor_y_arr\n    screen_coor_x_arr_copy[...] = screen_coor_x_arr\n    screen_coor_y_arr_copy[...] = screen_coor_y_arr\n    hover_duration_arr_copy[...] = hover_duration_arr\n    fqid_arr_copy[...] = fqid_arr\n    room_fqid_arr_copy[...] = room_fqid_arr\n    text_fqid_arr_copy[...] = text_fqid_arr\n    level_group_arr_copy[...] = level_group_arr\n\n    return np.asarray(\n        compute_stat_inner(\n            elapsed_time_arr_copy,\n            event_name_arr_copy,\n            name_arr_copy,\n            level_arr_copy,\n            page_arr_copy,\n            room_coor_x_arr_copy,\n            room_coor_y_arr_copy,\n            screen_coor_x_arr_copy,\n            screen_coor_y_arr_copy,\n            hover_duration_arr_copy,\n            fqid_arr_copy,\n            room_fqid_arr_copy,\n            text_fqid_arr_copy,\n            level_group_arr_copy,\n            arr_len\n        )\n    )\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:43:49.460482Z","iopub.execute_input":"2023-06-29T03:43:49.461048Z","iopub.status.idle":"2023-06-29T03:44:06.354388Z","shell.execute_reply.started":"2023-06-29T03:43:49.461022Z","shell.execute_reply":"2023-06-29T03:44:06.353343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom typing import Dict, List, Tuple","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.355483Z","iopub.execute_input":"2023-06-29T03:44:06.355873Z","iopub.status.idle":"2023-06-29T03:44:06.364176Z","shell.execute_reply.started":"2023-06-29T03:44:06.355838Z","shell.execute_reply":"2023-06-29T03:44:06.362181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DF_DTYPES = {\n    'elapsed_time': np.int32,\n    'event_name': 'category',\n    'name': 'category',\n    'level': np.uint8,\n    'room_coor_x': np.float32,\n    'room_coor_y': np.float32,\n    'screen_coor_x': np.float32,\n    'screen_coor_y': np.float32,\n    'hover_duration': np.float32,\n    'text': 'category',\n    'fqid': 'category',\n    'room_fqid': 'category',\n    'text_fqid': 'category',\n    'fullscreen': 'category',\n    'hq': 'category',\n    'music': 'category',\n    'level_group': 'category'\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.368116Z","iopub.execute_input":"2023-06-29T03:44:06.368660Z","iopub.status.idle":"2023-06-29T03:44:06.384870Z","shell.execute_reply.started":"2023-06-29T03:44:06.368617Z","shell.execute_reply":"2023-06-29T03:44:06.381504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open(f\"{MODELS_PATH}/column_description.json\", \"r\") as f:\n    COLUMN_DESCRIPTION = json.load(f)\n    COLUMN_DTYPES = {}\n    for k, d in COLUMN_DESCRIPTION.items():\n        if d['type'] == 'category':\n            COLUMN_DTYPES[k] = pd.CategoricalDtype(\n                categories=d['categories'],\n                ordered=d['ordered']\n            )\n        else:\n            COLUMN_DTYPES[k] = d['type']","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.389977Z","iopub.execute_input":"2023-06-29T03:44:06.390940Z","iopub.status.idle":"2023-06-29T03:44:06.419946Z","shell.execute_reply.started":"2023-06-29T03:44:06.390886Z","shell.execute_reply":"2023-06-29T03:44:06.417523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONTINUOUS_NUMBER_COLUMNS = [\n    \"room_coor_x\",\n    \"room_coor_y\",\n    \"screen_coor_x\",\n    \"screen_coor_y\",\n    \"hover_duration\",\n    \"elapsed_time\",\n]\nCATEGORY_COLUMNS = [\n    \"text_fqid\",\n    \"fqid\",\n    \"room_fqid\",\n    \"text\",\n    \"level_group\",\n    \"event_name\",\n    \"name\",\n    \"page\",\n    'level',\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.422535Z","iopub.execute_input":"2023-06-29T03:44:06.423033Z","iopub.status.idle":"2023-06-29T03:44:06.430802Z","shell.execute_reply.started":"2023-06-29T03:44:06.422995Z","shell.execute_reply":"2023-06-29T03:44:06.429735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DELTA_TIME = 'time_diff_with_prev_event'\n\ndef process_base_dataset(df: pd.DataFrame) -> pd.DataFrame:\n    # Ensuring reproducible results when inferencing on a subset of data\n\n    df = df.drop(['text'], axis=1)\n    \n    df = df.astype(COLUMN_DTYPES)\n            \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.432916Z","iopub.execute_input":"2023-06-29T03:44:06.435167Z","iopub.status.idle":"2023-06-29T03:44:06.448379Z","shell.execute_reply.started":"2023-06-29T03:44:06.435101Z","shell.execute_reply":"2023-06-29T03:44:06.445777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stringify(v) -> str:\n    if isinstance(v, str):\n        return v\n    if isinstance(v, float):\n        return f\"{v:.04f}\"\n    if isinstance(v, int):\n        return str(v)\n    return v","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.451379Z","iopub.execute_input":"2023-06-29T03:44:06.452280Z","iopub.status.idle":"2023-06-29T03:44:06.461705Z","shell.execute_reply.started":"2023-06-29T03:44:06.452236Z","shell.execute_reply":"2023-06-29T03:44:06.460893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def flatten_column_names(df: pd.DataFrame) -> pd.DataFrame:\n    df.columns = ['_'.join([stringify(k) for k in col]) for col in df.columns.values]\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.463004Z","iopub.execute_input":"2023-06-29T03:44:06.463443Z","iopub.status.idle":"2023-06-29T03:44:06.477854Z","shell.execute_reply.started":"2023-06-29T03:44:06.463418Z","shell.execute_reply":"2023-06-29T03:44:06.475868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMERICAL_METRICS = [\n    'std', 'mean', 'var', 'sem', 'median', 'sum', 'min', 'max',\n]\nNUMERICAL_QUANTILES = np.concatenate([np.arange(0.2, 0.81, 0.2), np.array([0.01, 0.99])])\nCATEGORICAL_METRICS = ['nunique']","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.482957Z","iopub.execute_input":"2023-06-29T03:44:06.483414Z","iopub.status.idle":"2023-06-29T03:44:06.495115Z","shell.execute_reply.started":"2023-06-29T03:44:06.483387Z","shell.execute_reply":"2023-06-29T03:44:06.491632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def perform_feature_engineering_michael(bdf: pd.DataFrame) -> pd.DataFrame:\n    BASE_GROUPING = ['session_id','level_group']\n    grouped = bdf.groupby(BASE_GROUPING)\n    NUMERICAL = CONTINUOUS_NUMBER_COLUMNS + [DELTA_TIME]\n    CATEGORICAL = CATEGORY_COLUMNS + []\n    CATEGORICAL.remove('text')\n\n    dfs = []\n\n    grouped = bdf.groupby(BASE_GROUPING + ['level', 'room_fqid'])\n\n    df = grouped.size().unstack().unstack()\n    df = flatten_column_names(df)\n    dfs.append(df)\n\n    df = grouped[[DELTA_TIME]].agg(['sum']).unstack().unstack()\n    df = flatten_column_names(df)\n    dfs.append(df)\n\n    dataset_df = pd.concat(dfs, axis=1)\n    dataset_df = dataset_df.fillna(-999)\n    dataset_df = dataset_df.astype(np.float32)\n    return dataset_df","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.498441Z","iopub.execute_input":"2023-06-29T03:44:06.499067Z","iopub.status.idle":"2023-06-29T03:44:06.511159Z","shell.execute_reply.started":"2023-06-29T03:44:06.499017Z","shell.execute_reply":"2023-06-29T03:44:06.509005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_dataset(bdf: pd.DataFrame) -> pd.DataFrame:\n    bdf = process_base_dataset(bdf)\n    df = perform_feature_engineering_michael(bdf)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.513746Z","iopub.execute_input":"2023-06-29T03:44:06.514258Z","iopub.status.idle":"2023-06-29T03:44:06.535464Z","shell.execute_reply.started":"2023-06-29T03:44:06.514213Z","shell.execute_reply":"2023-06-29T03:44:06.533115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEVEL_GROUP_TO_QUESTION_RANGE = {\n    '0-4': (1, 4),\n    '5-12': (4, 14),\n    '13-22': (14, 19)\n}\nQUESTION_NO_TO_LEVEL_GROUP = {}\nfor k, v in LEVEL_GROUP_TO_QUESTION_RANGE.items():\n    for i in range(*v):\n        QUESTION_NO_TO_LEVEL_GROUP[i] = k","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.538296Z","iopub.execute_input":"2023-06-29T03:44:06.538925Z","iopub.status.idle":"2023-06-29T03:44:06.550783Z","shell.execute_reply.started":"2023-06-29T03:44:06.538882Z","shell.execute_reply":"2023-06-29T03:44:06.548798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:06.553566Z","iopub.execute_input":"2023-06-29T03:44:06.554090Z","iopub.status.idle":"2023-06-29T03:44:07.882114Z","shell.execute_reply.started":"2023-06-29T03:44:06.554050Z","shell.execute_reply":"2023-06-29T03:44:07.878888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\nfor q_no in QUESTION_NO_TO_LEVEL_GROUP:\n    models[q_no] = CatBoostClassifier()\n    models[q_no].load_model(f\"{MODELS_PATH}/q{q_no}.cbm\")","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:07.887764Z","iopub.execute_input":"2023-06-29T03:44:07.888256Z","iopub.status.idle":"2023-06-29T03:44:08.157477Z","shell.execute_reply.started":"2023-06-29T03:44:07.888209Z","shell.execute_reply":"2023-06-29T03:44:08.155553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nHere you'll use the `best_threshold` calculate in the previous cell","metadata":{"id":"ezA40GQ4n2PH"}},{"cell_type":"code","source":"# Reference\n# https://www.kaggle.com/code/philculliton/basic-submission-demo\n# https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\n\nimport jo_wilder_310 as jo_wilder\nimport numpy as np\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n#     print(test.shape)\n#     print(test['session_id'].nunique())\n#     print(test['level_group'].nunique())\n#     print(sample_submission.shape)\n    test_df = test\n    level_group = test_df.level_group.values[0]\n    base_df = process_base_dataset(test_df)\n    x = compute_stat_copy(\n        base_df.elapsed_time.to_numpy(),\n        base_df.event_name.cat.codes.to_numpy(),\n        base_df['name'].cat.codes.to_numpy(),\n        base_df.level.cat.codes.to_numpy(),\n        base_df.page.cat.codes.to_numpy(),\n        base_df.room_coor_x.to_numpy(),\n        base_df.room_coor_y.to_numpy(),\n        base_df.screen_coor_x.to_numpy(),\n        base_df.screen_coor_y.to_numpy(),\n        base_df.hover_duration.to_numpy(),\n        base_df.fqid.cat.codes.to_numpy(),\n        base_df.room_fqid.cat.codes.to_numpy(),\n        base_df.text_fqid.cat.codes.to_numpy(),\n        base_df.level_group.cat.codes.to_numpy(),\n        base_df.shape[0]\n    )\n    x = x.reshape((1, -1))\n#     print(x.shape)\n    question_range = LEVEL_GROUP_TO_QUESTION_RANGE[level_group]\n#     sol = {}\n#     dfs = []\n    for q_no in range(*question_range):\n        model = models[q_no]\n#         session_id = x.index.astype('string') + f\"_q{q_no}\"\n        predictions = model.predict_proba(x)\n#         assert predictions.shape[0] == 1\n#         assert predictions.shape[1] == 2\n#         print(predictions.shape)\n#         print(sample_submission.loc[sample_submission.session_id.str.contains(f\"_q{q_no}\"), 'correct'].shape)\n        sample_submission.loc[sample_submission.session_id.str.contains(f\"_q{q_no}\"), 'correct'] = 1 if predictions[0, 1] >= THRESHOLD else 0\n        \n#         df = pd.DataFrame({\n#             'session_id': session_id,\n#             'correct': predictions\n#         })\n#         dfs.append(df)\n#     result_df = pd.concat(dfs).reset_index()\n#     sample_submission['session_id'] = result_df['session_id']\n#     sample_submission['correct'] = result_df['correct']\n    env.predict(sample_submission)\n#     grp = test_df.level_group.values[0]\n#     a,b = limits[grp]\n#     for t in range(a,b):\n#         gbtm = models[f'{grp}_{t}']\n#         test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_df.loc[:, test_df.columns != 'level_group'])\n#         predictions = gbtm.predict(test_ds)\n#         predictions = np.repeat(np.array([1]), test.si)\n#         mask = sample_submission.session_id.str.contains(f'q{t}')\n#         n_predictions = (predictions > best_threshold).astype(int)\n# #     sample_submission.loc[:,'correct'] = 1\n    \n#     env.predict(sample_submission)","metadata":{"id":"gHiXTnTVn2PI","execution":{"iopub.status.busy":"2023-06-29T03:44:08.159398Z","iopub.execute_input":"2023-06-29T03:44:08.159819Z","iopub.status.idle":"2023-06-29T03:44:10.810508Z","shell.execute_reply.started":"2023-06-29T03:44:08.159793Z","shell.execute_reply":"2023-06-29T03:44:10.809046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(test)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:10.813314Z","iopub.execute_input":"2023-06-29T03:44:10.814329Z","iopub.status.idle":"2023-06-29T03:44:10.826153Z","shell.execute_reply.started":"2023-06-29T03:44:10.814245Z","shell.execute_reply":"2023-06-29T03:44:10.822987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test['name'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:10.832134Z","iopub.execute_input":"2023-06-29T03:44:10.832707Z","iopub.status.idle":"2023-06-29T03:44:10.847500Z","shell.execute_reply.started":"2023-06-29T03:44:10.832625Z","shell.execute_reply":"2023-06-29T03:44:10.844304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head submission.csv","metadata":{"id":"iYBXokAyn2PI","execution":{"iopub.status.busy":"2023-06-29T03:44:10.849978Z","iopub.execute_input":"2023-06-29T03:44:10.850430Z","iopub.status.idle":"2023-06-29T03:44:11.124317Z","shell.execute_reply.started":"2023-06-29T03:44:10.850392Z","shell.execute_reply":"2023-06-29T03:44:11.123226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv('submission.csv')\n# print( df.shape )\n# df.head(60)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:11.125859Z","iopub.execute_input":"2023-06-29T03:44:11.127605Z","iopub.status.idle":"2023-06-29T03:44:11.135206Z","shell.execute_reply.started":"2023-06-29T03:44:11.127478Z","shell.execute_reply":"2023-06-29T03:44:11.133490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !free -m","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:11.137017Z","iopub.execute_input":"2023-06-29T03:44:11.137569Z","iopub.status.idle":"2023-06-29T03:44:11.151947Z","shell.execute_reply.started":"2023-06-29T03:44:11.137536Z","shell.execute_reply":"2023-06-29T03:44:11.150280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.Series([]).sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:11.153246Z","iopub.execute_input":"2023-06-29T03:44:11.154398Z","iopub.status.idle":"2023-06-29T03:44:11.168007Z","shell.execute_reply.started":"2023-06-29T03:44:11.154335Z","shell.execute_reply":"2023-06-29T03:44:11.166461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !diff submission.csv submission.old.csv","metadata":{"execution":{"iopub.status.busy":"2023-06-29T03:44:11.169362Z","iopub.execute_input":"2023-06-29T03:44:11.170147Z","iopub.status.idle":"2023-06-29T03:44:11.452291Z","shell.execute_reply.started":"2023-06-29T03:44:11.170110Z","shell.execute_reply":"2023-06-29T03:44:11.450339Z"},"trusted":true},"execution_count":null,"outputs":[]}]}