{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc,sys\nimport sys,joblib\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom catboost import CatBoostClassifier, Pool\nfrom xgboost import XGBClassifier\nfrom sklearn.cluster import KMeans\nimport jo_wilder\n\nclass CFG:\n    th = 0.605\n    model = \"xgb\" \n    ESTIMATE = False\n    full_data = True\n    mode = \"median\"\n    model_pth=\"/kaggle/input/psp-xgb-2023-05-08-lambda-0/xgb_2023-05-08-3.6/\" \n    model_pth_2=\"/kaggle/input/psp-xgb-2023-05-08-lambda-0/xgb_2023-05-08-3.1/\" \n    GLOBALS=False\n    DIST=False\n    CLUSTER_FEAT=False\n    EMBED=False\n    META_AGG=False\nprint (joblib.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-25T01:35:54.423694Z","iopub.execute_input":"2023-06-25T01:35:54.424116Z","iopub.status.idle":"2023-06-25T01:35:56.198716Z","shell.execute_reply.started":"2023-06-25T01:35:54.424034Z","shell.execute_reply":"2023-06-25T01:35:56.197892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0,CFG.model_pth) \nfrom fe_v2 import feature_engineer_kiki,time_feature,global_feats \n\ncluster_scrn_dict_lvl={'0-4': {'0-4': 6}, '5-12': {'5-12': 6}, '13-22': {'13-22': 6}}\nnot_null_mask = ((pl.col('screen_coor_x').is_not_null()) & (pl.col('screen_coor_y').is_not_null()) & (pl.col('next_x').is_not_null()) & (pl.col('next_y').is_not_null()))\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    ((pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)).fill_null(0).clip(0, 1e9).over([\"session_id\", \"level\"]) .alias(\"elapsed_time_diff\")),\n    ((pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) .abs().over([\"session_id\", \"level\"]) .alias('screen_coor_x_diff') ),\n    ((pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)).abs().over([\"session_id\", \"level\"]) .alias('screen_coor_y_diff') ), \n#     ((pl.col(\"screen_coor_x\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_x')),\n#     ((pl.col(\"screen_coor_y\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_y')), \n    #((pl.col(\"room_coor_x\").shift_and_fill(-1, fill_value=None)).alias('room_next_x')),\n    #((pl.col(\"room_coor_y\").shift_and_fill(-1, fill_value=None)).alias('room_next_y')), \n    pl.col(\"fqid\").fill_null(\"fqid_None\"), \n    ((pl.col(\"elapsed_time\").diff(n=5)) .fill_null(0).abs().over([\"session_id\", \"level_group\"]).alias(\"elapsed_time_diff2\")),\n    ((pl.col(\"elapsed_time\").diff(n=10)) .fill_null(0).abs().over([\"session_id\", \"level_group\"]).alias(\"elapsed_time_diff3\")),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\"),\n] \nif CFG.DIST:\n    columns = columns + [ ((pl.col(\"screen_coor_x\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_x')),\n    ((pl.col(\"screen_coor_y\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_y')), ]\ndef load_model(path):\n    if CFG.model == \"xgb\":\n        model = XGBClassifier()\n        model.load_model(path)\n    else:\n        model = CatBoostClassifier().load_model(path)\n    return model\n\nif CFG.full_data:\n    if CFG.mode==\"median\":\n        models_list = [load_model( f\"{CFG.model_pth}full_data_q{q}.cbm\") for q in range(1, 19)]\n        models_list_2 = [load_model( f\"{CFG.model_pth_2}full_data_q{q}.cbm\") for q in range(1, 19)]\n    else:\n        models_list = [load_model( f\"/kaggle/input/psp-train/full_data_mean_q{q}.cbm\") for q in range(1, 19)]   \nelse:    \n    models_list = [[load_model( f\"{CFG.model_pth}fold{fold}_q{q}.cbm\") for fold in range(5)] for q in range(1, 19)]  ","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:35:56.200162Z","iopub.execute_input":"2023-06-25T01:35:56.200425Z","iopub.status.idle":"2023-06-25T01:35:57.065690Z","shell.execute_reply.started":"2023-06-25T01:35:56.200401Z","shell.execute_reply":"2023-06-25T01:35:57.064594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer or do Infer Time estimate","metadata":{}},{"cell_type":"code","source":"TIME_ESTIMATION = CFG.ESTIMATE\n# CLEAR MEMORY\nimport gc \n_ = gc.collect()    ","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:35:57.068028Z","iopub.execute_input":"2023-06-25T01:35:57.068337Z","iopub.status.idle":"2023-06-25T01:35:57.189678Z","shell.execute_reply.started":"2023-06-25T01:35:57.068310Z","shell.execute_reply":"2023-06-25T01:35:57.188663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold_multiplier = 2\ndef count_outliers(x, multiplier):\n    threshold = x.mean() + multiplier * x.std()\n    return (x > threshold).sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:35:57.194031Z","iopub.execute_input":"2023-06-25T01:35:57.195909Z","iopub.status.idle":"2023-06-25T01:35:57.202836Z","shell.execute_reply.started":"2023-06-25T01:35:57.195874Z","shell.execute_reply":"2023-06-25T01:35:57.202094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_func(item1, item2):\n    if item1['level'] == item2['level']:\n        if abs(item1['index'] - item2['index']) > 50:\n            if item1['elapsed_time'] < item2['elapsed_time']:\n                return -1\n            elif item1['elapsed_time'] > item2['elapsed_time']:\n                return 1\n            else:\n                return 0\n        else:\n            if item1['index'] < item2['index']:\n                return -1\n            elif item1['index'] > item2['index']:\n                return 1\n            else:\n                return 0\n    else:\n        return item1['level'] - item2['level']\n\nimport functools\ndef sort_frame(df):\n    v = [{'idx': idx, 'index':row['index'], 'elapsed_time': row['elapsed_time'], 'level': row['level']} for idx, row in df.iterrows()]\n    v.sort(key=functools.cmp_to_key(compare_func))\n    mdf = pd.DataFrame(v)\n    assert len(mdf) == len(df)\n    return df.loc[mdf.idx]","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:35:57.206397Z","iopub.execute_input":"2023-06-25T01:35:57.208706Z","iopub.status.idle":"2023-06-25T01:35:57.219700Z","shell.execute_reply.started":"2023-06-25T01:35:57.208670Z","shell.execute_reply":"2023-06-25T01:35:57.218259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport pickle\n\n# Load importance dict\nf_read = open(f'{CFG.model_pth}importance_dict.pkl', 'rb')\nimportance_dict = pickle.load(f_read)\nf_read.close()\n\nf_read_2 = open(f'{CFG.model_pth_2}importance_dict.pkl', 'rb')\nimportance_dict_2 = pickle.load(f_read_2)\nf_read_2.close()\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\nhistorical_meta2 = defaultdict(list)\nCLUSTER_FEAT = False\npre_04 = {}\npre_512 = {}\n\ntfidfDict = {\"0-4\":None, \"5-12\":None, \"13-22\":None}\nclusterKdict = {\"0-4\":None, \"5-12\":None, \"13-22\":None}\n# Cache Kmeans models\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]\nif CFG.CLUSTER_FEAT:\n    for lvl in level_groups: \n        print(f\"Adding {lvl} clustering\")\n        tfidfDict[lvl] =joblib.load(f\"/kaggle/input/ps-fe-and-cluster-analysis/tfidf_txt_{lvl}.pkl\")  \n        clusterKdict[lvl] =joblib.load(f\"/kaggle/input/ps-fe-and-cluster-analysis/txt_tfidf_{lvl}_cluster.pkl\")  \nif CFG.EMBED:\n    labelsDict = joblib.load(f\"/kaggle/input/ps-fe-and-cluster-analysis/text_embed_clstr_1.01.pkl\")\n## pack iteration logic in a function\ndef iteration (test,sample_submission ): \n    # INFER TEST DATA \n    #test = sort_frame(test)\n    test = test.sort_values(by = 'elapsed_time')\n    session_id = test.session_id.values[0]\n    grp = test.level_group.values[0] \n    sample_submission['questions'] = sample_submission['session_id'].str.split('_').apply(lambda x: x[1])\n    sample_submission = sample_submission.sort_values(by = 'questions')\n    sample_submission = sample_submission[['session_id', 'correct']]\n    \n    if grp == '0-4':\n        suffix = '04'\n    elif grp == '5-12':\n        suffix = '512'\n    else:\n        suffix = '1322' \n    test = (pl.from_pandas(test)\n          .drop([\"fullscreen\", \"hq\", \"music\"])\n          .with_columns(columns)) \n    if CFG.EMBED: \n        lblDic = labelsDict[grp]  \n        pca1, pca2 = zip(*[lblDic[k] for k in test['text'].fill_null(\"\")  ])    \n        pca_series = pl.Series('pca_1', pca1)\n        pca_series_2 = pl.Series('pca_2', pca2)\n        test = test.with_columns([pca_series,pca_series_2]) \n    if CFG.CLUSTER_FEAT:\n        vec = tfidfDict[grp].transform(test['text'].fill_null(\"\"))\n        transformed_series = pl.Series('tfidf_clstr', clusterKdict[grp].predict(vec))\n        # add the new Series to the Polars DataFrame\n        test = test.with_columns(transformed_series)  \n     \n #     test = test.with_columns([\n#     ((pl.col(\"elapsed_time_diff\") - pl.first( \"elapsed_time_diff\"))/(pl.last(\"elapsed_time_diff\") - pl.first( \"elapsed_time_diff\"))*3 +1)\n#         .fill_null(0).over([\"session_id\", \"level\"]).alias(\"tw_score\"), \n#      (pl.when(not_null_mask).then(((pl.col('next_x') - pl.col('screen_coor_x'))**2 + (pl.col('next_y') - pl.col('screen_coor_y'))**2).sqrt().over([\"session_id\", \"level\"])).otherwise(None).abs().alias('distance')),\n#      (pl.when(not_null_mask).then(((pl.col('next_x') - pl.col('screen_coor_x')) + (pl.col('next_y') - pl.col('screen_coor_y'))).over([\"session_id\", \"level\"])).otherwise(None).abs().alias('manhattan')),     #(pl.when(not_null_mask).then(((pl.col('room_next_x') - pl.col('screen_coor_x'))**2 + (pl.col('room_next_y') - pl.col('room_coor_y'))**2).sqrt().over([\"session_id\", \"level\"])).otherwise(None).abs().alias('distance_room')),\n#      #(pl.when(not_null_mask).then(((pl.col('next_y') - pl.col('screen_coor_y')) / (pl.col('next_x') - pl.col('screen_coor_x'))).arctan().over([\"session_id\", \"level\"])).otherwise(None).alias('angle')) \n#     ])  \n#     test=test.with_columns([ \n#         pl.when(pl.col(\"tw_score\").is_infinite()).then(0).otherwise(pl.col(\"tw_score\")).keep_name(),\n#         #pl.col('angle').shift_and_fill(-1, fill_value=None).over([\"session_id\", \"level\"]).alias('prev_angle' )\n#     ])  \n    #test =test.with_columns([(pl.col(\"angle\")-pl.col(\"prev_angle\")).over([\"session_id\", \"level\"]).alias('angle_change') ])  \n    test_all = None\n    if CFG.GLOBALS:\n        if grp == '0-4':\n            pre_04[session_id] = test  \n        if grp == '5-12': \n            pre_512[session_id] = test \n            if len(pre_04[session_id]) > 0: \n                test_all = global_feats(pre_04[session_id]) \n        if grp == '13-22':\n            if len(pre_04[session_id]) > 0:\n                test_all = pl.concat([pre_04[session_id]]) \n                pre_04[session_id] = None\n            if len(pre_512[session_id]) > 0:\n                test_all = pl.concat([test_all,pre_512[session_id]]) \n                pre_512[session_id] = None\n            if test_all is not None:    \n                test_all = global_feats(test_all) \n    test = feature_engineer_kiki(test, grp, use_extra=True, feature_suffix=suffix) \n    if CFG.GLOBALS & (grp != '0-4') & (test_all is not None):\n        test = test.join(test_all,on=\"session_id\", how='left')  \n     \n    if not CFG.GLOBALS:\n        if grp == '0-4':\n            pre_04[session_id] = test\n        if grp == '5-12':\n            pre_512[session_id] = test \n        if grp == '5-12':\n            if len(pre_04[session_id]) > 0:\n                test = test.join(pre_04[session_id], on = \"session_id\", how = 'left')\n        if grp == '13-22':\n            if len(pre_04[session_id]) > 0:\n                test = test.join(pre_04[session_id], on = \"session_id\", how = 'left')\n                pre_04[session_id] = None\n            if len(pre_512[session_id]) > 0:\n                test = test.join(pre_512[session_id], on = \"session_id\", how = 'left')\n                pre_512[session_id] = None \n    test = test.to_pandas()\n    #test = time_feature(test)    \n    a,b = limits[grp]\n    fold=0 \n    for q in range(a,b):\n        clf = models_list[q-1] if CFG.full_data else models_list[q-1][fold]\n        clf_2 = models_list_2[q-1] if CFG.full_data else models_list_2[q-1][fold]\n        FEATURES = importance_dict[str(q)]\n        FEATURES_2 = importance_dict_2[str(q)]\n        #print(grp,len(FEATURES))\n        FEATURES_t = FEATURES.copy()\n        FEATURES_t_2 = FEATURES_2.copy()\n        meta_id = f'meta_{q}'\n        test_t = test.copy()\n        #print(q,test.shape)\n        if q-1>0:\n            meta_name=[]\n            for tt in range(1,q):\n                test_t[f'meta_{tt}'] = historical_meta2[session_id][tt-1]\n                FEATURES_t.append(f'meta_{tt}')   \n                FEATURES_t_2.append(f'meta_{tt}')   \n                meta_name.append(f'meta_{tt}')\n                \n            if CFG.META_AGG:\n                test_t['meta_sum'] = test_t[meta_name].apply(lambda x: x.sum(), axis=1)\n                test_t['meta_mean'] = test_t[meta_name].apply(lambda x: x.mean(), axis=1)\n                test_t['meta_outliers_count'] = test_t[meta_name].apply(lambda x: count_outliers(x, threshold_multiplier), axis=1)\n                FEATURES_t.append('meta_sum')\n                FEATURES_t.append('meta_mean')\n                # FEATURES.append('meta_std')\n                FEATURES_t.append('meta_outliers_count')\n        else:\n            test_t = test.copy() \n        p1 = clf.predict_proba(test_t[FEATURES_t].astype('float32'))[0,1] if q != 2 else np.ones(test_t.shape[0])\n        p2 = clf_2.predict_proba(test_t[FEATURES_t_2].astype('float32'))[0,1] if q != 2 else np.ones(test_t.shape[0])\n        p = 0.5 * p1 + 0.5 * p2\n        mask = sample_submission.session_id.str.contains(f'q{q}')\n        sample_submission.loc[mask,'correct'] = int( p > CFG.th ) \n        historical_meta2[session_id].append(p)\n    \n    env.predict(sample_submission) ","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:35:57.221048Z","iopub.execute_input":"2023-06-25T01:35:57.221414Z","iopub.status.idle":"2023-06-25T01:35:57.291520Z","shell.execute_reply.started":"2023-06-25T01:35:57.221324Z","shell.execute_reply":"2023-06-25T01:35:57.290525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nif TIME_ESTIMATION:\n    from tqdm.notebook import tqdm \n    \n    #generate a mock Env\n    class MockEnv:\n        def predict(self, sample_submission):\n            None\n\n    # set the true Env to debug mode\n    jo_wilder.make_env.__called__ = False\n    env = jo_wilder.make_env()\n    type(env)._state = type(type(env)._state).__dict__['INIT']\n    iter_test = env.iter_test()    \n    iters = []\n    ## load sample submissions in a list \n    count = 0\n    for n, (sample_submission, test) in enumerate(iter_test):\n        iters.append( (sample_submission, test) )\n        count += sample_submission.shape[0]\n        env.predict(sample_submission)\n    num_sids = count / 18\n    print(f\"num submission id in test set: {num_sids}\")\n    env = MockEnv()\n    # iterate over 4000 x 3 : 12000 submission ids (~ 11779 train set)\n    for nn in tqdm(range(4000), desc=\"Inference Time estimation\"):\n        for (sample_submission, test) in iters:    \n            iteration (sample_submission, test)\n        ## we don't want to wait  for 4000 iterations\n        if nn == 50:\n            break\n        \nelse:\n    env = jo_wilder.make_env()\n    iter_test = env.iter_test() \n    for sample_submission, test in iter_test:\n        iteration (sample_submission, test)\n\nfor (sample_submission, test) in iter_test:\n    iteration (sample_submission, test)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:35:57.292650Z","iopub.execute_input":"2023-06-25T01:35:57.292912Z","iopub.status.idle":"2023-06-25T01:36:13.926456Z","shell.execute_reply.started":"2023-06-25T01:35:57.292887Z","shell.execute_reply":"2023-06-25T01:36:13.925247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub.head() ","metadata":{"execution":{"iopub.status.busy":"2023-06-25T01:36:13.928050Z","iopub.execute_input":"2023-06-25T01:36:13.928383Z","iopub.status.idle":"2023-06-25T01:36:13.945035Z","shell.execute_reply.started":"2023-06-25T01:36:13.928323Z","shell.execute_reply":"2023-06-25T01:36:13.943973Z"},"trusted":true},"execution_count":null,"outputs":[]}]}