{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc,sys\nimport sys,joblib\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom catboost import CatBoostClassifier, Pool\nfrom xgboost import XGBClassifier\nfrom sklearn.cluster import KMeans\nimport jo_wilder\n\nclass CFG:\n    th = 0.61\n    model = \"xgb\" \n    ESTIMATE = False\n    full_data = True\n    mode = \"median\"\n    model_pth=\"/kaggle/input/psp-xgb-2023-05-08/xgb_2023-06-23-1/\" \n    GLOBALS=False\n    DIST=False\n    CLUSTER_FEAT=False\n    EMBED=False\n    META_AGG=False\nprint (joblib.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:27.991569Z","iopub.execute_input":"2023-06-25T23:09:27.991905Z","iopub.status.idle":"2023-06-25T23:09:27.998417Z","shell.execute_reply.started":"2023-06-25T23:09:27.991878Z","shell.execute_reply":"2023-06-25T23:09:27.997725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0,CFG.model_pth) \nfrom fe_v2 import feature_engineer_kiki,time_feature,global_feats \n\ncluster_scrn_dict_lvl={'0-4': {'0-4': 6}, '5-12': {'5-12': 6}, '13-22': {'13-22': 6}}\nnot_null_mask = ((pl.col('screen_coor_x').is_not_null()) & (pl.col('screen_coor_y').is_not_null()) & (pl.col('next_x').is_not_null()) & (pl.col('next_y').is_not_null()))\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    ((pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)).fill_null(0).clip(0, 1e9).over([\"session_id\", \"level\"]) .alias(\"elapsed_time_diff\")),\n    ((pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) .abs().over([\"session_id\", \"level\"]) .alias('screen_coor_x_diff') ),\n    ((pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)).abs().over([\"session_id\", \"level\"]) .alias('screen_coor_y_diff') ), \n#     ((pl.col(\"screen_coor_x\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_x')),\n#     ((pl.col(\"screen_coor_y\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_y')), \n    #((pl.col(\"room_coor_x\").shift_and_fill(-1, fill_value=None)).alias('room_next_x')),\n    #((pl.col(\"room_coor_y\").shift_and_fill(-1, fill_value=None)).alias('room_next_y')), \n    pl.col(\"fqid\").fill_null(\"fqid_None\"), \n    ((pl.col(\"elapsed_time\").diff(n=5)) .fill_null(0).abs().over([\"session_id\", \"level_group\"]).alias(\"elapsed_time_diff2\")),\n    ((pl.col(\"elapsed_time\").diff(n=10)) .fill_null(0).abs().over([\"session_id\", \"level_group\"]).alias(\"elapsed_time_diff3\")),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\"),\n] \nif CFG.DIST:\n    columns = columns + [ ((pl.col(\"screen_coor_x\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_x')),\n    ((pl.col(\"screen_coor_y\").shift_and_fill(-1, fill_value=None)).abs().over([\"session_id\", \"level\"]).alias('next_y')), ]\ndef load_model(path):\n    if CFG.model == \"xgb\":\n        model = XGBClassifier()\n        model.load_model(path)\n    else:\n        model = CatBoostClassifier().load_model(path)\n    return model\n\nif CFG.full_data:\n    if CFG.mode==\"median\":\n        models_list = [load_model( f\"{CFG.model_pth}full_data_q{q}.cbm\") for q in range(1, 19)]\n    else:\n        models_list = [load_model( f\"/kaggle/input/psp-train/full_data_mean_q{q}.cbm\") for q in range(1, 19)]   \nelse:    \n    models_list = [[load_model( f\"{CFG.model_pth}fold{fold}_q{q}.cbm\") for fold in range(5)] for q in range(1, 19)]  ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:27.999960Z","iopub.execute_input":"2023-06-25T23:09:28.000429Z","iopub.status.idle":"2023-06-25T23:09:28.092140Z","shell.execute_reply.started":"2023-06-25T23:09:28.000399Z","shell.execute_reply":"2023-06-25T23:09:28.091378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer or do Infer Time estimate","metadata":{}},{"cell_type":"code","source":"TIME_ESTIMATION = CFG.ESTIMATE\n# CLEAR MEMORY\nimport gc \n_ = gc.collect()    ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:28.093239Z","iopub.execute_input":"2023-06-25T23:09:28.093690Z","iopub.status.idle":"2023-06-25T23:09:28.204708Z","shell.execute_reply.started":"2023-06-25T23:09:28.093661Z","shell.execute_reply":"2023-06-25T23:09:28.203951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold_multiplier = 2\ndef count_outliers(x, multiplier):\n    threshold = x.mean() + multiplier * x.std()\n    return (x > threshold).sum()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:28.206516Z","iopub.execute_input":"2023-06-25T23:09:28.206979Z","iopub.status.idle":"2023-06-25T23:09:28.217662Z","shell.execute_reply.started":"2023-06-25T23:09:28.206942Z","shell.execute_reply":"2023-06-25T23:09:28.216637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_func(item1, item2):\n    if item1['level'] == item2['level']:\n        if abs(item1['index'] - item2['index']) > 50:\n            if item1['elapsed_time'] < item2['elapsed_time']:\n                return -1\n            elif item1['elapsed_time'] > item2['elapsed_time']:\n                return 1\n            else:\n                return 0\n        else:\n            if item1['index'] < item2['index']:\n                return -1\n            elif item1['index'] > item2['index']:\n                return 1\n            else:\n                return 0\n    else:\n        return item1['level'] - item2['level']\n\nimport functools\ndef sort_frame(df):\n    v = [{'idx': idx, 'index':row['index'], 'elapsed_time': row['elapsed_time'], 'level': row['level']} for idx, row in df.iterrows()]\n    v.sort(key=functools.cmp_to_key(compare_func))\n    mdf = pd.DataFrame(v)\n    assert len(mdf) == len(df)\n    return df.loc[mdf.idx]","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:28.218872Z","iopub.execute_input":"2023-06-25T23:09:28.219361Z","iopub.status.idle":"2023-06-25T23:09:28.231297Z","shell.execute_reply.started":"2023-06-25T23:09:28.219317Z","shell.execute_reply":"2023-06-25T23:09:28.230305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport pickle\n\n# Load importance dict\nf_read = open(f'{CFG.model_pth}importance_dict.pkl', 'rb')\nimportance_dict = pickle.load(f_read)\nf_read.close()\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\nhistorical_meta2 = defaultdict(list)\nCLUSTER_FEAT = False\npre_04 = {}\npre_512 = {}\n\ntfidfDict = {\"0-4\":None, \"5-12\":None, \"13-22\":None}\nclusterKdict = {\"0-4\":None, \"5-12\":None, \"13-22\":None}\n# Cache Kmeans models\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]\nif CFG.CLUSTER_FEAT:\n    for lvl in level_groups: \n        print(f\"Adding {lvl} clustering\")\n        tfidfDict[lvl] =joblib.load(f\"/kaggle/input/ps-fe-and-cluster-analysis/tfidf_txt_{lvl}.pkl\")  \n        clusterKdict[lvl] =joblib.load(f\"/kaggle/input/ps-fe-and-cluster-analysis/txt_tfidf_{lvl}_cluster.pkl\")  \nif CFG.EMBED:\n    labelsDict = joblib.load(f\"/kaggle/input/ps-fe-and-cluster-analysis/text_embed_clstr_1.01.pkl\")\n## pack iteration logic in a function\ndef iteration (test,sample_submission ): \n    # INFER TEST DATA \n    #test = sort_frame(test)\n    test = test.sort_values(by = 'elapsed_time')\n    session_id = test.session_id.values[0]\n    grp = test.level_group.values[0] \n    sample_submission['questions'] = sample_submission['session_id'].str.split('_').apply(lambda x: x[1])\n    sample_submission = sample_submission.sort_values(by = 'questions')\n    sample_submission = sample_submission[['session_id', 'correct']]\n    \n    if grp == '0-4':\n        suffix = '04'\n    elif grp == '5-12':\n        suffix = '512'\n    else:\n        suffix = '1322' \n    test = (pl.from_pandas(test)\n          .drop([\"fullscreen\", \"hq\", \"music\"])\n          .with_columns(columns)) \n    if CFG.EMBED: \n        lblDic = labelsDict[grp]  \n        pca1, pca2 = zip(*[lblDic[k] for k in test['text'].fill_null(\"\")  ])    \n        pca_series = pl.Series('pca_1', pca1)\n        pca_series_2 = pl.Series('pca_2', pca2)\n        test = test.with_columns([pca_series,pca_series_2]) \n    if CFG.CLUSTER_FEAT:\n        vec = tfidfDict[grp].transform(test['text'].fill_null(\"\"))\n        transformed_series = pl.Series('tfidf_clstr', clusterKdict[grp].predict(vec))\n        # add the new Series to the Polars DataFrame\n        test = test.with_columns(transformed_series)  \n     \n    test = test.with_columns([\n     ((pl.col(\"elapsed_time_diff\") - pl.first( \"elapsed_time_diff\"))/(pl.last(\"elapsed_time_diff\") - pl.first( \"elapsed_time_diff\"))*3 +1)\n        .fill_null(0).over([\"session_id\", \"level\"]).alias(\"tw_score\"), \n     ])\n#      (pl.when(not_null_mask).then(((pl.col('next_x') - pl.col('screen_coor_x'))**2 + (pl.col('next_y') - pl.col('screen_coor_y'))**2).sqrt().over([\"session_id\", \"level\"])).otherwise(None).abs().alias('distance')),\n#      (pl.when(not_null_mask).then(((pl.col('next_x') - pl.col('screen_coor_x')) + (pl.col('next_y') - pl.col('screen_coor_y'))).over([\"session_id\", \"level\"])).otherwise(None).abs().alias('manhattan')),     #(pl.when(not_null_mask).then(((pl.col('room_next_x') - pl.col('screen_coor_x'))**2 + (pl.col('room_next_y') - pl.col('room_coor_y'))**2).sqrt().over([\"session_id\", \"level\"])).otherwise(None).abs().alias('distance_room')),\n     #(pl.when(not_null_mask).then(((pl.col('next_y') - pl.col('screen_coor_y')) / (pl.col('next_x') - pl.col('screen_coor_x'))).arctan().over([\"session_id\", \"level\"])).otherwise(None).alias('angle')) \n    test=test.with_columns([ \n        pl.when(pl.col(\"tw_score\").is_infinite()).then(0).otherwise(pl.col(\"tw_score\")).keep_name(),\n#         #pl.col('angle').shift_and_fill(-1, fill_value=None).over([\"session_id\", \"level\"]).alias('prev_angle' )\n     ])  \n    #test =test.with_columns([(pl.col(\"angle\")-pl.col(\"prev_angle\")).over([\"session_id\", \"level\"]).alias('angle_change') ])  \n    test_all = None\n    if CFG.GLOBALS:\n        if grp == '0-4':\n            pre_04[session_id] = test  \n        if grp == '5-12': \n            pre_512[session_id] = test \n            if len(pre_04[session_id]) > 0: \n                test_all = global_feats(pre_04[session_id]) \n        if grp == '13-22':\n            if len(pre_04[session_id]) > 0:\n                test_all = pl.concat([pre_04[session_id]]) \n                pre_04[session_id] = None\n            if len(pre_512[session_id]) > 0:\n                test_all = pl.concat([test_all,pre_512[session_id]]) \n                pre_512[session_id] = None\n            if test_all is not None:    \n                test_all = global_feats(test_all) \n    test = feature_engineer_kiki(test, grp, use_extra=True, feature_suffix=suffix) \n    if CFG.GLOBALS & (grp != '0-4') & (test_all is not None):\n        test = test.join(test_all,on=\"session_id\", how='left')  \n     \n    if not CFG.GLOBALS:\n        if grp == '0-4':\n            pre_04[session_id] = test\n        if grp == '5-12':\n            pre_512[session_id] = test \n        if grp == '5-12':\n            if len(pre_04[session_id]) > 0:\n                test = test.join(pre_04[session_id], on = \"session_id\", how = 'left')\n        if grp == '13-22':\n            if len(pre_04[session_id]) > 0:\n                test = test.join(pre_04[session_id], on = \"session_id\", how = 'left')\n                pre_04[session_id] = None\n            if len(pre_512[session_id]) > 0:\n                test = test.join(pre_512[session_id], on = \"session_id\", how = 'left')\n                pre_512[session_id] = None \n    test = test.to_pandas()\n    #test = time_feature(test)    \n    a,b = limits[grp]\n    fold=0 \n    for q in range(a,b):\n        clf = models_list[q-1] if CFG.full_data else models_list[q-1][fold]\n        FEATURES = importance_dict[str(q)]\n        #print(grp,len(FEATURES))\n        FEATURES_t = FEATURES.copy()\n        meta_id = f'meta_{q}'\n        test_t = test.copy()\n        #print(q,test.shape)\n        if q-1>0:\n            meta_name=[]\n            for tt in range(1,q):\n                test_t[f'meta_{tt}'] = historical_meta2[session_id][tt-1]\n                FEATURES_t.append(f'meta_{tt}')   \n                meta_name.append(f'meta_{tt}')\n                \n            if CFG.META_AGG:\n                test_t['meta_sum'] = test_t[meta_name].apply(lambda x: x.sum(), axis=1)\n                test_t['meta_mean'] = test_t[meta_name].apply(lambda x: x.mean(), axis=1)\n                test_t['meta_outliers_count'] = test_t[meta_name].apply(lambda x: count_outliers(x, threshold_multiplier), axis=1)\n                FEATURES_t.append('meta_sum')\n                FEATURES_t.append('meta_mean')\n                # FEATURES.append('meta_std')\n                FEATURES_t.append('meta_outliers_count')\n        else:\n            test_t = test.copy() \n        p = clf.predict_proba(test_t[FEATURES_t].astype('float32'))[0,1] if q != 2 else np.ones(test_t.shape[0])\n        mask = sample_submission.session_id.str.contains(f'q{q}')\n        sample_submission.loc[mask,'correct'] = int( p > CFG.th ) \n        historical_meta2[session_id].append(p)\n    \n    env.predict(sample_submission) ","metadata":{"execution":{"iopub.status.busy":"2023-06-25T23:09:28.234386Z","iopub.execute_input":"2023-06-25T23:09:28.234715Z","iopub.status.idle":"2023-06-25T23:09:28.279259Z","shell.execute_reply.started":"2023-06-25T23:09:28.234678Z","shell.execute_reply":"2023-06-25T23:09:28.278466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nif TIME_ESTIMATION:\n    from tqdm.notebook import tqdm \n    \n    #generate a mock Env\n    class MockEnv:\n        def predict(self, sample_submission):\n            None\n\n    # set the true Env to debug mode\n    jo_wilder.make_env.__called__ = False\n    env = jo_wilder.make_env()\n    type(env)._state = type(type(env)._state).__dict__['INIT']\n    iter_test = env.iter_test()    \n    iters = []\n    ## load sample submissions in a list \n    count = 0\n    for n, (sample_submission, test) in enumerate(iter_test):\n        iters.append( (sample_submission, test) )\n        count += sample_submission.shape[0]\n        env.predict(sample_submission)\n    num_sids = count / 18\n    print(f\"num submission id in test set: {num_sids}\")\n    env = MockEnv()\n    # iterate over 4000 x 3 : 12000 submission ids (~ 11779 train set)\n    for nn in tqdm(range(4000), desc=\"Inference Time estimation\"):\n        for (sample_submission, test) in iters:    \n            iteration (sample_submission, test)\n        ## we don't want to wait  for 4000 iterations\n        if nn == 50:\n            break\n        \nelse:\n    env = jo_wilder.make_env()\n    iter_test = env.iter_test() \n    for sample_submission, test in iter_test:\n        iteration (sample_submission, test)\n\nfor (sample_submission, test) in iter_test:\n    iteration (sample_submission, test)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:28.280410Z","iopub.execute_input":"2023-06-25T23:09:28.281083Z","iopub.status.idle":"2023-06-25T23:09:38.855789Z","shell.execute_reply.started":"2023-06-25T23:09:28.281048Z","shell.execute_reply":"2023-06-25T23:09:38.854760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub.head() ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-06-25T23:09:38.856928Z","iopub.execute_input":"2023-06-25T23:09:38.857370Z","iopub.status.idle":"2023-06-25T23:09:38.872990Z","shell.execute_reply.started":"2023-06-25T23:09:38.857343Z","shell.execute_reply":"2023-06-25T23:09:38.872222Z"},"trusted":true},"execution_count":null,"outputs":[]}]}