{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### XGBoost Test\nA homework.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\ncategory = [\n    'event_name', 'name', 'fqid', 'room_fqid', 'text_fqid'\n]\n\nnumeric = [\n    'page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n    'hover_duration', 'elapsed_time_diff'\n]","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:58:07.908863Z","iopub.execute_input":"2023-05-21T17:58:07.909272Z","iopub.status.idle":"2023-05-21T17:58:08.017127Z","shell.execute_reply.started":"2023-05-21T17:58:07.909193Z","shell.execute_reply":"2023-05-21T17:58:08.015421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature engineering","metadata":{}},{"cell_type":"code","source":"dtypes={ \n    'elapsed_time':np.int32,\n    'event_name':'category',\n    'name':'category',\n    'level':np.uint8,\n    'page':'category',\n    'room_coor_x':np.float32,\n    'room_coor_y':np.float32,\n    'screen_coor_x':np.float32,\n    'screen_coor_y':np.float32,\n    'hover_duration':np.float32,\n     'text':'category',\n     'fqid':'category',\n     'room_fqid':'category',\n     'text_fqid':'category',\n     'fullscreen':'category',\n     'hq':'category',\n     'music':'category',\n     'level_group':'category'}\ntrain=pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:58:08.019304Z","iopub.execute_input":"2023-05-21T17:58:08.020101Z","iopub.status.idle":"2023-05-21T18:00:10.843721Z","shell.execute_reply.started":"2023-05-21T17:58:08.020060Z","shell.execute_reply":"2023-05-21T18:00:10.842238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group and preprocess for some column\n# calculate delta value for some column and fill none value\ndtypes={\n    'elapsed_time':pl.Int32,\n    'level':pl.Int8,\n    'page':pl.Float32,\n    'room_coor_x':pl.Float32,\n    'room_coor_y':pl.Float32,\n    'screen_coor_x':pl.Float32,\n    'screen_coor_y':pl.Float32,\n    'hover_duration':pl.Float32,\n}\ncolumns = [\n    (\n        (pl.col('elapsed_time') - pl.col('elapsed_time').shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over(['session_id', 'level'])\n        .alias('elapsed_time_diff')\n    ),\n    (\n        (pl.col('screen_coor_x') - pl.col('screen_coor_x').shift(1))\n        .abs().\n        over(['session_id', 'level'])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    (\n        (pl.col('room_coor_x') - pl.col('room_coor_x').shift(1))\n        .abs().\n        over(['session_id', 'level'])\n    ),\n    (\n        (pl.col(\"room_coor_y\") - pl.col(\"room_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n]\n\n# train = pl.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtypes=dtypes).drop(['fullscreen', 'hq', 'music']).with_columns(columns)\n# # train = pl.read_csv('train.csv').drop(['fullscreen', 'hq', 'music']).with_columns(columns)\n\ntrain = pl.from_pandas(train).drop(['fullscreen', 'hq', 'music']).with_columns(columns)\n\nfqid_lists = list(train['fqid'].unique())\ntext_fqid_lists = list(train['text_fqid'].unique())\nroom_fqid_lists = list(train['room_fqid'].unique())\nname_feature_lists = list(train['name'].unique())\nevent_name_feature_lists = list(train['event_name'].unique())\nlevels = list(train['level'].unique())\nlevel_groups = list(train['level_group'].unique())\nprint(levels)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:00:10.845579Z","iopub.execute_input":"2023-05-21T18:00:10.846083Z","iopub.status.idle":"2023-05-21T18:00:35.369422Z","shell.execute_reply.started":"2023-05-21T18:00:10.846049Z","shell.execute_reply":"2023-05-21T18:00:35.368414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def elapsed_diff_agg(column, column_value_list, feature_suffix):\n    elapsed_agg = [\n        *[pl.col(column).filter(pl.col(column) == c).count().alias(f\"{c}_{column}_counts_{feature_suffix}\") for c in column_value_list], # calculate number of different fqid\n\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(column) == c).std().alias(f\"{c}_{column}_elapsed_std_{feature_suffix}\") for\n          c in column_value_list],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(column) == c).mean().alias(f\"{c}_{column}_elapsed_mean_{feature_suffix}\") for\n          c in column_value_list],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(column) == c).max().alias(f\"{c}_{column}_elapsed_max_{feature_suffix}\") for\n          c in column_value_list],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(column) == c).min().alias(f\"{c}_{column}_elapsed_min_{feature_suffix}\") for\n          c in column_value_list],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(column) == c).sum().alias(f\"{c}_{column}_elapsed_sum_{feature_suffix}\") for\n          c in column_value_list], # calculate statistics value of elapsed time delta as we discuss\n    ]\n    return elapsed_agg\n\ndef feature_engineer(x, grp, feature_suffix, use_extra=True):\n    aggs = [\n        pl.col('index').count().alias(f\"session_number_{feature_suffix}\"), # the number of events of a session\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in category], # drop none and calcu number of unique value\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in numeric],\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in numeric],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in numeric],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in numeric],\n        *[pl.col(c).sum().alias(f\"{c}_sum_{feature_suffix}\") for c in numeric], # calculate statistic value of numeric values\n    ]\n    aggs.extend(elapsed_diff_agg('fqid', fqid_lists, feature_suffix))\n    aggs.extend(elapsed_diff_agg('text_fqid', text_fqid_lists, feature_suffix))\n    aggs.extend(elapsed_diff_agg('room_fqid', room_fqid_lists, feature_suffix))\n    aggs.extend(elapsed_diff_agg('name', name_feature_lists, feature_suffix))\n    aggs.extend(elapsed_diff_agg('event_name', event_name_feature_lists, feature_suffix))\n    aggs.extend(elapsed_diff_agg('level', levels, feature_suffix))\n    aggs.extend(elapsed_diff_agg('level_group', level_groups, feature_suffix))\n\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort('session_id')\n    \n    if use_extra:\n        if grp == '5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\")==\"Here's the log book.\")\n                                              |(pl.col(\"fqid\")=='logbook.page.bingo'))\n                    .apply(lambda s: s.max()-s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                                pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                                pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                                pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                                pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                                pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                                pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:00:35.372765Z","iopub.execute_input":"2023-05-21T18:00:35.375198Z","iopub.status.idle":"2023-05-21T18:00:35.421348Z","shell.execute_reply.started":"2023-05-21T18:00:35.375145Z","shell.execute_reply":"2023-05-21T18:00:35.419453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    return train","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:00:35.423546Z","iopub.execute_input":"2023-05-21T18:00:35.423970Z","iopub.status.idle":"2023-05-21T18:00:35.445236Z","shell.execute_reply.started":"2023-05-21T18:00:35.423935Z","shell.execute_reply":"2023-05-21T18:00:35.443652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Model","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom lightgbm import early_stopping\nfrom lightgbm import log_evaluation\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score, f1_score\n\ndf1 = train.filter(pl.col(\"level_group\")=='0-4')\ndf2 = train.filter(pl.col(\"level_group\")=='5-12')\ndf3 = train.filter(pl.col(\"level_group\")=='13-22')\nprint(df1.shape,df2.shape,df3.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:00:35.450151Z","iopub.execute_input":"2023-05-21T18:00:35.450581Z","iopub.status.idle":"2023-05-21T18:00:38.879011Z","shell.execute_reply.started":"2023-05-21T18:00:35.450544Z","shell.execute_reply":"2023-05-21T18:00:38.877963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:00:38.880120Z","iopub.execute_input":"2023-05-21T18:00:38.880487Z","iopub.status.idle":"2023-05-21T18:00:39.133967Z","shell.execute_reply.started":"2023-05-21T18:00:38.880454Z","shell.execute_reply":"2023-05-21T18:00:39.132697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df1.columns)\ndf1 = feature_engineer(df1, grp='0-4', feature_suffix='xgboost')\ndf1.set_index('session_id')\nprint(df1.shape)\ndf2 = feature_engineer(df2, grp='5-12', feature_suffix='xgboost')\ndf2.set_index('session_id')\nprint(df2.shape)\ndf3 = feature_engineer(df3, grp='13-22', feature_suffix='xgboost')\ndf3.set_index('session_id')\nprint(df3.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:00:39.135762Z","iopub.execute_input":"2023-05-21T18:00:39.136712Z","iopub.status.idle":"2023-05-21T18:01:34.164048Z","shell.execute_reply.started":"2023-05-21T18:00:39.136660Z","shell.execute_reply":"2023-05-21T18:01:34.161227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some cleaning...\nnull1 = df1.isnull().sum().sort_values(ascending=False) / len(df1)\nnull2 = df2.isnull().sum().sort_values(ascending=False) / len(df1)\nnull3 = df3.isnull().sum().sort_values(ascending=False) / len(df1)\n\ndrop1 = list(null1[null1>0.9].index)\ndrop2 = list(null2[null2>0.9].index)\ndrop3 = list(null3[null3>0.9].index)\nprint(len(drop1), len(drop2), len(drop3))\n\nfor col in df1.columns:\n    if df1[col].nunique()==1:\n        print(col)\n        drop1.append(col)\nprint(\"*********df1 DONE*********\")\nfor col in df2.columns:\n    if df2[col].nunique()==1:\n        print(col)\n        drop2.append(col)\nprint(\"*********df2 DONE*********\")\nfor col in df3.columns:\n    if df3[col].nunique()==1:\n        print(col)\n        drop3.append(col)\nprint(\"*********df3 DONE*********\")","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:01:34.166053Z","iopub.execute_input":"2023-05-21T18:01:34.167096Z","iopub.status.idle":"2023-05-21T18:01:36.982547Z","shell.execute_reply.started":"2023-05-21T18:01:34.167053Z","shell.execute_reply":"2023-05-21T18:01:36.981456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features1 = [c for c in df1.columns if c not in drop1 + ['level_group']]\nfeatures2 = [c for c in df2.columns if c not in drop2 + ['level_group']]\nfeatures3 = [c for c in df3.columns if c not in drop3 + ['level_group']]\n\nusers = df1.index.unique()","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:01:36.984017Z","iopub.execute_input":"2023-05-21T18:01:36.984619Z","iopub.status.idle":"2023-05-21T18:01:37.117697Z","shell.execute_reply.started":"2023-05-21T18:01:36.984583Z","shell.execute_reply":"2023-05-21T18:01:37.116442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"choose for some hyper params","metadata":{}},{"cell_type":"code","source":"estimators_xgb = [498, 448, 378, 364, 405, 495, 456, 249, 384, 405, 356, 262, 484, 381, 392, 248 ,248, 345]\nxgb_params = {\n    'booster': 'gbtree',\n    'tree_method': 'hist',\n    'objective': 'binary:logistic',\n    'eval_metric':'logloss',\n    'learning_rate': 0.02,\n    'alpha': 8,\n    'max_depth': 4,\n    'subsample':0.8,\n    'colsample_bytree': 0.5,\n    'seed': 2023\n}\n\ntargets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]))\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]))\npred_xgb = np.zeros((df1.shape[0],18))     ","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:01:37.119306Z","iopub.execute_input":"2023-05-21T18:01:37.120060Z","iopub.status.idle":"2023-05-21T18:01:38.670503Z","shell.execute_reply.started":"2023-05-21T18:01:37.120019Z","shell.execute_reply":"2023-05-21T18:01:38.669272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in range(1, 19):\n    if t <= 3:\n        grp = '0-4'\n        df = df1\n        features = features1\n    elif t <= 13:\n        grp = '5-12'\n        df = df2\n        features = features2\n    else:\n        grp = '13-22'\n        df = df3\n        features = features3\n    train_users = df['session_id']\n    train_y = targets.loc[targets.q == t].set_index('session').loc[train_users]\n\n    xgb_params['n_estimators'] = estimators_xgb[t-1]\n\n    clf =  XGBClassifier(**xgb_params)\n    clf.fit(df[features].astype('float32'), train_y['correct'], verbose = 0)\n    clf.save_model(f'XGB_question{t}.xgb')\n    \n    print(f'model XGB saved for question {t}')","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:01:38.671981Z","iopub.execute_input":"2023-05-21T18:01:38.672678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import KFold, GroupKFold\n# gkf = GroupKFold(n_splits=5)\n\n# oof = pd.DataFrame(data=np.zeros((len(users),18)), index=users)\n\n# print(train.columns)\n\n# for t in range(1, 19):\n#     if t <= 3:\n#         grp = '0-4'\n#         df = df1\n#         features = features1\n#     elif t <= 13:\n#         grp = '5-12'\n#         df = df2\n#         features = features2\n#     else:\n#         grp = '13-22'\n#         df = df3\n#         features = features3\n#     for i, (train_index, test_index) in enumerate(gkf.split(X=df, groups=df['session_id'])):\n\n#         valid_x = df.iloc[test_index]\n#         valid_users = valid_x['session_id']\n#         valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n\n#         clf = XGBClassifier()\n#         clf.load_model(f\"XGB_question{t}.xgb\")\n#         valid_users = valid_x.index.values\n\n#         oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[features].astype('float32'))[:,1]\n\n# true = oof.copy()\n# for k in range(18):\n#     # GET TRUE LABELS\n#     tmp = targets.loc[targets.q == k+1]\n#     true[k] = tmp.correct.values\n\n# scores = []; thresholds = []\n# best_score = 0; best_threshold = 0\n\n# for threshold in np.arange(0.4,0.81,0.005):\n#     print(f'{threshold:.03f}, ',end='')\n#     preds = (oof.values.reshape((-1))>threshold).astype('int')\n#     m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n#     scores.append(m)\n#     thresholds.append(threshold)\n#     if m>best_score:\n#         best_score = m\n#         best_threshold = threshold\n\n# import matplotlib.pyplot as plt\n\n# # PLOT THRESHOLD VS. F1_SCORE\n# plt.figure(figsize=(20,5))\n# plt.plot(thresholds,scores,'-o',color='blue')\n# plt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\n# plt.xlabel('Threshold',size=14)\n# plt.ylabel('Validation F1 Score',size=14)\n# plt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import jo_wilder_310 as jo_wilder\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\ncount = 0\n\nfor (test, sample_submission) in iter_test:\n    session_id = test.session_id.values[0]\n    grp = test['level_group'].values[0]\n    a,b = limits[grp]\n    if a == 1:\n        features = features1\n    elif a == 4:\n        features = features2\n    else:\n        features = features3\n\n    test = pl.from_pandas(test).drop(['fullscreen', 'hq', 'music']).with_columns(columns)\n    test = feature_engineer(test, grp, feature_suffix='xgboost')\n    test = test[features]\n    for t in range(a, b):\n        clf = XGBClassifier()\n        clf.load_model(f\"/kaggle/working/XGB_question{t}.xgb\")\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        p = clf.predict_proba(test.astype('float32'))[:, 1]\n        sample_submission.loc[mask, 'correct'] = int((p.item()) > 0.625)\n    env.predict(sample_submission)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}