{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Activation, Dense, Dropout, Embedding,Flatten,Conv1D,MaxPooling1D,BatchNormalization,GlobalMaxPooling1D\nfrom tensorflow.keras.optimizers import Adam\n# from tensorflow.keras.utils import to_categorical #One-hot encoder\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nimport gc\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom sklearn.metrics import f1_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-28T03:42:09.705310Z","iopub.execute_input":"2023-04-28T03:42:09.706386Z","iopub.status.idle":"2023-04-28T03:42:21.733681Z","shell.execute_reply.started":"2023-04-28T03:42:09.706340Z","shell.execute_reply":"2023-04-28T03:42:21.732250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",\n                usecols=['session_id','elapsed_time','level_group'])\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:42:21.736117Z","iopub.execute_input":"2023-04-28T03:42:21.737194Z","iopub.status.idle":"2023-04-28T03:43:59.613755Z","shell.execute_reply.started":"2023-04-28T03:42:21.737150Z","shell.execute_reply":"2023-04-28T03:43:59.612492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\ntmp = target.session_id.str.split(\"_\", expand =True)\ntarget[\"user_id\"] = tmp[0].astype(\"int\")\ntarget[\"q\"] = tmp[1].str.slice(1).astype(\"int\")\nprint(target.shape)\ntarget.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:43:59.615513Z","iopub.execute_input":"2023-04-28T03:43:59.616373Z","iopub.status.idle":"2023-04-28T03:44:01.823441Z","shell.execute_reply.started":"2023-04-28T03:43:59.616330Z","shell.execute_reply":"2023-04-28T03:44:01.822509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.session_id.nunique(), target.user_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:01.826461Z","iopub.execute_input":"2023-04-28T03:44:01.827212Z","iopub.status.idle":"2023-04-28T03:44:02.006887Z","shell.execute_reply.started":"2023-04-28T03:44:01.827172Z","shell.execute_reply":"2023-04-28T03:44:02.005284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FEATURE ENGINEER: 切割每個group有相同的資料長度","metadata":{}},{"cell_type":"code","source":"GP1 = '0-4'\nGP2 = '5-12'\nGP3 = '13-22'\n\nGROUP_DATA = {\n    GP1: {\n        \"data\": [],\n        'target': target[(target.q >= 1) & (target.q <= 3)] # 只有train data會用到這個 /test不會\n    },\n    GP2: {\n        \"data\": [],\n        'target': target[(target.q >= 4) & (target.q <= 13)]\n    },\n    GP3: {\n        \"data\":  [],\n        'target': target[(target.q >= 14) & (target.q <= 18)]\n    }\n}\n\n\n# cut_len = {}\ngrp=[GP1,GP2,GP3]\ndef feature_engineer(df):\n    df = df.loc[:,['session_id','elapsed_time','level_group']]\n#     event_duration\n    df.elapsed_time /= 1000 #轉為秒\n    event_duration = df.elapsed_time.diff().fillna(0).clip(0,3600).astype(\"float32\") #最大道3600秒也就是一小時\n    df['event_duration'] = event_duration.astype(\"float32\")\n    df.drop('elapsed_time', axis = 1, inplace =True)\n    \n    #每個level_group要pad_sequence的長度\n#     for level in grp:\n#         sequence_len = np.ceil(df[df[\"level_group\"] == level].groupby([\"session_id\",\"level_group\"]).agg(\"count\").mean()).astype(\"int\") #擷取每個level_group的事件長度為平均數\n#         cut_len[level] = sequence_len\n        \n    for level in grp:\n        GROUP_DATA[level][\"data\"] = df[df[\"level_group\"] ==level]\n        \n    for level in grp:\n        tmp = []\n        user_id = []\n        for session_id, info in GROUP_DATA[level][\"data\"].groupby(\"session_id\"):\n            tmp.append(info['event_duration'].astype(\"float32\"))\n            user_id.append(session_id)\n        GROUP_DATA[level][\"data\"] = pad_sequences(tmp, padding='post', maxlen = 500,dtype='float16')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:02.008482Z","iopub.execute_input":"2023-04-28T03:44:02.008868Z","iopub.status.idle":"2023-04-28T03:44:02.049596Z","shell.execute_reply.started":"2023-04-28T03:44:02.008835Z","shell.execute_reply":"2023-04-28T03:44:02.048518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GROUP_DATA[\"0-4\"][\"data\"]\n# tmp","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:07:22.966708Z","iopub.execute_input":"2023-04-28T04:07:22.967131Z","iopub.status.idle":"2023-04-28T04:07:22.971652Z","shell.execute_reply.started":"2023-04-28T04:07:22.967096Z","shell.execute_reply":"2023-04-28T04:07:22.970613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_engineer(train)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:02.051319Z","iopub.execute_input":"2023-04-28T03:44:02.051984Z","iopub.status.idle":"2023-04-28T03:44:33.112273Z","shell.execute_reply.started":"2023-04-28T03:44:02.051946Z","shell.execute_reply":"2023-04-28T03:44:33.111348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = feature_engineer(train)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.114067Z","iopub.execute_input":"2023-04-28T03:44:33.114757Z","iopub.status.idle":"2023-04-28T03:44:33.119709Z","shell.execute_reply.started":"2023-04-28T03:44:33.114718Z","shell.execute_reply":"2023-04-28T03:44:33.118124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GROUP_DATA[\"0-4\"][\"data\"].shape, len(GROUP_DATA[\"5-12\"][\"data\"]), len(GROUP_DATA[\"0-4\"][\"data\"])","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.121582Z","iopub.execute_input":"2023-04-28T03:44:33.121987Z","iopub.status.idle":"2023-04-28T03:44:33.135241Z","shell.execute_reply.started":"2023-04-28T03:44:33.121951Z","shell.execute_reply":"2023-04-28T03:44:33.134201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in grp:\n    np.save(f\"{i}_data\", GROUP_DATA[i][\"data\"])\nfor i in grp:\n    np.save(f\"{i}_target\", GROUP_DATA[i][\"target\"])","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.136770Z","iopub.execute_input":"2023-04-28T03:44:33.137377Z","iopub.status.idle":"2023-04-28T03:44:33.479188Z","shell.execute_reply.started":"2023-04-28T03:44:33.137336Z","shell.execute_reply":"2023-04-28T03:44:33.478175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.483837Z","iopub.execute_input":"2023-04-28T03:44:33.484995Z","iopub.status.idle":"2023-04-28T03:44:33.704776Z","shell.execute_reply.started":"2023-04-28T03:44:33.484938Z","shell.execute_reply":"2023-04-28T03:44:33.703190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for level in grp:\n    print(GROUP_DATA[level][\"data\"].shape, GROUP_DATA[level][\"target\"].user_id.nunique())","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.706390Z","iopub.execute_input":"2023-04-28T03:44:33.707773Z","iopub.status.idle":"2023-04-28T03:44:33.725833Z","shell.execute_reply.started":"2023-04-28T03:44:33.707726Z","shell.execute_reply":"2023-04-28T03:44:33.724644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GROUP_DATA[GP1][\"data\"][0][:5]","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.726982Z","iopub.execute_input":"2023-04-28T03:44:33.727978Z","iopub.status.idle":"2023-04-28T03:44:33.736161Z","shell.execute_reply.started":"2023-04-28T03:44:33.727936Z","shell.execute_reply":"2023-04-28T03:44:33.734844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GROUP_DATA[GP1][\"target\"].head(2)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.737840Z","iopub.execute_input":"2023-04-28T03:44:33.738439Z","iopub.status.idle":"2023-04-28T03:44:33.757837Z","shell.execute_reply.started":"2023-04-28T03:44:33.738398Z","shell.execute_reply":"2023-04-28T03:44:33.756499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.759233Z","iopub.execute_input":"2023-04-28T03:44:33.759597Z","iopub.status.idle":"2023-04-28T03:44:33.980704Z","shell.execute_reply.started":"2023-04-28T03:44:33.759561Z","shell.execute_reply":"2023-04-28T03:44:33.979419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conv1D model","metadata":{}},{"cell_type":"code","source":"def train_model(dim): \n    model = Sequential()\n    model.add(Embedding(input_dim=dim, output_dim=128, input_length=500))\n    model.add(Conv1D(filters=32, kernel_size=3, activation='relu'))\n#     model.add(MaxPooling1D(pool_size=3))\n#     model.add(Conv1D(filters = 32, kernel_size=3,activation=\"relu\"))\n    model.add(GlobalMaxPooling1D())\n    model.add(Dense(64, activation='relu'))\n    model.add(Dropout(0.25))\n    model.add(Dense(1, activation='sigmoid'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.982249Z","iopub.execute_input":"2023-04-28T03:44:33.982772Z","iopub.status.idle":"2023-04-28T03:44:33.991846Z","shell.execute_reply.started":"2023-04-28T03:44:33.982733Z","shell.execute_reply":"2023-04-28T03:44:33.990517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(100).summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:33.993997Z","iopub.execute_input":"2023-04-28T03:44:33.994456Z","iopub.status.idle":"2023-04-28T03:44:34.361924Z","shell.execute_reply.started":"2023-04-28T03:44:33.994401Z","shell.execute_reply":"2023-04-28T03:44:34.360775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"ALL_USER = train.session_id.unique()\nlen(ALL_USER)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:34.363091Z","iopub.execute_input":"2023-04-28T03:44:34.363438Z","iopub.status.idle":"2023-04-28T03:44:34.522061Z","shell.execute_reply.started":"2023-04-28T03:44:34.363404Z","shell.execute_reply":"2023-04-28T03:44:34.520771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame(np.zeros((len(ALL_USER),18)), index= ALL_USER,columns=range(1,19))\npred_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:34.523456Z","iopub.execute_input":"2023-04-28T03:44:34.523928Z","iopub.status.idle":"2023-04-28T03:44:34.554741Z","shell.execute_reply.started":"2023-04-28T03:44:34.523889Z","shell.execute_reply":"2023-04-28T03:44:34.553206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keep_model = {}\nques_grp = {GP1:(1,4),GP2:(4,14),GP3:(14,19)}\n# save_history = []\n\n# tb_callback = tf.keras.callbacks.TensorBoard('./logs', update_freq=1)\nfor level in grp:\n    a, b = ques_grp[level]\n    for Q in range(a,b):\n        model = train_model(int(GROUP_DATA[level][\"data\"].max())+1 #最大的數字要再+1曾能送到embedding因為embedding會試左閉又開，不包含傳入的.max()值\n                        )\n        model.compile(loss=\"binary_crossentropy\", optimizer = \"adam\",metrics = \"accuracy\")\n        grp_target = GROUP_DATA[level][\"target\"]\n        y_label = grp_target[grp_target.q == Q].correct\n        \n        \n        model_checkpoint_callback = ModelCheckpoint(\n                                    filepath=f'{level}_{Q}/checkpoint',\n                                    save_weights_only= False,\n                                    monitor='val_accuracy',\n                                    mode='max',\n                                    save_best_only=True)\n        \n        print(f\"NUM: {Q}\")\n        History = model.fit(GROUP_DATA[level][\"data\"],\n                            y_label,validation_split=0.1, \n                            epochs=3, \n                            batch_size= 256,\n                            verbose = 1,\n                            callbacks = [model_checkpoint_callback])\n#         save_history.append(History)\n        loss, acc = model.evaluate(GROUP_DATA[level][\"data\"],\n                                   y_label)\n        y_pred = model.predict(GROUP_DATA[level][\"data\"])\n#         keep_model[f\"{level}_{Q}\"] = model\n        pred_df[Q] = y_pred","metadata":{"execution":{"iopub.status.busy":"2023-04-28T03:44:34.556488Z","iopub.execute_input":"2023-04-28T03:44:34.557348Z","iopub.status.idle":"2023-04-28T04:04:22.879312Z","shell.execute_reply.started":"2023-04-28T03:44:34.557303Z","shell.execute_reply":"2023-04-28T04:04:22.875504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.882838Z","iopub.status.idle":"2023-04-28T04:04:22.885696Z","shell.execute_reply.started":"2023-04-28T04:04:22.885344Z","shell.execute_reply":"2023-04-28T04:04:22.885384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def plot_history(n):\n#     loss =save_history[n].history[\"loss\"]\n#     val_loss =save_history[n].history[\"val_loss\"]\n#     acc = save_history[n].history[\"accuracy\"]\n#     val_acc = save_history[n].history[\"val_accuracy\"]\n#     epoch = range(1,len(loss)+1)\n#     plt.figure(figsize = (8,3))\n#     plt.subplot(1,2,1)\n#     plt.plot(epoch,loss,label =\"loss\")\n#     plt.plot(epoch,val_loss,label =\"val_loss\")\n#     plt.legend()\n#     plt.subplot(1,2,2)\n#     plt.plot(epoch,acc,label =\"accuracy\")\n#     plt.plot(epoch,val_acc,label =\"val_accuracy\")\n#     plt.legend()\n#     plt.title(n+1)\n#     plt.show()\n    \n# for i in range(18):\n#     plot_history(i)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.890765Z","iopub.status.idle":"2023-04-28T04:04:22.893673Z","shell.execute_reply.started":"2023-04-28T04:04:22.893304Z","shell.execute_reply":"2023-04-28T04:04:22.893343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.898501Z","iopub.status.idle":"2023-04-28T04:04:22.901210Z","shell.execute_reply.started":"2023-04-28T04:04:22.900882Z","shell.execute_reply":"2023-04-28T04:04:22.900922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.905237Z","iopub.status.idle":"2023-04-28T04:04:22.906212Z","shell.execute_reply.started":"2023-04-28T04:04:22.905940Z","shell.execute_reply":"2023-04-28T04:04:22.905972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue =pred_df.copy()\nfor Q in range(1,19):\n    # GET TRUE LABELS\n    tmp = target.loc[target.q == Q].set_index('user_id').loc[ALL_USER]\n    true[Q] = tmp.correct.values\ntrue.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.907644Z","iopub.status.idle":"2023-04-28T04:04:22.913915Z","shell.execute_reply.started":"2023-04-28T04:04:22.913565Z","shell.execute_reply":"2023-04-28T04:04:22.913618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\nthresholds = []\nbest_score = 0\nbest_threshold = 0\n#values: Only the values in the DataFrame will be returned, the axes labels will be removed.\n#reshape(-1): 降至一維\nfor t in np.arange(0.4,0.8, 0.01):\n    pred = (pred_df.values.reshape(-1) > t).astype(\"int\")\n    s = f1_score(true.values.reshape(-1), pred,average='macro')\n    scores.append(s)\n    thresholds.append(t)\nbest_score = max(scores)\nbest_threshold = thresholds[scores.index(max(scores))]","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.915594Z","iopub.status.idle":"2023-04-28T04:04:22.916395Z","shell.execute_reply.started":"2023-04-28T04:04:22.916160Z","shell.execute_reply":"2023-04-28T04:04:22.916188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter(best_threshold, best_score, color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.917898Z","iopub.status.idle":"2023-04-28T04:04:22.918699Z","shell.execute_reply.started":"2023-04-28T04:04:22.918453Z","shell.execute_reply":"2023-04-28T04:04:22.918481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,19):\n    pred = (pred_df[i].values.reshape(-1) > best_threshold).astype(\"int\")\n    s = f1_score(true[i].values.reshape(-1), pred,average='macro')\n    print(f\"Q{i} F1_score: {s}\")\n\npred = (pred_df.values.reshape(-1) > best_threshold).astype(\"int\")\ns = f1_score(true.values.reshape(-1), pred,average='macro')\nprint(f\"OVERALL F1_score: {s}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.920083Z","iopub.status.idle":"2023-04-28T04:04:22.920838Z","shell.execute_reply.started":"2023-04-28T04:04:22.920610Z","shell.execute_reply":"2023-04-28T04:04:22.920637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer test data","metadata":{}},{"cell_type":"code","source":"# IMPORT KAGGLE API\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.922242Z","iopub.status.idle":"2023-04-28T04:04:22.923031Z","shell.execute_reply.started":"2023-04-28T04:04:22.922771Z","shell.execute_reply":"2023-04-28T04:04:22.922796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_engineer(level, df):\n    df = df.loc[:,['session_id','elapsed_time','level_group']]\n    df.elapsed_time /= 1000 #轉為秒\n    event_duration = df.elapsed_time.diff().fillna(0).clip(0,3600)#最大道3600秒也就是一小時\n    df['event_duration'] = event_duration.astype(\"float32\")\n    df.drop('elapsed_time', axis = 1, inplace =True)\n\n    GROUP_DATA[level][\"data\"] = df\n        \n    tmp = []\n    for session_id, info in GROUP_DATA[level][\"data\"].groupby(\"session_id\"):\n        tmp.append(info['event_duration'])\n    GROUP_DATA[level][\"data\"] = pad_sequences(tmp, padding='post', maxlen = 500,dtype='float16')","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.924418Z","iopub.status.idle":"2023-04-28T04:04:22.925186Z","shell.execute_reply.started":"2023-04-28T04:04:22.924952Z","shell.execute_reply":"2023-04-28T04:04:22.924978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor i, (test, sample_submission) in enumerate(iter_test): #第一次0-4 最後一次是13-22\n    level = test.level_group.values[0]\n    test_engineer(level, test)\n    # INFER TEST DATA    \n    a,b = limits[level]\n    for Q in range(a,b):\n        model = load_model(f'{level}_{Q}/checkpoint')\n        pred_value = model.predict(GROUP_DATA[level][\"data\"])\n        \n        mask = sample_submission.session_id.str.contains(f'q{Q}')\n        sample_submission.loc[mask,'correct'] = ( pred_value > best_threshold ).astype(\"int\")\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.926576Z","iopub.status.idle":"2023-04-28T04:04:22.927310Z","shell.execute_reply.started":"2023-04-28T04:04:22.927087Z","shell.execute_reply":"2023-04-28T04:04:22.927114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:04:22.928634Z","iopub.status.idle":"2023-04-28T04:04:22.929365Z","shell.execute_reply.started":"2023-04-28T04:04:22.929134Z","shell.execute_reply":"2023-04-28T04:04:22.929161Z"},"trusted":true},"execution_count":null,"outputs":[]}]}