{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🧑‍🎓 Predict Student Performance from Game Play\n\nThe goal of this competition is to predict student performance during game-based learning in real-time. You'll develop a model trained on one of the largest open datasets of game logs.","metadata":{}},{"cell_type":"markdown","source":"### ⚒️ Imports","metadata":{}},{"cell_type":"code","source":"import tensorflow_decision_forests as tfdf\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:21:15.537184Z","iopub.execute_input":"2023-04-18T13:21:15.537636Z","iopub.status.idle":"2023-04-18T13:21:15.543393Z","shell.execute_reply.started":"2023-04-18T13:21:15.537586Z","shell.execute_reply":"2023-04-18T13:21:15.542264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow_decision_forests as tfdf\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nprint(\"TensorFlow Decision Forests v\" + tfdf.__version__)\nprint(\"TensorFlow Addons v\" + tfa.__version__)\nprint(\"TensorFlow v\" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:21:17.251745Z","iopub.execute_input":"2023-04-18T13:21:17.252168Z","iopub.status.idle":"2023-04-18T13:21:17.464079Z","shell.execute_reply.started":"2023-04-18T13:21:17.252131Z","shell.execute_reply":"2023-04-18T13:21:17.463198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the Dataset\n\nSince the dataset is huge, some people may face memory errors while reading the dataset from the csv. To avoid this, we will try to optimize the memory used by Pandas to load and store the dataset.\n\nWhen Pandas loads a dataset, by default, it automatically detects the data types of the different columns. Irresepective of the maximum value that is stored in these columns, Pandas assigns int64 for numerical columns, float64 for float columns, object dtype for string columns etc.\n\nWe may be able to reduce the size of these columns in memory by downcasting numerical columns to smaller types (like int8, int32, float32 etc.), if their maximum values don't need the larger types for storage, (like int64, float64 etc.).\n\nSimilarly, Pandas automatically detects string columns as object datatype. To reduce memory usage of string columns which store categorical data, we specify their datatype as category.\n\nMany of the columns in this dataset can be downcast to smaller types.\n\nWe will provide a dict of dtypes for columns to pandas while reading the dataset.","metadata":{}},{"cell_type":"code","source":"# Reference: https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/384359\ndtypes={\n    'elapsed_time':np.int32,\n    'event_name':'category',\n    'name':'category',\n    'level':np.uint8,\n    'room_coor_x':np.float32,\n    'room_coor_y':np.float32,\n    'screen_coor_x':np.float32,\n    'screen_coor_y':np.float32,\n    'hover_duration':np.float32,\n    'text':'category',\n    'fqid':'category',\n    'room_fqid':'category',\n    'text_fqid':'category',\n    'fullscreen':'category',\n    'hq':'category',\n    'music':'category',\n    'level_group':'category'}\n\ndataset_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)\nprint(\"Full train dataset shape is {}\".format(dataset_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:21:17.884218Z","iopub.execute_input":"2023-04-18T13:21:17.884914Z","iopub.status.idle":"2023-04-18T13:23:34.051318Z","shell.execute_reply.started":"2023-04-18T13:21:17.884874Z","shell.execute_reply":"2023-04-18T13:23:34.050222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data is composed of 20 columns and 26296946 entries. We can see all 20 dimensions of our dataset by printing out the first 5 entries using the following code","metadata":{}},{"cell_type":"code","source":"# Display the first 5 examples\ndataset_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:23:34.053367Z","iopub.execute_input":"2023-04-18T13:23:34.054093Z","iopub.status.idle":"2023-04-18T13:23:34.094716Z","shell.execute_reply.started":"2023-04-18T13:23:34.054051Z","shell.execute_reply":"2023-04-18T13:23:34.093830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\nlabels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\nlabels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:23:38.529732Z","iopub.execute_input":"2023-04-18T13:23:38.530145Z","iopub.status.idle":"2023-04-18T13:23:39.673298Z","shell.execute_reply.started":"2023-04-18T13:23:38.530108Z","shell.execute_reply":"2023-04-18T13:23:39.672202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the first 5 examples\nlabels.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:23:45.427174Z","iopub.execute_input":"2023-04-18T13:23:45.427576Z","iopub.status.idle":"2023-04-18T13:23:45.438860Z","shell.execute_reply.started":"2023-04-18T13:23:45.427542Z","shell.execute_reply":"2023-04-18T13:23:45.437542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(3, 3))\nplot_df = labels.correct.value_counts()\nplot_df.plot(kind=\"bar\", color=['b', 'c'])","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:23:58.408102Z","iopub.execute_input":"2023-04-18T13:23:58.408520Z","iopub.status.idle":"2023-04-18T13:23:58.625626Z","shell.execute_reply.started":"2023-04-18T13:23:58.408485Z","shell.execute_reply":"2023-04-18T13:23:58.624632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 20))\nplt.subplots_adjust(hspace=0.5, wspace=0.5)\nplt.suptitle(\"\\\"Correct\\\" column values for each question\", fontsize=14, y=0.94)\nfor n in range(1,19):\n    #print(n, str(n))\n    ax = plt.subplot(6, 3, n)\n\n    # filter df and plot ticker on the new subplot axis\n    plot_df = labels.loc[labels.q == n]\n    plot_df = plot_df.correct.value_counts()\n    plot_df.plot(ax=ax, kind=\"bar\", color=['b', 'c'])\n    \n    # chart formatting\n    ax.set_title(\"Question \" + str(n))\n    ax.set_xlabel(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:24:07.024772Z","iopub.execute_input":"2023-04-18T13:24:07.025199Z","iopub.status.idle":"2023-04-18T13:24:08.759985Z","shell.execute_reply.started":"2023-04-18T13:24:07.025162Z","shell.execute_reply":"2023-04-18T13:24:08.758925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare the dataset","metadata":{}},{"cell_type":"code","source":"CATEGORICAL = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMERICAL = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:26:57.105591Z","iopub.execute_input":"2023-04-18T13:26:57.106024Z","iopub.status.idle":"2023-04-18T13:26:57.112204Z","shell.execute_reply.started":"2023-04-18T13:26:57.105986Z","shell.execute_reply":"2023-04-18T13:26:57.110929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\ndef feature_engineer(dataset_df):\n    dfs = []\n    for c in CATEGORICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMERICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('mean')\n        dfs.append(tmp)\n    for c in NUMERICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    dataset_df = pd.concat(dfs,axis=1)\n    dataset_df = dataset_df.fillna(-1)\n    dataset_df = dataset_df.reset_index()\n    dataset_df = dataset_df.set_index('session_id')\n    return dataset_df","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:26:57.338410Z","iopub.execute_input":"2023-04-18T13:26:57.338837Z","iopub.status.idle":"2023-04-18T13:26:57.351590Z","shell.execute_reply.started":"2023-04-18T13:26:57.338801Z","shell.execute_reply":"2023-04-18T13:26:57.350310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df = feature_engineer(dataset_df)\nprint(\"Full prepared dataset shape is {}\".format(dataset_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:26:57.674101Z","iopub.execute_input":"2023-04-18T13:26:57.674494Z","iopub.status.idle":"2023-04-18T13:27:38.087095Z","shell.execute_reply.started":"2023-04-18T13:26:57.674461Z","shell.execute_reply":"2023-04-18T13:27:38.085813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Explore created dataset","metadata":{}},{"cell_type":"code","source":"# Display the first 5 examples\ndataset_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:27:38.088909Z","iopub.execute_input":"2023-04-18T13:27:38.089384Z","iopub.status.idle":"2023-04-18T13:27:38.117593Z","shell.execute_reply.started":"2023-04-18T13:27:38.089344Z","shell.execute_reply":"2023-04-18T13:27:38.116646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:27:38.118835Z","iopub.execute_input":"2023-04-18T13:27:38.119775Z","iopub.status.idle":"2023-04-18T13:27:38.256638Z","shell.execute_reply.started":"2023-04-18T13:27:38.119723Z","shell.execute_reply":"2023-04-18T13:27:38.255759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Numerical data distribution","metadata":{}},{"cell_type":"code","source":"figure, axis = plt.subplots(3, 2, figsize=(10, 10))\n\nfor name, data in dataset_df.groupby('level_group'):\n    axis[0, 0].plot(range(1, len(data['room_coor_x_std'])+1), data['room_coor_x_std'], label=name)\n    axis[0, 1].plot(range(1, len(data['room_coor_y_std'])+1), data['room_coor_y_std'], label=name)\n    axis[1, 0].plot(range(1, len(data['screen_coor_x_std'])+1), data['screen_coor_x_std'], label=name)\n    axis[1, 1].plot(range(1, len(data['screen_coor_y_std'])+1), data['screen_coor_y_std'], label=name)\n    axis[2, 0].plot(range(1, len(data['hover_duration'])+1), data['hover_duration_std'], label=name)\n    axis[2, 1].plot(range(1, len(data['elapsed_time_std'])+1), data['elapsed_time_std'], label=name)\n    \n\naxis[0, 0].set_title('room_coor_x')\naxis[0, 1].set_title('room_coor_y')\naxis[1, 0].set_title('screen_coor_x')\naxis[1, 1].set_title('screen_coor_y')\naxis[2, 0].set_title('hover_duration')\naxis[2, 1].set_title('elapsed_time_std')\n\nfor i in range(3):\n    axis[i, 0].legend()\n    axis[i, 1].legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:27:38.258399Z","iopub.execute_input":"2023-04-18T13:27:38.259458Z","iopub.status.idle":"2023-04-18T13:27:40.818921Z","shell.execute_reply.started":"2023-04-18T13:27:38.259416Z","shell.execute_reply":"2023-04-18T13:27:40.818026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_dataset(dataset, test_ratio=0.20):\n    USER_LIST = dataset_df.index.unique()\n    split = int(len(USER_LIST) * (1 - 0.20))\n    return dataset.loc[USER_LIST[:split]], dataset.loc[USER_LIST[split:]]\n\ntrain_x, valid_x = split_dataset(dataset_df)\nprint(\"{} examples in training, {} examples in testing.\".format(\n    len(train_x), len(valid_x)))","metadata":{"execution":{"iopub.status.busy":"2023-04-18T13:27:40.820020Z","iopub.execute_input":"2023-04-18T13:27:40.821041Z","iopub.status.idle":"2023-04-18T13:27:41.118902Z","shell.execute_reply.started":"2023-04-18T13:27:40.821004Z","shell.execute_reply":"2023-04-18T13:27:41.118010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select Model","metadata":{}},{"cell_type":"code","source":"tfdf.keras.get_all_models()","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:02:43.487773Z","iopub.execute_input":"2023-04-18T14:02:43.489503Z","iopub.status.idle":"2023-04-18T14:02:43.501626Z","shell.execute_reply.started":"2023-04-18T14:02:43.489418Z","shell.execute_reply":"2023-04-18T14:02:43.500386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = tfdf.keras.GradientBoostedTreesModel(hyperparameter_template=\"benchmark_rank1\")","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:02:46.900519Z","iopub.execute_input":"2023-04-18T14:02:46.901010Z","iopub.status.idle":"2023-04-18T14:02:47.169657Z","shell.execute_reply.started":"2023-04-18T14:02:46.900971Z","shell.execute_reply":"2023-04-18T14:02:47.168714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch the unique list of user sessions in the validation dataset. We assigned \n# `session_id` as the index of our feature engineered dataset. Hence fetching \n# the unique values in the index column will give us a list of users in the \n# validation set.\nVALID_USER_LIST = valid_x.index.unique()\n\n# Create a dataframe for storing the predictions of each question for all users\n# in the validation set.\n# For this, the required size of the data frame is: \n# (no: of users in validation set  x no of questions).\n# We will initialize all the predicted values in the data frame to zero.\n# The dataframe's index column is the user `session_id`s. \nprediction_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\n\n# Create an empty dictionary to store the models created for each question.\nmodels = {}\n\n# Create an empty dictionary to store the evaluation score for each question.\nevaluation_dict ={}","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:02:48.996248Z","iopub.execute_input":"2023-04-18T14:02:48.997045Z","iopub.status.idle":"2023-04-18T14:02:49.008340Z","shell.execute_reply.started":"2023-04-18T14:02:48.997000Z","shell.execute_reply":"2023-04-18T14:02:49.007246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Iterate through questions 1 to 18 to train models for each question, evaluate\n# the trained model and store the predicted values.\nfor q_no in range(1,19):\n\n    # Select level group for the question based on the q_no.\n    if q_no<=3: grp = '0-4'\n    elif q_no<=13: grp = '5-12'\n    elif q_no<=22: grp = '13-22'\n    print(\"### q_no\", q_no, \"grp\", grp)\n    \n        \n    # Filter the rows in the datasets based on the selected level group. \n    train_df = train_x.loc[train_x.level_group == grp]\n    train_users = train_df.index.values\n    valid_df = valid_x.loc[valid_x.level_group == grp]\n    valid_users = valid_df.index.values\n\n    # Select the labels for the related q_no.\n    train_labels = labels.loc[labels.q==q_no].set_index('session').loc[train_users]\n    valid_labels = labels.loc[labels.q==q_no].set_index('session').loc[valid_users]\n\n    # Add the label to the filtered datasets.\n    train_df[\"correct\"] = train_labels[\"correct\"]\n    valid_df[\"correct\"] = valid_labels[\"correct\"]\n\n    # There's one more step required before we can train the model. \n    # We need to convert the datatset from Pandas format (pd.DataFrame)\n    # into TensorFlow Datasets format (tf.data.Dataset).\n    # TensorFlow Datasets is a high performance data loading library \n    # which is helpful when training neural networks with accelerators like GPUs and TPUs.\n    # We are omitting `level_group`, since it is not needed for training anymore.\n    train_ds = tfdf.keras.pd_dataframe_to_tf_dataset(train_df.loc[:, train_df.columns != 'level_group'], label=\"correct\")\n    valid_ds = tfdf.keras.pd_dataframe_to_tf_dataset(valid_df.loc[:, valid_df.columns != 'level_group'], label=\"correct\")\n\n    # We will now create the Gradient Boosted Trees Model with default settings. \n    # By default the model is set to train for a classification task.\n    gbtm = tfdf.keras.GradientBoostedTreesModel(verbose=0)\n    gbtm.compile(metrics=[\"accuracy\"])\n\n    # Train the model.\n    gbtm.fit(x=train_ds)\n\n    # Store the model\n    models[f'{grp}_{q_no}'] = gbtm\n\n    # Evaluate the trained model on the validation dataset and store the \n    # evaluation accuracy in the `evaluation_dict`.\n    inspector = gbtm.make_inspector()\n    inspector.evaluation()\n    evaluation = gbtm.evaluate(x=valid_ds,return_dict=True)\n    evaluation_dict[q_no] = evaluation[\"accuracy\"]         \n\n    # Use the trained model to make predictions on the validation dataset and \n    # store the predicted values in the `prediction_df` dataframe.\n    predict = gbtm.predict(x=valid_ds)\n    prediction_df.loc[valid_users, q_no-1] = predict.flatten()     ","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:02:49.273694Z","iopub.execute_input":"2023-04-18T14:02:49.274623Z","iopub.status.idle":"2023-04-18T14:05:33.771308Z","shell.execute_reply.started":"2023-04-18T14:02:49.274576Z","shell.execute_reply":"2023-04-18T14:05:33.769802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name, value in evaluation_dict.items():\n  print(f\"question {name}: accuracy {value:.4f}\")\n\nprint(\"\\nAverage accuracy\", sum(evaluation_dict.values())/18)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:33.775088Z","iopub.execute_input":"2023-04-18T14:05:33.777134Z","iopub.status.idle":"2023-04-18T14:05:33.784169Z","shell.execute_reply.started":"2023-04-18T14:05:33.777082Z","shell.execute_reply":"2023-04-18T14:05:33.782664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfdf.model_plotter.plot_model_in_colab(models['0-4_1'], tree_idx=0, max_depth=3)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:33.786440Z","iopub.execute_input":"2023-04-18T14:05:33.786885Z","iopub.status.idle":"2023-04-18T14:05:33.807315Z","shell.execute_reply.started":"2023-04-18T14:05:33.786829Z","shell.execute_reply":"2023-04-18T14:05:33.806059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#variable importance\ninspector = models['0-4_1'].make_inspector()\n\nprint(f\"Available variable importances:\")\nfor importance in inspector.variable_importances().keys():\n  print(\"\\t\", importance)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:33.809948Z","iopub.execute_input":"2023-04-18T14:05:33.810346Z","iopub.status.idle":"2023-04-18T14:05:33.819459Z","shell.execute_reply.started":"2023-04-18T14:05:33.810292Z","shell.execute_reply":"2023-04-18T14:05:33.818311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each line is: (feature name, (index of the feature), importance score)\ninspector.variable_importances()[\"NUM_AS_ROOT\"]","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:33.821140Z","iopub.execute_input":"2023-04-18T14:05:33.821700Z","iopub.status.idle":"2023-04-18T14:05:33.833361Z","shell.execute_reply.started":"2023-04-18T14:05:33.821660Z","shell.execute_reply":"2023-04-18T14:05:33.832440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a dataframe of required size:\n# (no: of users in validation set x no: of questions) initialized to zero values\n# to store true values of the label `correct`. \ntrue_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\nfor i in range(18):\n    # Get the true labels.\n    tmp = labels.loc[labels.q == i+1].set_index('session').loc[VALID_USER_LIST]\n    true_df[i] = tmp.correct.values\n\nmax_score = 0; best_threshold = 0\n\n# Loop through threshold values from 0.4 to 0.8 and select the threshold with \n# the highest `F1 score`.\nfor threshold in np.arange(0.4,0.8,0.01):\n    metric = tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=threshold)\n    y_true = tf.one_hot(true_df.values.reshape((-1)), depth=2)\n    y_pred = tf.one_hot((prediction_df.values.reshape((-1))>threshold).astype('int'), depth=2)\n    metric.update_state(y_true, y_pred)\n    f1_score = metric.result().numpy()\n    if f1_score > max_score:\n        max_score = f1_score\n        best_threshold = threshold\n        \nprint(\"Best threshold \", best_threshold, \"\\tF1 score \", max_score)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:33.834633Z","iopub.execute_input":"2023-04-18T14:05:33.835531Z","iopub.status.idle":"2023-04-18T14:05:35.080734Z","shell.execute_reply.started":"2023-04-18T14:05:33.835495Z","shell.execute_reply":"2023-04-18T14:05:35.079766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference\n# https://www.kaggle.com/code/philculliton/basic-submission-demo\n# https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\n\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    test_df = feature_engineer(test)\n    grp = test_df.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        gbtm = models[f'{grp}_{t}']\n        test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_df.loc[:, test_df.columns != 'level_group'])\n        predictions = gbtm.predict(test_ds)\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        n_predictions = (predictions > best_threshold).astype(int)\n        sample_submission.loc[mask,'correct'] = n_predictions.flatten()\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:35.082460Z","iopub.execute_input":"2023-04-18T14:05:35.083338Z","iopub.status.idle":"2023-04-18T14:05:37.006435Z","shell.execute_reply.started":"2023-04-18T14:05:35.083294Z","shell.execute_reply":"2023-04-18T14:05:37.005445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:05:37.007826Z","iopub.execute_input":"2023-04-18T14:05:37.009060Z","iopub.status.idle":"2023-04-18T14:05:38.189509Z","shell.execute_reply.started":"2023-04-18T14:05:37.009015Z","shell.execute_reply":"2023-04-18T14:05:38.188087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}