{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-02T13:16:02.569641Z","iopub.execute_input":"2023-05-02T13:16:02.570032Z","iopub.status.idle":"2023-05-02T13:16:02.578826Z","shell.execute_reply.started":"2023-05-02T13:16:02.570003Z","shell.execute_reply":"2023-05-02T13:16:02.578167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Importing Library ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow_decision_forests as tfdf\nimport pandas as pd\nimport numpy as np \nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:16:02.580611Z","iopub.execute_input":"2023-05-02T13:16:02.581185Z","iopub.status.idle":"2023-05-02T13:16:10.645853Z","shell.execute_reply.started":"2023-05-02T13:16:02.581157Z","shell.execute_reply":"2023-05-02T13:16:10.644980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading Data","metadata":{}},{"cell_type":"code","source":"dtypes={\n    'elapsed_time':np.int32,\n    'event_name':'category',\n    'name':'category',\n    'level':np.uint8,\n    'room_coor_x':np.float32,\n    'room_coor_y':np.float32,\n    'screen_coor_x':np.float32,\n    'screen_coor_y':np.float32,\n    'hover_duration':np.float32,\n    'text':'category',\n    'fqid':'category',\n    'room_fqid':'category',\n    'text_fqid':'category',\n    'fullscreen':'category',\n    'hq':'category',\n    'music':'category',\n    'level_group':'category'}\n# Load the data\ndataset_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)\ntrain_labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntest_data = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv',dtype=dtypes)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:16:10.646933Z","iopub.execute_input":"2023-05-02T13:16:10.647657Z","iopub.status.idle":"2023-05-02T13:18:18.062228Z","shell.execute_reply.started":"2023-05-02T13:16:10.647626Z","shell.execute_reply":"2023-05-02T13:18:18.061020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the head of each dataset\nprint(\"Train Data:\")\nprint(dataset_df.head())\nprint(\"\\nTrain Labels:\")\nprint(train_labels.head())\nprint(\"\\nTest Data:\")\nprint(test_data.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:18.065890Z","iopub.execute_input":"2023-05-02T13:18:18.066606Z","iopub.status.idle":"2023-05-02T13:18:18.105554Z","shell.execute_reply.started":"2023-05-02T13:18:18.066572Z","shell.execute_reply":"2023-05-02T13:18:18.104677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:18.107189Z","iopub.execute_input":"2023-05-02T13:18:18.107512Z","iopub.status.idle":"2023-05-02T13:18:18.139968Z","shell.execute_reply.started":"2023-05-02T13:18:18.107485Z","shell.execute_reply":"2023-05-02T13:18:18.138984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:18.141724Z","iopub.execute_input":"2023-05-02T13:18:18.142311Z","iopub.status.idle":"2023-05-02T13:18:18.416341Z","shell.execute_reply.started":"2023-05-02T13:18:18.142271Z","shell.execute_reply":"2023-05-02T13:18:18.415453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\nlabels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:18.417482Z","iopub.execute_input":"2023-05-02T13:18:18.418429Z","iopub.status.idle":"2023-05-02T13:18:19.070825Z","shell.execute_reply.started":"2023-05-02T13:18:18.418386Z","shell.execute_reply":"2023-05-02T13:18:19.069755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:19.072005Z","iopub.execute_input":"2023-05-02T13:18:19.072318Z","iopub.status.idle":"2023-05-02T13:18:19.081803Z","shell.execute_reply.started":"2023-05-02T13:18:19.072292Z","shell.execute_reply":"2023-05-02T13:18:19.080977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(3, 3))\nplot_df = labels.correct.value_counts()\nplot_df.plot(kind=\"bar\", color=['b', 'c'])","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:19.084591Z","iopub.execute_input":"2023-05-02T13:18:19.085551Z","iopub.status.idle":"2023-05-02T13:18:19.304650Z","shell.execute_reply.started":"2023-05-02T13:18:19.085517Z","shell.execute_reply":"2023-05-02T13:18:19.303934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 20))\nplt.subplots_adjust(hspace=0.5, wspace=0.5)\nplt.suptitle(\"\\\"Correct\\\" column values for each question\", fontsize=14, y=0.94)\nfor n in range(1,19):\n    #print(n, str(n))\n    ax = plt.subplot(6, 3, n)\n\n    # filter df and plot ticker on the new subplot axis\n    plot_df = labels.loc[labels.q == n]\n    plot_df = plot_df.correct.value_counts()\n    plot_df.plot(ax=ax, kind=\"bar\", color=['b', 'c'])\n    \n    # chart formatting\n    ax.set_title(\"Question \" + str(n))\n    ax.set_xlabel(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:19.305651Z","iopub.execute_input":"2023-05-02T13:18:19.306115Z","iopub.status.idle":"2023-05-02T13:18:21.475710Z","shell.execute_reply.started":"2023-05-02T13:18:19.306088Z","shell.execute_reply":"2023-05-02T13:18:21.474871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATEGORICAL = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMERICAL = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:21.476572Z","iopub.execute_input":"2023-05-02T13:18:21.476835Z","iopub.status.idle":"2023-05-02T13:18:21.481383Z","shell.execute_reply.started":"2023-05-02T13:18:21.476812Z","shell.execute_reply":"2023-05-02T13:18:21.480632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(dataset_df):\n    dfs = []\n    for c in CATEGORICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMERICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('mean')\n        dfs.append(tmp)\n    for c in NUMERICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    dataset_df = pd.concat(dfs,axis=1)\n    dataset_df = dataset_df.fillna(-1)\n    dataset_df = dataset_df.reset_index()\n    dataset_df = dataset_df.set_index('session_id')\n    return dataset_df","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:21.482147Z","iopub.execute_input":"2023-05-02T13:18:21.482403Z","iopub.status.idle":"2023-05-02T13:18:21.493038Z","shell.execute_reply.started":"2023-05-02T13:18:21.482380Z","shell.execute_reply":"2023-05-02T13:18:21.492299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df = feature_engineer(dataset_df)\nprint(\"Full prepared dataset shape is {}\".format(dataset_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:18:21.494588Z","iopub.execute_input":"2023-05-02T13:18:21.495195Z","iopub.status.idle":"2023-05-02T13:19:07.455739Z","shell.execute_reply.started":"2023-05-02T13:18:21.495167Z","shell.execute_reply":"2023-05-02T13:19:07.454533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:07.457881Z","iopub.execute_input":"2023-05-02T13:19:07.458515Z","iopub.status.idle":"2023-05-02T13:19:07.486600Z","shell.execute_reply.started":"2023-05-02T13:19:07.458481Z","shell.execute_reply":"2023-05-02T13:19:07.485648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:07.488020Z","iopub.execute_input":"2023-05-02T13:19:07.488305Z","iopub.status.idle":"2023-05-02T13:19:07.623693Z","shell.execute_reply.started":"2023-05-02T13:19:07.488281Z","shell.execute_reply":"2023-05-02T13:19:07.622768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure, axis = plt.subplots(3, 2, figsize=(10, 10))\n\nfor name, data in dataset_df.groupby('level_group'):\n    axis[0, 0].plot(range(1, len(data['room_coor_x_std'])+1), data['room_coor_x_std'], label=name)\n    axis[0, 1].plot(range(1, len(data['room_coor_y_std'])+1), data['room_coor_y_std'], label=name)\n    axis[1, 0].plot(range(1, len(data['screen_coor_x_std'])+1), data['screen_coor_x_std'], label=name)\n    axis[1, 1].plot(range(1, len(data['screen_coor_y_std'])+1), data['screen_coor_y_std'], label=name)\n    axis[2, 0].plot(range(1, len(data['hover_duration'])+1), data['hover_duration_std'], label=name)\n    axis[2, 1].plot(range(1, len(data['elapsed_time_std'])+1), data['elapsed_time_std'], label=name)\n    \n\naxis[0, 0].set_title('room_coor_x')\naxis[0, 1].set_title('room_coor_y')\naxis[1, 0].set_title('screen_coor_x')\naxis[1, 1].set_title('screen_coor_y')\naxis[2, 0].set_title('hover_duration')\naxis[2, 1].set_title('elapsed_time_std')\n\nfor i in range(3):\n    axis[i, 0].legend()\n    axis[i, 1].legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:07.624857Z","iopub.execute_input":"2023-05-02T13:19:07.625640Z","iopub.status.idle":"2023-05-02T13:19:10.357735Z","shell.execute_reply.started":"2023-05-02T13:19:07.625613Z","shell.execute_reply":"2023-05-02T13:19:10.356942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_dataset(dataset, test_ratio=0.20):\n    USER_LIST = dataset_df.index.unique()\n    split = int(len(USER_LIST) * (1 - 0.20))\n    return dataset.loc[USER_LIST[:split]], dataset.loc[USER_LIST[split:]]\n\ntrain_x, valid_x = split_dataset(dataset_df)\nprint(\"{} examples in training, {} examples in testing.\".format(\n    len(train_x), len(valid_x)))","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:10.359028Z","iopub.execute_input":"2023-05-02T13:19:10.359491Z","iopub.status.idle":"2023-05-02T13:19:10.427715Z","shell.execute_reply.started":"2023-05-02T13:19:10.359463Z","shell.execute_reply":"2023-05-02T13:19:10.426962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfdf.keras.get_all_models()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:10.429110Z","iopub.execute_input":"2023-05-02T13:19:10.429610Z","iopub.status.idle":"2023-05-02T13:19:10.435695Z","shell.execute_reply.started":"2023-05-02T13:19:10.429583Z","shell.execute_reply":"2023-05-02T13:19:10.434685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = tfdf.keras.GradientBoostedTreesModel(hyperparameter_template=\"benchmark_rank1\")","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:10.437322Z","iopub.execute_input":"2023-05-02T13:19:10.437625Z","iopub.status.idle":"2023-05-02T13:19:10.631850Z","shell.execute_reply.started":"2023-05-02T13:19:10.437599Z","shell.execute_reply":"2023-05-02T13:19:10.631020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch the unique list of user sessions in the validation dataset. We assigned \n\nVALID_USER_LIST = valid_x.index.unique()\n\n# Create a dataframe for storing the predictions of each question for all users\n\nprediction_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\n\n# Create an empty dictionary to store the models created for each question.\nmodels = {}\n\n# Create an empty dictionary to store the evaluation score for each question.\nevaluation_dict ={}","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:10.633241Z","iopub.execute_input":"2023-05-02T13:19:10.633756Z","iopub.status.idle":"2023-05-02T13:19:10.639872Z","shell.execute_reply.started":"2023-05-02T13:19:10.633727Z","shell.execute_reply":"2023-05-02T13:19:10.638805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Iterate through questions 1 to 18 to train models for each question, evaluate\n# the trained model and store the predicted values.\nfor q_no in range(1,19):\n\n    # Select level group for the question based on the q_no.\n    if q_no<=3: grp = '0-4'\n    elif q_no<=13: grp = '5-12'\n    elif q_no<=22: grp = '13-22'\n    print(\"### q_no\", q_no, \"grp\", grp)\n    \n        \n    # Filter the rows in the datasets based on the selected level group. \n    train_df = train_x.loc[train_x.level_group == grp]\n    train_users = train_df.index.values\n    valid_df = valid_x.loc[valid_x.level_group == grp]\n    valid_users = valid_df.index.values\n\n    # Select the labels for the related q_no.\n    train_labels = labels.loc[labels.q==q_no].set_index('session').loc[train_users]\n    valid_labels = labels.loc[labels.q==q_no].set_index('session').loc[valid_users]\n\n    # Add the label to the filtered datasets.\n    train_df[\"correct\"] = train_labels[\"correct\"]\n    valid_df[\"correct\"] = valid_labels[\"correct\"]\n\n    # There's one more step required before we can train the model. \n    # We need to convert the datatset from Pandas format (pd.DataFrame)\n    # into TensorFlow Datasets format (tf.data.Dataset).\n    # TensorFlow Datasets is a high performance data loading library \n    # which is helpful when training neural networks with accelerators like GPUs and TPUs.\n    # We are omitting `level_group`, since it is not needed for training anymore.\n    train_ds = tfdf.keras.pd_dataframe_to_tf_dataset(train_df.loc[:, train_df.columns != 'level_group'], label=\"correct\")\n    valid_ds = tfdf.keras.pd_dataframe_to_tf_dataset(valid_df.loc[:, valid_df.columns != 'level_group'], label=\"correct\")\n\n    # We will now create the Gradient Boosted Trees Model with default settings. \n    # By default the model is set to train for a classification task.\n    gbtm = tfdf.keras.GradientBoostedTreesModel(verbose=0)\n    gbtm.compile(metrics=[\"accuracy\"])\n\n    # Train the model.\n    gbtm.fit(x=train_ds)\n\n    # Store the model\n    models[f'{grp}_{q_no}'] = gbtm\n\n    # Evaluate the trained model on the validation dataset and store the \n    # evaluation accuracy in the `evaluation_dict`.\n    inspector = gbtm.make_inspector()\n    inspector.evaluation()\n    evaluation = gbtm.evaluate(x=valid_ds,return_dict=True)\n    evaluation_dict[q_no] = evaluation[\"accuracy\"]         \n\n    # Use the trained model to make predictions on the validation dataset and \n    # store the predicted values in the `prediction_df` dataframe.\n    predict = gbtm.predict(x=valid_ds)\n    prediction_df.loc[valid_users, q_no-1] = predict.flatten()","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:19:10.641451Z","iopub.execute_input":"2023-05-02T13:19:10.641745Z","iopub.status.idle":"2023-05-02T13:21:54.541703Z","shell.execute_reply.started":"2023-05-02T13:19:10.641721Z","shell.execute_reply":"2023-05-02T13:21:54.540703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name, value in evaluation_dict.items():\n    print(f\"question {name}: accuracy {value:.4f}\")\n\nprint(\"\\nAverage accuracy\", sum(evaluation_dict.values())/18)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:54.543165Z","iopub.execute_input":"2023-05-02T13:21:54.543452Z","iopub.status.idle":"2023-05-02T13:21:54.549350Z","shell.execute_reply.started":"2023-05-02T13:21:54.543428Z","shell.execute_reply":"2023-05-02T13:21:54.548588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfdf.model_plotter.plot_model_in_colab(models['0-4_1'], tree_idx=0, max_depth=3)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:54.550590Z","iopub.execute_input":"2023-05-02T13:21:54.551504Z","iopub.status.idle":"2023-05-02T13:21:54.570629Z","shell.execute_reply.started":"2023-05-02T13:21:54.551477Z","shell.execute_reply":"2023-05-02T13:21:54.569869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inspector = models['0-4_1'].make_inspector()\n\nprint(f\"Available variable importances:\")\nfor importance in inspector.variable_importances().keys():\n    print(\"\\t\", importance)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:54.574746Z","iopub.execute_input":"2023-05-02T13:21:54.575090Z","iopub.status.idle":"2023-05-02T13:21:54.583078Z","shell.execute_reply.started":"2023-05-02T13:21:54.575062Z","shell.execute_reply":"2023-05-02T13:21:54.582109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each line is: (feature name, (index of the feature), importance score)\ninspector.variable_importances()[\"NUM_AS_ROOT\"]","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:54.584556Z","iopub.execute_input":"2023-05-02T13:21:54.584836Z","iopub.status.idle":"2023-05-02T13:21:54.595429Z","shell.execute_reply.started":"2023-05-02T13:21:54.584812Z","shell.execute_reply":"2023-05-02T13:21:54.594452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a dataframe of required size:\n# (no: of users in validation set x no: of questions) initialized to zero values\n# to store true values of the label `correct`. \ntrue_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\nfor i in range(18):\n    # Get the true labels.\n    tmp = labels.loc[labels.q == i+1].set_index('session').loc[VALID_USER_LIST]\n    true_df[i] = tmp.correct.values\n\nmax_score = 0; best_threshold = 0\n\n# Loop through threshold values from 0.4 to 0.8 and select the threshold with \n# the highest `F1 score`.\nfor threshold in np.arange(0.4,0.8,0.01):\n    metric = tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=threshold)\n    y_true = tf.one_hot(true_df.values.reshape((-1)), depth=2)\n    y_pred = tf.one_hot((prediction_df.values.reshape((-1))>threshold).astype('int'), depth=2)\n    metric.update_state(y_true, y_pred)\n    f1_score = metric.result().numpy()\n    if f1_score > max_score:\n        max_score = f1_score\n        best_threshold = threshold\n        \nprint(\"Best threshold \", best_threshold, \"\\tF1 score \", max_score)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:54.596966Z","iopub.execute_input":"2023-05-02T13:21:54.597248Z","iopub.status.idle":"2023-05-02T13:21:55.707272Z","shell.execute_reply.started":"2023-05-02T13:21:54.597225Z","shell.execute_reply":"2023-05-02T13:21:55.706228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. Prepare the test dataset\ntest_dataset = feature_engineer(test_data)\n\n# 2. Make predictions on the test dataset\ntest_predictions = pd.DataFrame(data=np.zeros((len(test_dataset), 18)), index=test_dataset.index)\n\nfor q_no in range(1, 19):\n    if q_no <= 3:\n        grp = '0-4'\n    elif q_no <= 13:\n        grp = '5-12'\n    elif q_no <= 22:\n        grp = '13-22'\n    \n    test_df = test_dataset.loc[test_dataset.level_group == grp]\n    test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_df.loc[:, test_df.columns != 'level_group'])\n    gbtm = models[f'{grp}_{q_no}']\n    predict = gbtm.predict(x=test_ds)\n    \n    # Check the length of the predictions and the test_df, make sure they match\n    assert len(predict) == len(test_df), f\"Predictions and test_df length mismatch for group {grp} and question {q_no}\"\n\n    for idx, pred_value in zip(test_df.index, predict.flatten()):\n        test_predictions.loc[idx, q_no-1] = pred_value","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:55.708739Z","iopub.execute_input":"2023-05-02T13:21:55.709121Z","iopub.status.idle":"2023-05-02T13:21:57.374718Z","shell.execute_reply.started":"2023-05-02T13:21:55.709094Z","shell.execute_reply":"2023-05-02T13:21:57.373957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/predict-student-performance-from-game-play/jo_wilder')","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:57.377467Z","iopub.execute_input":"2023-05-02T13:21:57.377762Z","iopub.status.idle":"2023-05-02T13:21:57.382084Z","shell.execute_reply.started":"2023-05-02T13:21:57.377737Z","shell.execute_reply":"2023-05-02T13:21:57.381318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Create the directory if it does not exist.\nif not os.path.exists('/kaggle/output/working'):\n    os.makedirs('/kaggle/output/working')","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:21:57.383260Z","iopub.execute_input":"2023-05-02T13:21:57.384271Z","iopub.status.idle":"2023-05-02T13:21:57.398543Z","shell.execute_reply.started":"2023-05-02T13:21:57.384242Z","shell.execute_reply":"2023-05-02T13:21:57.397671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# 1. Prepare the test dataset\ntest_dataset = feature_engineer(test_data)\n\n# 2. Make predictions on the test dataset\ntest_predictions = pd.DataFrame(data=np.zeros((len(test_dataset), 18)), index=test_dataset.index)\n\n# Create an empty list to store rows for the submission DataFrame\nsubmission_rows = []\n\nfor q_no in range(1, 19):\n    if q_no <= 3:\n        grp = '0-4'\n    elif q_no <= 13:\n        grp = '5-12'\n    elif q_no <= 22:\n        grp = '13-22'\n    \n    test_df = test_dataset.loc[test_dataset.level_group == grp]\n    test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_df.loc[:, test_df.columns != 'level_group'])\n    gbtm = models[f'{grp}_{q_no}']\n    predict = gbtm.predict(x=test_ds)\n    \n    # Convert predicted probabilities to binary values (0 or 1) based on the best_threshold\n    binary_predict = (predict.flatten() > best_threshold).astype(int)\n    \n    for idx, pred_value in zip(test_df.index, binary_predict):\n        session_level = q_no  # Extract the session_level from the question number\n        submission_rows.append({\n            \"session_id\": f\"{idx}_q{q_no}\",\n            \"correct\": pred_value,\n            \"session_level\": session_level\n        })\n\n# Convert the list of rows into a DataFrame\nsubmission = pd.DataFrame(submission_rows)\n\n# Print the submission DataFrame\nprint(submission)\n\nsubmission.to_csv(\"/kaggle/working/submission_new.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T13:22:24.582050Z","iopub.execute_input":"2023-05-02T13:22:24.582418Z","iopub.status.idle":"2023-05-02T13:22:26.223451Z","shell.execute_reply.started":"2023-05-02T13:22:24.582391Z","shell.execute_reply":"2023-05-02T13:22:26.222603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}