{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-16T15:07:26.811654Z","iopub.execute_input":"2023-05-16T15:07:26.812634Z","iopub.status.idle":"2023-05-16T15:07:26.864786Z","shell.execute_reply.started":"2023-05-16T15:07:26.812580Z","shell.execute_reply":"2023-05-16T15:07:26.863281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:07:26.867177Z","iopub.execute_input":"2023-05-16T15:07:26.867817Z","iopub.status.idle":"2023-05-16T15:07:26.873426Z","shell.execute_reply.started":"2023-05-16T15:07:26.867777Z","shell.execute_reply":"2023-05-16T15:07:26.872065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory data analysis","metadata":{}},{"cell_type":"markdown","source":"As you know the training dataset is too large to handle, we try to take only the first column of the training dataset to know the total length and planning to break it into manageable ones.\n\n","metadata":{}},{"cell_type":"code","source":"tmp = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', usecols=[0])\ntmp = tmp.groupby('session_id').session_id.agg('count')\nprint(len(tmp))","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:07:26.875612Z","iopub.execute_input":"2023-05-16T15:07:26.876099Z","iopub.status.idle":"2023-05-16T15:08:54.388956Z","shell.execute_reply.started":"2023-05-16T15:07:26.876059Z","shell.execute_reply":"2023-05-16T15:08:54.386911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, totally we have around 2,62,96,946 rows (i.e Roughly around 26.3 million). It is really huge!!!!","metadata":{}},{"cell_type":"code","source":"pieces = 10\nchunk = int(np.ceil(len(tmp)/pieces))\n\nread = []\nskip = [0]\n\nfor k in range(pieces):\n    a = k*chunk\n    b = (k+1)*chunk\n    if b > len(tmp):\n        b = len(tmp)\n    r = tmp.iloc[a:b].sum()\n    read.append(r)\n    skip.append(skip[-1]+r)\n    \nprint(f'We have split the dataset into almost same 10 pieces {pieces}')\nprint(read)\nprint(skip)\n\n#Reference: https://www.kaggle.com/code/cdeotte/xgboost-baseline-0-680","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:08:54.391705Z","iopub.execute_input":"2023-05-16T15:08:54.392466Z","iopub.status.idle":"2023-05-16T15:08:54.405649Z","shell.execute_reply.started":"2023-05-16T15:08:54.392413Z","shell.execute_reply":"2023-05-16T15:08:54.404246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take only the first chunk of data for initial analysis","metadata":{}},{"cell_type":"code","source":"df1 = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', nrows = read[0])","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:08:54.407411Z","iopub.execute_input":"2023-05-16T15:08:54.407980Z","iopub.status.idle":"2023-05-16T15:09:03.971051Z","shell.execute_reply.started":"2023-05-16T15:08:54.407906Z","shell.execute_reply":"2023-05-16T15:09:03.969314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:03.973182Z","iopub.execute_input":"2023-05-16T15:09:03.973654Z","iopub.status.idle":"2023-05-16T15:09:04.015006Z","shell.execute_reply.started":"2023-05-16T15:09:03.973609Z","shell.execute_reply":"2023-05-16T15:09:04.013839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.index.values","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:04.016619Z","iopub.execute_input":"2023-05-16T15:09:04.017281Z","iopub.status.idle":"2023-05-16T15:09:04.028806Z","shell.execute_reply.started":"2023-05-16T15:09:04.017239Z","shell.execute_reply":"2023-05-16T15:09:04.027327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1[df1['session_id']== 20090312431273200 ][['level_group']].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:04.033513Z","iopub.execute_input":"2023-05-16T15:09:04.034454Z","iopub.status.idle":"2023-05-16T15:09:04.055711Z","shell.execute_reply.started":"2023-05-16T15:09:04.034398Z","shell.execute_reply":"2023-05-16T15:09:04.054678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Exploring the target class metrics for analysis","metadata":{}},{"cell_type":"code","source":"target = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:04.057037Z","iopub.execute_input":"2023-05-16T15:09:04.058097Z","iopub.status.idle":"2023-05-16T15:09:04.579612Z","shell.execute_reply.started":"2023-05-16T15:09:04.058050Z","shell.execute_reply":"2023-05-16T15:09:04.578281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Eventhough we have 26.3 million rows, we have only 0.4 million of data in target label. \n\nTwo third of the output is \"1\" and one third of the output is \"0\"","metadata":{}},{"cell_type":"code","source":"target['correct'].value_counts().plot(kind = 'bar')","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:04.581322Z","iopub.execute_input":"2023-05-16T15:09:04.581989Z","iopub.status.idle":"2023-05-16T15:09:04.835010Z","shell.execute_reply.started":"2023-05-16T15:09:04.581947Z","shell.execute_reply":"2023-05-16T15:09:04.833928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target['session'] = target['session_id'].apply(lambda x: int(x.split('_')[0]))\ntarget['q'] = target['session_id'].apply(lambda x: int(x.split('_')[1][1:]))","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:04.836586Z","iopub.execute_input":"2023-05-16T15:09:04.837229Z","iopub.status.idle":"2023-05-16T15:09:06.006686Z","shell.execute_reply.started":"2023-05-16T15:09:04.837180Z","shell.execute_reply":"2023-05-16T15:09:06.005420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target['session'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:06.008417Z","iopub.execute_input":"2023-05-16T15:09:06.009122Z","iopub.status.idle":"2023-05-16T15:09:06.028441Z","shell.execute_reply.started":"2023-05-16T15:09:06.009076Z","shell.execute_reply":"2023-05-16T15:09:06.026969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target[target['session']==20090312431273200]","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:06.030496Z","iopub.execute_input":"2023-05-16T15:09:06.030889Z","iopub.status.idle":"2023-05-16T15:09:06.055839Z","shell.execute_reply.started":"2023-05-16T15:09:06.030850Z","shell.execute_reply":"2023-05-16T15:09:06.054560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:06.057696Z","iopub.execute_input":"2023-05-16T15:09:06.058183Z","iopub.status.idle":"2023-05-16T15:09:06.067413Z","shell.execute_reply.started":"2023-05-16T15:09:06.058145Z","shell.execute_reply":"2023-05-16T15:09:06.065997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfig, axs = plt.subplots(nrows=3, ncols=6, figsize=(18, 9))\n\nfor i, ax in enumerate(axs.flat):\n    if i < 18:\n        q = target[target['q'] == i+1]\n        q['correct'].value_counts().plot(kind='bar', ax=ax)\n        ax.set_xlabel('Correct or Incorrect')\n        ax.set_ylabel('Counts')\n        ax.set_title(f'q = {i+1}')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:06.069237Z","iopub.execute_input":"2023-05-16T15:09:06.069760Z","iopub.status.idle":"2023-05-16T15:09:08.329317Z","shell.execute_reply.started":"2023-05-16T15:09:06.069709Z","shell.execute_reply":"2023-05-16T15:09:08.328217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CAT = [['event_name','name','text','fqid','room_fqid','text_fqid']]\n\nCAT = ['event_name','text','fqid','room_fqid']","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:08.330449Z","iopub.execute_input":"2023-05-16T15:09:08.330798Z","iopub.status.idle":"2023-05-16T15:09:08.336930Z","shell.execute_reply.started":"2023-05-16T15:09:08.330763Z","shell.execute_reply":"2023-05-16T15:09:08.335802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = ['event_name', 'fqid', 'room_fqid', 'text']\nNUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']\n\n# https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data\nEVENTS = ['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'checkpoint']","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:08.338428Z","iopub.execute_input":"2023-05-16T15:09:08.338786Z","iopub.status.idle":"2023-05-16T15:09:08.350419Z","shell.execute_reply.started":"2023-05-16T15:09:08.338751Z","shell.execute_reply":"2023-05-16T15:09:08.349248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    \n    dfs = []\n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    for c in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    train = train.drop(EVENTS,axis=1)\n        \n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:08.352088Z","iopub.execute_input":"2023-05-16T15:09:08.352930Z","iopub.status.idle":"2023-05-16T15:09:08.366832Z","shell.execute_reply.started":"2023-05-16T15:09:08.352871Z","shell.execute_reply":"2023-05-16T15:09:08.365670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/cdeotte/xgboost-baseline-0-680\n#chunk = int(np.ceil(len(tmp)/pieces))\n#read = []\n#[2684191, 2631991, 2638304, 2657670, 2644229, 2629801, 2596616, 2602258, 2619995, 2591891]\n#skip = [0]\n#[0, 2684191, 5316182, 7954486, 10612156, 13256385, 15886186, 18482802, 21085060, 23705055, 26296946]\n\n#%%time\n\nall_pieces = []\npieces = 10\nprint(f'To avoid the memory loss we try to split the data into {pieces} pieces')\nfor k in range(pieces):\n    SKIP = 0\n    if k>0: \n        SKIP = range(1, skip[k]+1)\n    train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',\n                        nrows = read[k],skiprows = SKIP)\n    df = feature_engineer(train)\n    all_pieces.append(df)\n\nprint('\\n')\ndel train; gc.collect() \ndf = pd.concat(all_pieces, axis = 0)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:09:08.370307Z","iopub.execute_input":"2023-05-16T15:09:08.370773Z","iopub.status.idle":"2023-05-16T15:17:19.515518Z","shell.execute_reply.started":"2023-05-16T15:09:08.370736Z","shell.execute_reply":"2023-05-16T15:17:19.513567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we have the processed data, it is not advised to have only this structure as the bible, I see lot of code are build on the above data only. But we have lot of oppourtunity to expand the dataset further. Example:we have different set of event names, instead of making in into only one column, we can make them into different labels.","metadata":{}},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:19.522641Z","iopub.execute_input":"2023-05-16T15:17:19.523185Z","iopub.status.idle":"2023-05-16T15:17:19.537252Z","shell.execute_reply.started":"2023-05-16T15:17:19.523139Z","shell.execute_reply":"2023-05-16T15:17:19.535767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:19.539398Z","iopub.execute_input":"2023-05-16T15:17:19.540331Z","iopub.status.idle":"2023-05-16T15:17:19.551774Z","shell.execute_reply.started":"2023-05-16T15:17:19.540279Z","shell.execute_reply":"2023-05-16T15:17:19.550309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model\n\nWe have lot of options for prediction. But I personally want to learn about Tensorflow, I have choosen this.\n\nSome reference: https://www.kaggle.com/code/gusthema/student-performance-w-tensorflow-decision-forests","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow_decision_forests as tfdf","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:19.554483Z","iopub.execute_input":"2023-05-16T15:17:19.554963Z","iopub.status.idle":"2023-05-16T15:17:31.952531Z","shell.execute_reply.started":"2023-05-16T15:17:19.554894Z","shell.execute_reply":"2023-05-16T15:17:31.950716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and Validation split\n\nHere, we can't use any library to split the dataset into train and test. Because all the library use the target variable as the mean for split the dataset into equal set withe same distribution as original dataset.","metadata":{}},{"cell_type":"code","source":"def split_dataset(dataset, test_ratio=0.20):\n    session_ids = dataset.index.unique()\n    n_sessions = len(session_ids)\n    n_test_sessions = int(n_sessions * test_ratio)\n    test_sessions = np.random.choice(session_ids, n_test_sessions, replace=False)\n    test_set = dataset.loc[test_sessions]\n    train_set = dataset.drop(test_sessions)\n    return train_set, test_set","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:31.954718Z","iopub.execute_input":"2023-05-16T15:17:31.955430Z","iopub.status.idle":"2023-05-16T15:17:31.962374Z","shell.execute_reply.started":"2023-05-16T15:17:31.955386Z","shell.execute_reply":"2023-05-16T15:17:31.961268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, val_x = split_dataset(df)\nprint(f\"{len(train_x)} examples in training, {len(val_x)} examples in validation.\")","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:31.964071Z","iopub.execute_input":"2023-05-16T15:17:31.964711Z","iopub.status.idle":"2023-05-16T15:17:32.075337Z","shell.execute_reply.started":"2023-05-16T15:17:31.964672Z","shell.execute_reply":"2023-05-16T15:17:32.074345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow_decision_forests as tfdf\n\nVALID_USER_LIST = val_x.index.unique()\n\n# Create a dataframe for storing the predictions of each question for all users\n# in the validation set.\n# For this, the required size of the data frame is: \n# (no: of users in validation set  x no of questions).\n# We will initialize all the predicted values in the data frame to zero.\n# The dataframe's index column is the user `session_id`s. \nprediction_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\n\n# Create an empty dictionary to store the models created for each question.\nmodels = {}\n\n# Create an empty dictionary to store the evaluation score for each question.\nevaluation_dict ={}","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:32.076810Z","iopub.execute_input":"2023-05-16T15:17:32.077423Z","iopub.status.idle":"2023-05-16T15:17:32.084649Z","shell.execute_reply.started":"2023-05-16T15:17:32.077386Z","shell.execute_reply":"2023-05-16T15:17:32.083461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Iterate through questions 1 to 18 to train models for each question, evaluate\n# the trained model and store the predicted values.\nfor q_no in range(1, 19):\n\n    # Select level group for the question based on the q_no.\n    if q_no <= 3:\n        grp = '0-4'\n    elif q_no <= 13:\n        grp = '5-12'\n    elif q_no <= 22:\n        grp = '13-22'\n    print(\"### q_no\", q_no, \"grp\", grp)\n\n    # Filter the rows in the datasets based on the selected level group. \n    train_df = train_x.loc[train_x.level_group == grp]\n    train_users = train_df.index.values\n    valid_df = val_x.loc[val_x.level_group == grp]\n    valid_users = valid_df.index.values\n\n    # Select the target for the related q_no.\n    train_target = target.loc[target.q == q_no].set_index('session').loc[train_users]\n    valid_target = target.loc[target.q == q_no].set_index('session').loc[valid_users]\n\n    # Add the target to the filtered datasets.\n    train_df[\"target\"] = train_target[\"correct\"]\n    valid_df[\"target\"] = valid_target[\"correct\"]\n\n    # There's one more step required before we can train the model. \n    # We need to convert the dataset from Pandas format (pd.DataFrame)\n    # into TensorFlow Datasets format (tf.data.Dataset).\n    # TensorFlow Datasets is a high performance data loading library \n    # which is helpful when training neural networks with accelerators like GPUs and TPUs.\n    # We are omitting `level_group`, since it is not needed for training anymore.\n    train_ds = tfdf.keras.pd_dataframe_to_tf_dataset(train_df.loc[:, train_df.columns != 'level_group'], label=\"target\")\n    valid_ds = tfdf.keras.pd_dataframe_to_tf_dataset(valid_df.loc[:, valid_df.columns != 'level_group'], label=\"target\")\n\n    # We will now create the Distributed Gradient Boosted Trees Model with default settings. \n    # By default the model is set to train for a classification task.\n    gbtm = tfdf.keras.GradientBoostedTreesModel(num_trees=1000, max_depth=8)\n    gbtm.compile(metrics=[\"accuracy\"])\n\n    # Train the model.\n    gbtm.fit(x=train_ds)\n\n    # Store the model.\n    models[f'{grp}_{q_no}'] = gbtm\n\n    # Evaluate the trained model on the validation dataset and store the \n    # evaluation accuracy in the `evaluation_dict`.\n    inspector = gbtm.make_inspector()\n    inspector.evaluation()\n    evaluation = gbtm.evaluate(x=valid_ds, return_dict=True)\n    evaluation_dict[q_no] = evaluation[\"accuracy\"]         \n\n    # Use the trained model to make predictions on the validation dataset and \n    # store the predicted values in the `prediction_df` dataframe.\n    predict = gbtm.predict(x=valid_ds)\n    prediction_df.loc[valid_users, q_no-1] = predict.flatten()\n","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:17:32.086406Z","iopub.execute_input":"2023-05-16T15:17:32.087101Z","iopub.status.idle":"2023-05-16T15:21:26.238584Z","shell.execute_reply.started":"2023-05-16T15:17:32.087062Z","shell.execute_reply":"2023-05-16T15:21:26.237150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a dataframe of required size:\n# (no: of users in validation set x no: of questions) initialized to zero values\n# to store true values of the label `correct`. \ntrue_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\nfor i in range(18):\n    # Get the true labels.\n    tmp = target.loc[target.q == i+1].set_index('session').loc[VALID_USER_LIST]\n    true_df[i] = tmp.correct.values\n\nmax_score = 0; best_threshold = 0\n\n# Loop through threshold values from 0.4 to 0.8 and select the threshold with \n# the highest `F1 score`.\nfor threshold in np.arange(0.4,0.8,0.01):\n    metric = tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=threshold)\n    y_true = tf.one_hot(true_df.values.reshape((-1)), depth=2)\n    y_pred = tf.one_hot((prediction_df.values.reshape((-1))>threshold).astype('int'), depth=2)\n    metric.update_state(y_true, y_pred)\n    f1_score = metric.result().numpy()\n    if f1_score > max_score:\n        max_score = f1_score\n        best_threshold = threshold\n        \nprint(\"Best threshold \", best_threshold, \"\\tF1 score \", max_score)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:21:26.240732Z","iopub.execute_input":"2023-05-16T15:21:26.241672Z","iopub.status.idle":"2023-05-16T15:21:27.676910Z","shell.execute_reply.started":"2023-05-16T15:21:26.241608Z","shell.execute_reply":"2023-05-16T15:21:27.675729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference\n# https://www.kaggle.com/code/philculliton/basic-submission-demo\n# https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    test_df = feature_engineer(test)\n    grp = test_df.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        gbtm = models[f'{grp}_{t}']\n        test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_df.loc[:, test_df.columns != 'level_group'])\n        predictions = gbtm.predict(test_ds)\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        n_predictions = (predictions > best_threshold).astype(int)\n        sample_submission.loc[mask,'correct'] = n_predictions.flatten()\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T15:21:27.679102Z","iopub.execute_input":"2023-05-16T15:21:27.679509Z","iopub.status.idle":"2023-05-16T15:21:47.718165Z","shell.execute_reply.started":"2023-05-16T15:21:27.679472Z","shell.execute_reply":"2023-05-16T15:21:47.716448Z"},"trusted":true},"execution_count":null,"outputs":[]}]}