{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score\nimport matplotlib.pyplot as plt","metadata":{"papermill":{"duration":1.027875,"end_time":"2023-02-07T00:59:59.180261","exception":false,"start_time":"2023-02-07T00:59:58.152386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:38:36.649038Z","iopub.execute_input":"2023-05-11T02:38:36.649939Z","iopub.status.idle":"2023-05-11T02:38:37.185270Z","shell.execute_reply.started":"2023-05-11T02:38:36.649866Z","shell.execute_reply":"2023-05-11T02:38:37.184232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data and Labels\nthis process was taken from the process describe in the following kaggle thread:\nhttps://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/397688\n\nThe idea is to read only the sessions as ids as chunks, then use those chunks to individually load in the data and reduce the number of rows with our synthesis function that will be defined later\n\nBy doing this we avoid handling a super large amount of data all at once, which keeps us from having any memory errors. We still manage to use all the memory available, but at least with this method we don't crash the program.","metadata":{"papermill":{"duration":0.004542,"end_time":"2023-02-07T00:59:59.189777","exception":false,"start_time":"2023-02-07T00:59:59.185235","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# get the user ids\nuser_ids = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",usecols=[0])\nuser_ids = user_ids.groupby('session_id').session_id.agg('count')\n\n# COMPUTE READS AND SKIPS\nPIECES = 10\nCHUNK = int(np.ceil(len(user_ids)/PIECES))\n\nreads = []\nskips = [0]\n\n# cycle through the chunks and add each piece size to reads and skips arrays\nfor k in range(PIECES):\n    start = k*CHUNK\n    end = (k+1)*CHUNK\n    if end>len(user_ids): end=len(user_ids)\n    read_section = user_ids.iloc[start:end].sum()\n    reads.append(read_section)\n    skips.append(skips[-1]+read_section)\n    \ndel user_ids; gc.collect()\n    \nprint('reads:\\n',reads)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-11T02:38:37.190310Z","iopub.execute_input":"2023-05-11T02:38:37.191158Z","iopub.status.idle":"2023-05-11T02:39:22.667980Z","shell.execute_reply.started":"2023-05-11T02:38:37.191123Z","shell.execute_reply":"2023-05-11T02:39:22.667022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example of a chunk\ntrain = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', nrows=reads[0])\nprint('Train size of first piece:', train.shape )\ntrain.head()","metadata":{"papermill":{"duration":59.284316,"end_time":"2023-02-07T01:00:58.478743","exception":false,"start_time":"2023-02-07T00:59:59.194427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:39:22.669211Z","iopub.execute_input":"2023-05-11T02:39:22.669999Z","iopub.status.idle":"2023-05-11T02:39:31.911564Z","shell.execute_reply.started":"2023-05-11T02:39:22.669932Z","shell.execute_reply":"2023-05-11T02:39:31.910681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Labels\nbecause it's much smaller we don't need to load this in chunks","metadata":{}},{"cell_type":"code","source":"# Label data\nlabels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\n# splitting session id/question pair into separate columns\nlabels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\nlabels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nprint( labels.shape )\nlabels.head()","metadata":{"papermill":{"duration":0.598155,"end_time":"2023-02-07T01:00:59.082015","exception":false,"start_time":"2023-02-07T01:00:58.48386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:39:31.914230Z","iopub.execute_input":"2023-05-11T02:39:31.915398Z","iopub.status.idle":"2023-05-11T02:39:33.528166Z","shell.execute_reply.started":"2023-05-11T02:39:31.915360Z","shell.execute_reply":"2023-05-11T02:39:33.527034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\nWe create basic aggregate features.\nbasic feature ideas taken from:\nhttps://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data","metadata":{"papermill":{"duration":0.005196,"end_time":"2023-02-07T01:00:59.092865","exception":false,"start_time":"2023-02-07T01:00:59.087669","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CATEGORIES = ['event_name', 'fqid', 'room_fqid', 'text']\nNUMBERS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']\n\n# https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data\nEVENTS = ['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'checkpoint']","metadata":{"papermill":{"duration":0.014685,"end_time":"2023-02-07T01:00:59.112856","exception":false,"start_time":"2023-02-07T01:00:59.098171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:39:33.529560Z","iopub.execute_input":"2023-05-11T02:39:33.530182Z","iopub.status.idle":"2023-05-11T02:39:33.536553Z","shell.execute_reply.started":"2023-05-11T02:39:33.530137Z","shell.execute_reply":"2023-05-11T02:39:33.535395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    engineered_dfs = []\n    # for every session/group pair in the given training df, do the following:\n    # get number of unique events for every cateogry\n    for category in CATEGORIES:\n        tmp = train.groupby(['session_id','level_group'])[category].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        engineered_dfs.append(tmp)\n    \n    # get the mean and standard deviation for each number event\n    for number in NUMBERS:\n        tmp = train.groupby(['session_id','level_group'])[number].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        engineered_dfs.append(tmp)\n    \n    # get standard deviation for each number event\n    for number in NUMBERS:\n        tmp = train.groupby(['session_id','level_group'])[number].agg('std')\n        tmp.name = tmp.name + '_std'\n        engineered_dfs.append(tmp)\n    \n    # convert every event type to an int (technically a bool value) for better aggregation\n    for event in EVENTS: \n        train[event] = (train.event_name == event).astype('int8')\n    \n    # get sum of event types for each event (note that this is basically a count, since we converted to 1/0 varlues excluding elapsed time)\n    for event in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[event].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        engineered_dfs.append(tmp)\n    \n    # drop default events\n    train = train.drop(EVENTS,axis=1)\n    \n    # combine all the engineered dataframes\n    df = pd.concat(engineered_dfs,axis=1)\n    df = df.fillna(-1) # magic value of -1\n    df = df.reset_index()\n    df = df.set_index('session_id') # this allows us to call df.loc[[list of session ids]]\n    return df","metadata":{"papermill":{"duration":0.017716,"end_time":"2023-02-07T01:00:59.136021","exception":false,"start_time":"2023-02-07T01:00:59.118305","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:39:33.538318Z","iopub.execute_input":"2023-05-11T02:39:33.538664Z","iopub.status.idle":"2023-05-11T02:39:33.552889Z","shell.execute_reply.started":"2023-05-11T02:39:33.538631Z","shell.execute_reply":"2023-05-11T02:39:33.551638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process training data as their separate pieces, reduce space by feature engineering\n# KENTATANAKA shared this code and is described https://www.kaggle.com/code/kentatanaka1data/the-training-dataset-into-small-chunks/notebook\nall_pieces = []\nprint(f'Processing train as {PIECES} pieces to avoid memory error... ')\nfor k in range(PIECES):\n    print(f'{k}, ',end='')\n    # SKIPS just helps manage which rows were processing\n    SKIPS = 0\n    if k > 0: \n        SKIPS = range(1,skips[k]+1)\n    train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',\n                        nrows=reads[k], skiprows=SKIPS)\n    df = feature_engineer(train)\n    all_pieces.append(df)\n    \n# Concatenate the now reduced dfs\nprint('\\n')\n\ndel train; gc.collect()\ndf = pd.concat(all_pieces, axis=0)\nprint('Shape of all train data after feature engineering:', df.shape )\ndf.head()","metadata":{"papermill":{"duration":34.516494,"end_time":"2023-02-07T01:01:33.658043","exception":false,"start_time":"2023-02-07T01:00:59.141549","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:39:33.554524Z","iopub.execute_input":"2023-05-11T02:39:33.554893Z","iopub.status.idle":"2023-05-11T02:46:51.048301Z","shell.execute_reply.started":"2023-05-11T02:39:33.554859Z","shell.execute_reply":"2023-05-11T02:46:51.047161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the idea of using validation to set our threshold is inspired by discussions and code throughout kaggle, but not copied.\n# https://www.kaggle.com/code/mayukh18/lgbm-regressor-ensemble-pipeline is an example, though this person uses KFold cross validation\n# which is something we'd hoped to implement in the future\n\n# all features excluding level group\nFEATURES = list(df.columns.drop(['level_group']))\n# getting the list of unique users and their count\nunique_users = df.index.unique()\nNUM_USERS = unique_users.size\n\n# set up our training and validation sets\nTRAIN_SIZE = 0.8\nTRAIN_INDEX = int(NUM_USERS * TRAIN_SIZE) # gets the index at the 80% point\n\n# get training session ids\ntrain_users = unique_users[:TRAIN_INDEX]\n# get validation session ids\nvalidation_users = unique_users[TRAIN_INDEX:]\n\n# get session data for training users and validation users\ntrain_df = df.loc[train_users]\ntrain_validation_df = df.loc[validation_users]\n\n# get labels according to the training and validation users\nlabels = labels.set_index('session')\nlabels_df = labels.loc[train_users]\nlabels_validation_df = labels.loc[validation_users]\n\n# set up an empty cross validation df\ncross_validation_df = pd.DataFrame(data=np.zeros((validation_users.size,18)), index=validation_users)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:46:51.049911Z","iopub.execute_input":"2023-05-11T02:46:51.050298Z","iopub.status.idle":"2023-05-11T02:46:51.641445Z","shell.execute_reply.started":"2023-05-11T02:46:51.050264Z","shell.execute_reply":"2023-05-11T02:46:51.640253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df; gc.collect() # garbage collection to manage our data as closely as we can","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:46:51.642983Z","iopub.execute_input":"2023-05-11T02:46:51.643436Z","iopub.status.idle":"2023-05-11T02:46:51.779930Z","shell.execute_reply.started":"2023-05-11T02:46:51.643403Z","shell.execute_reply":"2023-05-11T02:46:51.778482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a map of models, one for each question\nmodels = {}\n\n# xgboosting hyperparameters\n# params for this library are explained in https://xgboost.readthedocs.io/en/stable/python/python_intro.html#setting-parameters\nxgb_params = {\n    'objective': 'binary:logistic',\n    'eval_metric': 'logloss',\n    'learning_rate': 0.05,\n    'max_depth': 4,\n    'n_estimators': 1000,\n    'early_stopping_rounds': 50, # we could optimize this in the future by using an average round rate \n    'tree_method': 'hist',\n    'subsample': 0.8,\n    'colsample_bytree': 0.4,\n    'use_label_encoder': False\n}\n\n# iterate through all 18 questions\nprint('starting')\nfor question_number in range(1, 19):\n    print(f'{question_number}, ', end='')\n    # because we reduced the data according the level group and session id, we have to identify the level group we are in given a question\n    # this connects the labeled data to our training/validation data\n    if question_number<=3: level_group = '0-4'\n    elif question_number<=13: level_group = '5-12'\n    elif question_number<=22: level_group = '13-22'\n\n    # train data\n    train_x = train_df.loc[train_df.level_group == level_group]\n    valid_x = train_validation_df.loc[train_validation_df.level_group == level_group]\n\n    # test data relative to current question\n    train_y = labels_df.loc[labels_df.q == question_number]\n    valid_y = labels_validation_df.loc[labels_validation_df.q == question_number]\n    \n    # set up model - for this we just use the default xgb parameters\n    model = XGBClassifier(**xgb_params)\n    model.fit(train_x[FEATURES].astype('float32'), train_y['correct'],\n             eval_set=[(valid_x[FEATURES].astype('float32'), valid_y['correct'])], # eval set allows for early stopping\n             verbose=False)\n    \n    # this allows us to compute an f1 score\n    cross_validation_df.loc[validation_users, question_number - 1] = model.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n\n    # map model to question\n    models[question_number] = model\n        \n        \n        \n        \n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:46:51.781891Z","iopub.execute_input":"2023-05-11T02:46:51.782401Z","iopub.status.idle":"2023-05-11T02:47:24.979037Z","shell.execute_reply.started":"2023-05-11T02:46:51.782367Z","shell.execute_reply":"2023-05-11T02:47:24.977829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# probability score of each question model and user\nprint(cross_validation_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:47:24.980756Z","iopub.execute_input":"2023-05-11T02:47:24.981490Z","iopub.status.idle":"2023-05-11T02:47:24.997006Z","shell.execute_reply.started":"2023-05-11T02:47:24.981451Z","shell.execute_reply":"2023-05-11T02:47:24.996035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comput Cross Validation Score","metadata":{}},{"cell_type":"code","source":"# get the actual values for each of the predicted values as a binary matrix\ntrue_values = cross_validation_df.copy()\nfor k in range(18):\n    tmp = labels.loc[labels.q == k+1].loc[validation_users]\n    true_values[k] = tmp.correct.values","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:47:24.998680Z","iopub.execute_input":"2023-05-11T02:47:24.999349Z","iopub.status.idle":"2023-05-11T02:47:25.080608Z","shell.execute_reply.started":"2023-05-11T02:47:24.999312Z","shell.execute_reply":"2023-05-11T02:47:25.079481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find the best threshold for the known true values\nscores = []\nthresholds = []\nbest_score = 0; best_threshold = 0\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    # get predictions gerater than thre given threshold (as a list)\n    predictions = (cross_validation_df.values.reshape((-1))>threshold).astype('int')\n    # compute the f1 score given the actual values and the predictions\n    validation_score = f1_score(true_values.values.reshape((-1)), predictions, average='macro')   \n    \n    # track the scores for plotting purposes\n    scores.append(validation_score)\n    thresholds.append(threshold)\n    \n    # track best threshold (and score) so far\n    if validation_score > best_score:\n        best_score = validation_score\n        best_threshold = threshold","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:47:25.084451Z","iopub.execute_input":"2023-05-11T02:47:25.085137Z","iopub.status.idle":"2023-05-11T02:47:26.598147Z","shell.execute_reply.started":"2023-05-11T02:47:25.085094Z","shell.execute_reply":"2023-05-11T02:47:26.597003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot threshold/scores\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score (best score highlighted) = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T02:47:26.599487Z","iopub.execute_input":"2023-05-11T02:47:26.599912Z","iopub.status.idle":"2023-05-11T02:47:26.836655Z","shell.execute_reply.started":"2023-05-11T02:47:26.599878Z","shell.execute_reply":"2023-05-11T02:47:26.834025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{"papermill":{"duration":0.011075,"end_time":"2023-02-07T01:02:44.400918","exception":false,"start_time":"2023-02-07T01:02:44.389843","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# kaggle api\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\n# garbage collect no longer needed training/label sets\ndel train_df, labels_df; gc.collect()","metadata":{"papermill":{"duration":0.052132,"end_time":"2023-02-07T01:02:44.464739","exception":false,"start_time":"2023-02-07T01:02:44.412607","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:47:26.837996Z","iopub.execute_input":"2023-05-11T02:47:26.838336Z","iopub.status.idle":"2023-05-11T02:47:26.988953Z","shell.execute_reply.started":"2023-05-11T02:47:26.838305Z","shell.execute_reply":"2023-05-11T02:47:26.987662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level_map = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nprint(\"starting inference\")\nfor (test, sample_submission) in iter_test:\n    # reduce new data with feature engineering and synthesis \n    new_df = feature_engineer(test)\n    \n    # infer new data\n    level_group = test.level_group.values[0]\n    \n    # find the level group in which it's in\n    a,b = level_map[level_group]\n    \n    # cycle through questions in level group and predict accordingly\n    for question_number in range(a,b):\n        question_model = models[question_number]\n        # get probability question (given as a 1x2 matrix, left column = prob 0, right = prob 1)\n        prob_question_correct = question_model.predict_proba(new_df[FEATURES].astype('float32'))[0,1]\n        \n        # single out the user session/question number pair row so we can change the value\n        user_question_id = sample_submission.session_id.str.contains(f'q{question_number}')\n        \n        # assign 0 or 1 according to the best found f1-score based threshold\n        sample_submission.loc[user_question_id,'correct'] = int(prob_question_correct > best_threshold)\n    \n    # apply sample_submission changes to environment (not actual prediction, just submission)\n    env.predict(sample_submission)","metadata":{"papermill":{"duration":1.002014,"end_time":"2023-02-07T01:02:45.47927","exception":false,"start_time":"2023-02-07T01:02:44.477256","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:47:26.990757Z","iopub.execute_input":"2023-05-11T02:47:26.991455Z","iopub.status.idle":"2023-05-11T02:47:27.942598Z","shell.execute_reply.started":"2023-05-11T02:47:26.991419Z","shell.execute_reply":"2023-05-11T02:47:27.941604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA submission.csv","metadata":{"papermill":{"duration":0.011427,"end_time":"2023-02-07T01:02:45.502331","exception":false,"start_time":"2023-02-07T01:02:45.490904","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# verify submission changes\ndf = pd.read_csv('submission.csv')\nprint( df.shape )\ndf","metadata":{"papermill":{"duration":0.027432,"end_time":"2023-02-07T01:02:45.541022","exception":false,"start_time":"2023-02-07T01:02:45.51359","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-11T02:47:27.943985Z","iopub.execute_input":"2023-05-11T02:47:27.944294Z","iopub.status.idle":"2023-05-11T02:47:27.961816Z","shell.execute_reply.started":"2023-05-11T02:47:27.944263Z","shell.execute_reply":"2023-05-11T02:47:27.960874Z"},"trusted":true},"execution_count":null,"outputs":[]}]}