{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Student Performance from Game Play Using Transformers\n---\n\n","metadata":{"id":"iCTA6kl3n2O1","papermill":{"duration":0.02507,"end_time":"2023-06-24T09:19:00.258657","exception":false,"start_time":"2023-06-24T09:19:00.233587","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Import the Required Libraries","metadata":{"id":"zAXHC6-Tn2O5","papermill":{"duration":0.024035,"end_time":"2023-06-24T09:19:00.305433","exception":false,"start_time":"2023-06-24T09:19:00.281398","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd","metadata":{"papermill":{"duration":1.192477,"end_time":"2023-06-24T09:19:01.520367","exception":false,"start_time":"2023-06-24T09:19:00.327890","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:49.660416Z","iopub.execute_input":"2023-06-26T08:16:49.660950Z","iopub.status.idle":"2023-06-26T08:16:49.679558Z","shell.execute_reply.started":"2023-06-26T08:16:49.660918Z","shell.execute_reply":"2023-06-26T08:16:49.678551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\nstrategy = tf.distribute.TPUStrategy(resolver)\n\nprint('Number of devices: {}'.format(strategy.num_replicas_in_sync))","metadata":{"papermill":{"duration":490.666427,"end_time":"2023-06-24T09:27:12.211588","exception":false,"start_time":"2023-06-24T09:19:01.545161","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:49.682011Z","iopub.execute_input":"2023-06-26T08:16:49.682673Z","iopub.status.idle":"2023-06-26T08:16:49.691913Z","shell.execute_reply.started":"2023-06-26T08:16:49.682638Z","shell.execute_reply":"2023-06-26T08:16:49.690939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n# import tensorflow_addons as tfa\n# import tensorflow_decision_forests as tfdf\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import f1_score\n# from imblearn.over_sampling import RandomOverSampler\n# from imblearn.under_sampling import RandomUnderSampler\n# from imblearn.over_sampling import SMOTE\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport sys\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.preprocessing import StandardScaler\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import OrdinalEncoder,StandardScaler,MinMaxScaler\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom sklearn.metrics import f1_score\nfrom keras.callbacks import Callback,ModelCheckpoint\nfrom keras.models import Sequential,load_model\nfrom keras.layers import Dense, Dropout\nfrom keras.wrappers.scikit_learn import KerasClassifier\nimport keras.backend as K\nimport pickle\nimport os\nimport gc\n\nfrom keras.utils import register_keras_serializable","metadata":{"id":"IanlX-Eqn2O5","papermill":{"duration":2.451388,"end_time":"2023-06-24T09:27:14.688261","exception":false,"start_time":"2023-06-24T09:27:12.236873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:49.693396Z","iopub.execute_input":"2023-06-26T08:16:49.694121Z","iopub.status.idle":"2023-06-26T08:16:58.941143Z","shell.execute_reply.started":"2023-06-26T08:16:49.694085Z","shell.execute_reply":"2023-06-26T08:16:58.940105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.random import seed\nseed(1123) \nimport random as pyrandom\npyrandom.seed(1123)","metadata":{"papermill":{"duration":0.033638,"end_time":"2023-06-24T09:27:14.746927","exception":false,"start_time":"2023-06-24T09:27:14.713289","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:58.942663Z","iopub.execute_input":"2023-06-26T08:16:58.943568Z","iopub.status.idle":"2023-06-26T08:16:58.949354Z","shell.execute_reply.started":"2023-06-26T08:16:58.943529Z","shell.execute_reply":"2023-06-26T08:16:58.948082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce Memory Usage\n# reference : https://www.kaggle.com/code/arjanso/reducing-dataframe-memory-size-by-65 @ARJANGROEN\n\ndef reduce_memory_usage(df):\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype.name\n        if ((col_type != 'datetime64[ns]') & (col_type != 'category')):\n            if (col_type != 'object'):\n                c_min = df[col].min()\n                c_max = df[col].max()\n\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)\n\n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        pass\n            else:\n                df[col] = df[col].astype('category')\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage became: \",mem_usg,\" MB\")\n    \n    return df","metadata":{"papermill":{"duration":0.046834,"end_time":"2023-06-24T09:27:14.820317","exception":false,"start_time":"2023-06-24T09:27:14.773483","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:58.952703Z","iopub.execute_input":"2023-06-26T08:16:58.953889Z","iopub.status.idle":"2023-06-26T08:16:58.968709Z","shell.execute_reply.started":"2023-06-26T08:16:58.953853Z","shell.execute_reply":"2023-06-26T08:16:58.967740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"TensorFlow Decision Forests v\" + tfdf.__version__)\n# print(\"TensorFlow Addons v\" + tfa.__version__)\n# print(\"TensorFlow v\" + tf.__version__)","metadata":{"id":"gLpK2yAen2O7","papermill":{"duration":0.032866,"end_time":"2023-06-24T09:27:14.879447","exception":false,"start_time":"2023-06-24T09:27:14.846581","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:58.971139Z","iopub.execute_input":"2023-06-26T08:16:58.971894Z","iopub.status.idle":"2023-06-26T08:16:58.983441Z","shell.execute_reply.started":"2023-06-26T08:16:58.971860Z","shell.execute_reply":"2023-06-26T08:16:58.982519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/384359\ndtypes={\n    'elapsed_time':np.int32,\n    'event_name':'category',\n    'name':'category',\n    'level':np.uint8,\n    'room_coor_x':np.float32,\n    'room_coor_y':np.float32,\n    'screen_coor_x':np.float32,\n    'screen_coor_y':np.float32,\n    'hover_duration':np.float32,\n    'text':'category',\n    'fqid':'category',\n    'room_fqid':'category',\n    'text_fqid':'category',\n    'fullscreen':'category',\n    'hq':'category',\n    'music':'category',\n    'level_group':'category'}\n\ndataset_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', \n                         dtype=dtypes) #,nrows=1000000)\nprint(\"Full train dataset shape is {}\".format(dataset_df.shape))\n\n#809 812","metadata":{"papermill":{"duration":101.864757,"end_time":"2023-06-24T09:28:56.769358","exception":false,"start_time":"2023-06-24T09:27:14.904601","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:16:58.985452Z","iopub.execute_input":"2023-06-26T08:16:58.986343Z","iopub.status.idle":"2023-06-26T08:17:03.155997Z","shell.execute_reply.started":"2023-06-26T08:16:58.986306Z","shell.execute_reply":"2023-06-26T08:17:03.153973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nprint(sys.getsizeof(dataset_df))\nreduce_memory_usage(dataset_df)","metadata":{"papermill":{"duration":2.940757,"end_time":"2023-06-24T09:28:59.734547","exception":false,"start_time":"2023-06-24T09:28:56.793790","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.157799Z","iopub.execute_input":"2023-06-26T08:17:03.158109Z","iopub.status.idle":"2023-06-26T08:17:03.329291Z","shell.execute_reply.started":"2023-06-26T08:17:03.158083Z","shell.execute_reply":"2023-06-26T08:17:03.328112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.drop(['fullscreen','hq','music','text'],inplace=True, axis=1)","metadata":{"papermill":{"duration":2.004132,"end_time":"2023-06-24T09:29:01.765095","exception":false,"start_time":"2023-06-24T09:28:59.760963","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.330972Z","iopub.execute_input":"2023-06-26T08:17:03.331447Z","iopub.status.idle":"2023-06-26T08:17:03.393859Z","shell.execute_reply.started":"2023-06-26T08:17:03.331409Z","shell.execute_reply":"2023-06-26T08:17:03.392896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.info()","metadata":{"papermill":{"duration":0.041905,"end_time":"2023-06-24T09:29:01.833146","exception":false,"start_time":"2023-06-24T09:29:01.791241","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.395230Z","iopub.execute_input":"2023-06-26T08:17:03.395593Z","iopub.status.idle":"2023-06-26T08:17:03.449774Z","shell.execute_reply.started":"2023-06-26T08:17:03.395560Z","shell.execute_reply":"2023-06-26T08:17:03.448722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CATEGORICAL = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\n# NUMERICAL = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n#         'screen_coor_x', 'screen_coor_y', 'hover_duration']\n# def feature_engineer1(dataset_df):\n#     dfs = []\n#     for c in CATEGORICAL:\n#         tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('nunique')\n#         tmp.name = tmp.name + '_nunique'\n#         dfs.append(tmp)\n#     for c in NUMERICAL:\n#         tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('mean')\n#         dfs.append(tmp)\n#     for c in NUMERICAL:\n#         tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('std')\n#         tmp.name = tmp.name + '_std'\n#         dfs.append(tmp)\n#     dataset_df = pd.concat(dfs,axis=1)\n#     dataset_df = dataset_df.fillna(-1)\n#     dataset_df = dataset_df.reset_index()\n#     dataset_df = dataset_df.set_index('session_id')\n#     return dataset_df","metadata":{"papermill":{"duration":0.03598,"end_time":"2023-06-24T09:29:01.896776","exception":false,"start_time":"2023-06-24T09:29:01.860796","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.451584Z","iopub.execute_input":"2023-06-26T08:17:03.451939Z","iopub.status.idle":"2023-06-26T08:17:03.457236Z","shell.execute_reply.started":"2023-06-26T08:17:03.451907Z","shell.execute_reply":"2023-06-26T08:17:03.455969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset_df2 = feature_engineer1(dataset_df)\n# cols2 = ['level_group','elapsed_time_std','level_std','page_std','room_coor_x_std','room_coor_y_std',\n#          'screen_coor_x_std','screen_coor_y_std','hover_duration_std']\n# df2 = dataset_df2[cols2]","metadata":{"papermill":{"duration":0.032996,"end_time":"2023-06-24T09:29:01.956710","exception":false,"start_time":"2023-06-24T09:29:01.923714","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.459051Z","iopub.execute_input":"2023-06-26T08:17:03.459441Z","iopub.status.idle":"2023-06-26T08:17:03.468219Z","shell.execute_reply.started":"2023-06-26T08:17:03.459391Z","shell.execute_reply":"2023-06-26T08:17:03.467040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# d2=df2[(df2.index==20090312431273200) & (df2.level_group=='0-4')]\n# d2.drop('level_group',inplace=True, axis=1)\n# d2\n# d1=dataset_df[(dataset_df.session_id==20090312431273200) & (dataset_df.level_group=='0-4')]\n\n# d4= pd.concat([d2] * len(d1), ignore_index=True)\n# d3 = pd.concat([d1.reset_index(drop=True), d4.reindex(d1.index)], axis=1)","metadata":{"papermill":{"duration":0.03337,"end_time":"2023-06-24T09:29:02.015856","exception":false,"start_time":"2023-06-24T09:29:01.982486","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.474286Z","iopub.execute_input":"2023-06-26T08:17:03.474595Z","iopub.status.idle":"2023-06-26T08:17:03.480209Z","shell.execute_reply.started":"2023-06-26T08:17:03.474569Z","shell.execute_reply":"2023-06-26T08:17:03.479130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# d4= pd.concat([d2] * len(d1), ignore_index=True)\n# d3 = pd.concat([d1, d4.reindex(d1.index)], axis=1)\n\n\n# d3 = pd.concat([d1, d4.reindex(d1.index)], axis=1)\n# d3\n\n# d3 = pd.concat([d1.reset_index(drop=True), d2.reindex(d1.index)], axis=1)\n# d3","metadata":{"papermill":{"duration":0.033312,"end_time":"2023-06-24T09:29:02.074682","exception":false,"start_time":"2023-06-24T09:29:02.041370","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.481628Z","iopub.execute_input":"2023-06-26T08:17:03.482298Z","iopub.status.idle":"2023-06-26T08:17:03.499934Z","shell.execute_reply.started":"2023-06-26T08:17:03.482265Z","shell.execute_reply":"2023-06-26T08:17:03.498261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# d4","metadata":{"papermill":{"duration":0.034726,"end_time":"2023-06-24T09:29:02.137046","exception":false,"start_time":"2023-06-24T09:29:02.102320","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.501257Z","iopub.execute_input":"2023-06-26T08:17:03.501695Z","iopub.status.idle":"2023-06-26T08:17:03.510043Z","shell.execute_reply.started":"2023-06-26T08:17:03.501659Z","shell.execute_reply":"2023-06-26T08:17:03.507952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# d3 = pd.concat([d1.reset_index(drop=True), d4.reindex(d1.index)], axis=1)\n# d3","metadata":{"papermill":{"duration":0.032804,"end_time":"2023-06-24T09:29:02.197305","exception":false,"start_time":"2023-06-24T09:29:02.164501","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.511207Z","iopub.execute_input":"2023-06-26T08:17:03.511534Z","iopub.status.idle":"2023-06-26T08:17:03.521418Z","shell.execute_reply.started":"2023-06-26T08:17:03.511494Z","shell.execute_reply":"2023-06-26T08:17:03.520090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.sum(d3.isna())","metadata":{"papermill":{"duration":0.032259,"end_time":"2023-06-24T09:29:02.255119","exception":false,"start_time":"2023-06-24T09:29:02.222860","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.522857Z","iopub.execute_input":"2023-06-26T08:17:03.523224Z","iopub.status.idle":"2023-06-26T08:17:03.537858Z","shell.execute_reply.started":"2023-06-26T08:17:03.523188Z","shell.execute_reply":"2023-06-26T08:17:03.536324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATEGORICAL = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMERICAL = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']","metadata":{"papermill":{"duration":0.033361,"end_time":"2023-06-24T09:29:02.316238","exception":false,"start_time":"2023-06-24T09:29:02.282877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.548815Z","iopub.execute_input":"2023-06-26T08:17:03.549148Z","iopub.status.idle":"2023-06-26T08:17:03.559949Z","shell.execute_reply.started":"2023-06-26T08:17:03.549117Z","shell.execute_reply":"2023-06-26T08:17:03.558686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in CATEGORICAL:\n    dataset_df[column].cat.add_categories('undefined1').fillna('undefined1', inplace=True)","metadata":{"papermill":{"duration":0.394343,"end_time":"2023-06-24T09:29:02.736295","exception":false,"start_time":"2023-06-24T09:29:02.341952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.564735Z","iopub.execute_input":"2023-06-26T08:17:03.565351Z","iopub.status.idle":"2023-06-26T08:17:03.596250Z","shell.execute_reply.started":"2023-06-26T08:17:03.565325Z","shell.execute_reply":"2023-06-26T08:17:03.595270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in NUMERICAL:\n    dataset_df[column].fillna(0,inplace=True,axis=0) \n    \ndataset_df.isna().sum()","metadata":{"papermill":{"duration":1.548728,"end_time":"2023-06-24T09:29:04.311699","exception":false,"start_time":"2023-06-24T09:29:02.762971","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.600097Z","iopub.execute_input":"2023-06-26T08:17:03.600683Z","iopub.status.idle":"2023-06-26T08:17:03.706008Z","shell.execute_reply.started":"2023-06-26T08:17:03.600650Z","shell.execute_reply":"2023-06-26T08:17:03.705126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['fqid']=dataset_df['fqid'].cat.add_categories('None').fillna('None') ","metadata":{"papermill":{"duration":0.194436,"end_time":"2023-06-24T09:29:04.532133","exception":false,"start_time":"2023-06-24T09:29:04.337697","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.716194Z","iopub.execute_input":"2023-06-26T08:17:03.718593Z","iopub.status.idle":"2023-06-26T08:17:03.734181Z","shell.execute_reply.started":"2023-06-26T08:17:03.718556Z","shell.execute_reply":"2023-06-26T08:17:03.733270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['text_fqid']=dataset_df['text_fqid'].cat.add_categories('None').fillna('None')","metadata":{"papermill":{"duration":0.179511,"end_time":"2023-06-24T09:29:04.737343","exception":false,"start_time":"2023-06-24T09:29:04.557832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.738488Z","iopub.execute_input":"2023-06-26T08:17:03.741119Z","iopub.status.idle":"2023-06-26T08:17:03.754352Z","shell.execute_reply.started":"2023-06-26T08:17:03.741085Z","shell.execute_reply":"2023-06-26T08:17:03.753498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.isna().sum()","metadata":{"papermill":{"duration":0.808138,"end_time":"2023-06-24T09:29:05.572270","exception":false,"start_time":"2023-06-24T09:29:04.764132","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.758903Z","iopub.execute_input":"2023-06-26T08:17:03.761133Z","iopub.status.idle":"2023-06-26T08:17:03.814685Z","shell.execute_reply.started":"2023-06-26T08:17:03.761098Z","shell.execute_reply":"2023-06-26T08:17:03.813834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\ndr = '/kaggle/working/models'\n\nif not os.path.exists(dr):\n    os.mkdir(r'/kaggle/working/models')","metadata":{"papermill":{"duration":0.034332,"end_time":"2023-06-24T09:29:05.633309","exception":false,"start_time":"2023-06-24T09:29:05.598977","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.818947Z","iopub.execute_input":"2023-06-26T08:17:03.821249Z","iopub.status.idle":"2023-06-26T08:17:03.828078Z","shell.execute_reply.started":"2023-06-26T08:17:03.821213Z","shell.execute_reply":"2023-06-26T08:17:03.826924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nprint(sys.getsizeof(dataset_df))\nreduce_memory_usage(dataset_df)","metadata":{"papermill":{"duration":5.097435,"end_time":"2023-06-24T09:29:10.756492","exception":false,"start_time":"2023-06-24T09:29:05.659057","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:03.832523Z","iopub.execute_input":"2023-06-26T08:17:03.833214Z","iopub.status.idle":"2023-06-26T08:17:04.153621Z","shell.execute_reply.started":"2023-06-26T08:17:03.833181Z","shell.execute_reply":"2023-06-26T08:17:04.152701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# neg, pos = np.bincount(lab['correct'])\n# total = neg + pos\n# # Scaling by total/2 helps keep the loss to a similar magnitude.\n# # The sum of the weights of all examples stays the same.\n# weight_for_0 = (1 / neg) * (total / 2.0)\n# weight_for_1 = (1 / pos) * (total / 2.0)\n\n# class_weight = {0: weight_for_0, 1: weight_for_1}\n\n# print('Weight for class 0: {:.2f}'.format(weight_for_0))\n# print('Weight for class 1: {:.2f}'.format(weight_for_1))","metadata":{"papermill":{"duration":0.034048,"end_time":"2023-06-24T09:29:10.817429","exception":false,"start_time":"2023-06-24T09:29:10.783381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:04.158295Z","iopub.execute_input":"2023-06-26T08:17:04.158961Z","iopub.status.idle":"2023-06-26T08:17:04.167801Z","shell.execute_reply.started":"2023-06-26T08:17:04.158924Z","shell.execute_reply":"2023-06-26T08:17:04.166425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_multi_game_naive(\n    df: pd.DataFrame,\n    local: bool = True\n) -> pd.DataFrame:\n    \"\"\"Drop events not occurring at the first game play.\n    \n    Note: `groupby` should be done with `sort=False` for the new\n    training set, because `session_id` isn't sorted by default.\n    \n    Parameters:\n        df: input DataFrame\n        local: if False, the processing step is simplified based on the\n            properties of returned test DataFrame by time series API\n    \n    Return:\n        df: DataFrame with events occurring at the first game play only\n    \"\"\"\n    df = df.copy()\n    if local:\n        df[\"lv_diff\"] = df.groupby(\"session_id\", sort=False).apply(lambda x: x[\"level\"].diff().fillna(0)).values\n    else:\n        df[\"lv_diff\"] = df[\"level\"].diff().fillna(0)\n    reversed_lv_pts = df[\"lv_diff\"] < 0\n    df.loc[~reversed_lv_pts, \"lv_diff\"] = 0\n    if local:\n        df[\"multi_game_flag\"] = df.groupby(\"session_id\", sort=False)[\"lv_diff\"].cumsum().values\n    else:\n        df[\"multi_game_flag\"] = df[\"lv_diff\"].cumsum()\n    multi_game_mask = df[\"multi_game_flag\"] < 0\n    multi_game_rows = df[multi_game_mask].index\n    df = df.drop(multi_game_rows).reset_index(drop=True)\n    \n    # Drop redundant columns\n    df.drop([\"lv_diff\", \"multi_game_flag\"], axis=1, inplace=True)\n    \n    return df\n\ndef map_lvgp_order(lvgp_seq: pd.Series) -> pd.Series:\n    \"\"\"Map level_group sequence to level_group order sequence.\n    \n    Parameters:\n        lvgp_seq: level_group sequence\n    \n    Return:\n        lvgp_order_seq: level_group order sequence\n    \"\"\"\n    lvgp_order_seq = lvgp_seq.map(LVGP_ORDER)\n    \n    return lvgp_order_seq\n\n# def check_multi_game(df: pd.DataFrame) -> bool:\n#     \"\"\"Check if multiple game plays exist in any session.\n    \n#     Parameters:\n#         df: input DataFrame\n    \n#     Return:\n#         multi_game_exist: if True, multiple game plays exist in at\n#             least one of the session\n#     \"\"\"\n#     multi_game_exist = False\n#     for i, (sess_id, gp) in enumerate(df.groupby(\"session_id\", sort=False)):\n#         if ((not gp[\"level\"].is_monotonic_increasing)\n#             or (not gp[\"lvgp_order\"].is_monotonic_increasing)):\n#             multi_game_exist = True\n#             break\n            \n#     return multi_game_exist","metadata":{"papermill":{"duration":0.041028,"end_time":"2023-06-24T09:29:10.884872","exception":false,"start_time":"2023-06-24T09:29:10.843844","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:04.171584Z","iopub.execute_input":"2023-06-26T08:17:04.173128Z","iopub.status.idle":"2023-06-26T08:17:04.194182Z","shell.execute_reply.started":"2023-06-26T08:17:04.173090Z","shell.execute_reply":"2023-06-26T08:17:04.192687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df = drop_multi_game_naive(dataset_df,True)","metadata":{"papermill":{"duration":29.450894,"end_time":"2023-06-24T09:29:40.362121","exception":false,"start_time":"2023-06-24T09:29:10.911227","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:04.198361Z","iopub.execute_input":"2023-06-26T08:17:04.199661Z","iopub.status.idle":"2023-06-26T08:17:05.355775Z","shell.execute_reply.started":"2023-06-26T08:17:04.199597Z","shell.execute_reply":"2023-06-26T08:17:05.354735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LVGP_ORDER = {\"0-4\": 0, \"5-12\": 1, \"13-22\": 2}","metadata":{"papermill":{"duration":0.034257,"end_time":"2023-06-24T09:29:40.423471","exception":false,"start_time":"2023-06-24T09:29:40.389214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:05.357337Z","iopub.execute_input":"2023-06-26T08:17:05.357757Z","iopub.status.idle":"2023-06-26T08:17:05.363110Z","shell.execute_reply.started":"2023-06-26T08:17:05.357711Z","shell.execute_reply":"2023-06-26T08:17:05.362032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df[\"lvgp_order\"] = map_lvgp_order(dataset_df[\"level_group\"])","metadata":{"papermill":{"duration":0.05135,"end_time":"2023-06-24T09:29:40.500875","exception":false,"start_time":"2023-06-24T09:29:40.449525","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:05.365501Z","iopub.execute_input":"2023-06-26T08:17:05.365897Z","iopub.status.idle":"2023-06-26T08:17:05.377513Z","shell.execute_reply.started":"2023-06-26T08:17:05.365865Z","shell.execute_reply":"2023-06-26T08:17:05.376289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.info()","metadata":{"papermill":{"duration":0.041897,"end_time":"2023-06-24T09:29:40.569563","exception":false,"start_time":"2023-06-24T09:29:40.527666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:05.379803Z","iopub.execute_input":"2023-06-26T08:17:05.380278Z","iopub.status.idle":"2023-06-26T08:17:05.429500Z","shell.execute_reply.started":"2023-06-26T08:17:05.380238Z","shell.execute_reply":"2023-06-26T08:17:05.428610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset_df[\"event_comb\"] = dataset_df[\"event_name\"] + \"_\" + dataset_df[\"name\"]\n\ndataset_df[\"event_comb\"] = dataset_df[\"event_name\"].astype(str).str.cat(dataset_df[\"name\"].astype(str), sep=\"_\")\ndataset_df","metadata":{"papermill":{"duration":23.926908,"end_time":"2023-06-24T09:30:04.523114","exception":false,"start_time":"2023-06-24T09:29:40.596206","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:05.431081Z","iopub.execute_input":"2023-06-26T08:17:05.432007Z","iopub.status.idle":"2023-06-26T08:17:06.545559Z","shell.execute_reply.started":"2023-06-26T08:17:05.431970Z","shell.execute_reply":"2023-06-26T08:17:06.544466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df[(dataset_df.session_id==20090317080721164) & (dataset_df.level_group=='0-4')]","metadata":{"papermill":{"duration":0.172891,"end_time":"2023-06-24T09:30:04.725683","exception":false,"start_time":"2023-06-24T09:30:04.552792","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:06.547411Z","iopub.execute_input":"2023-06-26T08:17:06.548336Z","iopub.status.idle":"2023-06-26T08:17:06.593180Z","shell.execute_reply.started":"2023-06-26T08:17:06.548299Z","shell.execute_reply":"2023-06-26T08:17:06.592002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset_df.drop(['event_name','name'],inplace=True, axis=1)","metadata":{"papermill":{"duration":0.035047,"end_time":"2023-06-24T09:30:04.789018","exception":false,"start_time":"2023-06-24T09:30:04.753971","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:06.594747Z","iopub.execute_input":"2023-06-26T08:17:06.595222Z","iopub.status.idle":"2023-06-26T08:17:06.602387Z","shell.execute_reply.started":"2023-06-26T08:17:06.595184Z","shell.execute_reply":"2023-06-26T08:17:06.601156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['event_comb']=dataset_df['event_comb'].astype('category')","metadata":{"papermill":{"duration":3.141372,"end_time":"2023-06-24T09:30:07.958765","exception":false,"start_time":"2023-06-24T09:30:04.817393","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:06.604436Z","iopub.execute_input":"2023-06-26T08:17:06.604904Z","iopub.status.idle":"2023-06-26T08:17:06.764088Z","shell.execute_reply.started":"2023-06-26T08:17:06.604867Z","shell.execute_reply":"2023-06-26T08:17:06.763136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.info()","metadata":{"papermill":{"duration":0.047303,"end_time":"2023-06-24T09:30:08.036513","exception":false,"start_time":"2023-06-24T09:30:07.989210","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:06.766285Z","iopub.execute_input":"2023-06-26T08:17:06.766958Z","iopub.status.idle":"2023-06-26T08:17:06.817149Z","shell.execute_reply.started":"2023-06-26T08:17:06.766920Z","shell.execute_reply":"2023-06-26T08:17:06.816106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport keras.backend as K\nthreshold=0.5\n\ndef get_f1_1(y_true, y_pred):\n#     threshold = 0.63\n    \n#     y_pred = K.round(K.abs(y_pred))\n    \n#     y_pred = [1 if x > 0.5 else 0 for x in y_pred]\n    \n#     threshold = 0.5 #tf.constant(0.5)  # Threshold value\n    y_pred  = tf.cast(tf.math.greater(y_pred, threshold), tf.float32)\n    \n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n\n    true_positives = tf.cast(true_positives, tf.int64)\n    predicted_positives = tf.cast(predicted_positives, tf.int64)\n    possible_positives = tf.cast(possible_positives, tf.int64)\n\n    precision = tf.cast(true_positives, tf.float32) / (tf.cast(predicted_positives, tf.float32) + K.epsilon())\n    recall = tf.cast(true_positives, tf.float32) / (tf.cast(possible_positives, tf.float32) + K.epsilon())\n    f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon())\n\n    return f1_val \n    \n    \n    \ndef get_f1_2(y_true, y_pred):\n#     threshold = 0.63\n    \n#     y_pred1 = 1-y_pred\n#     y_pred1 = K.round(K.abs(y_pred1))\n    \n    y_pred1  = tf.cast(tf.math.greater(y_pred, threshold), tf.float32)\n    \n    y_pred1 = 1- y_pred1\n    \n    y_true1 = 1-y_true\n#     y_true1 = K.round(K.abs(y_true1))\n    \n    true_positives = K.sum(K.round(K.clip(y_true1 * y_pred1, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred1, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true1, 0, 1)))\n\n    true_positives = tf.cast(true_positives, tf.int64)\n    predicted_positives = tf.cast(predicted_positives, tf.int64)\n    possible_positives = tf.cast(possible_positives, tf.int64)\n\n    precision = tf.cast(true_positives, tf.float32) / (tf.cast(predicted_positives, tf.float32) + K.epsilon())\n    recall = tf.cast(true_positives, tf.float32) / (tf.cast(possible_positives, tf.float32) + K.epsilon())\n    f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon())\n\n    return f1_val \n    \n    \n#     return f1_val\n\n\n\ndef get_f1_macro(y_true, y_pred): \n     \n    \n    f1_val = (get_f1_1(y_true,y_pred) + get_f1_2(y_true, y_pred)) /2\n    \n    return f1_val\n\n","metadata":{"papermill":{"duration":0.048871,"end_time":"2023-06-24T09:30:08.115058","exception":false,"start_time":"2023-06-24T09:30:08.066187","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:06.826517Z","iopub.execute_input":"2023-06-26T08:17:06.827679Z","iopub.status.idle":"2023-06-26T08:17:06.842733Z","shell.execute_reply.started":"2023-06-26T08:17:06.827636Z","shell.execute_reply":"2023-06-26T08:17:06.841707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def delt_time_def(df):\n    df.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    df['d_time'] = df['elapsed_time'].diff(1) \n    df['d_time2'] = df['elapsed_time'].diff(10) \n    df['d_time'].fillna(0, inplace=True)\n    df['d_time2'].fillna(0, inplace=True)\n    df['delt_time'] = df['d_time'].clip(0, 103000)\n    df['delt_time_next'] = df['delt_time'].shift(-1)\n    return df\n\ndataset_df = delt_time_def(dataset_df)","metadata":{"papermill":{"duration":16.693301,"end_time":"2023-06-24T09:30:24.836621","exception":false,"start_time":"2023-06-24T09:30:08.143320","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:06.845067Z","iopub.execute_input":"2023-06-26T08:17:06.845955Z","iopub.status.idle":"2023-06-26T08:17:07.402245Z","shell.execute_reply.started":"2023-06-26T08:17:06.845918Z","shell.execute_reply":"2023-06-26T08:17:07.401145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_arr(dataset_df,grp):\n    \n    CATEGORICAL = ['event_name','name','fqid', 'room_fqid', 'text_fqid','lvgp_order','event_comb']\n    NUMERICAL = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration','d_time','d_time2',\n                 'delt_time','delt_time_next']\n    \n#     NUMERICAL = ['elapsed_time','level','page','hover_duration','d_time',\n#                  'delt_time','delt_time_next']\n    \n\n    # mber of rows for each session_id\n    max_rows = 400 #dataset_df['session_id'].value_counts().max()\n\n    # Create a list to store reshaped arrays\n    df_arrays = []\n\n    # df = dataset_df.copy()\n    # Iterate over unique session_id values\n    for session_id in dataset_df['session_id'].unique():\n        # Filter the rows for each session_id\n        session_rows = dataset_df[dataset_df['session_id'] == session_id].head(max_rows)\n        \n        \n#         d2=df2[(df2.index==session_id) & (df2.level_group==grp)]\n#         d2.drop('level_group',inplace=True, axis=1)\n        \n# #         d1=dataset_df[(dataset_df.session_id==20090312431273200) & (dataset_df.level_group=='0-4')]\n\n#         d4= pd.concat([d2] * len(session_rows), ignore_index=True)\n#         session_rows = pd.concat([session_rows, d4.reindex(session_rows.index)], axis=1)\n\n\n        \n#         d2=df2[(df2.index==session_id) & (df2.level_group==grp)]\n#         d2.drop('level_group',inplace=True, axis=1)\n        \n#         d4= pd.concat([d2] * len(session_rows), ignore_index=True)\n#         session_rows = pd.concat([session_rows, d4.reindex(session_rows.index)], axis=1)\n#         session_rows = session_rows.head(max_rows)\n        \n        \n        session_rows.drop('session_id',inplace=True, axis=1)\n        session_rows.drop('index',inplace=True, axis=1)     \n        \n    \n        sc = StandardScaler()   \n        session_rows[NUMERICAL] = sc.fit_transform(session_rows[NUMERICAL])        \n        \n        encoded_data = pd.get_dummies(session_rows, columns=CATEGORICAL)\n        session_rows = pd.concat([session_rows.drop(columns=CATEGORICAL), encoded_data], axis=1)\n        \n        c = session_rows.shape[1]        \n        r = session_rows.shape[0]\n        r1 = r // 10\n        \n        dr = r - r1 * 10\n        \n#         print(c,r,r1,dr)\n        \n#         print(session_rows.shape)\n        \n        if dr!=0:\n            session_rows = session_rows[:-dr].values\n        else: session_rows = session_rows.values\n        \n        \n        session_rows = session_rows.reshape((r1,10*c))\n\n        \n#         ohe = OneHotEncoder()\n        \n#         encoded_data = pd.get_dummies(session_rows, columns=CATEGORICAL)\n        \n# #         encoded_data = ohe.fit_transform(session_rows[CATEGORICAL]).toarray()\n#         session_rows[CATEGORICAL] = encoded_data\n        \n#         ohe = OneHotEncoder()\n#         session_rows[CATEGORICAL] = ohe.fit_transform(session_rows[CATEGORICAL])\n\n        # Pad the rows to match the maximum number of rows\n    #     padding_rows = max_rows - len(session_rows)\n    #     padded_rows = pd.concat([session_rows, pd.DataFrame(index=range(padding_rows), columns=session_rows.columns)])\n\n        # Append the reshaped array to the list\n        df_arrays.append(session_rows)\n    #     df_arrays = pd.concat([df_arrays,session_rows])\n    #     print(df_arrays)\n    #     asdf\n\n    # Convert the list of reshaped arrays to a numpy array\n    # df_arrays = np.array(df_arrays).nan_to_num(arr, nan=0.0)\n\n    # np.nan_to_num(arr, nan=0.0)\n\n    # Display the reshaped array\n    print(len(df_arrays))\n    return df_arrays\n","metadata":{"papermill":{"duration":0.042834,"end_time":"2023-06-24T09:30:24.907820","exception":false,"start_time":"2023-06-24T09:30:24.864986","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:07.404110Z","iopub.execute_input":"2023-06-26T08:17:07.404545Z","iopub.status.idle":"2023-06-26T08:17:07.417930Z","shell.execute_reply.started":"2023-06-26T08:17:07.404505Z","shell.execute_reply":"2023-06-26T08:17:07.416752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef pad_arrays(arrays):\n    max_rows = 40 #max(arr.shape[0] for arr in arrays)\n    padded_arrays = []\n\n    for arr in arrays:\n        num_rows = arr.shape[0]\n        pad_rows = max_rows - num_rows\n        padded_arr = np.pad(arr, ((0, pad_rows), (0, 0)), mode='constant')\n        padded_arrays.append(padded_arr)\n\n    return np.array(padded_arrays) ","metadata":{"papermill":{"duration":0.037151,"end_time":"2023-06-24T09:30:24.972025","exception":false,"start_time":"2023-06-24T09:30:24.934874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:07.420191Z","iopub.execute_input":"2023-06-26T08:17:07.420885Z","iopub.status.idle":"2023-06-26T08:17:07.431247Z","shell.execute_reply.started":"2023-06-26T08:17:07.420856Z","shell.execute_reply":"2023-06-26T08:17:07.430123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = ['0-4','5-12','13-22']\nlabels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\ndef split_dataset(dataset, test_ratio=0.20):\n    USER_LIST = dataset.index.unique()\n    split = int(len(USER_LIST) * (1 - 0.20))\n    return dataset.loc[USER_LIST[:split]], dataset.loc[USER_LIST[split:]]\n\n# train_x.set_index('session_id',drop=True,inplace=True)\n# valid_x.set_index('session_id',drop=True,inplace=True)\n \n\ndef create_dataset(dataset_df,grp):\n    dataset_df = dataset_df[dataset_df.level_group==grp]\n#     dataset_df = delt_time_def(dataset_df)\n    dataset_df.drop('level_group',inplace=True, axis=1)    \n     \n    \n     \n    dataset_df = dataset_df.reset_index(drop=True)\n    dataset_df = dataset_df.set_index('session_id',drop=False)\n    \n    users = dataset_df.index.values     \n    \n    df_arrays = convert_arr(dataset_df,grp)\n    df = pad_arrays(df_arrays) \n    return df, users\n\ndef create_ytrain(grp,q): \n    labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\n    labels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\n    labels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\n    labels=labels[labels.q==q]\n    y_train = labels.drop(['session_id','session','q'],axis=1).values.ravel() \n    \n    return y_train\n    \n     \ndef normalize_dataset(x_train,x_test):    \n#     x_train, x_test, y_train, y_test = train_test_split(x_train,y_train, test_size=0.2, \n#                                                         random_state=1123)\n    # Reshape x_train\n    batch_size1, m, n = x_train.shape\n    batch_size2, m, n = x_test.shape\n    x_train = np.reshape(x_train, (batch_size1 * m, n))\n    x_test  = np.reshape(x_test, (batch_size2 * m, n))\n\n    # Instantiate the StandardScaler\n    scaler = StandardScaler()\n\n    # Fit and transform on the reshaped x_train data\n    scaler.fit(x_train)\n\n    x_train,x_test = scaler.transform(x_train),scaler.transform(x_test)\n\n    # Reshape the scaled data back to the original shape\n    x_train= np.reshape(x_train, (batch_size1, m, n))\n    x_test= np.reshape(x_test, (batch_size2, m, n))\n\n\n    return x_train, x_test","metadata":{"papermill":{"duration":0.36804,"end_time":"2023-06-24T09:30:25.367628","exception":false,"start_time":"2023-06-24T09:30:24.999588","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:07.433503Z","iopub.execute_input":"2023-06-26T08:17:07.433921Z","iopub.status.idle":"2023-06-26T08:17:07.798599Z","shell.execute_reply.started":"2023-06-26T08:17:07.433888Z","shell.execute_reply":"2023-06-26T08:17:07.797495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n    # Normalization and Attention\n    x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n    x = layers.MultiHeadAttention(\n        key_dim=head_size, num_heads=num_heads, dropout=dropout,kernel_initializer=tf.keras.initializers.GlorotUniform(seed=1234)\n    )(x, x)\n    x = layers.Dropout(dropout)(x)\n    res = x + inputs\n    \n#     res = x\n\n    # Feed Forward Part\n    x = layers.LayerNormalization(epsilon=1e-6)(res)\n    x = layers.Dense(ff_dim,activation='relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n    x = layers.Dense(ff_dim/2, activation = 'relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n    x = layers.Dense(ff_dim/4, activation = 'relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n#                     kernel_regularizer=tf.keras.regularizers.L1(0.1))(x)\n    x = layers.Dense(16, activation = 'relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n#                     kernel_regularizer=tf.keras.regularizers.L1(0.1))(x)\n    \n#     x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n#     x = layers.Dropout(dropout)(x)\n#     x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n    x = layers.Dense(1,activation='relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n    return x + inputs\n#     return x  \n\n\n# def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n#     # Normalization and Attention\n#     x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n#     x = layers.MultiHeadAttention(\n#         key_dim=head_size, num_heads=num_heads, dropout=dropout,kernel_initializer=tf.keras.initializers.GlorotUniform(seed=1234)\n#     )(x, x)\n#     x = layers.Dropout(dropout)(x)\n#     res = x + inputs\n    \n# #     res = x\n\n#     # Feed Forward Part\n#     x = layers.LayerNormalization(epsilon=1e-6)(res)\n# #     x = layers.Dense(ff_dim,activation='relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n# #     x = layers.Dense(ff_dim/2, activation = 'relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n# #     x = layers.Dense(ff_dim/4, activation = 'relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234),\n# #                     kernel_regularizer=tf.keras.regularizers.L1(0.01))(x)\n# #     x = layers.Dense(16, activation = 'relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234),\n# #                     kernel_regularizer=tf.keras.regularizers.L1(0.01))(x)\n    \n#     x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\",kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n#     x = layers.Dropout(dropout)(x)\n#     x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1,kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n#     x = layers.Dense(1,activation='relu',kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n#     return x + res\n# #     return x \n\n# def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n#     # Normalization and Attention\n#     x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n#     x = layers.MultiHeadAttention(\n#         key_dim=head_size, num_heads=num_heads, dropout=dropout\n#     )(x, x)\n#     x = layers.Dropout(dropout)(x)\n#     res = x + inputs\n\n#     # Feed Forward Part\n#     x = layers.LayerNormalization(epsilon=1e-6)(res)\n#     x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n#     x = layers.Dropout(dropout)(x)\n#     x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n#     return x + res\n","metadata":{"papermill":{"duration":0.044547,"end_time":"2023-06-24T09:30:25.440229","exception":false,"start_time":"2023-06-24T09:30:25.395682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:07.800291Z","iopub.execute_input":"2023-06-26T08:17:07.800672Z","iopub.status.idle":"2023-06-26T08:17:07.815606Z","shell.execute_reply.started":"2023-06-26T08:17:07.800636Z","shell.execute_reply":"2023-06-26T08:17:07.814673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(\n    input_shape,\n    head_size,\n    num_heads,\n    ff_dim,\n    num_transformer_blocks,\n    mlp_units,\n    dropout=0,\n    mlp_dropout=0,\n):\n    inputs = keras.Input(shape=input_shape)\n    x = inputs\n    for _ in range(num_transformer_blocks):\n        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)\n\n    x = layers.GlobalAveragePooling1D(data_format=\"channels_last\")(x) #(data_format=\"channels_first\")(x)\n    \n    for dim in mlp_units:\n        x = layers.Dense(dim, activation=\"relu\",kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n#                         kernel_regularizer=tf.keras.regularizers.L1(0.1))(x)\n        x = layers.Dropout(mlp_dropout)(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\",kernel_initializer=tf.keras.initializers.HeNormal(seed=1234))(x)\n    return keras.Model(inputs, outputs)\n\n\n# def build_model(\n#     input_shape,\n#     head_size,\n#     num_heads,\n#     ff_dim,\n#     num_transformer_blocks,\n#     mlp_units,\n#     dropout=0,\n#     mlp_dropout=0,\n# ):\n#     inputs = keras.Input(shape=input_shape)\n#     x = inputs\n#     for _ in range(num_transformer_blocks):\n#         x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)\n\n#     x = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n#     for dim in mlp_units:\n#         x = layers.Dense(dim, activation=\"relu\")(x)\n#         x = layers.Dropout(mlp_dropout)(x)\n#     outputs = layers.Dense(n_classes, activation=\"softmax\")(x)\n#     return keras.Model(inputs, outputs)\n","metadata":{"papermill":{"duration":0.039806,"end_time":"2023-06-24T09:30:25.507723","exception":false,"start_time":"2023-06-24T09:30:25.467917","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:07.817443Z","iopub.execute_input":"2023-06-26T08:17:07.818185Z","iopub.status.idle":"2023-06-26T08:17:07.829584Z","shell.execute_reply.started":"2023-06-26T08:17:07.818149Z","shell.execute_reply":"2023-06-26T08:17:07.828507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.info()","metadata":{"papermill":{"duration":0.042866,"end_time":"2023-06-24T09:30:25.577739","exception":false,"start_time":"2023-06-24T09:30:25.534873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:17:07.830940Z","iopub.execute_input":"2023-06-26T08:17:07.831711Z","iopub.status.idle":"2023-06-26T08:17:07.925537Z","shell.execute_reply.started":"2023-06-26T08:17:07.831678Z","shell.execute_reply":"2023-06-26T08:17:07.924280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(1123)\n\ntrain_x, valid_x = split_dataset(dataset_df)\n# del dataset_df\n# gc.collect()\nprint(\"{} examples in training, {} examples in testing.\".format(len(train_x), len(valid_x)))\n\nlabels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\nlabels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\nlabels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\n\nlabels.set_index('session',inplace=True)\n# g1='0-4'\n\nx_train_g1,train_users_g1 = create_dataset(train_x,'0-4')\nx_test_g1,valid_users_g1 = create_dataset(valid_x,'0-4')\n\nx_train_g2,train_users_g2 = create_dataset(train_x,'5-12')\nx_test_g2,valid_users_g2 = create_dataset(valid_x,'5-12')\n\nx_train_g3,train_users_g3 = create_dataset(train_x,'13-22')\nx_test_g3,valid_users_g3 = create_dataset(valid_x,'13-22')","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:17:07.931141Z","iopub.execute_input":"2023-06-26T08:17:07.931487Z","iopub.status.idle":"2023-06-26T08:17:55.318395Z","shell.execute_reply.started":"2023-06-26T08:17:07.931432Z","shell.execute_reply":"2023-06-26T08:17:55.317055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del dataset_df\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:17:55.320157Z","iopub.execute_input":"2023-06-26T08:17:55.320833Z","iopub.status.idle":"2023-06-26T08:17:55.326092Z","shell.execute_reply.started":"2023-06-26T08:17:55.320795Z","shell.execute_reply":"2023-06-26T08:17:55.324798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models ={}\nevaluation_dict = {} \npd.options.mode.chained_assignment = None\n\nBATCH_SIZE_PER_REPLICA = 64\nBATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n\n# # with strategy.scope():\n# models = {}\n# evaluation_dict = {}\n# input_shape = (200, 308)\n\n# callbacks = [tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)]\ntf.random.set_seed(1123)\n\n    \n# del dataset_df\n# gc.collect()\n  \n# qns = []\n\npred_test = []\ny_true = []\n\nwith strategy.scope():\n    for q_no in range(1,19):\n    #         q_no=1\n    # Select level group for the question based on the q_no.\n        if q_no<=3: grp = 1\n        elif q_no<=13: grp = 2\n        elif q_no<=22: grp = 3\n        print(\"### q_no\", q_no, \"grp\", grp)\n\n\n    #         x_train,train_users = create_dataset(train_x,grp)\n    #         x_test,valid_users = create_dataset(valid_x,grp)\n        xt=f\"x_train_g{grp}\" \n        x_train = locals()[xt]\n\n        xv = f\"x_test_g{grp}\" \n        x_test = locals()[xv]\n\n        tu = f'train_users_g{grp}'\n        train_users = locals()[tu]\n\n        vu = f'valid_users_g{grp}'\n        valid_users = locals()[vu]\n\n\n\n\n        x_train = x_train.astype('float32')\n        x_test = x_test.astype('float32') \n\n        input_shape = x_train.shape[1:]\n\n        #     y_train = create_ytrain(grp,q_no)\n\n        print(\"shape of xtrain, xtest \\n\",x_train.shape,x_test.shape)\n\n        #     x_train, x_test = normalize_dataset(x_train,x_test)\n\n        train_x.set_index('session_id',drop=False,inplace=True)\n        valid_x.set_index('session_id',drop=False,inplace=True)\n\n        labels1 = labels.copy()\n        labels1=labels1[labels1.q==q_no] \n        y_train  = labels1.loc[train_users].drop_duplicates()\n        y_test   = labels1.loc[valid_users].drop_duplicates()\n\n        neg, pos = np.bincount(y_train['correct'])\n        total = neg + pos\n        # Scaling by total/2 helps keep the loss to a similar magnitude.\n        # The sum of the weights of all examples stays the same.\n        weight_for_0 = (1 / neg) * (total / 2.0)\n        weight_for_1 = (1 / pos) * (total / 2.0)\n\n        class_weight = {0: weight_for_0, 1: weight_for_1}\n\n        print('Weight for class 0: {:.2f}'.format(weight_for_0))\n        print('Weight for class 1: {:.2f}'.format(weight_for_1))\n\n        y_train=y_train.drop(['session_id','q'],axis=1).values\n        y_test=y_test.drop(['session_id','q'],axis=1).values\n\n        y_train = y_train.astype('float32')\n        y_test =  y_test.astype('float32')\n\n        model = build_model(\n        input_shape,\n        head_size=256,\n        num_heads=6,\n        ff_dim=6,\n        num_transformer_blocks=4,\n        mlp_units=[128],\n        mlp_dropout=0.4,\n        dropout=0.25,\n        )\n\n    #     model = build_model(\n    #     input_shape,\n    #     head_size=256,\n    #     num_heads=4,\n    #     ff_dim=4,\n    #     num_transformer_blocks=4,\n    #     mlp_units=[128],\n    #     mlp_dropout=0.4,\n    #     dropout=0.25,\n    # )\n\n\n        #     model = build_model(\n        #         input_shape,\n        #         head_size=256,\n        #         num_heads=6,\n        #         ff_dim=256,\n        #         num_transformer_blocks=6,\n        #         mlp_units=[128],\n        #         mlp_dropout=0.4,\n        #         dropout=0.25,\n        #     )\n        #     f1ametric=tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=0.5)\n\n        model.compile(\n            loss=\"binary_crossentropy\",\n        #         loss = custom_loss_function,\n        #         optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n            optimizer=keras.optimizers.Adam(learning_rate=0.0001), #,weight_decay=1e-4),\n\n            metrics=['accuracy',get_f1_1,get_f1_2,get_f1_macro],\n        #                 get_f1_12,get_f1_22,get_f1_macro2]\n        )\n\n        callbacks = [keras.callbacks.EarlyStopping(monitor= 'val_get_f1_macro',patience=10, \n                                                   start_from_epoch=15,baseline=0.2,\n                                                   restore_best_weights=True,mode='max')]\n\n        #     ros = RandomOverSampler()\n        #     x_train, y_train = ros.fit_resample(x_train,y_train)\n\n        #     print(\"null \",np.sum(np.isnan(x_train)))\n        #     dfsdaf\n\n    #     print(\"xtrain\",x_train.shape)\n    #     print(\"xtest\",x_test.shape)\n    #     print(\"ytrain\",y_train.shape)\n    #     print(\"ytest\",y_test.shape)\n    #     asdfa\n\n        model.fit(\n            x_train,\n            y_train,\n            validation_data=(x_test, y_test),\n            epochs=200,\n            batch_size= BATCH_SIZE,\n            callbacks=callbacks,\n            class_weight=class_weight\n        )\n\n        filename = str('tpu'+'_'+f'{grp}_{q_no}')\n\n        os.chdir(r'/kaggle/working/models')\n        pickle.dump(model, open(filename, 'wb'))\n        models['tpu'+'_'+f'{grp}_{q_no}'] = filename \n        y_pred = model.predict(x_test)\n\n        pred_test.append(y_pred.flatten().tolist())\n        y_true.append(y_test.flatten().tolist()) \n\n\n        y_pred1 = [1 if x > 0.63 else 0 for x in y_pred.flatten().tolist()]\n\n        print('f1 score macro for qn',q_no,f1_score(y_pred1,\n                                                    y_test.flatten().tolist(),average='macro'))\n        print('f1 score majority',q_no,f1_score(y_pred1,\n                                                y_test.flatten().tolist(),average=None))\n\n        del model\n        del x_train, x_test, y_train, y_test, labels1\n        gc.collect()\n\n\npred_test1 = [1 if x > 0.5 else 0 for x in pred_test[0]]\n\nf1macro_overall1 = f1_score(y_true[0], pred_test1,average='macro')\nf1macro_overall2 = f1_score(y_true[0], pred_test1,average=None)\n\nprint(\"f1 macro overall\",f1macro_overall1,f1macro_overall2) \n\npred_test2 = [1 if x > 0.63 else 0 for x in pred_test[0]]\n\nf1macro_overall3 = f1_score(y_true[0], pred_test2,average='macro')\nf1macro_overall4 = f1_score(y_true[0], pred_test2,average=None)\n\nprint(\"f1 macro overall\",f1macro_overall3,f1macro_overall4) ","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:17:55.327819Z","iopub.execute_input":"2023-06-26T08:17:55.328370Z","iopub.status.idle":"2023-06-26T08:21:11.426296Z","shell.execute_reply.started":"2023-06-26T08:17:55.328337Z","shell.execute_reply":"2023-06-26T08:21:11.424500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models ={}\n# evaluation_dict = {} \n# pd.options.mode.chained_assignment = None\n\n# # BATCH_SIZE_PER_REPLICA = 64\n# # BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n\n# # # with strategy.scope():\n# # models = {}\n# # evaluation_dict = {}\n# # input_shape = (200, 308)\n\n# # callbacks = [tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)]\n# tf.random.set_seed(1123)\n\n# train_x, valid_x = split_dataset(dataset_df)\n# print(\"{} examples in training, {} examples in testing.\".format(len(train_x), len(valid_x)))\n\n# labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n\n# labels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\n# labels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\n\n# labels.set_index('session',inplace=True)\n    \n# # del dataset_df\n# gc.collect()\n \n# # [5, 8, 10, 13, 15]\n\n# # with strategy.scope():\n# # qns = [14,15,16,17,18]\n# # qns=[16]\n# # qns = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]\n\n# # qns = [1,3,4,6,7,9,11,14,16,17]\n# # qns = []\n\n# # with strategy.scope():\n# for q_no in range(1,9):\n# #         q_no=1\n# # Select level group for the question based on the q_no.\n#     if q_no<=3: grp = '0-4'\n#     elif q_no<=13: grp = '5-12'\n#     elif q_no<=22: grp = '13-22'\n#     print(\"### q_no\", q_no, \"grp\", grp)\n\n\n#     x_train,train_users = create_dataset(train_x,grp)\n#     x_test,valid_users = create_dataset(valid_x,grp)\n\n#     x_train = x_train.astype('float32')\n#     x_test = x_test.astype('float32')\n\n#     #     x_train = x_train[~np.isnan(x_train).any(axis=1)]\n#     #     x_test = x_test[~np.isnan(x_test).any(axis=1)]\n\n#     input_shape = x_train.shape[1:]\n\n#     #     y_train = create_ytrain(grp,q_no)\n\n#     print(\"shape of xtrain, xtest \\n\",x_train.shape,x_test.shape)\n\n#     #     x_train, x_test = normalize_dataset(x_train,x_test)\n\n#     train_x.set_index('session_id',drop=False,inplace=True)\n#     valid_x.set_index('session_id',drop=False,inplace=True)\n\n#     labels1 = labels.copy()\n#     labels1=labels1[labels1.q==q_no] \n#     y_train  = labels1.loc[train_users].drop_duplicates()\n#     y_test   = labels1.loc[valid_users].drop_duplicates()\n\n#     neg, pos = np.bincount(y_train['correct'])\n#     total = neg + pos\n#     # Scaling by total/2 helps keep the loss to a similar magnitude.\n#     # The sum of the weights of all examples stays the same.\n#     weight_for_0 = (1 / neg) * (total / 2.0)\n#     weight_for_1 = (1 / pos) * (total / 2.0)\n\n#     class_weight = {0: weight_for_0, 1: weight_for_1}\n\n#     print('Weight for class 0: {:.2f}'.format(weight_for_0))\n#     print('Weight for class 1: {:.2f}'.format(weight_for_1))\n\n#     y_train=y_train.drop(['session_id','q'],axis=1).values\n#     y_test=y_test.drop(['session_id','q'],axis=1).values\n\n#     y_train = y_train.astype('float32')\n#     y_test =  y_test.astype('float32')\n\n#     model = build_model(\n#     input_shape,\n#     head_size=256,\n#     num_heads=4,\n#     ff_dim=4,\n#     num_transformer_blocks=4,\n#     mlp_units=[128],\n#     mlp_dropout=0.4,\n#     dropout=0.25,\n#     )\n\n\n#     #     model = build_model(\n#     #         input_shape,\n#     #         head_size=256,\n#     #         num_heads=6,\n#     #         ff_dim=256,\n#     #         num_transformer_blocks=6,\n#     #         mlp_units=[128],\n#     #         mlp_dropout=0.4,\n#     #         dropout=0.25,\n#     #     )\n#     #     f1ametric=tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=0.5)\n\n#     model.compile(\n#         loss=\"binary_crossentropy\",\n#     #         loss = custom_loss_function,\n#     #         optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n#         optimizer=keras.optimizers.Adam(learning_rate=0.01), #,weight_decay=1e-4),\n\n#         metrics=['accuracy',get_f1_1,get_f1_2,get_f1_macro],\n#     #                 get_f1_12,get_f1_22,get_f1_macro2]\n#     )\n\n#     callbacks = [keras.callbacks.EarlyStopping(monitor= 'val_loss',patience=10, \n#                                                start_from_epoch=10,baseline=0.2,\n#                                                restore_best_weights=True,mode='max')]\n\n#     #     ros = RandomOverSampler()\n#     #     x_train, y_train = ros.fit_resample(x_train,y_train)\n\n#     #     print(\"null \",np.sum(np.isnan(x_train)))\n#     #     dfsdaf\n\n#     model.fit(\n#         x_train,\n#         y_train,\n#         validation_data=(x_test, y_test),\n#         epochs=50,\n#         batch_size=BATCH_SIZE,\n#         callbacks=callbacks,\n#         class_weight=class_weight\n#     )\n\n#     filename = str('tpu'+'_'+f'{grp}_{q_no}')\n\n#     os.chdir(r'/kaggle/working/models')\n#     pickle.dump(model, open(filename, 'wb'))\n#     models['tpu'+'_'+f'{grp}_{q_no}'] = filename\n\n# #         #         y_pred = model(x_test, training=False)\n# #         y_pred = model.predict(x_test)\n# #         y_pred = [1 if x > 0.5 else 0 for x in y_pred]\n# #         print('f1 score macro  ',f1_score(y_test, y_pred, average='macro'))\n# #         print('f1 score ',f1_score(y_test, y_pred))\n# #         evaluation_dict[q_no] = f1_score(y_test, y_pred, average='macro')\n\n#     del model\n#     del x_train, x_test, y_train, y_test, labels1\n#     gc.collect()\n","metadata":{"papermill":{"duration":6689.715274,"end_time":"2023-06-24T11:21:55.321750","exception":false,"start_time":"2023-06-24T09:30:25.606476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.427975Z","iopub.status.idle":"2023-06-26T08:21:11.428558Z","shell.execute_reply.started":"2023-06-26T08:21:11.428279Z","shell.execute_reply":"2023-06-26T08:21:11.428304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Epoch 1/50\n# 23/23 [==============================] - 25s 260ms/step - loss: 0.7151 - accuracy: 0.5098 - get_f1_1: 0.5594 - get_f1_2: 0.2861 - get_f1_macro: 0.4227 - val_loss: 0.6435 - val_accuracy: 0.6992 - val_get_f1_1: 0.8024 - val_get_f1_2: 0.3407 - val_get_f1_macro: 0.5715\n                                        \n                                        \n# Epoch 20/50\n# 23/23 [==============================] - 5s 201ms/step - loss: 0.6564 - accuracy: 0.6201 - get_f1_1: 0.7118 - get_f1_2: 0.4205 - get_f1_macro: 0.5661 - val_loss: 0.6424 - val_accuracy: 0.6685 - val_get_f1_1: 0.7646 - val_get_f1_2: 0.4215 - val_get_f1_macro: 0.5931\n                                        ","metadata":{"papermill":{"duration":0.800801,"end_time":"2023-06-24T11:21:56.891588","exception":false,"start_time":"2023-06-24T11:21:56.090787","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.430416Z","iopub.status.idle":"2023-06-26T08:21:11.431241Z","shell.execute_reply.started":"2023-06-26T08:21:11.430972Z","shell.execute_reply":"2023-06-26T08:21:11.430996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Epoch 1/50\n# 23/23 [==============================] - 23s 249ms/step - loss: 0.7073 - accuracy: 0.3841 - get_f1_1: 0.3118 - get_f1_2: 0.4308 - get_f1_macro: 0.3713 - val_loss: 0.6961 - val_accuracy: 0.4485 - val_get_f1_1: 0.4282 - val_get_f1_2: 0.4683 - val_get_f1_macro: 0.4482\n# Epoch 2/50\n# 23/23 [==============================] - 4s 190ms/step - loss: 0.6939 - accuracy: 0.4937 - get_f1_1: 0.5425 - get_f1_2: 0.4381 - get_f1_macro: 0.4903 - val_loss: 0.6832 - val_accuracy: 0.5627 - val_get_f1_1: 0.6173 - val_get_f1_2: 0.4858 - val_get_f1_macro: 0.5515\n\n\n\n\n\n# Epoch 1/50\n# 23/23 [==============================] - 36s 468ms/step - loss: 0.7062 - accuracy: 0.5744 - get_f1_1: 0.6878 - get_f1_2: 0.2364 - get_f1_macro: 0.4621 - val_loss: 0.6230 - val_accuracy: 0.7214 - val_get_f1_1: 0.8346 - val_get_f1_2: 0.0000e+00 - val_get_f1_macro: 0.4173\n# Epoch 2/50\n# 23/23 [==============================] - 8s 369ms/step - loss: 0.6220 - accuracy: 0.6945 - get_f1_1: 0.8158 - get_f1_2: 0.0905 - get_f1_macro: 0.4532 - val_loss: 0.5882 - val_accuracy: 0.7298 - val_get_f1_1: 0.8399 - val_get_f1_2: 0.0000e+00 - val_get_f1_macro: 0.4200\n# Epoch 3/50\n# 23/23 [==============================] - 9s 396ms/step - loss: 0.5933 - accuracy: 0.7065 - get_f1_1: 0.8251 - get_f1_2: 0.0724 - get_f1_macro: 0.4487 - val_loss: 0.5513 - val_accuracy: 0.7298 - val_get_f1_1: 0.8399 - val_get_f1_2: 0.0000e+00 - val_get_f1_macro: 0.4200\n# Epoch 4/50\n# 23/23 [==============================] - 8s 367ms/step - loss: 0.5840 - accuracy: 0.7107 - get_f1_1: 0.8275 - get_f1_2: 0.0766 - get_f1_macro: 0.4520 - val_loss: 0.5508 - val_accuracy: 0.7326 - val_get_f1_1: 0.8414 - val_get_f1_2: 0.0741 - val_get_f1_macro: 0.4577\n# Epoch 5/50\n# 23/23 [==============================] - 9s 391ms/step - loss: 0.5837 - accuracy: 0.7135 - get_f1_1: 0.8270 - get_f1_2: 0.0819 - get_f1_macro: 0.4544 - val_loss: 0.5463 - val_accuracy: 0.7270 - val_get_f1_1: 0.8368 - val_get_f1_2: 0.0974 - val_get_f1_macro: 0.4671\n# Epoch 6/50\n# 23/23 [==============================] - 8s 366ms/step - loss: 0.5745 - accuracy: 0.7156 - get_f1_1: 0.8271 - get_f1_2: 0.1458 - get_f1_macro: 0.4864 - val_loss: 0.5424 - val_accuracy: 0.7326 - val_get_f1_1: 0.8412 - val_get_f1_2: 0.0145 - val_get_f1_macro: 0.4278\n# Epoch 7/50\n# 23/23 [==============================] - 8s 369ms/step - loss: 0.5619 - accuracy: 0.7107 - get_f1_1: 0.8242 - get_f1_2: 0.0991 - get_f1_macro: 0.4617 - val_loss: 0.5366 - val_accuracy: 0.7242 - val_get_f1_1: 0.8351 - val_get_f1_2: 0.1064 - val_get_f1_macro: 0.4708\n# Epoch 8/50\n# 23/23 [==============================] - 9s 391ms/step - loss: 0.5645 - accuracy: 0.7135 - get_f1_1: 0.8280 - get_f1_2: 0.1372 - get_f1_macro: 0.4826 - val_loss: 0.5360 - val_accuracy: 0.7270 - val_get_f1_1: 0.8368 - val_get_f1_2: 0.1078 - val_get_f1_macro: 0.4723\n# Epoch 9/50\n# 23/23 [==============================] - 9s 388ms/step - loss: 0.5612 - accuracy: 0.7212 - get_f1_1: 0.8355 - get_f1_2: 0.1517 - get_f1_macro: 0.4936 - val_loss: 0.5411 - val_accuracy: 0.7214 - val_get_f1_1: 0.8296 - val_get_f1_2: 0.1535 - val_get_f1_macro: 0.4915\n# Epoch 10/50\n# 23/23 [==============================] - 9s 391ms/step - loss: 0.5532 - accuracy: 0.7226 - get_f1_1: 0.8355 - get_f1_2: 0.1928 - get_f1_macro: 0.5141 - val_loss: 0.5378 - val_accuracy: 0.7382 - val_get_f1_1: 0.8443 - val_get_f1_2: 0.0980 - val_get_f1_macro: 0.4711\n# Epoch 11/50\n# 23/23 [==============================] - 9s 388ms/step - loss: 0.5455 - accuracy: 0.7226 - get_f1_1: 0.8306 - get_f1_2: 0.1727 - get_f1_macro: 0.5016 - val_loss: 0.5470 - val_accuracy: 0.7409 - val_get_f1_1: 0.8462 - val_get_f1_2: 0.0960 - val_get_f1_macro: 0.4711\n# Epoch 12/50\n# 23/23 [==============================] - 9s 388ms/step - loss: 0.5468 - accuracy: 0.7289 - get_f1_1: 0.8354 - get_f1_2: 0.1663 - get_f1_macro: 0.5008 - val_loss: 0.5452 - val_accuracy: 0.7465 - val_get_f1_1: 0.8469 - val_get_f1_2: 0.2048 - val_get_f1_macro: 0.5259\n# Epoch 13/50\n# 23/23 [==============================] - 8s 364ms/step - loss: 0.5470 - accuracy: 0.7226 - get_f1_1: 0.8263 - get_f1_2: 0.3186 - get_f1_macro: 0.5724 - val_loss: 0.5461 - val_accuracy: 0.7354 - val_get_f1_1: 0.8433 - val_get_f1_2: 0.0476 - val_get_f1_macro: 0.4455\n# Epoch 14/50\n# 23/23 [==============================] - 8s 370ms/step - loss: 0.5452 - accuracy: 0.7268 - get_f1_1: 0.8294 - get_f1_2: 0.1685 - get_f1_macro: 0.4989 - val_loss: 0.5287 - val_accuracy: 0.7549 - val_get_f1_1: 0.8523 - val_get_f1_2: 0.2091 - val_get_f1_macro: 0.5307\n# Epoch 15/50\n# 23/23 [==============================] - 9s 388ms/step - loss: 0.5314 - accuracy: 0.7331 - get_f1_1: 0.8385 - get_f1_2: 0.1905 - get_f1_macro: 0.5145 - val_loss: 0.5498 - val_accuracy: 0.7521 - val_get_f1_1: 0.8430 - val_get_f1_2: 0.3147 - val_get_f1_macro: 0.5788\n# Epoch 16/50\n# 23/23 [==============================] - 8s 363ms/step - loss: 0.5331 - accuracy: 0.7444 - get_f1_1: 0.8460 - get_f1_2: 0.2973 - get_f1_macro: 0.5717 - val_loss: 0.5394 - val_accuracy: 0.7382 - val_get_f1_1: 0.8438 - val_get_f1_2: 0.0550 - val_get_f1_macro: 0.4494\n# Epoch 17/50\n# 23/23 [==============================] - 8s 367ms/step - loss: 0.5178 - accuracy: 0.7381 - get_f1_1: 0.8381 - get_f1_2: 0.2304 - get_f1_macro: 0.5342 - val_loss: 0.5280 - val_accuracy: 0.7493 - val_get_f1_1: 0.8490 - val_get_f1_2: 0.1953 - val_get_f1_macro: 0.5221\n# Epoch 18/50\n# 20/23 [=========================>....] - ETA: 1s - loss: 0.5185 - accuracy: 0.7391 - get_f1_1: 0.8379 - get_f1_2: 0.2684 - get_f1_macro: 0.5532","metadata":{"papermill":{"duration":0.846396,"end_time":"2023-06-24T11:21:58.526970","exception":false,"start_time":"2023-06-24T11:21:57.680574","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.432777Z","iopub.status.idle":"2023-06-26T08:21:11.433599Z","shell.execute_reply.started":"2023-06-26T08:21:11.433322Z","shell.execute_reply":"2023-06-26T08:21:11.433345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.771095,"end_time":"2023-06-24T11:22:00.156123","exception":false,"start_time":"2023-06-24T11:21:59.385028","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub = pd.DataFrame()\n# # sub['session_id']=0\n\n# limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\n# grp = ['0-4','5-12','13-22']\n\n# for session_id in valid_x['session_id'].unique():\n#     grps = tr[tr.session_id==session_id].level_group.unique()\n#     for grp in grps:\n#         m,n = limits[grp]\n#         for t in range(m,n):\n            \n#             sub = pd.concat([sub, pd.DataFrame({'session_id': str(session_id) + '_q'+f'{t}'},index=[len(sub)])], \n#                             ignore_index=True) \n              \n              \n\n\n#             #  print(test_data.shape)\n# # # dsfasf\n# # for i in range(a):\n    \n# #     test_data = test_data[i,:,:]\n# #     sess = test_data.session_id\n# #     grp = test_data.level_group.values[0] \n# #     m,n = limits[grp]\n# #     for t in range(m,n):\n# #         sub = sub.append({'session_id': str(sess) + '_q'+t}, ignore_index=True)\n    \n    \n    ","metadata":{"papermill":{"duration":0.791542,"end_time":"2023-06-24T11:22:01.757201","exception":false,"start_time":"2023-06-24T11:22:00.965659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.435111Z","iopub.status.idle":"2023-06-26T08:21:11.435922Z","shell.execute_reply.started":"2023-06-26T08:21:11.435667Z","shell.execute_reply":"2023-06-26T08:21:11.435691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.825902,"end_time":"2023-06-24T11:22:03.377933","exception":false,"start_time":"2023-06-24T11:22:02.552031","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.770576,"end_time":"2023-06-24T11:22:04.922129","exception":false,"start_time":"2023-06-24T11:22:04.151553","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = build_model(\n# input_shape,\n# head_size=256,\n# num_heads=4,\n# ff_dim=4,\n# num_transformer_blocks=4,\n# mlp_units=[128],\n# mlp_dropout=0.4,\n# dropout=0.25,\n# )\n\n# model.compile(\n#     loss=\"binary_crossentropy\",\n#     optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n#     metrics=[\"accuracy\",get_f1],\n# )\n\n\n\n# model.fit(\n#     x_train,\n#     y_train,\n#     validation_data=(x_test,y_test),\n# #     validation_split=0.2,\n#     epochs=10,\n#     batch_size=100,\n#     callbacks=callbacks,\n# )\n\n# #     models[f'{grp}_{q_no}'] = model\n\n# filename= str(f'{grp}_{q_no}')\n\n# os.chdir(r'/kaggle/working/models')\n\n# pickle.dump(model,open(filename, 'wb')) \n\n# models[f'{grp}_{q_no}'] = filename\n\n\n\n# #     print(\"model for \",u,\"saved in file name \",filename) \n\n# y_pred = model.predict(x_test)\n# y_pred = [1 if x > 0.5 else 0 for x in y_pred]\n# f1_score(y_pred, y_test)\n# evaluation_dict[q_no] = f1_score(y_test, y_pred,average='macro')\n\n# # del model\n# # del x_train,x_test,y_train,y_test\n# # gc.collect()","metadata":{"papermill":{"duration":0.768631,"end_time":"2023-06-24T11:22:06.592690","exception":false,"start_time":"2023-06-24T11:22:05.824059","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.437543Z","iopub.status.idle":"2023-06-26T08:21:11.438426Z","shell.execute_reply.started":"2023-06-26T08:21:11.438171Z","shell.execute_reply":"2023-06-26T08:21:11.438195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # import tensorflow as tf\n\n# # resolver = tf.distribute.cluster_resolver.TPUClusterResolver()\n# # tf.config.experimental_connect_to_cluster(resolver)\n# # tf.tpu.experimental.initialize_tpu_system(resolver)\n# # strategy = tf.distribute.TPUStrategy(resolver)\n\n# # print('Number of devices: {}'.format(strategy.num_replicas_in_sync))\n\n# BATCH_SIZE_PER_REPLICA = 100\n# BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n\n# # # with strategy.scope():\n# # models = {}\n# # evaluation_dict = {}\n# # input_shape = (200, 308)\n\n# # callbacks = [tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)]\n    \n# with strategy.scope():\n# # #     for q_no in range(1, 19):\n# #         if q_no <= 3:\n# #             grp = '0-4'\n# #         elif q_no <= 13:\n# #             grp = '5-12'\n# #         elif q_no <= 22:\n# #             grp = '13-22'\n# #         print(\"### q_no\", q_no, \"grp\", grp)\n\n# # #         x_train = create_xtrain(dataset_df, grp)\n# # #         y_train = create_ytrain(grp, q_no)\n\n# # #         x_train, x_test, y_train, y_test = create_dataset(x_train, y_train)\n\n# # #     with strategy.scope():\n#         model = build_model(\n#             input_shape,\n#             head_size=256,\n#             num_heads=4,\n#             ff_dim=4,\n#             num_transformer_blocks=4,\n#             mlp_units=[128],\n#             mlp_dropout=0.4,\n#             dropout=0.25,\n#         )\n\n#         model.compile(\n#             loss=\"binary_crossentropy\",\n#             optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n#             metrics=[\"accuracy\", get_f1],\n#         )\n\n#         model.fit(\n#             x_train,\n#             y_train,\n#             validation_data=(x_test, y_test),\n#             epochs=10,\n#             batch_size=BATCH_SIZE,\n#             callbacks=callbacks,\n#         )\n\n#         filename = str('tpu'+'_'+f'{grp}_{q_no}')\n\n#         os.chdir(r'/kaggle/working/models')\n#         pickle.dump(model, open(filename, 'wb'))\n#         models['tpu'+'_'+f'{grp}_{q_no}'] = filename\n\n# #         y_pred = model(x_test, training=False)\n#         y_pred = model.predict(x_test)\n#         y_pred = [1 if x > 0.5 else 0 for x in y_pred]\n#         evaluation_dict[q_no] = f1_score(y_test, y_pred, average='macro')\n\n# #         del model\n# #         del x_train, x_test, y_train, y_test\n# #         gc.collect()","metadata":{"papermill":{"duration":0.838239,"end_time":"2023-06-24T11:22:08.209196","exception":false,"start_time":"2023-06-24T11:22:07.370957","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.440051Z","iopub.status.idle":"2023-06-26T08:21:11.440921Z","shell.execute_reply.started":"2023-06-26T08:21:11.440649Z","shell.execute_reply":"2023-06-26T08:21:11.440674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install tensorflow_decision_forests\n# import tensorflow_decision_forests as tfdf","metadata":{"papermill":{"duration":0.772737,"end_time":"2023-06-24T11:22:09.754978","exception":false,"start_time":"2023-06-24T11:22:08.982241","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.442427Z","iopub.status.idle":"2023-06-26T08:21:11.443239Z","shell.execute_reply.started":"2023-06-26T08:21:11.442974Z","shell.execute_reply":"2023-06-26T08:21:11.442998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred = model(x_test,training=False)\n# y_pred","metadata":{"papermill":{"duration":0.82374,"end_time":"2023-06-24T11:22:11.335505","exception":false,"start_time":"2023-06-24T11:22:10.511765","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.444803Z","iopub.status.idle":"2023-06-26T08:21:11.445634Z","shell.execute_reply.started":"2023-06-26T08:21:11.445357Z","shell.execute_reply":"2023-06-26T08:21:11.445381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred1 = model.predict(x_test)\n# y_pred1","metadata":{"papermill":{"duration":0.781826,"end_time":"2023-06-24T11:22:12.972977","exception":false,"start_time":"2023-06-24T11:22:12.191151","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.447244Z","iopub.status.idle":"2023-06-26T08:21:11.448180Z","shell.execute_reply.started":"2023-06-26T08:21:11.447872Z","shell.execute_reply":"2023-06-26T08:21:11.447898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred.numpy().ravel()\n# y_pred1.ravel()\n","metadata":{"papermill":{"duration":0.799243,"end_time":"2023-06-24T11:22:14.618321","exception":false,"start_time":"2023-06-24T11:22:13.819078","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.449766Z","iopub.status.idle":"2023-06-26T08:21:11.450884Z","shell.execute_reply.started":"2023-06-26T08:21:11.450637Z","shell.execute_reply":"2023-06-26T08:21:11.450660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred1 = [1 if x > 0.5 else 0 for x in y_pred1]","metadata":{"papermill":{"duration":0.814491,"end_time":"2023-06-24T11:22:16.196574","exception":false,"start_time":"2023-06-24T11:22:15.382083","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.452423Z","iopub.status.idle":"2023-06-26T08:21:11.453261Z","shell.execute_reply.started":"2023-06-26T08:21:11.452996Z","shell.execute_reply":"2023-06-26T08:21:11.453031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import f1_score\n# f1_score(y_pred1,y_test)\n\n#0.8459793294681119\n#0.8038089903374878","metadata":{"papermill":{"duration":0.78696,"end_time":"2023-06-24T11:22:17.749152","exception":false,"start_time":"2023-06-24T11:22:16.962192","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.454712Z","iopub.status.idle":"2023-06-26T08:21:11.455545Z","shell.execute_reply.started":"2023-06-26T08:21:11.455247Z","shell.execute_reply":"2023-06-26T08:21:11.455271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# metric = tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=0.5)\n# y_true = tf.one_hot(true_df.values.reshape((-1)), depth=2)\n# y_pred = tf.one_hot((prediction_df.values.reshape((-1))>threshold).astype('int'), depth=2)\n# metric.update_state(y_true, y_pred)\n# f1_score = metric.result().numpy()\n# if f1_score > max_score:\n#     max_score = f1_score\n#     best_threshold = threshold\n        \n# print(\"Best threshold \", best_threshold, \"\\tF1 score \", max_score)","metadata":{"papermill":{"duration":0.802796,"end_time":"2023-06-24T11:22:19.403001","exception":false,"start_time":"2023-06-24T11:22:18.600205","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.457133Z","iopub.status.idle":"2023-06-26T08:21:11.458025Z","shell.execute_reply.started":"2023-06-26T08:21:11.457735Z","shell.execute_reply":"2023-06-26T08:21:11.457761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow import keras\n# from tensorflow.keras import layers\n\n# @register_keras_serializable\n# class TransformerModel(keras.Model):\n#     def __init__(\n#         self,\n#         input_shape,\n#         head_size,\n#         num_heads,\n#         ff_dim,\n#         num_transformer_blocks,\n#         mlp_units,\n#         dropout=0,\n#         mlp_dropout=0,\n#     ):\n#         super(TransformerModel, self).__init__()\n\n#         self.transformer_blocks = []\n#         for _ in range(num_transformer_blocks):\n#             self.transformer_blocks.append(\n#                 self.build_transformer_block(head_size, num_heads, ff_dim, dropout)\n#             )\n\n#         self.pooling_layer = layers.GlobalAveragePooling1D(data_format=\"channels_first\")\n\n#         self.mlp_layers = []\n#         for dim in mlp_units:\n#             self.mlp_layers.append(layers.Dense(dim, activation=\"relu\"))\n#             self.mlp_layers.append(layers.Dropout(mlp_dropout))\n\n#         self.output_layer = layers.Dense(1, activation=\"sigmoid\")\n\n#     def build_transformer_block(self, head_size, num_heads, ff_dim, dropout):\n#         inputs = keras.Input(shape=input_shape)\n#         x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n#         attention_output = layers.MultiHeadAttention(\n#             num_heads=num_heads,\n#             key_dim=head_size,\n#             dropout=dropout\n#         )(x, x)\n#         attention_output = layers.Dropout(dropout)(attention_output)\n#         attention_output = layers.Add()([x, attention_output])\n\n#         x = layers.LayerNormalization(epsilon=1e-6)(attention_output)\n#         x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n#         x = layers.Dropout(dropout)(x)\n#         x = layers.Conv1D(filters=input_shape[-1], kernel_size=1)(x)\n#         transformer_output = layers.Add()([attention_output, x])\n\n#         return keras.Model(inputs, transformer_output)\n\n#     def call(self, inputs, training=False):\n#         x = inputs\n#         for transformer_block in self.transformer_blocks:\n#             x = transformer_block(x, training=training)\n#         x = self.pooling_layer(x)\n#         for layer in self.mlp_layers:\n#             x = layer(x, training=training)\n#         outputs = self.output_layer(x)\n#         return outputs\n\n\n# # # Instantiate the model\n# # input_shape = (200, 10)\n# # model = TransformerModel(\n# #     input_shape,\n# #     head_size=256,\n# #     num_heads=4,\n# #     ff_dim=4,\n# #     num_transformer_blocks=4,\n# #     mlp_units=[128],\n# #     mlp_dropout=0.4,\n# #     dropout=0.25,\n# # )\n\n# # # Compile the model\n# # model.compile(\n# #     loss=\"binary_crossentropy\",\n# #     optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n# #     metrics=[\"accuracy\"]\n# # )\n\n# # # Train the model\n# # model.fit(\n# #     x_train,\n# #     y_train,\n# #     batch_size=32,\n# #     epochs=10,\n# #     validation_data=(x_test, y_test)\n# # )\n\n# # # Make predictions using __call__ method\n# # predictions = model(x_test)\n\n\n\n# # # Define model hyperparameters\n# # input_shape = ...\n# # head_size = ...\n# # num_heads = ...\n# # ff_dim = ...\n# # num_transformer_blocks = ...\n# # mlp_units = ...\n\n# # # Create an instance of the TransformerModel class\n# # model = TransformerModel(\n# #     input_shape,\n# #     head_size,\n# #     num_heads,\n# #     ff_dim,\n# #     num_transformer_blocks,\n# #     mlp_units,\n# #     dropout=0,\n# #     mlp_dropout=0,\n# # )\n\n# # Compile the model and define loss and optimizer\n# # model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n\n# # # Train the model\n# # model.fit(train_X, train_y, epochs=10, batch_size=64)\n\n# # # Make predictions\n# # predictions = model.predict(test_X)","metadata":{"papermill":{"duration":0.833133,"end_time":"2023-06-24T11:22:21.037840","exception":false,"start_time":"2023-06-24T11:22:20.204707","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.459627Z","iopub.status.idle":"2023-06-26T08:21:11.460496Z","shell.execute_reply.started":"2023-06-26T08:21:11.460204Z","shell.execute_reply":"2023-06-26T08:21:11.460229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models ={}\n# evaluation_dict = {}\n\n# drops [5, 8, 10, 13, 15]\n\n# # input_shape = x_train.shape[1:]\n# input_shape = (200,308)\n\n\n# # model.summary()\n\n# callbacks = [keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)]\n\n# for q_no in range(1,4):\n#     # Select level group for the question based on the q_no.\n#     if q_no<=3: grp = '0-4'\n#     elif q_no<=13: grp = '5-12'\n#     elif q_no<=22: grp = '13-22'\n#     print(\"### q_no\", q_no, \"grp\", grp)\n    \n#     x_train = create_xtrain(dataset_df,grp)\n#     y_train = create_ytrain(grp,q_no)\n    \n#     x_train, x_test,y_train,y_test = create_dataset(x_train,y_train)\n    \n#     model = TransformerModel(\n#     input_shape,\n#     head_size=256,\n#     num_heads=4,\n#     ff_dim=4,\n#     num_transformer_blocks=4,\n#     mlp_units=[128],\n#     mlp_dropout=0.4,\n#     dropout=0.25,\n# )\n\n#     model.compile(\n#         loss=\"binary_crossentropy\",\n#         optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n#         metrics=[\"accuracy\",get_f1],\n#     )\n    \n    \n\n#     model.fit(\n#         x_train,\n#         y_train,\n#         validation_data=(x_test,y_test),\n#     #     validation_split=0.2,\n#         epochs=10,\n#         batch_size=100,\n#         callbacks=callbacks,\n#     )\n    \n# #     models[f'{grp}_{q_no}'] = model\n    \n#     filename= str(f'{grp}_{q_no}')\n    \n#     os.chdir(r'/kaggle/working/models')\n    \n#     pickle.dump(model,open(filename, 'wb')) \n\n#     models[f'{grp}_{q_no}'] = filename\n    \n    \n    \n# #     print(\"model for \",u,\"saved in file name \",filename) \n    \n#     y_pred = model(x_test)\n#     y_pred = [1 if x > 0.5 else 0 for x in y_pred]\n#     f1_score(y_pred, y_test)\n#     evaluation_dict[q_no] = f1_score(y_test, y_pred,average='macro')\n    \n#     del model\n#     del x_train,x_test,y_train,y_test\n#     gc.collect()","metadata":{"papermill":{"duration":0.794439,"end_time":"2023-06-24T11:22:22.602722","exception":false,"start_time":"2023-06-24T11:22:21.808283","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.461991Z","iopub.status.idle":"2023-06-26T08:21:11.462819Z","shell.execute_reply.started":"2023-06-26T08:21:11.462563Z","shell.execute_reply":"2023-06-26T08:21:11.462587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \n\n\n# def feature_engineer(test_df):\n# #     test_df = pd.read_csv(f,usecols=cols)\n#     test_df = test_df[cols]\n#     test_df.fillna(0,inplace=True)\n#     test_df.drop('level_group',inplace=True, axis=1)\n#     test_df = test_df.reset_index(drop=True)\n#     test_df = test_df.set_index('session_id',drop=False)\n#     df = convert_arr(test_df)\n#     test_data = pad_arrays(df)\n#     return test_data","metadata":{"papermill":{"duration":0.79309,"end_time":"2023-06-24T11:22:24.226852","exception":false,"start_time":"2023-06-24T11:22:23.433762","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.464329Z","iopub.status.idle":"2023-06-26T08:21:11.465164Z","shell.execute_reply.started":"2023-06-26T08:21:11.464896Z","shell.execute_reply":"2023-06-26T08:21:11.464919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Reference\n# # https://www.kaggle.com/code/philculliton/basic-submission-demo\n# # https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\n\n# # import jo_wilder\n# # # jo_wilder.make_env.func_dict['__called__'] = False\n# # env = jo_wilder.make_env()\n\n# # iter_test = env.iter_test()\n# import os\n# os.chdir(r'/kaggle/working/')\n\n# import jo_wilder_310\n# jo_wilder_310.make_env.func_dict['__called__'] = False\n# env = jo_wilder_310.make_env()\n\n# iter_test = env.iter_test()\n\n# limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\n# for (test, sample_submission) in iter_test:\n#     grp = test.level_group.values[0]\n    \n#     test_data = feature_engineer(test)\n#     test_data2 = feature_engineer1(test)\n#     test_data2 = test_data2.loc[:, test_data2.columns != 'level_group'] \n# #     xgbmodels = [5,6,7,8,9,10,11,13,14,15,16,17]   \n#     xgbmodels = [5,10,13,15] \n#     best_threshold = 0.63\n#     a,b = limits[grp]\n#     for t in range(a,b):\n        \n# #         filename1=r'/kaggle/input/predict-student-perf-transformer-2'\n# #         filename2=r'/kaggle/input/student-performance-xgb-model'\n        \n#         if t in xgbmodels:  \n            \n#             xgbmodel = xgb.Booster()\n#             os.chdir(r'/kaggle/input/student-performance-xgb-model')\n#             filename = 'models/xg_'+f'{grp}_{t}'\n#             xgbmodel.load_model(filename) \n            \n#             d_test = xgb.DMatrix(test_data2)  \n#             pred = xgbmodel.predict(d_test) \n#             predictions = [1 if x > best_threshold else 0 for x in pred]\n            \n# #             print(\"predicted using xgb models\",predictions)\n#         else:\n# #             os.chdir(r'/kaggle/input/std-pred-perf-gpumodels/')\n#             os.chdir(r'/kaggle/input/std-pred-perf-gpumodels/models')\n# #     /kaggle/input/std-perf-transformerclassmodel/models/0-4_1\n#             filename = f'{grp}_{t}'\n    \n# #             print(\"filename \",filename)\n# #             adfaf\n        \n        \n# #             tr_model = pickle.load(open(filename, 'rb'))\n             \n#             pred= model(test_data)\n            \n#             predictions = [1 if x > 0.5 else 0 for x in pred]\n# #             print(\"predicted using trf models /n\",predictions)\n             \n         \n#         mask = sample_submission.session_id.str.contains(f'q{t}')\n         \n# #         n_predictions = (predictions > best_threshold).astype(int)\n#         sample_submission.loc[mask,'correct'] = predictions\n# #         print(\"sample \\n\",sample_submission, \"t is \",t, \"grp is \",grp)\n#     os.chdir(r'/kaggle/working/')\n#     env.predict(sample_submission)","metadata":{"papermill":{"duration":0.797833,"end_time":"2023-06-24T11:22:27.489890","exception":false,"start_time":"2023-06-24T11:22:26.692057","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.469168Z","iopub.status.idle":"2023-06-26T08:21:11.470059Z","shell.execute_reply.started":"2023-06-26T08:21:11.469778Z","shell.execute_reply":"2023-06-26T08:21:11.469804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ! head submission.csv","metadata":{"papermill":{"duration":0.81515,"end_time":"2023-06-24T11:22:29.074515","exception":false,"start_time":"2023-06-24T11:22:28.259365","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.471512Z","iopub.status.idle":"2023-06-26T08:21:11.472322Z","shell.execute_reply.started":"2023-06-26T08:21:11.472066Z","shell.execute_reply":"2023-06-26T08:21:11.472090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for name, value in evaluation_dict.items():\n#   print(f\"question {name}: accuracy {value:.4f}\")\n\n# print(\"\\nAverage accuracy\", sum(evaluation_dict.values())/18)\n\n# # Average accuracy 0.49796176251447516\n# # Average accuracy 0.5709902502264197 - ros\n# # Average accuracy 0.552894545015699 - rus\n# # Average accuracy 0.5603555485307106 - smote","metadata":{"papermill":{"duration":0.766567,"end_time":"2023-06-24T11:22:30.597858","exception":false,"start_time":"2023-06-24T11:22:29.831291","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.473775Z","iopub.status.idle":"2023-06-26T08:21:11.474589Z","shell.execute_reply.started":"2023-06-26T08:21:11.474314Z","shell.execute_reply":"2023-06-26T08:21:11.474337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Create a dataframe of required size:\n# # (no: of users in validation set x no: of questions) initialized to zero values\n# # to store true values of the label `correct`. \n# true_df = pd.DataFrame(data=np.zeros((len(VALID_USER_LIST),18)), index=VALID_USER_LIST)\n# for i in range(18):\n#     # Get the true labels.\n#     tmp = labels.loc[labels.q == i+1].set_index('session').loc[VALID_USER_LIST]\n#     true_df[i] = tmp.correct.values\n\n# max_score = 0; best_threshold = 0\n\n# # Loop through threshold values from 0.4 to 0.8 and select the threshold with \n# # the highest `F1 score`.\n# for threshold in np.arange(0.4,0.8,0.01):\n#     metric = tfa.metrics.F1Score(num_classes=2,average=\"macro\",threshold=threshold)\n#     y_true = tf.one_hot(true_df.values.reshape((-1)), depth=2)\n#     y_pred = tf.one_hot((prediction_df.values.reshape((-1))>threshold).astype('int'), depth=2)\n#     metric.update_state(y_true, y_pred)\n#     f1_score = metric.result().numpy()\n    \n#     if f1_score > max_score:\n#         max_score = f1_score\n#         best_threshold = threshold\n        \n# print(\"Best threshold \", best_threshold, \"\\tF1 score \", max_score)\n\n# # Best threshold  0.6300000000000002 \tF1 score  0.6738222\n# # Best threshold  0.4700000000000001 \tF1 score  0.5874005 for rus\n# # 0.62 for ros\n# # Best threshold  0.5500000000000002 \tF1 score  0.644594 smote","metadata":{"papermill":{"duration":0.859385,"end_time":"2023-06-24T11:22:32.335310","exception":false,"start_time":"2023-06-24T11:22:31.475925","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.476062Z","iopub.status.idle":"2023-06-26T08:21:11.476879Z","shell.execute_reply.started":"2023-06-26T08:21:11.476618Z","shell.execute_reply":"2023-06-26T08:21:11.476642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred1 = tf.one_hot((prediction_df.values.reshape((-1))>0.5).astype('int'), depth=2)\n# # ypred = calibrated_clf.predict(X_valid)\n# print(\"classification report \\n\",classification_report(y_pred1,y_true))\n# print(\"confusion matrix \\n\",confusion_matrix(y_pred1[:,1],y_true[:,1]))\n    ","metadata":{"papermill":{"duration":0.841751,"end_time":"2023-06-24T11:22:33.942929","exception":false,"start_time":"2023-06-24T11:22:33.101178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.478483Z","iopub.status.idle":"2023-06-26T08:21:11.479293Z","shell.execute_reply.started":"2023-06-26T08:21:11.479045Z","shell.execute_reply":"2023-06-26T08:21:11.479069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nHere you'll use the `best_threshold` calculate in the previous cell","metadata":{"id":"ezA40GQ4n2PH","papermill":{"duration":0.771588,"end_time":"2023-06-24T11:22:35.468937","exception":false,"start_time":"2023-06-24T11:22:34.697349","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# def feature_engineer(test_df):\n# #     test_df = pd.read_csv(f,usecols=cols)\n#     test_df = test_df[cols]\n#     test_df.fillna(0,inplace=True)\n#     test_df.drop('level_group',inplace=True, axis=1)\n#     test_df = test_df.reset_index()\n#     test_df = test_df.set_index('session_id',drop=False)\n#     df = convert_arr(test_df)\n#     test_data = pad_arrays(df)\n#     return test_data","metadata":{"papermill":{"duration":0.775484,"end_time":"2023-06-24T11:22:37.052855","exception":false,"start_time":"2023-06-24T11:22:36.277371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.480768Z","iopub.status.idle":"2023-06-26T08:21:11.481603Z","shell.execute_reply.started":"2023-06-26T08:21:11.481324Z","shell.execute_reply":"2023-06-26T08:21:11.481348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# os.chdir(r'/kaggle/input/predict-student-performance-from-game-play')","metadata":{"papermill":{"duration":0.885081,"end_time":"2023-06-24T11:22:38.745264","exception":false,"start_time":"2023-06-24T11:22:37.860183","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.483096Z","iopub.status.idle":"2023-06-26T08:21:11.483961Z","shell.execute_reply.started":"2023-06-26T08:21:11.483692Z","shell.execute_reply":"2023-06-26T08:21:11.483717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import jo_wilder_310\n# jo_wilder_310.make_env.func_dict['__called__'] = False\n# env = jo_wilder_310.make_env()\n\n# iter_test = env.iter_test()","metadata":{"papermill":{"duration":0.766048,"end_time":"2023-06-24T11:22:40.273052","exception":false,"start_time":"2023-06-24T11:22:39.507004","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.485596Z","iopub.status.idle":"2023-06-26T08:21:11.486399Z","shell.execute_reply.started":"2023-06-26T08:21:11.486147Z","shell.execute_reply":"2023-06-26T08:21:11.486170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Reference\n# # https://www.kaggle.com/code/philculliton/basic-submission-demo\n# # https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\n\n# # import jo_wilder\n# # # jo_wilder.make_env.func_dict['__called__'] = False\n# # env = jo_wilder.make_env()\n\n# # iter_test = env.iter_test()\n# import os\n# os.chdir(r'/kaggle/working/')\n\n# import jo_wilder_310\n# jo_wilder_310.make_env.func_dict['__called__'] = False\n# env = jo_wilder_310.make_env()\n\n# iter_test = env.iter_test()\n\n# limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\n# for (test, sample_submission) in iter_test:\n# #     print(test)\n# #     fadf\n    \n#     grp = test.level_group.values[0]\n    \n#     test_data = feature_engineer(test)\n     \n    \n#     a,b = limits[grp]\n#     for t in range(a,b):\n# #         xgbclassifier = models[f'{grp}_{t}']\n        \n# #         test_data = test_df.loc[:, test_df.columns != 'level_group']\n# #         test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_df.loc[:, test_df.columns != 'level_group'])\n        \n    \n#         pred= model.predict(test_data)\n#         predictions = [1 if x > 0.5 else 0 for x in pred]\n#         mask = sample_submission.session_id.str.contains(f'q{t}')\n# #         n_predictions = (predictions > best_threshold).astype(int)\n#         sample_submission.loc[mask,'correct'] = predictions #.flatten()\n    \n#     env.predict(sample_submission)","metadata":{"id":"gHiXTnTVn2PI","papermill":{"duration":0.763892,"end_time":"2023-06-24T11:22:41.840887","exception":false,"start_time":"2023-06-24T11:22:41.076995","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.488027Z","iopub.status.idle":"2023-06-26T08:21:11.488862Z","shell.execute_reply.started":"2023-06-26T08:21:11.488608Z","shell.execute_reply":"2023-06-26T08:21:11.488632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ! head submission.csv","metadata":{"id":"iYBXokAyn2PI","papermill":{"duration":0.834426,"end_time":"2023-06-24T11:22:43.431875","exception":false,"start_time":"2023-06-24T11:22:42.597449","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-26T08:21:11.490314Z","iopub.status.idle":"2023-06-26T08:21:11.491126Z","shell.execute_reply.started":"2023-06-26T08:21:11.490859Z","shell.execute_reply":"2023-06-26T08:21:11.490882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.817262,"end_time":"2023-06-24T11:22:45.073906","exception":false,"start_time":"2023-06-24T11:22:44.256644","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.797684,"end_time":"2023-06-24T11:22:46.626358","exception":false,"start_time":"2023-06-24T11:22:45.828674","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}