{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-14T17:20:24.869828Z","iopub.execute_input":"2023-05-14T17:20:24.870266Z","iopub.status.idle":"2023-05-14T17:20:24.941472Z","shell.execute_reply.started":"2023-05-14T17:20:24.870215Z","shell.execute_reply":"2023-05-14T17:20:24.940591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\ntrain = pd.read_parquet('/kaggle/input/how-to-get-32gb-ram/train.parquet')\nprint(train.shape)\ntrain.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets = pd.read_parquet('/kaggle/input/how-to-get-32gb-ram/train_labels.parquet')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]) )\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nprint(targets.shape)\ntargets.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y',\n        'screen_coor_x', 'screen_coor_y', 'hover_duration']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    dfs = []\n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df\n\ndf = feature_engineer(train)\nprint(df.shape)\ndf.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES = [c for c in df.columns if c != 'level_group']\nALL_USERS = df.index.unique()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5) \noof = pd.DataFrame(data=np.zeros((len(ALL_USERS), 18)), index=ALL_USERS)\nmodels = {}\n\n# CV score на 5 группах\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df, groups=df.index)):\n    print('#' * 25)\n    print('### Fold', i + 1)\n    print('#' * 25)\n\n    # Вопросы 1 - 18\n    for t in range(1, 19):\n        print(t, ', ', end='')\n\n        # Определиние группы вопросов\n        if t <= 3:\n            grp = '0-4'\n        elif t <= 13:\n            grp = '5-12'\n        elif t <= 22:\n            grp = '13-22'\n\n        # Тренировочные данные\n        train_x = df.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q == t].set_index('session').loc[train_users]\n\n        # Валидация данных\n        valid_x = df.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q == t].set_index('session').loc[valid_users]\n\n        # Обучение\n        clf = RandomForestClassifier()\n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'])\n\n        # Сохранение модели, предсказание OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t - 1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:, 1]\n\n    print()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Метки в фрейм (18 столбцов)\ntrue = oof.copy()\nfor k in range(18):\n    # Получаем метки\n    tmp = targets.loc[targets.q == k + 1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Лучший порог для конвертирования вероятностей\nscores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')\n    scores.append(m)\n    thresholds.append(threshold)\n    if m > best_score:\n        best_score = m\n        best_threshold = threshold\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# График\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Порог',size=14)\nplt.ylabel('F1 Score',size=14)\nplt.title(f'Пороги и F1_Score.  Лучший F1_Score = {best_score:.3f} при лучшем пороге = {best_threshold:.3}',size=18)\nplt.show()\n\nprint('Для оптимального порога')\nfor k in range(18):\n    # F1 для каждого вопроса\n    m = f1_score(true[k].values, (oof[k].values > best_threshold).astype('int'), average='macro')\n    print(f'Q{k}: F1 =', m)\n\n# Общий F1\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1)) > best_threshold).astype('int'), average='macro')\nprint('Общий F1 =', m)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-14T19:02:56.16327Z","iopub.execute_input":"2023-05-14T19:02:56.163765Z","iopub.status.idle":"2023-05-14T19:18:43.362147Z","shell.execute_reply.started":"2023-05-14T19:02:56.163715Z","shell.execute_reply":"2023-05-14T19:18:43.360687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\nlimits = {'0-4': (1, 4), '5-12': (4, 14), '13-22': (14, 19)}\n\nfor (sample_submission, test) in iter_test:\n\n    df = feature_engineer(test)\n    grp = test.level_group.values[0]\n    a, b = limits[grp]\n    for t in range(a, b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURES].astype('float32'))[:, 1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask, 'correct'] = int(p.item() > best_threshold)\n\n    env.predict(sample_submission)\n\n#Сдача\ndf = pd.read_csv('submission.csv')\nprint(df.shape)\ndf.head()\nprint(df.correct.mean())","metadata":{"execution":{"iopub.status.busy":"2023-05-14T19:52:48.934293Z","iopub.execute_input":"2023-05-14T19:52:48.934815Z","iopub.status.idle":"2023-05-14T19:52:49.473934Z","shell.execute_reply.started":"2023-05-14T19:52:48.934773Z","shell.execute_reply":"2023-05-14T19:52:49.472358Z"},"trusted":true},"execution_count":null,"outputs":[]}]}