{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Try Polars\nThis notebook is just for learning polars","metadata":{}},{"cell_type":"code","source":"!pip install -qq /kaggle/input/polars-installer/polars-0.16.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:00.595828Z","iopub.execute_input":"2023-02-03T05:09:00.596434Z","iopub.status.idle":"2023-02-03T05:09:35.061929Z","shell.execute_reply.started":"2023-02-03T05:09:00.596319Z","shell.execute_reply":"2023-02-03T05:09:35.060533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# import libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport pickle\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport cv2\nimport matplotlib.pyplot as plt\nimport time\nimport math\nimport sys\n\nimport polars as pl\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier, early_stopping, log_evaluation\n\nfrom sklearn.metrics import matthews_corrcoef, confusion_matrix, roc_auc_score\nfrom sklearn.model_selection import GroupKFold\nfrom scipy.optimize import minimize\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-02-03T05:09:35.065042Z","iopub.execute_input":"2023-02-03T05:09:35.065557Z","iopub.status.idle":"2023-02-03T05:09:37.016874Z","shell.execute_reply.started":"2023-02-03T05:09:35.065505Z","shell.execute_reply":"2023-02-03T05:09:37.015595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set Configurations","metadata":{}},{"cell_type":"code","source":"CFG = {\n    \"DEBUG\" : False,\n    \"n_folds\" : 5,\n    \"train_folds\" : [0,1,2,3,4],\n}\n\nCFG[\"lgb_params\"] = { 'boosting_type' :'gbdt',\n                      'objective': 'binary',\n                      'random_state': 21,\n                      'learning_rate': 0.03,\n                      }\n\nif CFG[\"DEBUG\"]:\n    CFG[\"EXP_CATEGORY\"] = \"DEBUG\"\n    CFG[\"sample_num\"] = 1000\n\nCFG[\"INPUT_DIR\"] = \"/kaggle/input/\"\nCFG[\"OUTPUT_DIR\"] = \"/kaggle/working/\"\nCFG[\"BASE_DIR\"] = os.path.join(CFG[\"INPUT_DIR\"], \"nfl-player-contact-detection\")\nCFG[\"TRAIN_HELMET_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"train_baseline_helmets.csv\")\nCFG[\"TRAIN_TRACKING_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"train_player_tracking.csv\")\nCFG[\"TRAIN_VIDEO_META_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"train_video_metadata.csv\")\nCFG[\"TRAIN_LABEL_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"train_labels.csv\")\n\nCFG[\"TEST_HELMET_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"test_baseline_helmets.csv\")\nCFG[\"TEST_TRACKING_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"test_player_tracking.csv\")\nCFG[\"TEST_VIDEO_META_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"test_video_metadata.csv\")\nCFG[\"TEST_LABEL_CSV\"] = os.path.join(CFG[\"BASE_DIR\"], \"sample_submission.csv\")\n\nCFG[\"EXP_DIR\"] = os.path.join(CFG[\"OUTPUT_DIR\"], \"EXP\")","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:37.01881Z","iopub.execute_input":"2023-02-03T05:09:37.019175Z","iopub.status.idle":"2023-02-03T05:09:37.032056Z","shell.execute_reply.started":"2023-02-03T05:09:37.019143Z","shell.execute_reply":"2023-02-03T05:09:37.031179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/EXP","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:37.035474Z","iopub.execute_input":"2023-02-03T05:09:37.036117Z","iopub.status.idle":"2023-02-03T05:09:38.161823Z","shell.execute_reply.started":"2023-02-03T05:09:37.036067Z","shell.execute_reply":"2023-02-03T05:09:38.160453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL Functions","metadata":{}},{"cell_type":"code","source":"# lgboost code\ndef fit_lgboost(df_, FEATURE_COLS):\n    kf = GroupKFold(n_splits=CFG[\"n_folds\"])\n    X = df_[FEATURE_COLS]\n    y = df_[\"contact\"]\n    oof_pred = np.zeros(len(y), dtype=np.float32)\n    generator = kf.split(X, y, df_[\"game_play\"])\n    for fold, (idx_train, idx_valid) in enumerate(generator):\n        print(\"-\")\n        print(f\"fold {fold}\")\n        x_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n        x_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n        display(pd.Series(y_valid).value_counts())\n\n        params = CFG[\"lgb_params\"]\n        model = LGBMClassifier(**params).fit(x_train, y_train, \n                                   eval_set=[(x_train, y_train), (x_valid, y_valid)],\n                                   callbacks=[early_stopping(100), log_evaluation(100)],\n                                   eval_metric=['auc','binary_logloss'])\n\n        model_path = os.path.join(CFG[\"EXP_DIR\"], f'lgb_fold{fold}.model')\n        pickle.dump(model, open(model_path, 'wb'))\n\n        pred_i = model.predict_proba(x_valid)[:, 1]\n        oof_pred[x_valid.index] = pred_i\n        score = round(roc_auc_score(y_valid, pred_i), 5)\n        print(f'Performance of the prediction: {score}\\n')\n        del model; gc.collect()\n    oof_pred_file = os.path.join(CFG[\"EXP_DIR\"], 'oof_pred')\n    np.save(oof_pred_file, oof_pred)\n    score = round(roc_auc_score(y, oof_pred), 5)\n    print(f'All Performance of the prediction: {score}')\n    return oof_pred","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.163771Z","iopub.execute_input":"2023-02-03T05:09:38.164334Z","iopub.status.idle":"2023-02-03T05:09:38.178671Z","shell.execute_reply.started":"2023-02-03T05:09:38.164266Z","shell.execute_reply":"2023-02-03T05:09:38.177191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Functions","metadata":{}},{"cell_type":"code","source":"def add_feature_cols(df_, FEATURE_COLS, remove_col_list):\n    additional_cols = list(df_.columns)\n    additional_cols = [col for col in additional_cols if not col in remove_col_list]\n    FEATURE_COLS.extend(additional_cols)\n    return FEATURE_COLS","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.180607Z","iopub.execute_input":"2023-02-03T05:09:38.181481Z","iopub.status.idle":"2023-02-03T05:09:38.199245Z","shell.execute_reply.started":"2023-02-03T05:09:38.18144Z","shell.execute_reply":"2023-02-03T05:09:38.197962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge Functions","metadata":{}},{"cell_type":"code","source":"def target_merge_tracking(target_df, tracking_df, FEATURE_COLS,\n                          TRACKING_COLS = [\"game_play\", \"nfl_player_id\", \"step\", \n                                           \"x_position\", \"y_position\", \"datetime\",\n                                           \"speed\",\"distance\",\"direction\",\"orientation\",\n                                           \"acceleration\",\"sa\", \"team\", \"jersey_number\"]):\n    target_df = target_df.with_column(pl.concat_str([pl.col(\"game_play\"),\n                                                     pl.col(\"step\").cast(str),\n                                                     pl.col(\"nfl_player_id_1\"),\n                                                    ], sep='_').alias('game_step_player_1'))\n    target_df = target_df.with_column(pl.concat_str([pl.col(\"game_play\"),\n                                                     pl.col(\"step\").cast(str),\n                                                     pl.col(\"nfl_player_id_2\"),\n                                                    ], sep='_').alias('game_step_player_2'))\n\n    tracking_df = tracking_df.select(TRACKING_COLS)\n    tracking_df = tracking_df.with_column(pl.concat_str([pl.col(\"game_play\"),\n                                                         pl.col(\"step\").cast(str),\n                                                         pl.col(\"nfl_player_id\"),\n                                                        ], sep='_').alias('game_step_player'))\n\n    tracking_df = tracking_df.drop([\"game_play\", \"step\", \"nfl_player_id\", \"datetime\"])\n\n    # merge tracking to target\n    for player_id in [1,2]:\n        tracking_player = tracking_df.select([pl.all().suffix(f\"_{player_id}\")])\n        target_df = target_df.join(tracking_player, on=[f\"game_step_player_{player_id}\"], how=\"left\")\n        # add features col\n        FEATURE_COLS = add_feature_cols(tracking_player, FEATURE_COLS,\n                                        [f\"game_step_player_{player_id}\", f\"frame_{player_id}\", f\"datetime_{player_id}\"])\n    # drop col\n    target_df = target_df.drop([\"game_step_player_1\", \"game_step_player_2\"])\n    print(len(target_df.columns))\n    print(\"original length\", len(target_df))\n    return target_df, FEATURE_COLS","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.200969Z","iopub.execute_input":"2023-02-03T05:09:38.201393Z","iopub.status.idle":"2023-02-03T05:09:38.215947Z","shell.execute_reply.started":"2023-02-03T05:09:38.201352Z","shell.execute_reply":"2023-02-03T05:09:38.214381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create feature function","metadata":{}},{"cell_type":"code","source":"def create_trackmerged_ftr(target_df, FEATURE_COLS):\n    target_df = target_df.with_column((np.sqrt(np.square(pl.col(\"x_position_1\")-pl.col(\"x_position_2\")) \\\n                                              + np.square(pl.col(\"y_position_1\")-pl.col(\"y_position_2\"))) \\\n                                      ).alias(\"players_dis\"))\n    target_df = target_df.with_column(pl.col(\"players_dis\").fill_null(0))\n    FEATURE_COLS.append(\"players_dis\")   \n    return target_df, FEATURE_COLS","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.217859Z","iopub.execute_input":"2023-02-03T05:09:38.218385Z","iopub.status.idle":"2023-02-03T05:09:38.232512Z","shell.execute_reply.started":"2023-02-03T05:09:38.218337Z","shell.execute_reply":"2023-02-03T05:09:38.231448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def target_merge_helmet(target_df, helmet_df, FEATURE_COLS):\n    print(\"original length\", len(target_df))\n    # set merge-key (game_frame_player_1,2) to merge helmet_df\n    target_df = target_df.with_column(((pl.col(\"step\").cast(int)/10*59.94+5*59.94).cast(int)+1).alias(\"frame\"))\n    target_df = target_df.with_column(pl.concat_str([pl.col(\"game_play\"),\n                                                     pl.col(\"frame\").cast(str),\n                                                     pl.col(\"nfl_player_id_1\"),\n                                                    ], sep='_').alias('game_frame_player_1'))\n    target_df = target_df.with_column(pl.concat_str([pl.col(\"game_play\"),\n                                                     pl.col(\"frame\").cast(str),\n                                                     pl.col(\"nfl_player_id_2\"),\n                                                    ], sep='_').alias('game_frame_player_2'))\n    # set merge key\n    helmet_df = helmet_df.with_column(pl.concat_str([pl.col(\"game_play\"),\n                                                     pl.col(\"frame\").cast(str),\n                                                     pl.col(\"nfl_player_id\"),\n                                                    ], sep='_').alias('game_frame_player'))\n\n    # merge target df & helmet_df\n    player_views = [[1, \"Endzone\"],[2, \"Endzone\"], [1, \"Sideline\"],[2, \"Sideline\"]]\n    for player_id, view in player_views:\n        helmet_view = helmet_df.filter(pl.col(\"view\")==view)\n        helmet_view = helmet_view[[\"game_frame_player\", \"left\", \"width\", \"top\", \"height\"]]\n        helmet_view = helmet_view.select(pl.all().suffix(f\"_{player_id}\"))\n        helmet_view = helmet_view.select([pl.col(helmet_view.columns[0]), pl.col(helmet_view.columns[1:]).prefix(f\"{view[0]}_\")])\n        target_df = target_df.join(helmet_view, on=f\"game_frame_player_{player_id}\", how=\"left\")  \n        # add features col\n        FEATURE_COLS = add_feature_cols(helmet_view, FEATURE_COLS, [f\"game_frame_player_{player_id}\"])\n\n    print(len(target_df.columns))\n    print(\"original length\", len(target_df))\n    return target_df, FEATURE_COLS","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.234295Z","iopub.execute_input":"2023-02-03T05:09:38.234659Z","iopub.status.idle":"2023-02-03T05:09:38.251485Z","shell.execute_reply.started":"2023-02-03T05:09:38.234609Z","shell.execute_reply":"2023-02-03T05:09:38.250369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_categorical_ftr(target_df, FEATURE_COLS):\n    target_df = target_df.with_column((pl.col(\"nfl_player_id_2\")== \"G\").cast(int).alias(\"is_ground\"))\n    # set team \n    target_df = target_df.with_column(((pl.col(\"team_1\")==\"home\").cast(int)).alias(\"team_1\"))\n    target_df = target_df.with_column(((pl.col(\"team_2\")==\"home\").cast(int)).alias(\"team_2\"))\n    target_df = target_df.with_column(pl.col('nfl_player_id_2').str.replace(\"G\", \"99999\"))\n    \n    target_df = target_df.with_column(pl.col([\"nfl_player_id_1\", \"nfl_player_id_2\"]).cast(int))\n    \n    target_df = target_df.fill_null(0)\n    target_df = target_df.fill_nan(0)\n    FEATURE_COLS.extend([\"is_ground\"])\n    return target_df, FEATURE_COLS","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.256579Z","iopub.execute_input":"2023-02-03T05:09:38.256966Z","iopub.status.idle":"2023-02-03T05:09:38.270607Z","shell.execute_reply.started":"2023-02-03T05:09:38.256935Z","shell.execute_reply":"2023-02-03T05:09:38.269364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Load Target","metadata":{}},{"cell_type":"code","source":"target_dtypes = {'contact_id':str, \n                'game_play':str,\n                'datetime':str,\n                'step':int,\n                'nfl_player_id_1':str,\n                'nfl_player_id_2':str,\n                'contact':int,\n                }\ntarget_df = pl.read_csv(CFG[\"TRAIN_LABEL_CSV\"], dtypes=target_dtypes)    \n\nFEATURE_COLS = [\"nfl_player_id_1\", \"nfl_player_id_2\", \"step\"]","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:38.272455Z","iopub.execute_input":"2023-02-03T05:09:38.272847Z","iopub.status.idle":"2023-02-03T05:09:40.696364Z","shell.execute_reply.started":"2023-02-03T05:09:38.272809Z","shell.execute_reply":"2023-02-03T05:09:40.695368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge tracking_df","metadata":{}},{"cell_type":"code","source":"%%time\ntracking_df = pl.read_csv(CFG[\"TRAIN_TRACKING_CSV\"])\ntarget_df, FEATURE_COLS = target_merge_tracking(target_df, tracking_df, FEATURE_COLS)\ntarget_df, FEATURE_COLS = create_trackmerged_ftr(target_df, FEATURE_COLS)\nprint(len(target_df))\n\ndel tracking_df","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:40.697655Z","iopub.execute_input":"2023-02-03T05:09:40.69864Z","iopub.status.idle":"2023-02-03T05:09:45.149422Z","shell.execute_reply.started":"2023-02-03T05:09:40.698557Z","shell.execute_reply":"2023-02-03T05:09:45.147839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exclude distance > 2","metadata":{}},{"cell_type":"code","source":"target_df = target_df.filter(pl.col(\"players_dis\") <= 2)\nprint(len(target_df))","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:45.151086Z","iopub.execute_input":"2023-02-03T05:09:45.151609Z","iopub.status.idle":"2023-02-03T05:09:45.479869Z","shell.execute_reply.started":"2023-02-03T05:09:45.151578Z","shell.execute_reply":"2023-02-03T05:09:45.478618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge helmet df","metadata":{}},{"cell_type":"code","source":"%%time\nhelmet_df = pl.read_csv(CFG[\"TRAIN_HELMET_CSV\"])\ntarget_df, FEATURE_COLS = target_merge_helmet(target_df, helmet_df, FEATURE_COLS)\ntarget_df, FEATURE_COLS = get_categorical_ftr(target_df, FEATURE_COLS)\nprint(len(target_df))\nprint(len(FEATURE_COLS))\ndel helmet_df","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:45.481552Z","iopub.execute_input":"2023-02-03T05:09:45.482752Z","iopub.status.idle":"2023-02-03T05:09:52.290314Z","shell.execute_reply.started":"2023-02-03T05:09:45.482692Z","shell.execute_reply":"2023-02-03T05:09:52.289075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"target_df = target_df.to_pandas()\noof_pred = fit_lgboost(target_df, FEATURE_COLS)","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:09:52.291903Z","iopub.execute_input":"2023-02-03T05:09:52.293129Z","iopub.status.idle":"2023-02-03T05:11:05.482949Z","shell.execute_reply.started":"2023-02-03T05:09:52.293054Z","shell.execute_reply":"2023-02-03T05:11:05.481308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_df[\"pred\"] = oof_pred\ndisplay(target_df[\"pred\"].hist(bins=20))","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:11:05.484813Z","iopub.execute_input":"2023-02-03T05:11:05.485401Z","iopub.status.idle":"2023-02-03T05:11:05.787395Z","shell.execute_reply.started":"2023-02-03T05:11:05.485334Z","shell.execute_reply":"2023-02-03T05:11:05.786009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/nfl-player-contact-detection/train_labels.csv\")\ntrain = pd.merge(train, target_df[[\"contact_id\", \"pred\"]], on=\"contact_id\", how=\"left\")\ntrain = train.fillna(0)\nprint(len(train))\n\ndef func(x_list):\n    score = matthews_corrcoef(train['contact'], train[\"pred\"]>x_list[0])\n    return -score\nx0 = [0.5]\nresult = minimize(func, x0,  method=\"nelder-mead\")\nopt_threshold = result.x[0]\nscore = round(matthews_corrcoef(train['contact'], train[\"pred\"]>opt_threshold), 5)\nthreshold = round(opt_threshold, 5)\nprint(f\"score:{score}, threshold:{threshold}\")\n\ndel train","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:11:05.789082Z","iopub.execute_input":"2023-02-03T05:11:05.789458Z","iopub.status.idle":"2023-02-03T05:12:30.059862Z","shell.execute_reply.started":"2023-02-03T05:11:05.789423Z","shell.execute_reply":"2023-02-03T05:12:30.05827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"CFG[\"features\"] = FEATURE_COLS\nCFG[\"threshold\"] = threshold","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:30.06205Z","iopub.execute_input":"2023-02-03T05:12:30.062498Z","iopub.status.idle":"2023-02-03T05:12:30.069087Z","shell.execute_reply.started":"2023-02-03T05:12:30.062464Z","shell.execute_reply":"2023-02-03T05:12:30.06752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df =pl.read_csv(CFG[\"TEST_LABEL_CSV\"])\nexpand_cols = [\"game_id\", \"play_id\", \"step\", \"nfl_player_id_1\", \"nfl_player_id_2\"]\ntest_df = test_df.with_column([pl.col(\"contact_id\").str.split_exact(\"_\", 4)\n                                    .struct.rename_fields(expand_cols).alias(\"key\")]).unnest(\"key\")\ntest_df = test_df.with_column(pl.concat_str([pl.col(\"game_id\"),\n                                             pl.col(\"play_id\"),\n                                             ], sep='_').alias('game_play'))\ntest_df = test_df.drop([\"game_id\", \"play_id\"])\n\nprint(len(test_df))","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:30.071026Z","iopub.execute_input":"2023-02-03T05:12:30.071405Z","iopub.status.idle":"2023-02-03T05:12:30.124678Z","shell.execute_reply.started":"2023-02-03T05:12:30.071369Z","shell.execute_reply":"2023-02-03T05:12:30.123453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge tracking","metadata":{}},{"cell_type":"code","source":"FEATURE_TEST = [\"nfl_player_id_1\", \"nfl_player_id_2\", \"step\"]\ntracking_df = pl.read_csv(CFG[\"TEST_TRACKING_CSV\"])\ntest_df, FEATURE_TEST = target_merge_tracking(test_df, tracking_df, FEATURE_TEST)\ntest_df, FEATURE_TEST = create_trackmerged_ftr(test_df, FEATURE_TEST)\nprint(len(test_df))\n\ndel tracking_df","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:30.12624Z","iopub.execute_input":"2023-02-03T05:12:30.126602Z","iopub.status.idle":"2023-02-03T05:12:30.208848Z","shell.execute_reply.started":"2023-02-03T05:12:30.126568Z","shell.execute_reply":"2023-02-03T05:12:30.207549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exclude distance","metadata":{}},{"cell_type":"code","source":"test_df = test_df.filter(pl.col(\"players_dis\") <= 2)\nprint(len(test_df))","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:30.210875Z","iopub.execute_input":"2023-02-03T05:12:30.21183Z","iopub.status.idle":"2023-02-03T05:12:30.22171Z","shell.execute_reply.started":"2023-02-03T05:12:30.211782Z","shell.execute_reply":"2023-02-03T05:12:30.220839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge helmet","metadata":{}},{"cell_type":"code","source":"helmet_df = pl.read_csv(CFG[\"TEST_HELMET_CSV\"])\ntest_df, FEATURE_TEST = target_merge_helmet(test_df, helmet_df, FEATURE_TEST)\ntest_df, FEATURE_TEST = get_categorical_ftr(test_df, FEATURE_TEST)\nprint(len(test_df))\n\ndel helmet_df","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:30.223217Z","iopub.execute_input":"2023-02-03T05:12:30.224242Z","iopub.status.idle":"2023-02-03T05:12:35.499445Z","shell.execute_reply.started":"2023-02-03T05:12:30.224208Z","shell.execute_reply":"2023-02-03T05:12:35.49816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.to_pandas()\ntest_df[\"step\"] = test_df[\"step\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:35.501212Z","iopub.execute_input":"2023-02-03T05:12:35.501657Z","iopub.status.idle":"2023-02-03T05:12:35.522171Z","shell.execute_reply.started":"2023-02-03T05:12:35.501614Z","shell.execute_reply":"2023-02-03T05:12:35.52053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_path = glob.glob(os.path.join(CFG[\"EXP_DIR\"], f'lgb_fold*.model'))\nprint(len(models_path))\npreds = []\nfor model_path in models_path:\n    model = pickle.load(open(model_path, 'rb'))\n    pred = np.array(model.predict_proba(test_df[CFG[\"features\"]])[:, 1])\n    preds.append(pred)\npreds = np.mean(preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:35.523852Z","iopub.execute_input":"2023-02-03T05:12:35.524283Z","iopub.status.idle":"2023-02-03T05:12:35.656692Z","shell.execute_reply.started":"2023-02-03T05:12:35.524247Z","shell.execute_reply":"2023-02-03T05:12:35.655616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"contact\"] = preds\ndisplay(test_df[\"contact\"].hist(bins=20))\ntest_df[\"contact\"] = (test_df[\"contact\"].values > CFG[\"threshold\"]).astype(np.int)\ntest_df = test_df[[\"contact_id\", \"contact\"]]\nprint(test_df[\"contact\"].value_counts())\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:35.658625Z","iopub.execute_input":"2023-02-03T05:12:35.659368Z","iopub.status.idle":"2023-02-03T05:12:36.057883Z","shell.execute_reply.started":"2023-02-03T05:12:35.659327Z","shell.execute_reply":"2023-02-03T05:12:36.056636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/EXP","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:36.059717Z","iopub.execute_input":"2023-02-03T05:12:36.06008Z","iopub.status.idle":"2023-02-03T05:12:37.209534Z","shell.execute_reply.started":"2023-02-03T05:12:36.060049Z","shell.execute_reply":"2023-02-03T05:12:37.20783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/nfl-player-contact-detection/sample_submission.csv')\nprint(len(sub))\nsub = sub.drop(\"contact\", axis=1).merge(test_df[['contact_id', 'contact']], how='left', on='contact_id')\nsub['contact'] = sub['contact'].fillna(0).astype('int')\nprint(len(sub))\ndisplay(sub[\"contact\"].value_counts())\n\nsub[[\"contact_id\", \"contact\"]].to_csv(\"submission.csv\", index=False)\n\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:12:37.211571Z","iopub.execute_input":"2023-02-03T05:12:37.21196Z","iopub.status.idle":"2023-02-03T05:12:37.714539Z","shell.execute_reply.started":"2023-02-03T05:12:37.21192Z","shell.execute_reply":"2023-02-03T05:12:37.713362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}