{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":27783,"databundleVersionId":2555459,"sourceType":"competition"},{"sourceId":7643477,"sourceType":"datasetVersion","datasetId":4455097}],"dockerImageVersionId":30357,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nCode Competition 提出用\n\"\"\"\n\nimport datetime as dt\nimport pickle\nimport warnings\nfrom collections import defaultdict\nfrom typing import Any, Callable, Dict, Iterable, List, Sequence, Tuple\n\nimport lightgbm as lgb\nimport mlb\nimport numpy as np\nimport pandas as pd\nimport pyarrow\nimport pyarrow.csv\nfrom dateutil.relativedelta import relativedelta\nfrom sklearn.metrics import mean_absolute_error\n\npd.options.display.float_format = \"{:10.4f}\".format\nwarnings.filterwarnings(\"ignore\")\n\n\n\"\"\"\n一回しか呼び出せないため、先頭において実行しておく\n\"\"\"\nenv = mlb.make_env()  # initialize the environment\niter_test = env.iter_test()  # iterator which loops over each date in test set\n\n# %%\n\n\ndef unpack(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)\n\n\ndef extract_data(input_df, col, show=False):\n    ret_df = pd.DataFrame()\n    for i in np.arange(len(input_df)):\n        if show:\n            print(\"\\r{}/{}\".format(i, len(input_df)), end=\"\")\n        try:\n            ret_df = pd.concat([ret_df, unpack(input_df[col].iloc[i])])\n        except:\n            if show:\n                print(\"Json Parse Error\")\n    if show:\n        print(\"\")\n        print(ret_df.shape)\n        print(ret_df.head())\n    return ret_df\n\n\n# ROOT = \"../data\"\nROOT = \"/kaggle/input/mlb-player-digital-engagement-forecasting\"\ntrain_path = ROOT + \"/train_updated.csv\"\n# https://issues.apache.org/jira/browse/ARROW-13314\n# https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.slice\ncsv_ro = pyarrow.csv.ReadOptions(block_size=2**25)\ntable = pyarrow.csv.read_csv(train_path, read_options=csv_ro)\ntrain_df = table.to_pandas()\n\n# json_ro = pyarrow.csv.ReadOptions(block_size=2**25)\n# %%\n\n# 処理時間の高速化\n# 季節性を考慮して最低でも1年間+検証データで13ヶ月分を確保.\ntrain = train_df[train_df.date >= 20200401].reset_index(drop=True)\nprint(train_df[train_df.date >= 20200401].date.nunique() / 365)\n\n\n# %%\n\"\"\"\njson をパース\n\"\"\"\ndf_engagement = extract_data(train, col=\"nextDayPlayerEngagement\", show=True)\n\n# %%\n\"\"\"\n前処理\n\"\"\"\ndf_engagement[\"date_playerId\"] = (\n    df_engagement[\"engagementMetricsDate\"].str.replace(\"-\", \"\")\n    + \"_\"\n    + df_engagement[\"playerId\"].astype(str)\n)\n\ndf_engagement.head()\n# 推論実施日に関する特徴量を作成\n# 推論実施日のカラム作成（推論実施日=推論対象日の前日）✨\n# ここで入ってるdate は推論対象日なので、推論実施日にするために-1 日する\ndf_engagement[\"date\"] = pd.to_datetime(\n    df_engagement.engagementMetricsDate, format=\"%Y-%m-%d\"\n) + dt.timedelta(days=-1)\ndf_engagement[\"dayofweek\"] = df_engagement[\"date\"].dt.dayofweek\ndf_engagement[\"yearmonth\"] = df_engagement[\"date\"].astype(str).apply(lambda s: s[:7])\n\n\n\"\"\"\nPlayers の情報を読み込んで、予測対象か否かのカラム01 へ変換.\n\"\"\"\n\ndf_players = pd.read_csv(ROOT + \"/players.csv\")\ndf_players = df_players[(df_players.playerId != False)]\n\n# <NA> のままでも np.where 判定できると思う.\ndf_players.loc[\n    df_players.playerForTestSetAndFuturePreds.isna(), \"playerForTestSetAndFuturePreds\"\n] = False\ndf_players.playerForTestSetAndFuturePreds = (\n    df_players.playerForTestSetAndFuturePreds.astype(bool)\n)\ndf_players.playerForTestSetAndFuturePreds = np.where(\n    df_players.playerForTestSetAndFuturePreds, 1, 0\n)\n\n\n# プレイヤー情報が一意だから、playerId でmerge しても大丈夫そう？\nassert all(\n    (df_players.groupby(\"playerId\").agg(func=\"count\").playerName == 1).values.tolist()\n)\n# 念の為、drop\ndf_players = df_players.drop_duplicates([\"playerId\"]).reset_index()\ndf_train = pd.merge(df_engagement, df_players, on=\"playerId\", how=\"left\")\nuse_cols = [\n    \"playerId\",\n    \"dayofweek\",\n    \"birthCity\",\n    \"birthStateProvince\",\n    \"birthCountry\",\n    \"heightInches\",\n    \"weight\",\n    \"primaryPositionCode\",\n    \"primaryPositionName\",\n    \"playerForTestSetAndFuturePreds\",\n]\ntarget_cols = [\"target1\", \"target2\", \"target3\", \"target4\"]\nid_cols = [\n    \"engagementMetricsDate\",\n    \"playerId\",\n    \"date_playerId\",\n    \"date\",\n    \"yearmonth\",\n    \"playerForTestSetAndFuturePreds\",\n]\nx_train = df_train[use_cols]\ny_train = df_train[target_cols]\nid_train = df_train[id_cols]\n\n# LightGBM ではobject 型は使えないため、category 型へconvert しておく.\ncat_cols = [\n    \"playerId\",\n    \"dayofweek\",\n    \"birthCity\",\n    \"birthStateProvince\",\n    \"birthCountry\",\n    \"primaryPositionCode\",\n    \"primaryPositionName\",\n]\nfor col in cat_cols:\n    x_train[col] = x_train[col].astype(\"category\")\n\n# %%\n\n\"\"\"\n時系列用のバリデーションを作成\n１年間の学習データと、その翌月をテストデータとした\nデータセットを１ヶ月ごとにずらしたものを３つ用意\n\"\"\"\nlist_cv_month = []\nymf = \"%Y-%m\"\nfor start in [\"2020-05\", \"2020-06\", \"2020-07\"]:\n    # 一年後を計算\n    # 年月単位では、relativedelta を使う\n    # https://dateutil.readthedocs.io/en/latest/\n    end_date = dt.datetime.strptime(start, ymf) + relativedelta(years=1)\n    end = end_date.strftime(ymf)\n    # end はexclusive で含まれなため注意\n    months = pd.date_range(start=start, end=end, freq=\"M\").format(\n        formatter=lambda x: x.strftime(ymf)\n    )\n    # exclusive で含まれない最後の月をvalid として使う\n    valid_date = end_date\n    valid = valid_date.strftime(ymf)\n    list_cv_month.append([months, [valid]])\n\n# ここで、list_cv_month を一度確認する\n\n# valid は予測対象の選手のみを扱う\n# 学習データは多い方がいいので含める\n# 学習データから、index で取り出せるようにここでは、.index に対してslicing\ncv = []\nfor month_tr, month_va in list_cv_month:\n    cv.append(\n        [\n            id_train.index[id_train.yearmonth.isin(month_tr)],\n            id_train.index[\n                (id_train.yearmonth.isin(month_va))\n                & (id_train.playerForTestSetAndFuturePreds == 1)\n            ],\n        ]\n    )\n\n# %%\n\n\ndef lightgbm_hy_params(trial=None, update_params=None, is_optimize=False) -> dict:\n    print(\"---\")\n    print(\"lightgbm params\")\n    print(\"{}, {}, {}\".format(trial, update_params, is_optimize))\n    print(\"---\")\n\n    # ハイパーパラメータ\n    params_base = {\n        \"boosting_type\": \"gbdt\",\n        \"objective\": \"regression_l1\",\n        \"metric\": \"mean_absolute_error\",\n        \"learning_rate\": 0.05,\n        # \"num_iterations\": 100000,\n        \"num_leaves\": 32,\n        \"subsample\": 0.7,\n        \"subsample_freq\": 1,\n        \"feature_fraction\": 0.8,\n        \"min_data_in_leaf\": 50,\n        \"min_sum_hessian_in_leaf\": 50,\n        \"n_estimators\": 1000,\n        \"random_state\": 123,\n        \"importance_type\": \"gain\",\n    }\n\n    if update_params is not None:\n        params_base.update(update_params)\n\n    if is_optimize and trial is not None:\n        # 探索するハイパーパラメータ\n        params_tuning = {\n            \"num_leaves\": trial.suggest_int(\"num_leavels\", 8, 256),\n            \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 5, 200),\n            \"min_sum_hessian_in_leaf\": trial.suggest_float(\n                \"min_sum_hessian_in_leaf\", 1e-5, 1e-2, log=True\n            ),\n            \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.5, 1.0),\n            \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.5, 1.0),\n            \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-2, 1e2, log=True),\n            \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-2, 1e2, log=True),\n        }\n        params_tuning.update(params_base)\n\n    return params_base\n\n\ntargets = [f\"target{n}\" for n in list(np.arange(4) + 1)]\n\ntrained_models: Dict[int, List[Any]] = defaultdict(list)\n\n\ndef train_cv(\n    input_x, input_y, input_id, cv: List[List[pd.Index]], mode=\"load\"\n) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n    \"\"\"train with cross validation\"\"\"\n    metrics = []\n    df_valid_pred = pd.DataFrame()\n    df_imp = pd.DataFrame()\n\n    for k in np.arange(len(cv)):\n        for target in targets:\n            \"\"\"\n            target の種類ごとにモデルを作成・予測\n            \"\"\"\n            print(f\"{k}-fold\")\n            idx_tr, idx_va = cv[k][0], cv[k][1]\n            x_tr, y_tr, _ = (\n                input_x.loc[idx_tr, :],\n                input_y.loc[idx_tr, target],\n                input_id.loc[idx_tr, :],\n            )\n            x_va, y_va, id_va = (\n                input_x.loc[idx_va, :],\n                input_y.loc[idx_va, target],\n                input_id.loc[idx_va, :],\n            )\n\n            fname_lgb = f\"./kfold_model_pickle/model_lgb_fold{k}_{target}.pickle\"\n            if mode == \"load\":\n                # fit済みのモデルを読み込む\n                # すでにdump されていない場合、file not found でexception\n                # を出してしまうが、今のところ補足したりしてない\n                print(\"model load.\")\n                with open(fname_lgb, \"rb\") as f:\n                    model = pickle.load(f)\n            else:\n                verbose_eval = 0\n                # https://qiita.com/c60evaporator/items/2b7a2820d575e212bcf4\n                callbacks = [\n                    # early_stopping用コールバック関数\n                    lgb.early_stopping(stopping_rounds=100, verbose=True),\n                    # コマンドライン出力用コールバック関数\n                    lgb.log_evaluation(verbose_eval),\n                ]\n                params = lightgbm_hy_params()\n                model = lgb.LGBMRegressor(**params)\n                model.fit(\n                    x_tr,\n                    y_tr,\n                    eval_set=[(x_tr, y_tr), (x_va, y_va)],\n                    callbacks=callbacks,\n                )\n\n                trained_models[k].append(model)\n\n            y_va_pred = model.predict(x_va)\n            pred_df = pd.DataFrame(\n                {\n                    \"target\": target,\n                    \"nfold\": k,\n                    \"true\": y_va,\n                    \"pred\": y_va_pred,\n                }\n            )\n            tmp_valid_pred = pd.concat(\n                [id_va, pred_df],\n                axis=1,\n            )\n            df_valid_pred = pd.concat(\n                [df_valid_pred, tmp_valid_pred], axis=0, ignore_index=True\n            )\n\n            metric_va = mean_absolute_error(y_va, y_va_pred)\n            metrics.append([k, target, metric_va])\n\n            print(\n                \"[mae] kfold: {}, target: {}, va_prob: {:.2f}\".format(\n                    k, target, metric_va\n                )\n            )\n\n            _df_imp = pd.DataFrame(\n                {\n                    \"nfold\": k,\n                    \"target\": target,\n                    \"col\": x_train.columns,\n                    \"imp\": model.feature_importances_,\n                }\n            )\n            df_imp = pd.concat([df_imp, _df_imp], axis=0, ignore_index=True)\n\n    print(\"-\" * 20, \"result\", \"-\" * 20)\n\n    df_metrics = pd.DataFrame(metrics, columns=[\"nfold\", \"target\", \"mae\"])\n    print(\"MCMAE: {:.4f}\".format(df_metrics.mae.mean()))\n\n    # valid 推論値\n    \"\"\"\n    ここで各行の \"target{n}_fold{k}_{pred|true}\" が１つだけ値があり、他がNaN になっているが、\n    これは正しい.\n    cv はそれぞれ異なる、月３ヶ月分をvalid として用いているから、\n    engagementMetricsDate をindex として、該当の日付がある時点で、それは１つのバリデーション（cv のうちの１つ）の結果しか持っていないことを意味するから、それ以外の fold はNaN\n    target は同じfold 内で4つ存在するから、これは同じ位置にある.\n    \"\"\"\n    df_valid_pred_all = pd.pivot_table(\n        df_valid_pred,\n        index=[\n            \"engagementMetricsDate\",\n            \"playerId\",\n            \"date_playerId\",\n            \"date\",\n            \"yearmonth\",\n            \"playerForTestSetAndFuturePreds\",\n        ],\n        columns=[\"target\", \"nfold\"],\n        values=[\"true\", \"pred\"],\n        aggfunc=np.sum,\n    )\n\n    df_valid_pred_all.columns = [\n        \"{}_fold{}_{}\".format(j, k, i) for i, j, k in df_valid_pred_all.columns\n    ]\n    df_valid_pred_all = df_valid_pred_all.reset_index(drop=False)\n\n    return df_valid_pred_all, df_metrics, df_imp\n\n\n# %%\n\"\"\"\ntrain\n\"\"\"\ndf_valid_pred, df_metrics, df_imp = train_cv(\n    input_x=x_train, input_y=y_train, input_id=id_train, cv=cv, mode=None\n)\n\n\n# %%\n\"\"\"\n入力データを整形し、モデルの入力値の形式に変換\n\"\"\"\n\n\ndef make_dataset_for_predict(input_test: pd.DataFrame, input_prediction: pd.DataFrame):\n    test = input_test.copy()\n    prediction = input_prediction.copy()\n\n    prediction.reset_index(drop=False, inplace=True)\n    prediction[\"date\"] = pd.to_datetime(prediction.date, format=\"%Y%m%d\")\n    prediction[\"dayofweek\"] = prediction[\"date\"].dt.dayofweek\n    prediction[\"yearmonth\"] = prediction.date.astype(str).apply(lambda s: s[:7])\n    prediction[\"engagementMetricsDate\"] = prediction.date_playerId.astype(str).apply(\n        lambda s: s[:8]\n    )\n    prediction[\"engagementMetricsDate\"] = pd.to_datetime(\n        prediction.engagementMetricsDate, format=\"%Y%m%d\"\n    )\n    prediction[\"playerId\"] = prediction.date_playerId.apply(lambda x: int(x[9:]))\n\n    df_players = pd.read_csv(ROOT + \"/players.csv\")\n    df_players = df_players[(df_players.playerId != False)]\n\n    # <NA> のままでも np.where 判定できると思う.\n    df_players.loc[\n        df_players.playerForTestSetAndFuturePreds.isna(),\n        \"playerForTestSetAndFuturePreds\",\n    ] = False\n    df_players.playerForTestSetAndFuturePreds = (\n        df_players.playerForTestSetAndFuturePreds.astype(bool)\n    )\n    df_players.playerForTestSetAndFuturePreds = np.where(\n        df_players.playerForTestSetAndFuturePreds, 1, 0\n    )\n\n    df_train = pd.merge(prediction, df_players, on=\"playerId\", how=\"left\")\n    use_cols = [\n        \"playerId\",\n        \"dayofweek\",\n        \"birthCity\",\n        \"birthStateProvince\",\n        \"birthCountry\",\n        \"heightInches\",\n        \"weight\",\n        \"primaryPositionCode\",\n        \"primaryPositionName\",\n        \"playerForTestSetAndFuturePreds\",\n    ]\n    target_cols = [\"target1\", \"target2\", \"target3\", \"target4\"]\n    id_cols = [\n        \"engagementMetricsDate\",\n        \"playerId\",\n        \"date_playerId\",\n        \"date\",\n        \"yearmonth\",\n        \"playerForTestSetAndFuturePreds\",\n    ]\n    x_test = df_train[use_cols]\n    y_test = df_train[target_cols]\n    id_test = df_train[id_cols]\n\n    # LightGBM ではobject 型は使えないため、category 型へconvert しておく.\n    cat_cols = [\n        \"playerId\",\n        \"dayofweek\",\n        \"birthCity\",\n        \"birthStateProvince\",\n        \"birthCountry\",\n        \"primaryPositionCode\",\n        \"primaryPositionName\",\n    ]\n    for col in cat_cols:\n        x_test[col] = x_test[col].astype(\"category\")\n\n    return x_test, y_test, id_test\n\n\n# %%\ndef predict_lgb(input_x, input_id, list_n_fold=[0, 1, 2]):\n    \"\"\"\n    fit済みのモデルがKaggle Notebook にupload ずみで読み込むだけ\n    の状態を前提にする\n    \"\"\"\n    targets = [f\"target{n}\" for n in list(np.arange(4) + 1)]\n    df_test_pred = id_test.copy()\n\n    for k in list_n_fold:\n        for i, target in enumerate(targets):\n            \"\"\"\n            target の種類ごとにモデルを予測\n            \"\"\"\n            print(\"model load.\")\n            model = trained_models[k][i]\n            pred = model.predict(input_x)\n            df_test_pred[\"{}_fold{}\".format(target, k)] = pred\n\n    # 各fold の平均値\n    # fold 数はlist_n_fold で決まる. 学習時と同じ= upload 済みの1target に対するcv 数\n    # mean(axis=1) 同じ行のカラムのmean を計算\n    for target in target_cols:\n        df_test_pred[target] = df_test_pred[\n            df_test_pred.columns[df_test_pred.columns.str.contains(target)]\n        ].mean(axis=1)\n\n    return df_test_pred\n\n\n# %%\n\nfor test_df, prediction_df in iter_test:\n    test = test_df.copy()\n    prediction = prediction_df.copy()\n\n    x_test, y_test, id_test = make_dataset_for_predict(\n        input_test=test_df, input_prediction=prediction_df\n    )\n    df_test_pred = predict_lgb(input_x=x_test, input_id=id_test)\n    df_submit = df_test_pred[[\"date_playerId\"] + target_cols]\n\n    # 後処理\n    # 欠損値埋め、0-100 の範囲外をデータクリッピング\n    # 基本的には0~100 に収まっているはず.\n    for i, col in enumerate(target_cols):\n        df_submit[col] = df_submit[col].fillna(0.0)\n        df_submit[col] = df_submit[col].clip(0, 100)\n\n    env.predict(df_submit)\n\nprint(\"Done.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-18T03:52:36.440655Z","iopub.execute_input":"2024-02-18T03:52:36.441289Z","iopub.status.idle":"2024-02-18T03:59:46.038102Z","shell.execute_reply.started":"2024-02-18T03:52:36.441165Z","shell.execute_reply":"2024-02-18T03:59:46.036901Z"},"trusted":true},"execution_count":null,"outputs":[]}]}