{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport pickle\nimport gc\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\nfrom catboost import Pool\nfrom sklearn.metrics import mean_absolute_error\n\npd.set_option('display.max_columns', 100)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:42:09.103216Z","iopub.execute_input":"2021-07-31T06:42:09.103653Z","iopub.status.idle":"2021-07-31T06:42:09.109962Z","shell.execute_reply.started":"2021-07-31T06:42:09.103613Z","shell.execute_reply":"2021-07-31T06:42:09.108814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trainの中身をcsv, pickleにしてくれている親切なデータセットがあったので使います\nbase_dir = '../input/mlb-player-digital-engagement-forecasting/'\n# train_dir = '../input/mlb-pdef-train-dataset/'\ntrain_dir = '../input/mlb-updated-dataset/'","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:40.412886Z","iopub.execute_input":"2021-07-31T06:39:40.413376Z","iopub.status.idle":"2021-07-31T06:39:40.424941Z","shell.execute_reply.started":"2021-07-31T06:39:40.413327Z","shell.execute_reply":"2021-07-31T06:39:40.423942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# このtarget1～4を予測するのがコンペの目的\n# target_df = pd.read_pickle(train_dir + 'nextDayPlayerEngagement_train.pkl')\ntarget_df = pd.read_csv(train_dir + 'nextDayPlayerEngagement_train.csv')\ntarget_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:40.436782Z","iopub.execute_input":"2021-07-31T06:39:40.437162Z","iopub.status.idle":"2021-07-31T06:39:43.605050Z","shell.execute_reply.started":"2021-07-31T06:39:40.437120Z","shell.execute_reply":"2021-07-31T06:39:43.603894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target encoding以外の特徴量はこのnotebookを参照\n# https://www.kaggle.com/mlconsult/1-38-lb-lightgbm-with-target-statistics\n\nplayers = pd.read_csv(base_dir + 'players.csv')\n# rosters = pd.read_pickle(train_dir + 'rosters_train.pkl')\nrosters = pd.read_csv(train_dir + 'rosters_train.csv')\n# scores = pd.read_pickle(train_dir + 'playerBoxScores_train.pkl')\nscores = pd.read_csv(train_dir + 'playerBoxScores_train.csv')\nscores = scores.groupby(['playerId', 'date']).sum().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:43.606718Z","iopub.execute_input":"2021-07-31T06:39:43.607016Z","iopub.status.idle":"2021-07-31T06:39:47.376577Z","shell.execute_reply.started":"2021-07-31T06:39:43.606987Z","shell.execute_reply":"2021-07-31T06:39:47.375495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 効きそうなカラムを抽出\ntargets_cols = ['playerId', 'target1', 'target2', 'target3', 'target4', 'date']\nplayers_cols = ['playerId', 'primaryPositionName',\n               'birthCity', 'birthStateProvince','birthCountry','playerForTestSetAndFuturePreds']\nrosters_cols = ['playerId', 'teamId', 'status', 'date']\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n               'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n               'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n               'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n               'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n               'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n               'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n               'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n               'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n               'groundOutsPitching', 'runsPitching', 'doublesPitching',\n               'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n               'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n               'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n               'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n               'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n               'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n               'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n               'inheritedRunnersScored', 'catchersInterferencePitching',\n               'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n               'assists', 'putOuts', 'errors', 'chances', 'date']\n\ntarget_stat_cols = ['playerId', 'target1_mean', 'target1_median', 'target1_std', 'target1_max', 'target1_min', \n                       'target2_mean', 'target2_median', 'target2_std', 'target2_max', 'target2_min', \n                       'target3_mean', 'target3_median', 'target3_std', 'target3_max', 'target3_min', \n                       'target4_mean', 'target4_median', 'target4_std', 'target4_max', 'target4_min']\n\nfeature_cols = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n                'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n                'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n                'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n                'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n                'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n                'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n                'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n                'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n                'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n                'groundOutsPitching', 'runsPitching', 'doublesPitching',\n                'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n                'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n                'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n                'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n                'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n                'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n                'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n                'inheritedRunnersScored', 'catchersInterferencePitching',\n                'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n                'assists', 'putOuts', 'errors', 'chances', \n                'target1_mean', 'target1_median', 'target1_std', 'target1_max', 'target1_min', \n                'target2_mean', 'target2_median', 'target2_std', 'target2_max', 'target2_min', \n                'target3_mean', 'target3_median', 'target3_std', 'target3_max', 'target3_min', \n                'target4_mean', 'target4_median', 'target4_std', 'target4_max', 'target4_min',\n                'status','primaryPositionName','birthCity', 'birthStateProvince','birthCountry',]\n\ncategory_cols = [\n                'primaryPositionName',\n                 'birthCity', 'birthStateProvince', 'birthCountry',\n                 'gamesPlayedBatting',\n#                  'positionCode',\n#                  'locationName',\n#                  'label_leagueId', \n#                  'label_divisionId',\n                 'status',\n                 'gamesPlayedPitching', 'gamesStartedPitching',\n                 'completeGamesPitching', 'shutoutsPitching',\n                 'winsPitching', 'lossesPitching',\n                 'saveOpportunities',\n                 'saves', 'holds', 'blownSaves'\n#                 'awardId',\n#                 'divisionChamp', 'divisionLeader', 'wildCardLeader',\n#                 'label_typeCode'\n                ]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:47.378774Z","iopub.execute_input":"2021-07-31T06:39:47.379179Z","iopub.status.idle":"2021-07-31T06:39:47.396448Z","shell.execute_reply.started":"2021-07-31T06:39:47.379138Z","shell.execute_reply":"2021-07-31T06:39:47.395271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 結合\ndf = target_df[targets_cols]\n\ndf = df.merge(players[players_cols], on=['playerId'], how='left')\ndf = df.merge(rosters[rosters_cols], on=['playerId', 'date'], how='left')\ndf = df.merge(scores[scores_cols], on=['playerId', 'date'], how='left')\n\n# label encoding\nplayer2num = {c: i for i, c in enumerate(df['playerId'].unique())}\nposition2num = {c: i for i, c in enumerate(df['primaryPositionName'].unique())}\nteamid2num = {c: i for i, c in enumerate(df['teamId'].unique())}\nstatus2num = {c: i for i, c in enumerate(df['status'].unique())}\ndf['label_playerId'] = df['playerId'].map(player2num)\ndf['label_primaryPositionName'] = df['primaryPositionName'].map(position2num)\ndf['label_teamId'] = df['teamId'].map(teamid2num)\ndf['label_status'] = df['status'].map(status2num)\n\n\ndel rosters, scores\ngc.collect()\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:47.398270Z","iopub.execute_input":"2021-07-31T06:39:47.398614Z","iopub.status.idle":"2021-07-31T06:39:52.643176Z","shell.execute_reply.started":"2021-07-31T06:39:47.398581Z","shell.execute_reply":"2021-07-31T06:39:52.642047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('player2num.pkl', 'wb') as f:\n    pickle.dump(player2num, f)\n\nwith open('position2num.pkl', 'wb') as f:\n    pickle.dump(position2num, f)\n\nwith open('teamid2num.pkl', 'wb') as f:\n    pickle.dump(teamid2num, f)\n\nwith open('status2num.pkl', 'wb') as f:\n    pickle.dump(status2num, f)\n    \ndel player2num, position2num, teamid2num, status2num","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:52.644917Z","iopub.execute_input":"2021-07-31T06:39:52.645375Z","iopub.status.idle":"2021-07-31T06:39:52.665157Z","shell.execute_reply.started":"2021-07-31T06:39:52.645323Z","shell.execute_reply":"2021-07-31T06:39:52.664104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seasonのみのモデルを作成するため4~9月のデータのみ抜き出し\nprint('before:', df.shape)\ndf['month'] = df['date'].astype('str').str[4:6].astype('int')\ndf = df[(4<=df['month']) & (df['month']<=9) & (df['playerForTestSetAndFuturePreds']==True)]\nprint('after:', df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:39:52.666489Z","iopub.execute_input":"2021-07-31T06:39:52.666933Z","iopub.status.idle":"2021-07-31T06:40:00.090261Z","shell.execute_reply.started":"2021-07-31T06:39:52.666898Z","shell.execute_reply":"2021-07-31T06:40:00.089216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target encoding用データセット作成\n# 直近のデータで作成するが、これでは特徴量の意味が変わると思いつつ一番精度が良い\ntarget_stat_df = df.loc[df['date']>=20210331, ['playerId', 'target1', 'target2', 'target3', 'target4']]\ntarget_stat_df = target_stat_df.groupby('playerId').agg(['mean', 'median', 'std', 'max', 'min'])","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:00.091544Z","iopub.execute_input":"2021-07-31T06:40:00.091860Z","iopub.status.idle":"2021-07-31T06:40:00.183592Z","shell.execute_reply.started":"2021-07-31T06:40:00.091827Z","shell.execute_reply":"2021-07-31T06:40:00.182529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_stat_df = target_stat_df.reset_index()\ntarget_stat_df.columns = target_stat_cols","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:00.186171Z","iopub.execute_input":"2021-07-31T06:40:00.186500Z","iopub.status.idle":"2021-07-31T06:40:00.192828Z","shell.execute_reply.started":"2021-07-31T06:40:00.186466Z","shell.execute_reply":"2021-07-31T06:40:00.191992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_stat_df.to_pickle('target_stat_df.pkl')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:00.194191Z","iopub.execute_input":"2021-07-31T06:40:00.194649Z","iopub.status.idle":"2021-07-31T06:40:00.207436Z","shell.execute_reply.started":"2021-07-31T06:40:00.194614Z","shell.execute_reply":"2021-07-31T06:40:00.206649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.merge(target_stat_df[target_stat_cols], how='left', on='playerId')\n\ndel target_stat_df\ngc.collect()\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:00.208403Z","iopub.execute_input":"2021-07-31T06:40:00.208820Z","iopub.status.idle":"2021-07-31T06:40:01.653000Z","shell.execute_reply.started":"2021-07-31T06:40:00.208791Z","shell.execute_reply":"2021-07-31T06:40:01.651852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[category_cols].info()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:01.654245Z","iopub.execute_input":"2021-07-31T06:40:01.654618Z","iopub.status.idle":"2021-07-31T06:40:02.590389Z","shell.execute_reply.started":"2021-07-31T06:40:01.654586Z","shell.execute_reply":"2021-07-31T06:40:02.589369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[category_cols].head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:02.591859Z","iopub.execute_input":"2021-07-31T06:40:02.592471Z","iopub.status.idle":"2021-07-31T06:40:02.668275Z","shell.execute_reply.started":"2021-07-31T06:40:02.592422Z","shell.execute_reply":"2021-07-31T06:40:02.667112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CatboostようにNaNを補完＋文字列\ndf['primaryPositionName'] = df['primaryPositionName'].fillna(0)\ndf['birthCity'] = df['birthCity'].fillna(0)\ndf['birthStateProvince'] = df['birthStateProvince'].fillna(0)\ndf['birthCountry'] = df['birthCountry'].fillna(0)\ndf['status'] = df['status'].fillna(0)\ndf['gamesPlayedBatting'] = df['gamesPlayedBatting'].fillna(0).astype(int)\ndf['gamesPlayedBatting'] = df['gamesPlayedBatting'].fillna(0).astype(int)\ndf['gamesPlayedPitching'] = df['gamesPlayedPitching'].fillna(0).astype(int)\ndf['gamesStartedPitching'] = df['gamesStartedPitching'].fillna(0).astype(int)\ndf['completeGamesPitching'] = df['completeGamesPitching'].fillna(0).astype(int)\ndf['shutoutsPitching'] = df['shutoutsPitching'].fillna(0).astype(int)\ndf['winsPitching'] = df['winsPitching'].fillna(0).astype(int)\ndf['lossesPitching'] = df['lossesPitching'].fillna(0).astype(int)\ndf['saveOpportunities'] = df['saveOpportunities'].fillna(0).astype(int)\ndf['saves'] = df['saves'].fillna(0).astype(int)\ndf['holds'] = df['holds'].fillna(0).astype(int)\ndf['blownSaves'] = df['blownSaves'].fillna(0).astype(int)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:02.669914Z","iopub.execute_input":"2021-07-31T06:40:02.670601Z","iopub.status.idle":"2021-07-31T06:40:05.150400Z","shell.execute_reply.started":"2021-07-31T06:40:02.670546Z","shell.execute_reply":"2021-07-31T06:40:05.149551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[category_cols].info()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:05.151817Z","iopub.execute_input":"2021-07-31T06:40:05.152438Z","iopub.status.idle":"2021-07-31T06:40:05.684229Z","shell.execute_reply.started":"2021-07-31T06:40:05.152387Z","shell.execute_reply":"2021-07-31T06:40:05.683451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[category_cols].head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:05.685768Z","iopub.execute_input":"2021-07-31T06:40:05.686181Z","iopub.status.idle":"2021-07-31T06:40:05.752992Z","shell.execute_reply.started":"2021-07-31T06:40:05.686135Z","shell.execute_reply":"2021-07-31T06:40:05.751970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_X = df[feature_cols]\ndf_y = df[['target1', 'target2', 'target3', 'target4']]\n\n_index = (df['date'] < 20210401)\nx_train = df_X.loc[_index].reset_index(drop=True)\ny_train = df_y.loc[_index].reset_index(drop=True)\nx_valid = df_X.loc[~_index].reset_index(drop=True)\ny_valid = df_y.loc[~_index].reset_index(drop=True)\n\nprint('training data shape:' , x_train.shape, y_train.shape)\nprint('validation data shape:' , x_valid.shape, y_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T06:40:05.754330Z","iopub.execute_input":"2021-07-31T06:40:05.754652Z","iopub.status.idle":"2021-07-31T06:40:06.581663Z","shell.execute_reply.started":"2021-07-31T06:40:05.754621Z","shell.execute_reply":"2021-07-31T06:40:06.580389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostRegressor\nfrom catboost import Pool\n\ndef fit_cat(x_train, y_train, x_valid, y_valid, params: dict=None, verbose=100):\n\n    oof_pred = np.zeros(len(y_valid), dtype=np.float32)\n    model = CatBoostRegressor(\n                n_estimators=2000,\n                learning_rate=0.05,\n                loss_function='MAE',\n                eval_metric='MAE',\n                max_bin=50,\n                subsample=0.9,\n                colsample_bylevel=0.5,\n                verbose=100)\n    model.fit(x_train, y_train, cat_features=category_cols,\n                         use_best_model=True,\n                         eval_set=(x_valid, y_valid),\n#                          early_stopping_rounds=25,\n                          plot=True\n                )\n    oof_pred = model.predict(x_valid)\n    score = mean_absolute_error(oof_pred, y_valid)\n    print('mae:', score)\n    return oof_pred, model, score\n\n# training xgbm\nparams = {\n'boosting_type': 'gbdt',\n'objective':'mae',\n'subsample': 0.5,\n'subsample_freq': 1,\n'learning_rate': 0.03,\n'num_leaves': 2**11-1,\n'min_data_in_leaf': 2**12-1,\n'feature_fraction': 0.5,\n'max_bin': 100,\n'n_estimators': 2500,\n'boost_from_average': False,\n\"random_seed\":42,\n    }\n\noof1, model1_cat, score1 = fit_cat(\n    x_train, y_train['target1'],\n    x_valid, y_valid['target1'],\n    params\n    )\noof2, model2_cat, score2 = fit_cat(\n    x_train, y_train['target2'],\n    x_valid, y_valid['target2'],\n    params\n    )\noof3, model3_cat, score3 = fit_cat(\n    x_train, y_train['target3'],\n    x_valid, y_valid['target3'],\n    params\n    )\noof4, model4_cat, score4 = fit_cat(\n    x_train, y_train['target4'],\n    x_valid, y_valid['target4'],\n    params\n    )\n\nscore = (score1+score2+score3+score4) / 4\nprint(f'score: {score}')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T07:05:35.807822Z","iopub.execute_input":"2021-07-31T07:05:35.808251Z","iopub.status.idle":"2021-07-31T08:11:24.720689Z","shell.execute_reply.started":"2021-07-31T07:05:35.808216Z","shell.execute_reply":"2021-07-31T08:11:24.718377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import Pool\nimport sklearn.metrics\ndef objective(trial):\n    train_pool = Pool(x_train, y_train['target1'], cat_features=category_cols)\n    test_pool = Pool(x_valid, y_valid['target1'], cat_features=category_cols)\n\n    # パラメータの指定\n    params = {\n        'iterations' : trial.suggest_int('iterations', 50, 300),                         \n        'depth' : trial.suggest_int('depth', 4, 10),                                       \n        'learning_rate' : trial.suggest_loguniform('learning_rate', 0.01, 0.3),               \n        'random_strength' :trial.suggest_int('random_strength', 0, 100),                       \n        'bagging_temperature' :trial.suggest_loguniform('bagging_temperature', 0.01, 100.00), \n        'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']),\n        'od_wait' :trial.suggest_int('od_wait', 10, 50)\n    }\n\n    # 学習\n    model = CatBoostRegressor(**params)\n    model.fit(train_pool)\n    # 予測\n    preds = model.predict(test_pool)\n    pred_labels = np.rint(preds)\n    # 精度の計算\n    score = mean_absolute_error(preds, y_valid['target1'])\n    return score\n#     print('mae:', score)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-31T08:20:17.400953Z","iopub.execute_input":"2021-07-31T08:20:17.401375Z","iopub.status.idle":"2021-07-31T08:20:17.411648Z","shell.execute_reply.started":"2021-07-31T08:20:17.401341Z","shell.execute_reply":"2021-07-31T08:20:17.410871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\nstudy = optuna.create_study()\nstudy.optimize(objective, n_trials=100)\nprint(study.best_trial)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T08:20:18.361075Z","iopub.execute_input":"2021-07-31T08:20:18.361660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}