{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T07:49:19.560445Z","iopub.execute_input":"2022-07-06T07:49:19.560886Z","iopub.status.idle":"2022-07-06T07:49:19.799375Z","shell.execute_reply.started":"2022-07-06T07:49:19.560853Z","shell.execute_reply":"2022-07-06T07:49:19.798106Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.Introduce\n一些基础的知识介绍: https://www.kaggle.com/code/jiashenliu/introduction-to-financial-concepts-and-data (也作为以下内容的参考)","metadata":{}},{"cell_type":"markdown","source":"## 1.1 Testing data","metadata":{}},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/optiver-realized-volatility-prediction/sample_submission.csv\")\nsample","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:19.801672Z","iopub.execute_input":"2022-07-06T07:49:19.802089Z","iopub.status.idle":"2022-07-06T07:49:19.817510Z","shell.execute_reply.started":"2022-07-06T07:49:19.802052Z","shell.execute_reply":"2022-07-06T07:49:19.816263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"+ row_id: ‘-’的左边为stock_id, 右边为time_id. 例如'0-4'就是在time_id为4时的stock_id为0的股票\n- target: 就是我们需要预测的东西, 这里为已实现波动率(Realized Volatility), 可以通过以下这个表格得出","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"../input/optiver-realized-volatility-prediction/test.csv\")\ntest","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:19.819665Z","iopub.execute_input":"2022-07-06T07:49:19.820179Z","iopub.status.idle":"2022-07-06T07:49:19.836582Z","shell.execute_reply.started":"2022-07-06T07:49:19.820131Z","shell.execute_reply":"2022-07-06T07:49:19.835279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"book_testparquet = pd.read_parquet(\"../input/optiver-realized-volatility-prediction/book_test.parquet/stock_id=0\")\nbook_testparquet","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:19.840131Z","iopub.execute_input":"2022-07-06T07:49:19.840635Z","iopub.status.idle":"2022-07-06T07:49:19.865417Z","shell.execute_reply.started":"2022-07-06T07:49:19.840588Z","shell.execute_reply":"2022-07-06T07:49:19.864151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"订单表, 具体的特征介绍我放在train data中.","metadata":{}},{"cell_type":"code","source":"trade_testparquet = pd.read_parquet(\"../input/optiver-realized-volatility-prediction/trade_test.parquet/stock_id=0\")\ntrade_testparquet","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:19.867649Z","iopub.execute_input":"2022-07-06T07:49:19.868619Z","iopub.status.idle":"2022-07-06T07:49:19.891009Z","shell.execute_reply.started":"2022-07-06T07:49:19.868565Z","shell.execute_reply":"2022-07-06T07:49:19.889702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"交易表, 具体的特征介绍我放在train data中.","metadata":{}},{"cell_type":"markdown","source":"## 1.2 Train data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/optiver-realized-volatility-prediction/train.csv\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:19.893281Z","iopub.execute_input":"2022-07-06T07:49:19.894242Z","iopub.status.idle":"2022-07-06T07:49:20.022168Z","shell.execute_reply.started":"2022-07-06T07:49:19.894187Z","shell.execute_reply":"2022-07-06T07:49:20.021039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train.csv:\n+ stock_id: 股票序号\n+ time_id: 时间序号(每一个time_id的时长为10mins)\n+ target: 需要预测的下个时间窗口(下一个10mins)的已实现波动率","metadata":{}},{"cell_type":"code","source":"book_example = pd.read_parquet('../input/optiver-realized-volatility-prediction/book_train.parquet/stock_id=0')\nbook_example","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:20.024197Z","iopub.execute_input":"2022-07-06T07:49:20.025036Z","iopub.status.idle":"2022-07-06T07:49:20.095424Z","shell.execute_reply.started":"2022-07-06T07:49:20.024958Z","shell.execute_reply":"2022-07-06T07:49:20.094529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"book_train.parquet: 订单表\n+ time_id: 时间序号(每一个time_id的时长为10mins)\n+ seconds_in_bucket: 在当前时间序号中的秒, 0-600.\n+ bid_price1: 第一买价\n+ ask_price1: 第一卖价\n+ bid_price2: 第二买价\n+ ask_price2: 第二卖价\n+ bid_size1: 第一买量(就是挂第一买价的数量)\n+ ask_size1: 第一卖量(挂第一卖价的数量)\n+ bid_size2: 第二买量\n+ ask_size2: 第二买量","metadata":{}},{"cell_type":"code","source":"trade_example = pd.read_parquet(\"../input/optiver-realized-volatility-prediction/trade_train.parquet/stock_id=0\")\ntrade_example","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:49:20.096771Z","iopub.execute_input":"2022-07-06T07:49:20.097603Z","iopub.status.idle":"2022-07-06T07:49:20.122388Z","shell.execute_reply.started":"2022-07-06T07:49:20.097569Z","shell.execute_reply":"2022-07-06T07:49:20.121313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"trade_train.parquet: 交易表\n+ stock_id: 同上\n+ time_id: 同上\n+ seconds_in_bucket: 同上\n+ price: 平均成交价\n+ size: 成交量\n+ order_count: 成交的订单量","metadata":{}},{"cell_type":"markdown","source":"## 1.3 与订单表有关的特征","metadata":{}},{"cell_type":"markdown","source":"### 1.3.1 bid/ask spread\nBidAskSpread = BestOffer/BestBid−1\n\n最优卖价与最优买价之间的差除以最优买价","metadata":{}},{"cell_type":"markdown","source":"### 1.3.2 Weighted averaged price\n基于Order Size的加权平均价(WAP)，在本次比赛中也是基于WAP计算已实现波动率(Realized Volatility)作为预测目标.\n\nWAP = (BidPrice1∗AskSize1 + AskPrice1∗BidSize1)/(BidSize1 + AskSize1)","metadata":{}},{"cell_type":"markdown","source":"### 1.3.3 Log returns\n称St为股票S在时间t的价格，我们可以将t1和t2之间的对数回报定义为:\n\n![image.png](https://s2.loli.net/2022/07/06/1goSZ8j2DnpEiG3.png)","metadata":{}},{"cell_type":"markdown","source":"### 1.3.4 Realized volatility\n计算所有连续的账面更新中的对数收益，并将已实现的波动率σ定义为对数收益的平方根.\n\n![image.png](https://s2.loli.net/2022/07/06/OWVlrtQAhknzI2N.png)\n\n其中r是基于WAP计算的对数收益率.","metadata":{}},{"cell_type":"markdown","source":"# 2. EDA\n以下部分参考了: \n\nhttps://www.kaggle.com/code/chumajin/optiver-realized-eda-for-starter-english-version/notebook\n\n\nhttps://www.kaggle.com/code/gunesevitan/optiver-realized-volatility-prediction-eda","metadata":{}},{"cell_type":"markdown","source":"## 3. Code\n参考：\n\nhttps://www.kaggle.com/code/tommy1028/lightgbm-starter-with-feature-engineering-idea\n\nhttps://www.kaggle.com/code/jiashenliu/introduction-to-financial-concepts-and-data","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport time\nimport glob\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\n\n# Parallel processing\nfrom joblib import Parallel\nfrom joblib import delayed\n\n# Preprocess\nfrom sklearn import preprocessing\nfrom sklearn import model_selection\n\n# Evaluation\nfrom sklearn.metrics import r2_score\n\n# Visullize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Modeling\nimport lightgbm as lgb\n\n# Others\nimport warnings\nwarnings.simplefilter(\"ignore\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import needed packages\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom joblib import Parallel\nfrom joblib import delayed\nimport lightgbm as lgbm\nfrom sklearn.model_selection import KFold\n\n# set the data files path\npath = '../input/optiver-realized-volatility-prediction/'","metadata":{"execution":{"iopub.status.busy":"2022-07-06T11:59:55.402244Z","iopub.execute_input":"2022-07-06T11:59:55.402822Z","iopub.status.idle":"2022-07-06T11:59:55.408419Z","shell.execute_reply.started":"2022-07-06T11:59:55.402781Z","shell.execute_reply":"2022-07-06T11:59:55.407142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### data processing","metadata":{}},{"cell_type":"code","source":"# WAP\ndef wap_calc1(df):\n    return (df['bid_price1'] * df['ask_size1'] + df['ask_price1'] * df['bid_size1']) / (df['bid_size1'] + df['ask_size1'])\n\ndef wap_calc2(df):\n    return (df['bid_price2'] * df['ask_size2'] + df['ask_price2'] * df['bid_size2']) / (df['bid_size2'] + df['ask_size2'])\n\n#　Log Return\ndef log_return(list_stock_prices):\n    return np.log(list_stock_prices).diff() \n\n# Realized Volatility\ndef realized_volatility(series_log_return):\n    return np.sqrt(np.sum(series_log_return**2))\n\n# RMSPE\ndef rmspe_calc(y_true, y_pred):\n    return  (np.sqrt(np.mean(np.square((y_true - y_pred) / y_true))))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:10:19.752830Z","iopub.execute_input":"2022-07-06T10:10:19.753503Z","iopub.status.idle":"2022-07-06T10:10:19.761357Z","shell.execute_reply.started":"2022-07-06T10:10:19.753468Z","shell.execute_reply":"2022-07-06T10:10:19.760164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def book_preprocessing(stock_id, trainortest):\n    # read data\n    df = pd.read_parquet(path + 'book_' + trainortest + '.parquet/stock_id=' + str(stock_id))\n    \n    # set stock_id\n    df['stock_id'] = stock_id\n    \n    # WAP\n    df['wap1'] = wap_calc1(df)\n    df['wap2'] = wap_calc2(df)\n    \n    # log return\n    df['log_return1'] = df.groupby(['time_id'])['wap1'].apply(log_return).fillna(0)\n    df['log_return2'] = df.groupby(['time_id'])['wap2'].apply(log_return).fillna(0)    \n    df_realized_volatility = pd.DataFrame(df.groupby(['stock_id','time_id'])[['log_return1','log_return2']].agg(realized_volatility)).reset_index()\n    \n    # return this stock's realized_volatility\n    return df_realized_volatility","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:10:22.074921Z","iopub.execute_input":"2022-07-06T10:10:22.075314Z","iopub.status.idle":"2022-07-06T10:10:22.083523Z","shell.execute_reply.started":"2022-07-06T10:10:22.075284Z","shell.execute_reply":"2022-07-06T10:10:22.082591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trade_preprocessing(stock_id, trainortest):\n    # read data\n    df = pd.read_parquet(path + 'trade_' + trainortest + '.parquet/stock_id=' + str(stock_id))\n    \n    # by order\n    df = df.sort_values(by=['time_id', 'seconds_in_bucket']).reset_index(drop=True)\n    \n    # set stock_id\n    df['stock_id'] = stock_id\n    \n    # log return\n    df['trade_log_return'] = df.groupby(by = ['time_id'])['price'].apply(log_return).fillna(0)\n    df_realized_volatility = pd.DataFrame(df.groupby(['stock_id','time_id'])[['trade_log_return']].agg(realized_volatility).reset_index())\n    \n    return df_realized_volatility","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:10:24.485987Z","iopub.execute_input":"2022-07-06T10:10:24.494633Z","iopub.status.idle":"2022-07-06T10:10:24.523850Z","shell.execute_reply.started":"2022-07-06T10:10:24.494580Z","shell.execute_reply":"2022-07-06T10:10:24.522981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge book and trade on stock_id and time_id\ndef get_stock_data(stock_id, trainortest):\n    \n    # get data\n    book = book_preprocessing(stock_id, trainortest)\n    trade = trade_preprocessing(stock_id, trainortest)\n    \n    # merge\n    stock_data = book.merge(trade, on=['stock_id', 'time_id'], how='left').fillna(-1)\n    \n    return stock_data\n\n# get the whole dataset(use stock_id list)\ndef get_data_set(stock_ids, trainortest):\n    # Parallel process of get_stock_data\n    stock_data = Parallel(n_jobs=-1)(\n        delayed(get_stock_data)(stock_id, trainortest) \n        for stock_id in stock_ids\n    )\n    # concat stock_datas\n    df_stock = pd.concat(stock_data, ignore_index = True)\n\n    return df_stock","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:10:26.467443Z","iopub.execute_input":"2022-07-06T10:10:26.467783Z","iopub.status.idle":"2022-07-06T10:10:26.475440Z","shell.execute_reply.started":"2022-07-06T10:10:26.467753Z","shell.execute_reply":"2022-07-06T10:10:26.474325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(path + 'train.csv')\n# creat row_id(like test data)\ntrain['row_id'] = train['stock_id'].astype(str) + '-' + train['time_id'].astype(str)\ndisplay(train.head())\nprint('train data shape:', train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T11:38:58.733399Z","iopub.execute_input":"2022-07-06T11:38:58.733738Z","iopub.status.idle":"2022-07-06T11:38:59.852682Z","shell.execute_reply.started":"2022-07-06T11:38:58.733709Z","shell.execute_reply":"2022-07-06T11:38:59.851691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = get_data_set(train['stock_id'].unique(), 'train')\n\n# Merge train with df_train\ntrain = pd.merge(train, df_train, on = ['stock_id', 'time_id'], how = 'left')\ndisplay(train.head(5))\nprint('Train shape:', train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T11:39:04.057825Z","iopub.execute_input":"2022-07-06T11:39:04.058452Z","iopub.status.idle":"2022-07-06T11:48:42.006625Z","shell.execute_reply.started":"2022-07-06T11:39:04.058415Z","shell.execute_reply":"2022-07-06T11:48:42.004330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(path + 'test.csv')\ntest['row_id'] = test['stock_id'].astype(str) + '-' + test['time_id'].astype(str)\ndisplay(test.head())\nprint('test data shape:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:07:05.773855Z","iopub.execute_input":"2022-07-06T12:07:05.774540Z","iopub.status.idle":"2022-07-06T12:07:05.797008Z","shell.execute_reply.started":"2022-07-06T12:07:05.774504Z","shell.execute_reply":"2022-07-06T12:07:05.796117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = get_data_set(test['stock_id'].unique(), 'test')\ntest = pd.merge(test, df_test, on = ['stock_id', 'time_id'], how = 'left').fillna(0)\ndisplay(test.head(5))\nprint('Test shape:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:08:38.110334Z","iopub.execute_input":"2022-07-06T12:08:38.110689Z","iopub.status.idle":"2022-07-06T12:08:38.806668Z","shell.execute_reply.started":"2022-07-06T12:08:38.110660Z","shell.execute_reply":"2022-07-06T12:08:38.805171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Light GBM","metadata":{}},{"cell_type":"code","source":"# Parameters of Light GBM\nparameters_lgbm = {\n        \"objective\": \"rmse\", \n        \"metric\": \"rmse\", \n        \"boosting_type\": \"gbdt\",\n        \"objective\": \"regression\",\n        \"learning_rate\": 0.01,\n        # Regularization parameter\n        \"lambda_l1\": 1,\n        \"lambda_l2\": 1,\n        # Random number of features used each time a new tree is created\n        \"feature_fraction\": 0.7,\n        # How many samples are used at random for each bagging session\n        \"bagging_fraction\": 0.7,\n}\n\n# loss function\ndef feval_RMSPE(preds, train_data):\n    labels = train_data.get_label()\n    return 'RMSPE', round(rmspe_calc(y_true = labels, y_pred = preds),5), False","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:04:31.835508Z","iopub.execute_input":"2022-07-06T12:04:31.835856Z","iopub.status.idle":"2022-07-06T12:04:31.843084Z","shell.execute_reply.started":"2022-07-06T12:04:31.835828Z","shell.execute_reply":"2022-07-06T12:04:31.842026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train function\ndef light_gbm(X_train, y_train, X_val ,y_val):\n    \n    # Create dataset\n    train_data = lgbm.Dataset(X_train, label=y_train, categorical_feature=['stock_id'], weight=1/np.power(y_train,2))\n    val_data = lgbm.Dataset(X_val, label=y_val, categorical_feature=['stock_id'], weight=1/np.power(y_val,2))\n    \n    # train model\n    model = lgbm.train(parameters_lgbm, \n                      train_data, \n                      num_boost_round=5000, \n                      valid_sets=val_data, \n                      feval=feval_RMSPE,\n                      verbose_eval= 250,\n                      early_stopping_rounds=500\n                     )\n    \n    # predict validation data\n    preds_val = model.predict(X_val)\n\n    # RMSPE calculation\n    score = round(rmspe_calc(y_true = y_val, y_pred = preds_val),5)\n    \n    # delete dataset\n    del train_data, val_data\n    \n    return score, model","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:12:35.767516Z","iopub.execute_input":"2022-07-06T12:12:35.768099Z","iopub.status.idle":"2022-07-06T12:12:35.776477Z","shell.execute_reply.started":"2022-07-06T12:12:35.768062Z","shell.execute_reply":"2022-07-06T12:12:35.774977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cross Validation","metadata":{}},{"cell_type":"code","source":"# ref https://www.kaggle.com/corochann/permutation-importance-for-feature-selection-part1\ndef calc_model_importance(model, feature_names=None, importance_type='gain'):\n    importance_df = pd.DataFrame(model.feature_importance(importance_type=importance_type),\n                                 index=feature_names,\n                                 columns=['importance']).sort_values('importance')\n    return importance_df\n\n\ndef plot_importance(importance_df, title='',\n                    save_filepath=None, figsize=(8, 12)):\n    fig, ax = plt.subplots(figsize=figsize)\n    importance_df.plot.barh(ax=ax)\n    if title:\n        plt.title(title)\n    plt.tight_layout()\n    if save_filepath is None:\n        plt.show()\n    else:\n        plt.savefig(save_filepath)\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T11:58:20.411234Z","iopub.execute_input":"2022-07-06T11:58:20.411588Z","iopub.status.idle":"2022-07-06T11:58:20.419308Z","shell.execute_reply.started":"2022-07-06T11:58:20.411559Z","shell.execute_reply":"2022-07-06T11:58:20.418090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\ngain_importance_list = []\nsplit_importance_list = []\nscores = 0.0\n\n# divide X and y\nX = train.drop(['row_id','target'],axis=1)\ny = train['target']\n\n# K-flods\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n    \n    print(f'CV trial : {fold + 1} / 5')\n    \n    # divide dataset into train and validation data\n    X_train, y_train = X.loc[train_idx], y[train_idx]\n    X_val, y_val = X.loc[val_idx], y[val_idx]\n    \n    # train with Light GBM\n    rmspe_score, model = light_gbm(X_train, y_train, X_val ,y_val)\n    \n    # Summary \n    print(f'Performance of the　prediction: , RMSPE: {rmspe_score}')\n    print('-'*50)\n    \n    # keep scores and models\n    scores += rmspe_score / 5\n    models.append(model)\n    \n    # calc model feature importance\n    feature_names = X_train.columns.values.tolist()\n    gain_importance_df = calc_model_importance(\n        model, feature_names=feature_names, importance_type='gain')\n    gain_importance_list.append(gain_importance_df)\n\n    split_importance_df = calc_model_importance(\n        model, feature_names=feature_names, importance_type='split')\n    split_importance_list.append(split_importance_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:22:54.709165Z","iopub.execute_input":"2022-07-06T12:22:54.709730Z","iopub.status.idle":"2022-07-06T12:26:42.361955Z","shell.execute_reply.started":"2022-07-06T12:22:54.709696Z","shell.execute_reply":"2022-07-06T12:26:42.360978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:31:24.295308Z","iopub.execute_input":"2022-07-06T12:31:24.295882Z","iopub.status.idle":"2022-07-06T12:31:24.302019Z","shell.execute_reply.started":"2022-07-06T12:31:24.295847Z","shell.execute_reply":"2022-07-06T12:31:24.300993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_mean_importance(importance_df_list):\n    mean_importance = np.mean(\n        np.array([df['importance'].values for df in importance_df_list]), axis=0)\n    mean_df = importance_df_list[0].copy()\n    mean_df['importance'] = mean_importance\n    return mean_df","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:28:49.074427Z","iopub.execute_input":"2022-07-06T12:28:49.075288Z","iopub.status.idle":"2022-07-06T12:28:49.081519Z","shell.execute_reply.started":"2022-07-06T12:28:49.075224Z","shell.execute_reply":"2022-07-06T12:28:49.080554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_gain_df = calc_mean_importance(gain_importance_list)\nplot_importance(mean_gain_df, title='Model feature importance by gain')\nmean_gain_df = mean_gain_df.reset_index().rename(columns={'index': 'feature_names'})\nmean_gain_df.to_csv('gain_importance_mean.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:29:07.464916Z","iopub.execute_input":"2022-07-06T12:29:07.465518Z","iopub.status.idle":"2022-07-06T12:29:07.733823Z","shell.execute_reply.started":"2022-07-06T12:29:07.465483Z","shell.execute_reply":"2022-07-06T12:29:07.732971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_split_df = calc_mean_importance(split_importance_list)\nplot_importance(mean_split_df, title='Model feature importance by split')\nmean_split_df = mean_split_df.reset_index().rename(columns={'index': 'feature_names'})\nmean_split_df.to_csv('split_importance_mean.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:29:33.713546Z","iopub.execute_input":"2022-07-06T12:29:33.713881Z","iopub.status.idle":"2022-07-06T12:29:33.941196Z","shell.execute_reply.started":"2022-07-06T12:29:33.713852Z","shell.execute_reply":"2022-07-06T12:29:33.940271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{}},{"cell_type":"code","source":"y_pred = test[['row_id']]\nX_test = test.drop(['row_id'], axis = 1)\n\ntarget = np.zeros(len(X_test))\n\nfor model in models:\n    pred = model.predict(X_test[X_val.columns], num_iteration=model.best_iteration)\n    target += pred / len(models)\n\ny_pred = y_pred.assign(target = target)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-06T12:39:55.156918Z","iopub.execute_input":"2022-07-06T12:39:55.157394Z","iopub.status.idle":"2022-07-06T12:39:55.240565Z","shell.execute_reply.started":"2022-07-06T12:39:55.157356Z","shell.execute_reply":"2022-07-06T12:39:55.239654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save models\ni = 0\nfor model in models:\n    # load model: lgbm.Booster(path)\n    model.save_model('Light GBM_' + str(i) + '.h5')\n    i += 1","metadata":{"execution":{"iopub.status.busy":"2022-07-06T13:06:31.107476Z","iopub.execute_input":"2022-07-06T13:06:31.107835Z","iopub.status.idle":"2022-07-06T13:06:31.988959Z","shell.execute_reply.started":"2022-07-06T13:06:31.107799Z","shell.execute_reply":"2022-07-06T13:06:31.987965Z"},"trusted":true},"execution_count":null,"outputs":[]}]}