{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# パッケージのインポート（Import Packages）","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport jpx_tokyo_market_prediction\nfrom lightgbm import LGBMRegressor\nimport optuna.integration.lightgbm as lgb\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport datetime\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:16.196997Z","iopub.execute_input":"2022-07-04T01:46:16.197473Z","iopub.status.idle":"2022-07-04T01:46:16.204713Z","shell.execute_reply.started":"2022-07-04T01:46:16.197438Z","shell.execute_reply":"2022-07-04T01:46:16.203704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データ読込（Load Data）","metadata":{}},{"cell_type":"code","source":"prices = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv\", parse_dates=[\"Date\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:16.639134Z","iopub.execute_input":"2022-07-04T01:46:16.63964Z","iopub.status.idle":"2022-07-04T01:46:17.302089Z","shell.execute_reply.started":"2022-07-04T01:46:16.639601Z","shell.execute_reply":"2022-07-04T01:46:17.30079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"prices","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:17.304223Z","iopub.execute_input":"2022-07-04T01:46:17.304742Z","iopub.status.idle":"2022-07-04T01:46:17.336877Z","shell.execute_reply.started":"2022-07-04T01:46:17.304696Z","shell.execute_reply":"2022-07-04T01:46:17.335993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"列の意味（column information）<br>\n(1) RowId: 日付_銘柄。ユニークID（Date_SecuritiesCode, Unique ID of price records）<br>\n(2) Date：日付（Trade date）<br>\n(3) SecuritiesCode: 銘柄コード。2000種類存在。（Local securities code, 2000）<br>\n(4) Open:始値（First traded price on a day）<br>\n(5) High:高値（Highest traded price on a day）<br>\n(6) Low :安値（Lowest traded price on a day）<br>\n(7) Close:終値（Last traded price on a day）<br>\n(8) Volume:出来高（Number of traded stocks on a day）<br>\n(9) AdjustmentFactor：分割／併合による株価の変動。終値から次の日の始まりまでの間に補正される。（Change in stock price due to a split/reverse split. Correction from the closing price to the beginning of the next day.）<br>\n(10) ExpectedDividend:予想配当金額。（Projected dividend amount）<br>\n(11) SupervisionFlag: 監理・整理銘柄フラグ。上場廃止リスク高い。（Flag for supervised and delisted stocks. High risk of delisting.）<br>\n(12) Target:目標変数。１日後と2日後の差額から得られる収益率（Target variable; rate of return derived from the difference between one and two days later）<br>","metadata":{}},{"cell_type":"markdown","source":"## 欠損値の確認（Check missing values）","metadata":{}},{"cell_type":"code","source":"#列ごとに欠損値の数を確認（Check the number of missing values per column）\nprices.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:17.699718Z","iopub.execute_input":"2022-07-04T01:46:17.700207Z","iopub.status.idle":"2022-07-04T01:46:17.748725Z","shell.execute_reply.started":"2022-07-04T01:46:17.700173Z","shell.execute_reply":"2022-07-04T01:46:17.747955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ","metadata":{}},{"cell_type":"code","source":"#\"Open\"（始値）, \"High\"（高値）, \"Low\"（安値）, \"Close\"（終値）の欠損部分について調査（Check about missing values of \"Open\", \"High\", \"Low\", \"Close\"）\nprint((prices[\"Open\"].isnull() == (prices[\"Volume\"]==0)).all())\nprint((prices[\"High\"].isnull() == (prices[\"Volume\"]==0)).all())\nprint((prices[\"Low\"].isnull() == (prices[\"Volume\"]==0)).all())\nprint((prices[\"Close\"].isnull() == (prices[\"Volume\"]==0)).all())\n#\"Volume\"(出来高)はその日に売買された合計金額を示す。（\"Volume\" represents the total amount traded on that day.）\n#ゆえに「出来高=0」はその日に売買されていないことを示し、始値、高値、安値、終値はnullとなる（So, \"Volume = 0\" means that there was no trading on that day, and the opening, high, low, and closing prices are null.）","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:18.106094Z","iopub.execute_input":"2022-07-04T01:46:18.10746Z","iopub.status.idle":"2022-07-04T01:46:18.126085Z","shell.execute_reply.started":"2022-07-04T01:46:18.107398Z","shell.execute_reply":"2022-07-04T01:46:18.12517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\"ExpectedDividend\"(予想配当金額)の欠損部分について調査（Check about missing values of \"\"ExpectedDividend\"）\nprices[(~prices[\"ExpectedDividend\"].isnull())][\"Date\"].value_counts()\n#特定の日のみ値が存在。決算時期に値が埋まるのではないか？（Value exists only on certain days. The value may be filled at the time of closing.）","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:18.28671Z","iopub.execute_input":"2022-07-04T01:46:18.287418Z","iopub.status.idle":"2022-07-04T01:46:18.299912Z","shell.execute_reply.started":"2022-07-04T01:46:18.287379Z","shell.execute_reply":"2022-07-04T01:46:18.298702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#銘柄コード別に日付に欠損がないかを確認（Check for missing dates）\nprices[\"SecuritiesCode\"].value_counts().sort_values()\n#1413は21日欠損。8806は20日欠損。4699は1日欠損（1413 misses 21days, 8806 misses 20days, 4699 misses 1day）","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:18.601793Z","iopub.execute_input":"2022-07-04T01:46:18.602453Z","iopub.status.idle":"2022-07-04T01:46:18.617526Z","shell.execute_reply.started":"2022-07-04T01:46:18.602413Z","shell.execute_reply":"2022-07-04T01:46:18.616299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#欠損している日付を確認（Check for missing dates）\npd.pivot(prices, index=\"Date\", columns=\"SecuritiesCode\", values=\"Volume\")[[1413, 8806, 4699, 1375]].tail(30)\n#1413は2022-04-25～2022-05-27（計21日）で欠損（1413 misses 2022-04-25～2022-05-27）\n#8806は2022-04-26～2022-05-27（計20日）で欠損（8806 misses 2022-04-26～2022-05-27）\n#4699は2022-05-27（計1日）で欠損（4699 misses 2022-05-27）","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:18.985005Z","iopub.execute_input":"2022-07-04T01:46:18.985459Z","iopub.status.idle":"2022-07-04T01:46:19.064097Z","shell.execute_reply.started":"2022-07-04T01:46:18.985425Z","shell.execute_reply":"2022-07-04T01:46:19.062554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 特徴量生成・選択（Feature generation and selection）","metadata":{}},{"cell_type":"code","source":"def MA(series, window=25):\n    return series.rolling(window, min_periods=1).mean()\n\ndef DMA(series, window=25):\n    return series/MA(series, window) - 1\n\ndef divergence(series, window=25):\n    std = series.rolling(window,min_periods=1).std()\n    mean = series.rolling(window,min_periods=1).mean()\n    return (series-mean) / std    \n\ndef rsi(series, n=14):\n    return (series - series.shift(1)).rolling(n).apply(lambda s:s[s>0].sum()/abs(s).sum())\n\ndef stochastic(series, k=14, n=3, m=3):\n    _min = series.rolling(k).min()\n    _max = series.rolling(k).max()\n    _k = (series - _min)/(_max - _min)\n    _d1 = _k.rolling(n).mean()\n    _d2 = _d1.rolling(m).mean()\n    return pd.DataFrame({\n                    \"%K\":_k,\n                    \"FAST-%D\":_d1,\n                    \"SLOW-%D\":_d2,\n                    },index=series.index)\n    # return _k, _d1, _d2\n\ndef psy(series, n=14):\n    return (series - series.shift(1)).rolling(n).apply(lambda s:(s>=0).mean())\n\ndef ICH(series):\n    conv = series.rolling(9).apply(lambda s:(s.max()+s.min())/2)\n    base = series.rolling(26).apply(lambda s:(s.max()+s.min())/2)\n    pre1 = ((conv + base)/2).shift(25)\n    pre2 = d.Close_adj.rolling(52).apply(lambda s:(s.max()+s.min())/2).shift(25)\n    lagg = d.Close_adj.shift(25)\n    return conv, base, pre1, pre2, lagg\n\ndef roc(series, window=14):\n    return series/series.shift(window) - 1\n\nclass FeatureBase():\n    def create_feature(self, d):\n        assert False, \"NotImplemented\"\n        \nclass MAFeature(FeatureBase):\n    def create_feature(self, d):\n        return self._create_feature(d[\"Close_adj\"])\n\n    def _create_feature(self, series, window1=5, window2=25):\n        ma1 = MA(series, window1).rename(\"MA1\")\n        ma2 = MA(series, window2).rename(\"MA2\")\n        diff = ma1 - ma2\n        cross = pd.Series(\n                        np.where((diff>0) & (diff<0).shift().fillna(False), 1,\n                            np.where((diff<0) & (diff>0).shift().fillna(False), -1, 0\n                                )\n                        ),\n                        index = series.index, name=\"MA_Cross\"\n                )\n        return pd.concat([ma1, ma2, cross], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:19.772498Z","iopub.execute_input":"2022-07-04T01:46:19.772886Z","iopub.status.idle":"2022-07-04T01:46:19.795507Z","shell.execute_reply.started":"2022-07-04T01:46:19.772855Z","shell.execute_reply":"2022-07-04T01:46:19.794343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def holiday(d):\n    return pd.DataFrame({\n        \"before_holiday\":(d[\"Date\"] != d[\"Date\"].shift(-1) - datetime.timedelta(days=1)) | (d[\"weekday\"]==4),\n        \"after_holiday\":(d[\"Date\"] != d[\"Date\"].shift(1) + datetime.timedelta(days=1)) | (d[\"weekday\"]==0)\n    }, index=d.index)\ndef make_features(df):\n    df = df[[\n        \"Date\",\"SecuritiesCode\",\"Open\",\"Close\",\"AdjustmentFactor\",\n        \"Volume\"\n    ]].copy()\n    df[\"weekday\"] = df[\"Date\"].dt.weekday\n    df = df.join(df.groupby(\"SecuritiesCode\").apply(holiday))\n    df[\"Volume_ratio\"] = df[\"Volume\"]/df.groupby(\"SecuritiesCode\")[\"Volume\"].rolling(window=15, min_periods=1).mean().reset_index(\"SecuritiesCode\",drop=True)\n    df[\"Close_adj\"] = df.groupby(\"SecuritiesCode\").apply(lambda d:d[\"Close\"]/d[\"AdjustmentFactor\"].cumprod().shift().fillna(1)).reset_index(\"SecuritiesCode\",drop=True)\n    df[[\"MA1\", \"MA2\", \"MA_Cross\"]] = df.groupby(\"SecuritiesCode\").apply(lambda d: MAFeature()._create_feature(d.Close_adj))# .join(df[\"Target\"].shift(-1)).groupby(\"MA_Cross\").describe()\n    df[\"Diff\"] = (df[\"Close\"] - df[\"Open\"]) / df[[\"Close\",\"Open\"]].mean(axis=1)\n    df[\"Diff_MA1\"] = df[\"Close_adj\"] - df[\"MA1\"]\n    df[\"Diff_MA2\"] = df[\"Close_adj\"] - df[\"MA2\"]\n    for i in range(1, 3):\n        df[\"MA_Cross_lag_{:}\".format(i)] = df.groupby(\"SecuritiesCode\")[\"MA_Cross\"].shift(i)\n\n    df[\"DivMA\"] = df.groupby(\"SecuritiesCode\")[\"Close_adj\"].apply(DMA)\n    df[\"Div\"] = df.groupby(\"SecuritiesCode\")[\"Close_adj\"].apply(divergence)\n    df[\"Rsi\"] = df.groupby(\"SecuritiesCode\")[\"Close_adj\"].apply(rsi)\n    df = df.join(df.groupby(\"SecuritiesCode\")[\"Close_adj\"].apply(stochastic))\n    \n    ##################以下、特徴量を追加してください（Add features）#######################\n    \n    ##################以上、特徴量を追加してください（Add features）#######################\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:20.223125Z","iopub.execute_input":"2022-07-04T01:46:20.223611Z","iopub.status.idle":"2022-07-04T01:46:20.241332Z","shell.execute_reply.started":"2022-07-04T01:46:20.223578Z","shell.execute_reply":"2022-07-04T01:46:20.239805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = [\n    \"Diff\", \"Close_adj\",\"Volume_ratio\",\n    \"before_holiday\", \"after_holiday\",\n    \"Diff_MA1\", \"Diff_MA2\",\"MA_Cross\",\n    'MA_Cross_lag_1', 'MA_Cross_lag_2',\n    \"DivMA\", \"Div\", \"Rsi\", \"%K\", \"FAST-%D\",\"SLOW-%D\",\n]","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:20.649059Z","iopub.execute_input":"2022-07-04T01:46:20.650202Z","iopub.status.idle":"2022-07-04T01:46:20.656275Z","shell.execute_reply.started":"2022-07-04T01:46:20.650158Z","shell.execute_reply":"2022-07-04T01:46:20.654899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデル生成（Build model）","metadata":{}},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv\", parse_dates=[\"Date\"])\ndf = make_features(df).join(df.Target)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:46:22.091285Z","iopub.execute_input":"2022-07-04T01:46:22.091717Z","iopub.status.idle":"2022-07-04T01:48:39.613856Z","shell.execute_reply.started":"2022-07-04T01:46:22.091685Z","shell.execute_reply":"2022-07-04T01:48:39.612342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(X, y):\n#     fast\n    model=LGBMRegressor(boosting_type=\"dart\",\n                        num_leaves=31,max_depth=12,\n                        learning_rate=0.2, n_estimators=100,\n                        random_state=0)\n#     model=LGBMRegressor(boosting_type=\"dart\",\n#                     num_leaves=31,max_depth=12,\n#                     learning_rate=0.02, n_estimators=1000,\n#                     random_state=0)\n    model.fit(X,y)\n    # model.score(X,y)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:48:39.616454Z","iopub.execute_input":"2022-07-04T01:48:39.616973Z","iopub.status.idle":"2022-07-04T01:48:39.624108Z","shell.execute_reply.started":"2022-07-04T01:48:39.616923Z","shell.execute_reply":"2022-07-04T01:48:39.622777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodels = {}\nnum = 0\nfor code, d in df.groupby(\"SecuritiesCode\"):\n    d = d[~d.Target.isnull()]\n    X = d[columns]\n    y = d.Target\n    model = train_model(X, y)\n    models[code] = model","metadata":{"execution":{"iopub.status.busy":"2022-07-04T02:00:15.548939Z","iopub.execute_input":"2022-07-04T02:00:15.549387Z","iopub.status.idle":"2022-07-04T02:01:37.347596Z","shell.execute_reply.started":"2022-07-04T02:00:15.549354Z","shell.execute_reply":"2022-07-04T02:01:37.346784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 精度評価(Evaluation)","metadata":{}},{"cell_type":"code","source":"import warnings, gc\nimport numpy as np \nimport pandas as pd\nimport matplotlib.colors\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\n# from datetime import datetime, timedelta\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom sklearn.metrics import mean_squared_error,mean_absolute_error\nfrom lightgbm import LGBMRegressor\nfrom decimal import ROUND_HALF_UP, Decimal\nwarnings.filterwarnings(\"ignore\")\nimport plotly.figure_factory as ff","metadata":{"execution":{"iopub.status.busy":"2022-07-04T01:55:53.664392Z","iopub.execute_input":"2022-07-04T01:55:53.664886Z","iopub.status.idle":"2022-07-04T01:55:55.303484Z","shell.execute_reply.started":"2022-07-04T01:55:53.664849Z","shell.execute_reply":"2022-07-04T01:55:55.302622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"参考ノートブック（Reference notebook）<br>\nhttps://www.kaggle.com/code/kellibelcher/jpx-stock-market-analysis-prediction-with-lgbm<br>\nアンサンブルモデル用に精度評価のコードを修正（Modify evaluation code for ensemble models）<br>","metadata":{}},{"cell_type":"code","source":"def calc_spread_return_sharpe(df: pd.DataFrame, portfolio_size: int = 200, toprank_weight_ratio: float = 2) -> float:\n    \"\"\"\n    Args:\n        df (pd.DataFrame): predicted results\n        portfolio_size (int): # of equities to buy/sell\n        toprank_weight_ratio (float): the relative weight of the most highly ranked stock compared to the least.\n    Returns:\n        (float): sharpe ratio\n    \"\"\"\n    def _calc_spread_return_per_day(df, portfolio_size, toprank_weight_ratio):\n        \"\"\"\n        Args:\n            df (pd.DataFrame): predicted results\n            portfolio_size (int): # of equities to buy/sell\n            toprank_weight_ratio (float): the relative weight of the most highly ranked stock compared to the least.\n        Returns:\n            (float): spread return\n        \"\"\"\n        assert df['Rank'].min() == 0\n        assert df['Rank'].max() == len(df['Rank']) - 1\n#         weights = np.linspace(start=toprank_weight_ratio, stop=1, num=portfolio_size)\n        df_len = len(df)\n        if df_len>=portfolio_size:\n            weights = np.linspace(start=toprank_weight_ratio, stop=1, num=portfolio_size)\n            purchase = (df.sort_values(by='Rank')['Target'][:portfolio_size] * weights).sum() / weights.mean()\n        else:\n            weights = np.linspace(start=toprank_weight_ratio, stop=1, num=df_len)\n            purchase = (df.sort_values(by='Rank')['Target'][:df_len] * weights).sum() / weights.mean()\n        short = (df.sort_values(by='Rank', ascending=False)['Target'][:portfolio_size] * weights).sum() / weights.mean()\n        return purchase - short\n\n    buf = df.groupby('Date').apply(_calc_spread_return_per_day, portfolio_size, toprank_weight_ratio)\n    sharpe_ratio = buf.mean() / buf.std()\n    return sharpe_ratio","metadata":{"execution":{"iopub.status.busy":"2022-07-03T06:41:59.458054Z","iopub.execute_input":"2022-07-03T06:41:59.458592Z","iopub.status.idle":"2022-07-03T06:41:59.470129Z","shell.execute_reply.started":"2022-07-03T06:41:59.45856Z","shell.execute_reply":"2022-07-03T06:41:59.469304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nts_fold = TimeSeriesSplit(n_splits=5, gap=0)\ndf_input = df.copy().sort_values(['Date','SecuritiesCode'])\ny=df_input['Target'].to_numpy()\nX=df_input.drop(['Target'],axis=1)\n\nfeat_importance=pd.DataFrame()\nsharpe_ratio=[]\nresult = {}\n    \nfor fold, (train_idx, val_idx) in enumerate(ts_fold.split(X, y)):\n    \n    print(\"\\n========================== Fold {} ==========================\".format(fold+1))\n    df_train = df_input.iloc[train_idx,:]\n    df_valid = df_input.iloc[val_idx,:]\n    X_train, y_train = X.iloc[train_idx,:], y[train_idx]\n    X_valid, y_val = X.iloc[val_idx,:], y[val_idx]\n    \n    print(\"Train Date range: {} to {}\".format(X_train.Date.min(),X_train.Date.max()))\n    print(\"Valid Date range: {} to {}\".format(X_valid.Date.min(),X_valid.Date.max()))\n    \n\n    models = {}\n    for code, d in df_train.groupby(\"SecuritiesCode\"):\n        X_tmp = d[columns]\n        y_tmp = d.Target\n        model = train_model(X_tmp[~y_tmp.isnull()], y_tmp[~y_tmp.isnull()])\n        models[code] = model\n        \n    \n    rank=[]\n    df_valid_tmp = df_valid.copy()\n    for i, d in df_valid_tmp.groupby(\"SecuritiesCode\"):\n        df_valid_tmp.loc[d.index, \"pred\"]=models[i].predict(d[columns])\n    for date, d in df_valid_tmp.groupby(\"Date\"):\n        df_valid_tmp.loc[d.index, \"Rank\"]= (d.pred.rank(method=\"first\", ascending=False)-1).astype(int)\n    sharpe=calc_spread_return_sharpe(df_valid_tmp)\n    result[fold] =  df_valid_tmp\n    \n    sharpe_ratio.append(sharpe)\n    print(\"Valid Sharpe: {}\".format(sharpe))\n    \nprint(\"\\nAverage cross-validation Sharpe Ratio: {:.4f}, standard deviation = {:.2f}.\".format(np.mean(sharpe_ratio),np.std(sharpe_ratio)))","metadata":{"execution":{"iopub.status.busy":"2022-07-03T06:41:59.471527Z","iopub.execute_input":"2022-07-03T06:41:59.472218Z","iopub.status.idle":"2022-07-03T06:46:16.318469Z","shell.execute_reply.started":"2022-07-03T06:41:59.472173Z","shell.execute_reply":"2022-07-03T06:46:16.317377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 予測（Predict）","metadata":{}},{"cell_type":"code","source":"import jpx_tokyo_market_prediction\nenv = jpx_tokyo_market_prediction.make_env()   # 環境初期化\niter_test = env.iter_test()    # テストファイルをループするイテレーター","metadata":{"execution":{"iopub.status.busy":"2022-07-04T02:01:44.350756Z","iopub.execute_input":"2022-07-04T02:01:44.351234Z","iopub.status.idle":"2022-07-04T02:01:44.39205Z","shell.execute_reply.started":"2022-07-04T02:01:44.351189Z","shell.execute_reply":"2022-07-04T02:01:44.389861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = df.copy()\nfor (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    display(prices)\n    prices[\"Date\"] = pd.to_datetime(prices[\"Date\"])\n#     data = prices.drop_duplicates([\"SecuritiesCode\", \"Date\"], keep=\"last\").sort_values([\"SecuritiesCode\", \"Date\"]).reset_index(drop=True)\n    data = data.append(prices).drop_duplicates([\"SecuritiesCode\", \"Date\"], keep=\"last\").sort_values([\"SecuritiesCode\", \"Date\"]).reset_index(drop=True)\n    data = make_features(data)\n    \n    d = sample_prediction[[\"Date\",\"SecuritiesCode\"]].reset_index()\n    d[\"Date\"] = pd.to_datetime(d[\"Date\"])\n    d = d.merge(data, on=[\"Date\",\"SecuritiesCode\"])\n    for code, _d in d.groupby(\"SecuritiesCode\"):\n        d.loc[_d.index, \"Pred\"] = models[code].predict(_d[columns])#予測\n    d = d.sort_values(by=\"Pred\", ascending=False).set_index(\"index\")\n    d[\"Rank\"] = np.arange(0,2000)\n    \n    d = d.sort_index()\n    sample_prediction[\"Rank\"] = d[\"Rank\"]\n    \n    submission = sample_prediction[[\"Date\",\"SecuritiesCode\",\"Rank\"]]\n    env.predict(submission)# 予測結果の提出","metadata":{"execution":{"iopub.status.busy":"2022-07-04T02:01:37.351863Z","iopub.execute_input":"2022-07-04T02:01:37.353933Z","iopub.status.idle":"2022-07-04T02:01:37.407586Z","shell.execute_reply.started":"2022-07-04T02:01:37.353889Z","shell.execute_reply":"2022-07-04T02:01:37.406218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}