{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n- 直近のReturnの逆順にRank付けするモデル\n- `ExpectedDividend`が入っている場合はマイナスになる可能性が高いので、下位にするルールを入れる\n","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"# libraries\nimport os\nfrom decimal import ROUND_HALF_UP, Decimal#float型の計算を正確に行うため\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm#処理状況の可視化\nimport warnings\n\nwarnings.filterwarnings('ignore')","metadata":{"papermill":{"duration":2.039395,"end_time":"2022-04-02T13:51:41.227973","exception":false,"start_time":"2022-04-02T13:51:39.188578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T05:16:04.062791Z","iopub.execute_input":"2022-07-05T05:16:04.063153Z","iopub.status.idle":"2022-07-05T05:16:04.069264Z","shell.execute_reply.started":"2022-07-05T05:16:04.063112Z","shell.execute_reply":"2022-07-05T05:16:04.067920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set base_dir to load data\nbase_dir = \"../input/jpx-tokyo-stock-exchange-prediction\"\n\ntrain_files_dir = f\"{base_dir}/train_files\"\nsupplemental_files_dir = f\"{base_dir}/supplemental_files\"","metadata":{"papermill":{"duration":0.053109,"end_time":"2022-04-02T13:51:41.466618","exception":false,"start_time":"2022-04-02T13:51:41.413509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T05:16:05.446235Z","iopub.execute_input":"2022-07-05T05:16:05.446531Z","iopub.status.idle":"2022-07-05T05:16:05.451970Z","shell.execute_reply.started":"2022-07-05T05:16:05.446500Z","shell.execute_reply":"2022-07-05T05:16:05.450749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# functions","metadata":{}},{"cell_type":"code","source":"#分割・逆分割による過去の価格差を小さくするために、AdjustmentFactorの値を用いて、株価を修正する\ndef adjust_price(price):\n    \"\"\"\n    Args:\n        price (pd.DataFrame)  : pd.DataFrame include stock_price\n    Returns:\n        price DataFrame (pd.DataFrame): stock_price with generated AdjustedClose\n    \"\"\"\n    # transform Date column into datetime\n    price.loc[: ,\"Date\"] = pd.to_datetime(price.loc[: ,\"Date\"], format=\"%Y-%m-%d\")\n\n    def generate_adjusted_close(df):\n        \"\"\"\n        Args:\n            df (pd.DataFrame)  : stock_price for a single SecuritiesCode\n        Returns:\n            df (pd.DataFrame): stock_price with AdjustedClose for a single SecuritiesCode\n        \"\"\"\n        # sort data to generate CumulativeAdjustmentFactor\n        df = df.sort_values(\"Date\", ascending=False)#降順(最新のものが先頭)\n        # generate CumulativeAdjustmentFactor\n        df.loc[:, \"CumulativeAdjustmentFactor\"] = df[\"AdjustmentFactor\"].cumprod()#cumprodは累積積を求める関数\n        # generate AdjustedClose\n        df.loc[:, \"AdjustedClose\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Close\"]\n        ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding=ROUND_HALF_UP)#四捨五入\n        ))\n        # reverse order\n        df = df.sort_values(\"Date\")#昇順に戻す\n        # to fill AdjustedClose, replace 0 into np.nan\n        df.loc[df[\"AdjustedClose\"] == 0, \"AdjustedClose\"] = np.nan\n        # forward fill AdjustedClose\n        df.loc[:, \"AdjustedClose\"] = df.loc[:, \"AdjustedClose\"].ffill()#ffill:前(上)の値に置換\n        return df\n\n    # generate AdjustedClose\n    price = price.sort_values([\"SecuritiesCode\", \"Date\"])\n    price = price.groupby(\"SecuritiesCode\").apply(generate_adjusted_close).reset_index(drop=True)\n\n    price.set_index(\"Date\", inplace=True)\n    return price","metadata":{"papermill":{"duration":0.060398,"end_time":"2022-04-02T13:51:41.755972","exception":false,"start_time":"2022-04-02T13:51:41.695574","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T05:16:08.389070Z","iopub.execute_input":"2022-07-05T05:16:08.389370Z","iopub.status.idle":"2022-07-05T05:16:08.401586Z","shell.execute_reply.started":"2022-07-05T05:16:08.389315Z","shell.execute_reply":"2022-07-05T05:16:08.400505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_for_predict(price, code):\n    \"\"\"\n    Args:\n        price (pd.DataFrame)  : pd.DataFrame include stock_price\n        code (int)  : A local code for a listed company\n    Returns:\n        feature DataFrame (pd.DataFrame)\n    \"\"\"\n    close_col = \"AdjustedClose\"\n    feats = price.loc[price[\"SecuritiesCode\"] == code, [\"SecuritiesCode\", close_col, \"ExpectedDividend\"]].copy()\n\n    # calculate return using AdjustedClose\n    feats[\"return_1day\"] = feats[close_col].pct_change(1)\n    \n    # ExpectedDividend\n    feats[\"ExpectedDividend\"] = feats[\"ExpectedDividend\"].mask(feats[\"ExpectedDividend\"] > 0, 1)\n\n    # filling data for nan and inf\n    feats = feats.fillna(0)\n    feats = feats.replace([np.inf, -np.inf], 0)\n    # drop AdjustedClose column\n    feats = feats.drop([close_col], axis=1)\n\n    return feats","metadata":{"papermill":{"duration":0.069843,"end_time":"2022-04-02T13:52:12.754025","exception":false,"start_time":"2022-04-02T13:52:12.684182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T05:16:09.376523Z","iopub.execute_input":"2022-07-05T05:16:09.376848Z","iopub.status.idle":"2022-07-05T05:16:09.384521Z","shell.execute_reply.started":"2022-07-05T05:16:09.376813Z","shell.execute_reply":"2022-07-05T05:16:09.383669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"# load stock price data\ndf_price_raw = pd.read_csv(f\"{train_files_dir}/stock_prices.csv\")\nprice_cols = [\n    \"Date\",\n    \"SecuritiesCode\",\n    \"Close\",\n    \"AdjustmentFactor\",\n    \"ExpectedDividend\"\n]\ndf_price_raw = df_price_raw[price_cols]\n\n# forecasting phase leaderboard:\ndf_price_supplemental = pd.read_csv(f\"{supplemental_files_dir}/stock_prices.csv\")\ndf_price_supplemental = df_price_supplemental[price_cols]\ndf_price_raw = pd.concat([df_price_raw, df_price_supplemental])\n\n# filter data to reduce culculation cost \ndf_price_raw = df_price_raw.loc[df_price_raw[\"Date\"] >= \"2022-07-01\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T05:16:14.925059Z","iopub.execute_input":"2022-07-05T05:16:14.925448Z","iopub.status.idle":"2022-07-05T05:16:23.768990Z","shell.execute_reply.started":"2022-07-05T05:16:14.925396Z","shell.execute_reply":"2022-07-05T05:16:23.767825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load Time Series API\nimport jpx_tokyo_market_prediction\n# make Time Series API environment (this function can be called only once in a session)\nenv = jpx_tokyo_market_prediction.make_env()\n# get iterator to fetch data day by day\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T05:16:29.417213Z","iopub.execute_input":"2022-07-05T05:16:29.418078Z","iopub.status.idle":"2022-07-05T05:16:29.447135Z","shell.execute_reply.started":"2022-07-05T05:16:29.418028Z","shell.execute_reply":"2022-07-05T05:16:29.446135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counter = 0\n# fetch data day by day\nfor (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    current_date = prices[\"Date\"].iloc[0]\n    sample_prediction_date = sample_prediction[\"Date\"].iloc[0]\n    print(f\"current_date: {current_date}, sample_prediction_date: {sample_prediction_date}\")\n\n    if counter == 0:\n        # to avoid data leakage\n        df_price_raw = df_price_raw.loc[df_price_raw[\"Date\"] < current_date]#current_date以前のデータにする\n\n    # to generate AdjustedClose, increment price data\n    df_price_raw = pd.concat([df_price_raw, prices[price_cols]])\n    # generate AdjustedClose\n    df_price = adjust_price(df_price_raw)\n\n    # get target SecuritiesCodes\n    codes = sorted(prices[\"SecuritiesCode\"].unique())\n\n    # generate feature\n    feature = pd.concat([get_features_for_predict(df_price, code) for code in codes])\n    # filter feature for this iteration\n    feature = feature.loc[feature.index == current_date]\n\n    # prediction\n    feature.loc[:, \"predict\"] = feature[\"return_1day\"] + feature[\"ExpectedDividend\"]*100\n\n    # set rank by predict\n    feature = feature.sort_values(\"predict\", ascending=True).drop_duplicates(subset=['SecuritiesCode'])\n    feature.loc[:, \"Rank\"] = np.arange(len(feature))\n    feature_map = feature.set_index('SecuritiesCode')['Rank'].to_dict()\n    sample_prediction['Rank'] = sample_prediction['SecuritiesCode'].map(feature_map)\n\n    # check Rank\n    assert sample_prediction[\"Rank\"].notna().all()\n    assert sample_prediction[\"Rank\"].min() == 0\n    assert sample_prediction[\"Rank\"].max() == len(sample_prediction[\"Rank\"]) - 1\n\n    # register your predictions\n    env.predict(sample_prediction)\n    counter += 1","metadata":{"execution":{"iopub.status.busy":"2022-07-05T05:16:30.442169Z","iopub.execute_input":"2022-07-05T05:16:30.442812Z","iopub.status.idle":"2022-07-05T05:17:06.231808Z","shell.execute_reply.started":"2022-07-05T05:16:30.442764Z","shell.execute_reply":"2022-07-05T05:17:06.230837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-07-05T05:17:14.124635Z","iopub.execute_input":"2022-07-05T05:17:14.124954Z","iopub.status.idle":"2022-07-05T05:17:14.902014Z","shell.execute_reply.started":"2022-07-05T05:17:14.124916Z","shell.execute_reply":"2022-07-05T05:17:14.900722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! tail submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-07-05T05:17:24.242871Z","iopub.execute_input":"2022-07-05T05:17:24.243344Z","iopub.status.idle":"2022-07-05T05:17:25.009051Z","shell.execute_reply.started":"2022-07-05T05:17:24.243295Z","shell.execute_reply":"2022-07-05T05:17:25.007846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}