{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction of a simple model submission\n\nThis notebook will show how to submit your simple model (built in [here](https://www.kaggle.com/code/jsmithperera/train-lgbm))\n\nIt is built from from https://www.kaggle.com/code/smeitoma/Train_Demo\n\nIf you want to know how this notebooks takes a model as input, check this: https://www.kaggle.com/discussions/getting-started/333636","metadata":{"papermill":{"duration":0.016397,"end_time":"2022-04-04T01:00:57.212294","exception":false,"start_time":"2022-04-04T01:00:57.195897","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys\nfrom decimal import ROUND_HALF_UP, Decimal\n\nimport numpy as np\nimport pandas as pd\nfrom lightgbm import Booster, LGBMRegressor\nimport datetime as dt","metadata":{"papermill":{"duration":2.040052,"end_time":"2022-04-04T01:00:59.298616","exception":false,"start_time":"2022-04-04T01:00:57.258564","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-27T05:22:58.242402Z","iopub.execute_input":"2022-06-27T05:22:58.243159Z","iopub.status.idle":"2022-06-27T05:22:58.247751Z","shell.execute_reply.started":"2022-06-27T05:22:58.243097Z","shell.execute_reply":"2022-06-27T05:22:58.246942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = \"../input/jpx-tokyo-stock-exchange-prediction\"\n\n# There are three types of folders.\n# We use stock_price.csv in the train_files folder to check our model in public leaderboard in this notebook.\ntrain_files_dir = f\"{base_dir}/train_files\"\n\n# for forecasting phase leaderboard, you may want to include stock_price.csv in the supplemental_files folder.\n# You can remove \"forecasting phase leaderboard\" comments in this notebook to use stock_price.csv in the supplemental_files folder.\n# forecasting phase leaderboard:\n#supplemental_files_dir = f\"{base_dir}/supplemental_files\"\n\n# model parameters generated by https://www.kaggle.com/code/smeitoma/train-demo\nmodel_file = \"../input/simplemodel/simple-model.txt\"","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.022705,"end_time":"2022-04-04T01:00:59.377977","exception":false,"start_time":"2022-04-04T01:00:59.355272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-27T05:23:01.869055Z","iopub.execute_input":"2022-06-27T05:23:01.869383Z","iopub.status.idle":"2022-06-27T05:23:01.876307Z","shell.execute_reply.started":"2022-06-27T05:23:01.86934Z","shell.execute_reply":"2022-06-27T05:23:01.875249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_adjusted_close(df):\n    \"\"\"\n    Args:\n        df (pd.DataFrame)  : stock_price for a single SecuritiesCode\n    Returns:\n        df (pd.DataFrame): stock_price with AdjustedClose for a single SecuritiesCode\n    \"\"\"\n    # sort data to generate CumulativeAdjustmentFactor\n    df = df.sort_values(\"Date\", ascending=False)\n    \n    # generate CumulativeAdjustmentFactor\n    df.loc[:, \"CumulativeAdjustmentFactor\"] = df[\"AdjustmentFactor\"].cumprod()\n    \n    # generate AdjustedClose\n    df.loc[:, \"AdClose\"] = (\n        df[\"CumulativeAdjustmentFactor\"] * df[\"Close\"]\n    ).map(lambda x: float(\n        Decimal(str(x)).quantize(Decimal('0.1'), rounding=ROUND_HALF_UP)\n    ))\n    \n    df.loc[:, \"AdOpen\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Open\"]\n    ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n    ))\n        \n    df.loc[:, \"AdHigh\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"High\"]\n    ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n    ))\n        \n    df.loc[:, \"AdLow\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Low\"]\n    ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n    ))\n    \n    # reverse order\n    df = df.sort_values(\"Date\")\n    \n    # to fill AdjustedClose, replace 0 into np.nan\n    df.loc[df[\"AdClose\"] == 0, \"AdClose\"] = np.nan\n    df.loc[df[\"AdOpen\"] == 0, \"AdOpen\"] = np.nan\n    df.loc[df[\"AdHigh\"] == 0, \"AdHigh\"] = np.nan\n    df.loc[df[\"AdLow\"] == 0, \"AdLow\"] = np.nan\n    \n    # forward fill AdjustedClose\n    df.loc[:, \"AdClose\"] = df.loc[:, \"AdClose\"].ffill()\n    df.loc[:, \"AdOpen\"] = df.loc[:, \"AdOpen\"].ffill()\n    df.loc[:, \"AdHigh\"] = df.loc[:, \"AdHigh\"].ffill()\n    df.loc[:, \"AdLow\"] = df.loc[:, \"AdLow\"].ffill()\n    \n    return df","metadata":{"papermill":{"duration":0.02829,"end_time":"2022-04-04T01:00:59.462393","exception":false,"start_time":"2022-04-04T01:00:59.434103","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-26T19:37:58.091186Z","iopub.execute_input":"2022-06-26T19:37:58.091936Z","iopub.status.idle":"2022-06-26T19:37:58.100108Z","shell.execute_reply.started":"2022-06-26T19:37:58.091896Z","shell.execute_reply":"2022-06-26T19:37:58.099262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_price(price):\n    \"\"\"\n    Args:\n        price (pd.DataFrame)  : pd.DataFrame include stock_price\n    Returns:\n        price DataFrame (pd.DataFrame): stock_price with generated AdjustedClose\n    \"\"\"\n    # copy to edit\n    price = price.copy()\n    \n    # transform Date column into datetime\n    price.loc[: ,\"Date\"] = pd.to_datetime(price.loc[: ,\"Date\"], format=\"%Y-%m-%d\")\n\n    # generate AdjustedClose\n    price = price.sort_values([\"SecuritiesCode\", \"Date\"])\n    price = price.groupby(\"SecuritiesCode\").apply(generate_adjusted_close).reset_index(drop=True)\n\n    price.set_index(\"Date\", inplace=True)\n    return price","metadata":{"papermill":{"duration":0.026,"end_time":"2022-04-04T01:00:59.504975","exception":false,"start_time":"2022-04-04T01:00:59.478975","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-26T19:38:06.21372Z","iopub.execute_input":"2022-06-26T19:38:06.214035Z","iopub.status.idle":"2022-06-26T19:38:06.221164Z","shell.execute_reply.started":"2022-06-26T19:38:06.214003Z","shell.execute_reply":"2022-06-26T19:38:06.220049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_change_rate_base(price, column_name, periods):\n    for period in periods:\n        price[f\"{column_name}_pct_{period}\"] = price[column_name].pct_change(period)\n    return price\n\ndef calc_volatility_base(price, column_name, periods):\n    for period in periods:\n        price[f\"{column_name}_vol_{period}\"] = np.log(price[column_name]).diff().rolling(window=period, min_periods=1).std()\n    return price\n\ndef calc_moving_average_rate_base(price, column_name, periods):\n    for period in periods:\n        price[f\"{column_name}_mov_{period}\"] = price[column_name].pct_change(period).rolling(window=period, min_periods=1).mean()\n    return price","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:38:10.196486Z","iopub.execute_input":"2022-06-26T19:38:10.197329Z","iopub.status.idle":"2022-06-26T19:38:10.204938Z","shell.execute_reply.started":"2022-06-26T19:38:10.197276Z","shell.execute_reply":"2022-06-26T19:38:10.204201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_for_predict(price, code):\n    \"\"\"\n    Args:\n        price (pd.DataFrame)  : pd.DataFrame include stock_price\n        code (int)  : A local code for a listed company\n    Returns:\n        feature DataFrame (pd.DataFrame)\n    \"\"\"\n    close_col = \"AdClose\"\n    open_col = \"AdOpen\"\n    high_col = \"AdHigh\"\n    low_col = \"AdLow\"\n    \n    feats = price.loc[price[\"SecuritiesCode\"] == code, \n                      [\"SecuritiesCode\", high_col, low_col, open_col, close_col,\"17SectorCode\",\"33SectorCode\"]].copy()\n\n    periods = [10,21,63]\n    feats = calc_change_rate_base(feats,close_col, periods)\n    feats = calc_volatility_base(feats,close_col, periods)\n    feats = calc_moving_average_rate_base(feats,close_col, periods)\n    \n    #feats = calc_change_rate_base(feats,open_col, periods)\n   \n    # calculate last 3 months historical volatility using AdjustedClose\n    feats[\"HLRolling\"] = ((feats[\"AdHigh\"] - feats[\"AdLow\"])/ feats[\"AdLow\"]).rolling(21).std()\n    \n    feats[\"CLRolling\"] = ((feats[\"AdClose\"]-feats[\"AdLow\"])/ feats[\"AdLow\"]).rolling(21).std()\n    \n    feats[\"HCRolling\"] = ((feats[\"AdHigh\"]-feats[\"AdClose\"])/ feats[\"AdClose\"]).rolling(21).std()\n  \n    feats[\"kurt\"] = feats[\"AdClose\"].kurtosis()\n    #feats = feats.dropna()\n    \n    feats.reset_index(inplace=True)\n    feats[\"month\"] = feats[\"Date\"].dt.month\n    feats[\"day\"] = feats[\"Date\"].dt.day\n    feats[\"dow\"] = feats[\"Date\"].dt.dayofweek\n    feats.set_index('Date',inplace=True, drop = True)\n\n    # filling data for nan and inf\n    feats = feats.fillna(0)\n    feats = feats.replace([np.inf, -np.inf], 0)\n    \n    # drop AdClose, High and Low columns\n    feats = feats.drop([close_col], axis = 1)\n    feats = feats.drop([open_col], axis=1)\n    feats = feats.drop([high_col], axis = 1)\n    feats = feats.drop([low_col], axis = 1)\n\n    return feats","metadata":{"papermill":{"duration":0.029881,"end_time":"2022-04-04T01:00:59.551544","exception":false,"start_time":"2022-04-04T01:00:59.521663","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-26T19:38:18.386292Z","iopub.execute_input":"2022-06-26T19:38:18.386948Z","iopub.status.idle":"2022-06-26T19:38:18.397625Z","shell.execute_reply.started":"2022-06-26T19:38:18.386913Z","shell.execute_reply":"2022-06-26T19:38:18.396602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load stock price data\ndf_price_raw = pd.read_csv(f\"{train_files_dir}/stock_prices.csv\")\n\nprice_cols = [\n    \"Date\",\n    \"SecuritiesCode\",\n    \"High\", \"Low\",\n    \"Open\",\"Close\",\n    \"AdjustmentFactor\",\n]\n\ndf_price_raw = df_price_raw[price_cols]","metadata":{"papermill":{"duration":6.919867,"end_time":"2022-04-04T01:01:06.488257","exception":false,"start_time":"2022-04-04T01:00:59.56839","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-27T05:23:19.576418Z","iopub.execute_input":"2022-06-27T05:23:19.576727Z","iopub.status.idle":"2022-06-27T05:23:26.482934Z","shell.execute_reply.started":"2022-06-27T05:23:19.576694Z","shell.execute_reply":"2022-06-27T05:23:26.482227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting sectors for each stock\nlista = pd.read_csv(f\"{base_dir}/stock_list.csv\")\nsectores = lista[[\"SecuritiesCode\",\"17SectorCode\",\"33SectorCode\"]]\n\nsector = sectores.loc[sectores['17SectorCode'] != '-']\nsector = sector.loc[sectores['33SectorCode'] != '-']\n\ndf = sector[sector.set_index('SecuritiesCode').index.isin(df_price_raw.set_index('SecuritiesCode').index)]\n\ndf_price_raw = df_price_raw.merge(df, how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:39:05.925138Z","iopub.execute_input":"2022-06-26T19:39:05.926251Z","iopub.status.idle":"2022-06-26T19:39:06.910426Z","shell.execute_reply.started":"2022-06-26T19:39:05.926193Z","shell.execute_reply":"2022-06-26T19:39:06.909511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del lista, sectores, sector","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_price_supplemental = pd.read_csv(f\"{supplemental_files_dir}/stock_prices.csv\")\n#df_price_supplemental = df_price_supplemental[price_cols]\n#df_price_raw = pd.concat([df_price_raw, df_price_supplemental])\n\ndf_price_raw = df_price_raw[df_price_raw[\"Date\"] >=\"2021-02-01\"]\n\nprint(df_price_raw.head(2))\nprint(df_price_raw.isna().all())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load pre-trained model\npred_model = Booster(model_file = model_file)","metadata":{"papermill":{"duration":0.050804,"end_time":"2022-04-04T01:01:06.60765","exception":false,"start_time":"2022-04-04T01:01:06.556846","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-10T05:16:38.01876Z","iopub.execute_input":"2022-06-10T05:16:38.019349Z","iopub.status.idle":"2022-06-10T05:16:38.03791Z","shell.execute_reply.started":"2022-06-10T05:16:38.019292Z","shell.execute_reply":"2022-06-10T05:16:38.037011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load Time Series API\nimport jpx_tokyo_market_prediction\n# make Time Series API environment (this function can be called only once in a session)\nenv = jpx_tokyo_market_prediction.make_env()\n# get iterator to fetch data day by day\niter_test = env.iter_test()","metadata":{"papermill":{"duration":0.046751,"end_time":"2022-04-04T01:01:06.67382","exception":false,"start_time":"2022-04-04T01:01:06.627069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-10T05:16:42.173501Z","iopub.execute_input":"2022-06-10T05:16:42.173831Z","iopub.status.idle":"2022-06-10T05:16:42.208957Z","shell.execute_reply.started":"2022-06-10T05:16:42.173798Z","shell.execute_reply":"2022-06-10T05:16:42.207628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature column names\n\n#feat_cols = [\n#    \"17SectorCode\",\"33SectorCode\",\n#    \"AdClose_pct_10\",\"AdClose_pct_21\",\"AdClose_pct_63\",\n#    \"AdClose_vol_10\",\"AdClose_vol_21\",\"AdClose_vol_63\",\n#    \"AdClose_mov_10\",\"AdClose_mov_21\",\"AdClose_mov_63\",\n#    \"AdOpen_pct_10\",\"AdOpen_pct_21\",\"AdOpen_pct_63\",\n#    \"HLRolling\",\"CLRolling\",\"HCRolling\",\n#    \"month\", \"day\", \"dow\"\n#]\n\nfeat_cols = [\n    \"17SectorCode\",\"33SectorCode\",\n    \"AdClose_pct_10\",\"AdClose_pct_21\",\"AdClose_pct_63\",\n    \"AdClose_vol_10\",\"AdClose_vol_21\",\"AdClose_vol_63\",\n    \"AdClose_mov_10\",\"AdClose_mov_21\",\"AdClose_mov_63\",\n    \"HLRolling\",\"CLRolling\",\"HCRolling\",\"kurt\",\n    \"month\", \"day\", \"dow\"\n]","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:39:50.067551Z","iopub.execute_input":"2022-06-26T19:39:50.068028Z","iopub.status.idle":"2022-06-26T19:39:50.072379Z","shell.execute_reply.started":"2022-06-26T19:39:50.067993Z","shell.execute_reply":"2022-06-26T19:39:50.071771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counter = 0\n# fetch data day by day\nfor (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    current_date = prices[\"Date\"].iloc[0]\n    sample_prediction_date = sample_prediction[\"Date\"].iloc[0]\n    print(f\"current_date: {current_date}, sample_prediction_date: {sample_prediction_date}\")\n\n    if counter == 0:\n        # to avoid data leakage\n        df_price_raw = df_price_raw.loc[df_price_raw[\"Date\"] < current_date]\n\n    # filter data to reduce culculation cost\n    threshold = (pd.Timestamp(current_date) - pd.offsets.BDay(80)).strftime(\"%Y-%m-%d\")\n    print(f\"threshold: {threshold}\")\n    \n    df_price_raw = df_price_raw.loc[df_price_raw[\"Date\"] >= threshold]\n    \n    prices = prices.merge(df, how=\"left\")\n\n    if counter == 0:\n        price_cols.append(\"17SectorCode\")\n        price_cols.append(\"33SectorCode\")\n    \n    # to generate AdjustedClose, increment price data\n    df_price_raw = pd.concat([df_price_raw, prices[price_cols]])\n    \n    # generate AdjustedClose\n    df_price = adjust_price(df_price_raw)\n\n    # get target SecuritiesCodes\n    codes = sorted(prices[\"SecuritiesCode\"].unique())\n    \n    # generate feature\n    feature = pd.concat([get_features_for_predict(df_price, code) for code in codes])\n    \n    if counter == 0:\n        print(\"10=\",feature.loc[feature.index == current_date].shape)\n    \n    feature['month'] = pd.Categorical(feature['month'])\n    feature['day'] = pd.Categorical(feature['day'])\n    feature['dow'] = pd.Categorical(feature['dow'])\n    \n    feature['17SectorCode'] = pd.Categorical(feature['17SectorCode'])\n    feature['33SectorCode'] = pd.Categorical(feature['33SectorCode'])\n    \n    # filter feature for this iteration\n    feature = feature.loc[feature.index == current_date]\n\n    # prediction\n    feature.loc[:, \"predict\"] = pred_model.predict(feature[feat_cols])\n            \n    # set rank by predict\n    feature = feature.sort_values(\"predict\", ascending=False).drop_duplicates(subset=['SecuritiesCode'])\n    \n    feature.loc[:,\"Rank\"] = np.arange(len(feature))\n    \n    feature_map = feature.set_index('SecuritiesCode')['Rank'].to_dict()\n    \n    sample_prediction['Rank'] = sample_prediction['SecuritiesCode'].map(feature_map)\n\n    # check Rank\n    assert sample_prediction[\"Rank\"].notna().all()\n    assert sample_prediction[\"Rank\"].min() == 0\n    assert sample_prediction[\"Rank\"].max() == len(sample_prediction[\"Rank\"]) - 1\n\n    # register your predictions\n    env.predict(sample_prediction)\n    counter += 1","metadata":{"papermill":{"duration":53.209828,"end_time":"2022-04-04T01:01:59.902468","exception":false,"start_time":"2022-04-04T01:01:06.69264","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head submission.csv","metadata":{"papermill":{"duration":0.027014,"end_time":"2022-04-04T01:01:59.949216","exception":false,"start_time":"2022-04-04T01:01:59.922202","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-02T04:21:55.480422Z","iopub.execute_input":"2022-06-02T04:21:55.481585Z","iopub.status.idle":"2022-06-02T04:21:56.25331Z","shell.execute_reply.started":"2022-06-02T04:21:55.481525Z","shell.execute_reply":"2022-06-02T04:21:56.252101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! tail submission.csv","metadata":{"papermill":{"duration":0.025284,"end_time":"2022-04-04T01:01:59.994875","exception":false,"start_time":"2022-04-04T01:01:59.969591","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-02T04:22:27.17607Z","iopub.execute_input":"2022-06-02T04:22:27.177456Z","iopub.status.idle":"2022-06-02T04:22:27.930617Z","shell.execute_reply.started":"2022-06-02T04:22:27.177395Z","shell.execute_reply":"2022-06-02T04:22:27.929357Z"},"trusted":true},"execution_count":null,"outputs":[]}]}