{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook generates a simple model for stock's returns based on past returns and  volatilities. It was copied from https://www.kaggle.com/code/smeitoma/Train_Demo \n\nSome ideas taken from https://www.kaggle.com/code/kotaromiura/jpx-lgbm-demo","metadata":{"papermill":{"duration":0.045711,"end_time":"2022-04-02T13:51:39.052709","exception":false,"start_time":"2022-04-02T13:51:39.006998","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nfrom decimal import ROUND_HALF_UP, Decimal\n\nimport numpy as np\nimport pandas as pd\nfrom lightgbm import LGBMRegressor\nfrom tqdm import tqdm\nimport datetime as dt\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"papermill":{"duration":2.039395,"end_time":"2022-04-02T13:51:41.227973","exception":false,"start_time":"2022-04-02T13:51:39.188578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:53:35.566159Z","iopub.execute_input":"2022-07-04T19:53:35.566715Z","iopub.status.idle":"2022-07-04T19:53:38.746115Z","shell.execute_reply.started":"2022-07-04T19:53:35.566625Z","shell.execute_reply":"2022-07-04T19:53:38.745382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = \"../input/jpx-tokyo-stock-exchange-prediction\"\n\ntrain_files_dir = f\"{base_dir}/train_files\"\nsupplemental_files_dir = f\"{base_dir}/supplemental_files\"","metadata":{"papermill":{"duration":0.053109,"end_time":"2022-04-02T13:51:41.466618","exception":false,"start_time":"2022-04-02T13:51:41.413509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:53:41.647904Z","iopub.execute_input":"2022-07-04T19:53:41.648163Z","iopub.status.idle":"2022-07-04T19:53:41.652380Z","shell.execute_reply.started":"2022-07-04T19:53:41.648129Z","shell.execute_reply":"2022-07-04T19:53:41.651603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating AdjustedClose price\n\nCalculates AdjustedClose prices,to take into account stocks splits or reverse splits. This has to be done in reverse order. For example, if a company split the stock 5 to 1, the stock price has to be 0.2 of the original price, to keep company valuation the same. Then the past prices have to be multiplied by 0.2, make pears to pears, not orange juice to pears.","metadata":{"papermill":{"duration":0.046403,"end_time":"2022-04-02T13:51:41.558974","exception":false,"start_time":"2022-04-02T13:51:41.512571","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def adjust_price(price):\n    \"\"\"\n    Args:\n        price (pd.DataFrame)  : pd.DataFrame include stock_price\n    Returns:\n        price DataFrame (pd.DataFrame): stock_price with generated AdjustedClose\n    \"\"\"\n    # transform Date column into datetime\n    price.loc[: ,\"Date\"] = pd.to_datetime(price.loc[: ,\"Date\"], format=\"%Y-%m-%d\")\n\n    def generate_adjusted_close(df):\n        \"\"\"\n        Args:\n            df (pd.DataFrame)  : stock_price for a single SecuritiesCode\n        Returns:\n            df (pd.DataFrame): stock_price with AdjustedClose for a single SecuritiesCode\n        \"\"\"\n        # sort data to generate CumulativeAdjustmentFactor\n        df = df.sort_values(\"Date\", ascending = False)\n        \n        # generate CumulativeAdjustmentFactor\n        df.loc[:, \"CumulativeAdjustmentFactor\"] = df[\"AdjustmentFactor\"].cumprod()\n        \n        # generate AdjustedClose\n        df.loc[:, \"AdClose\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Close\"]\n        ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n        ))\n        \n        df.loc[:, \"AdOpen\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Open\"]\n        ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n        ))\n        \n        df.loc[:, \"AdHigh\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"High\"]\n        ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n        ))\n        \n        df.loc[:, \"AdLow\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Low\"]\n        ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding = ROUND_HALF_UP)\n        ))\n        \n        # reverse order\n        df = df.sort_values(\"Date\")\n        \n        # to fill AdjustedClose, replace 0 into np.nan\n        df.loc[df[\"AdClose\"] == 0, \"AdClose\"] = np.nan\n        df.loc[df[\"AdOpen\"] == 0, \"AdOpen\"] = np.nan\n        df.loc[df[\"AdHigh\"] == 0, \"AdHigh\"] = np.nan\n        df.loc[df[\"AdLow\"] == 0, \"AdLow\"] = np.nan\n        \n        # forward fill AdjustedClose\n        df.loc[:, \"AdClose\"] = df.loc[:, \"AdClose\"].ffill()\n        df.loc[:, \"AdOpen\"] = df.loc[:, \"AdOpen\"].ffill()\n        df.loc[:, \"AdHigh\"] = df.loc[:, \"AdHigh\"].ffill()\n        df.loc[:, \"AdLow\"] = df.loc[:, \"AdLow\"].ffill()\n        \n        return df\n\n    # generate AdjustedClose\n    price = price.sort_values([\"SecuritiesCode\", \"Date\"])\n    price = price.groupby(\"SecuritiesCode\").apply(generate_adjusted_close).reset_index(drop=True)\n\n    price.set_index(\"Date\", inplace=True)\n    return price","metadata":{"papermill":{"duration":0.060398,"end_time":"2022-04-02T13:51:41.755972","exception":false,"start_time":"2022-04-02T13:51:41.695574","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:53:45.156652Z","iopub.execute_input":"2022-07-04T19:53:45.157088Z","iopub.status.idle":"2022-07-04T19:53:45.173067Z","shell.execute_reply.started":"2022-07-04T19:53:45.157055Z","shell.execute_reply":"2022-07-04T19:53:45.172226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_price = pd.read_csv(f\"{train_files_dir}/stock_prices.csv\")\ndf_price_supplemental = pd.read_csv(f\"{supplemental_files_dir}/stock_prices.csv\")\n\ndf_price = pd.concat([df_price, df_price_supplemental])\n\n# generate AdjustedClose\ndf_price = adjust_price(df_price)","metadata":{"papermill":{"duration":27.990189,"end_time":"2022-04-02T13:52:09.791984","exception":false,"start_time":"2022-04-02T13:51:41.801795","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:53:50.308880Z","iopub.execute_input":"2022-07-04T19:53:50.309142Z","iopub.status.idle":"2022-07-04T19:54:42.414091Z","shell.execute_reply.started":"2022-07-04T19:53:50.309111Z","shell.execute_reply":"2022-07-04T19:54:42.413343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_price.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T17:48:07.021239Z","iopub.execute_input":"2022-07-04T17:48:07.021780Z","iopub.status.idle":"2022-07-04T17:48:07.045075Z","shell.execute_reply.started":"2022-07-04T17:48:07.021741Z","shell.execute_reply":"2022-07-04T17:48:07.044420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adding 17SectorCode and 33SectorCode to df_price\n\nlista = pd.read_csv(f\"{base_dir}/stock_list.csv\")\nsectores = lista[[\"SecuritiesCode\",\"17SectorCode\", \"33SectorCode\"]]\n\nsector = sectores.loc[sectores['17SectorCode'] != '-']\nsector = sector.loc[sectores['33SectorCode'] != '-']\n\ndf = sector.loc[sector.set_index('SecuritiesCode').index.isin(df_price.set_index('SecuritiesCode').index)]\n\ndf_price = df_price.reset_index().merge(df, how=\"left\").set_index('Date')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:54:46.953793Z","iopub.execute_input":"2022-07-04T19:54:46.954496Z","iopub.status.idle":"2022-07-04T19:54:49.607320Z","shell.execute_reply.started":"2022-07-04T19:54:46.954457Z","shell.execute_reply":"2022-07-04T19:54:49.606573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_price.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-03T03:40:16.982581Z","iopub.execute_input":"2022-07-03T03:40:16.982837Z","iopub.status.idle":"2022-07-03T03:40:17.004775Z","shell.execute_reply.started":"2022-07-03T03:40:16.982812Z","shell.execute_reply":"2022-07-03T03:40:17.004041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del lista, sectores, sector, df","metadata":{"execution":{"iopub.status.busy":"2022-07-04T03:51:53.789549Z","iopub.execute_input":"2022-07-04T03:51:53.789852Z","iopub.status.idle":"2022-07-04T03:51:53.794102Z","shell.execute_reply.started":"2022-07-04T03:51:53.78982Z","shell.execute_reply":"2022-07-04T03:51:53.793304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-processing for model building\n\nThis notebook presents a simple model using LightGBM.\n\nFirst, the features are generated using the price change and historical volatility described above. Changing the past periods to 1, 2 and 3 improves the model from 99th percentile to 67% percentile. ","metadata":{"papermill":{"duration":0.056286,"end_time":"2022-04-02T13:52:12.515867","exception":false,"start_time":"2022-04-02T13:52:12.459581","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def calc_change_rate_base(price, column_name, periods):\n    for period in periods:\n        price[f\"{column_name}_pct_{period}\"] = price[column_name].pct_change(period)\n    return price\n\ndef calc_volatility_base(price, column_name, periods):\n    for period in periods:\n        price[f\"{column_name}_vol_{period}\"] = np.log(price[column_name]).diff().rolling(window=period, min_periods=1).std()\n    return price\n\ndef calc_moving_average_rate_base(price, column_name, periods):\n    for period in periods:\n        price[f\"{column_name}_mov_{period}\"] = price[column_name].pct_change(period).rolling(window=period, min_periods=1).mean()\n    return price","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:54:54.047392Z","iopub.execute_input":"2022-07-04T19:54:54.048154Z","iopub.status.idle":"2022-07-04T19:54:54.055831Z","shell.execute_reply.started":"2022-07-04T19:54:54.048097Z","shell.execute_reply":"2022-07-04T19:54:54.055011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_for_predict(price, code):\n    \"\"\"\n    Args:\n        price (pd.DataFrame): pd.DataFrame include stock_price\n        code (int): A local code for a listed company\n    Returns:4\n        feature DataFrame (pd.DataFrame)\n    \"\"\"\n    close_col = \"AdClose\"\n    open_col = \"AdOpen\"\n    \n    feats = price.loc[price[\"SecuritiesCode\"] == code, \n                      [\"SecuritiesCode\",\"AdHigh\",\"AdLow\",open_col,close_col,'17SectorCode','33SectorCode']].copy()\n    \n    periods = [10,21,63]\n    feats = calc_change_rate_base(feats,close_col, periods)\n    feats = calc_volatility_base(feats,close_col, periods)\n    feats = calc_moving_average_rate_base(feats,close_col, periods)\n    \n    #feats = calc_change_rate_base(feats,open_col, periods)\n    \n    # Additional features\n    feats[\"HLRolling\"] = ((feats[\"AdHigh\"]-feats[\"AdLow\"])/ feats[\"AdLow\"]).rolling(21).std()\n    \n    feats[\"CLRolling\"] = ((feats[\"AdClose\"]-feats[\"AdLow\"])/ feats[\"AdLow\"]).rolling(21).std()\n    \n    feats[\"HCRolling\"] = ((feats[\"AdHigh\"]-feats[\"AdClose\"])/ feats[\"AdClose\"]).rolling(21).std()\n    \n    #feats[\"skw\"] = feats[\"AdClose\"].skew()\n    feats[\"kurt\"] = feats[\"AdClose\"].kurtosis()\n    \n    #feats = feats.dropna()\n    \n    feats.reset_index(inplace = True)\n    feats[\"month\"] = feats[\"Date\"].dt.month\n    feats[\"day\"] = feats[\"Date\"].dt.day\n    feats[\"dow\"] = feats[\"Date\"].dt.dayofweek\n    feats.set_index('Date',inplace = True)\n    \n    # filling data for nan and inf\n    #feats = feats.fillna(0)\n    feats = feats.replace([np.inf, -np.inf], 0)\n    \n    # drop AdjustedClose column\n    feats = feats.drop([close_col], axis=1)\n    feats = feats.drop([open_col], axis=1)\n    feats = feats.drop(\"AdHigh\", axis=1)\n    feats = feats.drop(\"AdLow\", axis=1)\n    \n    return feats","metadata":{"papermill":{"duration":0.069843,"end_time":"2022-04-02T13:52:12.754025","exception":false,"start_time":"2022-04-02T13:52:12.684182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:54:57.482703Z","iopub.execute_input":"2022-07-04T19:54:57.483574Z","iopub.status.idle":"2022-07-04T19:54:57.495373Z","shell.execute_reply.started":"2022-07-04T19:54:57.483522Z","shell.execute_reply":"2022-07-04T19:54:57.494474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fetch prediction target SecuritiesCodes\ncodes = sorted(df_price[\"SecuritiesCode\"].unique())\nlen(codes)","metadata":{"papermill":{"duration":0.079997,"end_time":"2022-04-02T13:52:12.891901","exception":false,"start_time":"2022-04-02T13:52:12.811904","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:55:01.078323Z","iopub.execute_input":"2022-07-04T19:55:01.078624Z","iopub.status.idle":"2022-07-04T19:55:01.102175Z","shell.execute_reply.started":"2022-07-04T19:55:01.078587Z","shell.execute_reply":"2022-07-04T19:55:01.101343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate feature\nbuff = []\nfor code in tqdm(codes):\n    feat = get_features_for_predict(df_price, code)\n    buff.append(feat)\nfeature = pd.concat(buff)","metadata":{"papermill":{"duration":26.036168,"end_time":"2022-04-02T13:52:38.984984","exception":false,"start_time":"2022-04-02T13:52:12.948816","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:55:03.457992Z","iopub.execute_input":"2022-07-04T19:55:03.458502Z","iopub.status.idle":"2022-07-04T19:55:54.943755Z","shell.execute_reply.started":"2022-07-04T19:55:03.458462Z","shell.execute_reply":"2022-07-04T19:55:54.942876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature.tail(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:36:44.924436Z","iopub.execute_input":"2022-07-04T19:36:44.925225Z","iopub.status.idle":"2022-07-04T19:36:44.942851Z","shell.execute_reply.started":"2022-07-04T19:36:44.925188Z","shell.execute_reply":"2022-07-04T19:36:44.942080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature['17SectorCode'] = pd.Categorical(feature['17SectorCode'])\nfeature['33SectorCode'] = pd.Categorical(feature['33SectorCode'])\nfeature['month'] = pd.Categorical(feature['month'])\nfeature['day'] = pd.Categorical(feature['day'])\nfeature['dow'] = pd.Categorical(feature['dow'])","metadata":{"papermill":{"duration":0.162156,"end_time":"2022-04-02T13:52:39.295762","exception":false,"start_time":"2022-04-02T13:52:39.133606","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:56:01.302573Z","iopub.execute_input":"2022-07-04T19:56:01.303294Z","iopub.status.idle":"2022-07-04T19:56:01.797570Z","shell.execute_reply.started":"2022-07-04T19:56:01.303258Z","shell.execute_reply":"2022-07-04T19:56:01.796753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label creation\n\nNext, we obtain the labels to be used for training the model (this is where we load and split the label data).","metadata":{"papermill":{"duration":0.145502,"end_time":"2022-04-02T13:52:39.586544","exception":false,"start_time":"2022-04-02T13:52:39.441042","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_label(price, code):\n    \"\"\" Labelizer\n    Args:\n        price (pd.DataFrame): dataframe of stock_price.csv\n        code (int): Local Code in the universe\n    Returns:\n        df (pd.DataFrame): label data\n    \"\"\"\n    df = price.loc[price[\"SecuritiesCode\"] == code].copy()\n    \n    df.loc[:, \"label\"] = df[\"Target\"]\n\n    return df.loc[:, [\"SecuritiesCode\", \"label\"]] ","metadata":{"papermill":{"duration":0.154664,"end_time":"2022-04-02T13:52:40.181476","exception":false,"start_time":"2022-04-02T13:52:40.026812","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:56:04.741491Z","iopub.execute_input":"2022-07-04T19:56:04.741747Z","iopub.status.idle":"2022-07-04T19:56:04.746459Z","shell.execute_reply.started":"2022-07-04T19:56:04.741720Z","shell.execute_reply":"2022-07-04T19:56:04.745774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split data into TRAIN and TEST\nTRAIN_END = \"2021-11-25\"\n# We put a week gap between TRAIN_END and TEST_START\n# to avoid leakage of test data information from label\nTEST_START = \"2022-03-01\"\n\ndef get_features_and_label(price, codes, features):\n    \"\"\"\n    Args:\n        price (pd.DataFrame): loaded price data\n        codes  (array) : target codes\n        feature (pd.DataFrame): features\n    Returns:\n        train_X (pd.DataFrame): training data\n        train_y (pd.DataFrame): label for train_X\n        test_X (pd.DataFrame): test data\n        test_y (pd.DataFrame): label for test_X\n    \"\"\"\n    # to store splited data\n    trains_X, tests_X = [], []\n    trains_y, tests_y = [], []\n\n    # generate feature one by one\n    for code in tqdm(codes):\n\n        feats = features[features[\"SecuritiesCode\"] == code].dropna()\n        labels = get_label(price, code).dropna()\n\n        if feats.shape[0] > 0 and labels.shape[0] > 0:\n            # align label and feature indexes\n            labels = labels.loc[labels.index.isin(feats.index)]\n            feats = feats.loc[feats.index.isin(labels.index)]\n\n            assert (labels.loc[:, \"SecuritiesCode\"] == feats.loc[:, \"SecuritiesCode\"]).all()\n            labels = labels.loc[:, \"label\"]\n\n            # split data into TRAIN and TEST\n            _train_X = feats[: TRAIN_END]\n            _test_X = feats[TEST_START:]\n\n            _train_y = labels[: TRAIN_END]\n            _test_y = labels[TEST_START:]\n            \n            assert len(_train_X) == len(_train_y)\n            assert len(_test_X) == len(_test_y)\n\n            # store features\n            trains_X.append(_train_X)\n            tests_X.append(_test_X)\n            # store labels\n            trains_y.append(_train_y)\n            tests_y.append(_test_y)\n            \n    # combine features for each codes\n    train_X = pd.concat(trains_X)\n    test_X = pd.concat(tests_X)\n    # combine label for each codes\n    train_y = pd.concat(trains_y)\n    test_y = pd.concat(tests_y)\n\n    return train_X, train_y, test_X, test_y","metadata":{"papermill":{"duration":0.15998,"end_time":"2022-04-02T13:52:40.488053","exception":false,"start_time":"2022-04-02T13:52:40.328073","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:56:08.215230Z","iopub.execute_input":"2022-07-04T19:56:08.215897Z","iopub.status.idle":"2022-07-04T19:56:08.226265Z","shell.execute_reply.started":"2022-07-04T19:56:08.215860Z","shell.execute_reply":"2022-07-04T19:56:08.225524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate feature/label\ntrain_X, train_y, test_X, test_y = get_features_and_label(\n    df_price, codes, feature\n)\n\ntrain_X.head(2)","metadata":{"papermill":{"duration":37.867923,"end_time":"2022-04-02T13:53:18.500899","exception":false,"start_time":"2022-04-02T13:52:40.632976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T19:56:12.575239Z","iopub.execute_input":"2022-07-04T19:56:12.576127Z","iopub.status.idle":"2022-07-04T19:56:47.563690Z","shell.execute_reply.started":"2022-07-04T19:56:12.576078Z","shell.execute_reply":"2022-07-04T19:56:47.563029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_cols = [\n    \"17SectorCode\",\"33SectorCode\",\n    \"AdClose_pct_10\",\"AdClose_pct_21\",\"AdClose_pct_63\",\n    \"AdClose_vol_10\",\"AdClose_vol_21\",\"AdClose_vol_63\",\n    \"AdClose_mov_10\",\"AdClose_mov_21\",\"AdClose_mov_63\",\n    \"HLRolling\",\"CLRolling\",\"HCRolling\", \"kurt\",\n    \"month\", \"day\", \"dow\"\n]","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:56:54.840654Z","iopub.execute_input":"2022-07-04T19:56:54.840916Z","iopub.status.idle":"2022-07-04T19:56:54.845436Z","shell.execute_reply.started":"2022-07-04T19:56:54.840888Z","shell.execute_reply":"2022-07-04T19:56:54.844675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Taken from https://www.kaggle.com/code/bjoernholzhauer/lightgbm-tuning-with-optuna/notebook\n\n#import optuna\n#import optuna.integration.lightgbm as lgb\n#from sklearn.model_selection import RepeatedKFold\n\n#import warnings\n#warnings.simplefilter(action='ignore', category=FutureWarning)\n#warnings.simplefilter(action='ignore', category=UserWarning)\n\n#rkf = RepeatedKFold(n_splits=2, n_repeats=3, random_state=42)\n\n#params = {\n#        \"objective\": \"regression\",\n#        \"metric\": \"rmse\",\n#        \"verbose\" : 1,\n#        \"boosting_type\": \"gbdt\",\n#        \"random_state\" : 4222,\n#        \"force_row_wise\" : 'true'\n#    }    \n\n#X = np.array(train_X[feat_cols])\n#y = np.array(train_y).flatten()\n\n#dtrain = lgb.Dataset(X, label = y, categorical_feature = 'auto')    \n\n#tuner = lgb.LightGBMTunerCV(\n#    params, dtrain, \n#    verbose_eval = None,\n#    time_budget = 6000,\n#    folds = rkf,\n#    num_boost_round = 10,\n#    shuffle = False\n#)\n\n#tuner.run()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:57:15.673997Z","iopub.execute_input":"2022-07-04T19:57:15.674289Z","iopub.status.idle":"2022-07-04T20:27:38.327830Z","shell.execute_reply.started":"2022-07-04T19:57:15.674256Z","shell.execute_reply":"2022-07-04T20:27:38.327103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tuner.best_params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building a simple model\n\nUsing the created features and labels, build a model using the following procedure","metadata":{"papermill":{"duration":0.273296,"end_time":"2022-04-02T13:53:19.062225","exception":false,"start_time":"2022-04-02T13:53:18.788929","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Best parameters before kurtosis\nbest_params = {\n    'objective': 'regression',\n    'metric': 'rmse', \n    \"verbosity\": -1,\n    \"boosting_type\": \"gbdt\",\n    \"lambda_l1\": 0.000562875351775625,\n    'lambda_l2': 1.1029402737711809e-06,\n    'num_leaves': 255,\n    'feature_fraction': 1.0,\n    'bagging_fraction': 1.0,\n    'bagging_freq': 0,\n    'min_child_samples': 20\n}\n\n# Best parameters after kurtosis. Only difference: L2\nbest_params = {\n    'objective': 'regression',\n    'metric': 'rmse', \n    \"verbosity\": -1,\n    \"boosting_type\": \"gbdt\",\n    'force_row_wise': 'true',\n    'feature_pre_filter': False,\n    'lambda_l1': 0.000618003956709747,\n    'lambda_l2': 5.8327638813779385e-06,\n    'num_leaves': 255,\n    'feature_fraction': 1.0,\n    'bagging_fraction': 1.0,\n    'bagging_freq': 0,\n    'min_child_samples': 20\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-04T20:41:35.063979Z","iopub.execute_input":"2022-07-04T20:41:35.064254Z","iopub.status.idle":"2022-07-04T20:41:35.071778Z","shell.execute_reply.started":"2022-07-04T20:41:35.064223Z","shell.execute_reply":"2022-07-04T20:41:35.070092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialize model\npred_model = LGBMRegressor(**best_params)\n\n# train\npred_model.fit(train_X[feat_cols].values, train_y)\n\n# prepare result data\nresult = test_X[[\"SecuritiesCode\"]].copy()\n\n# predict\nresult.loc[:, \"predict\"] = pred_model.predict(test_X[feat_cols])\n\n# actual result\nresult.loc[:, \"Target\"] = test_y.values\n\ndef set_rank(df):\n    \"\"\"\n    Args:\n        df (pd.DataFrame): including predict column\n    Returns:\n        df (pd.DataFrame): df with Rank\n    \"\"\"\n    # sort records to set Rank\n    df = df.sort_values(\"predict\", ascending=False)\n    # set Rank starting from 0\n    df.loc[:, \"Rank\"] = np.arange(len(df[\"predict\"]))\n    return df\n\nresult = result.sort_values([\"Date\", \"predict\"], ascending=[True, False])\nresult = result.groupby(\"Date\").apply(set_rank)","metadata":{"papermill":{"duration":7.86376,"end_time":"2022-04-02T13:53:28.327193","exception":false,"start_time":"2022-04-02T13:53:20.463433","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T20:41:38.915465Z","iopub.execute_input":"2022-07-04T20:41:38.915717Z","iopub.status.idle":"2022-07-04T20:42:05.121431Z","shell.execute_reply.started":"2022-07-04T20:41:38.915688Z","shell.execute_reply":"2022-07-04T20:42:05.120522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T21:17:32.845735Z","iopub.execute_input":"2022-07-01T21:17:32.846012Z","iopub.status.idle":"2022-07-01T21:17:32.858915Z","shell.execute_reply.started":"2022-07-01T21:17:32.845976Z","shell.execute_reply":"2022-07-01T21:17:32.858249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_imp = pd.DataFrame([train_X[feat_cols].columns, pred_model.feature_importances_]).T\nfeature_imp.columns = ['feature', 'imp']\nfeature_imp.sort_values(by='imp', ascending=False, inplace=True)\nfeature_imp = feature_imp.iloc[0:20]\nfeature_imp['imp'] = pd.to_numeric(feature_imp['imp'])\nsns.barplot(x = 'imp', y = 'feature', data = feature_imp)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T20:40:54.782903Z","iopub.execute_input":"2022-07-04T20:40:54.783581Z","iopub.status.idle":"2022-07-04T20:40:55.161202Z","shell.execute_reply.started":"2022-07-04T20:40:54.783540Z","shell.execute_reply":"2022-07-04T20:40:55.160512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation\n\nInput the output of the forecasts of the constructed model into the evaluation function and plot the daily returns.\n\nThe evaluation function for this competition is as follows.\n\nPlease read [here](https://www.kaggle.com/code/smeitoma/jpx-competition-metric-definition) to know the evaluation function more.","metadata":{"papermill":{"duration":0.269682,"end_time":"2022-04-02T13:53:29.426688","exception":false,"start_time":"2022-04-02T13:53:29.157006","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def calc_spread_return_sharpe(df: pd.DataFrame, portfolio_size: int = 200, toprank_weight_ratio: float = 2) -> float:\n    \"\"\"\n    Args:\n        df (pd.DataFrame): predicted results\n        portfolio_size (int): # of equities to buy/sell\n        toprank_weight_ratio (float): the relative weight of the most highly ranked stock compared to the least.\n    Returns:\n        (float): sharpe ratio\n    \"\"\"\n    def _calc_spread_return_per_day(df, portfolio_size, toprank_weight_ratio):\n        \"\"\"\n        Args:\n            df (pd.DataFrame): predicted results\n            portfolio_size (int): # of equities to buy/sell\n            toprank_weight_ratio (float): the relative weight of the most highly ranked stock compared to the least.\n        Returns:\n            (float): spread return\n        \"\"\"\n        assert df['Rank'].min() == 0\n        assert df['Rank'].max() == len(df['Rank']) - 1\n        \n        weights = np.linspace(start = toprank_weight_ratio, stop = 1, num=portfolio_size)\n        \n        purchase = (df.sort_values(by='Rank')['Target'][:portfolio_size] * weights).sum() / weights.mean()\n        \n        short = (df.sort_values(by='Rank', ascending=False)['Target'][:portfolio_size] * weights).sum() / weights.mean()\n        \n        return purchase - short\n\n    buf = df.groupby('Date').apply(_calc_spread_return_per_day, portfolio_size, toprank_weight_ratio)\n    sharpe_ratio = buf.mean() / buf.std()\n    \n    return sharpe_ratio","metadata":{"papermill":{"duration":0.27819,"end_time":"2022-04-02T13:53:30.512824","exception":false,"start_time":"2022-04-02T13:53:30.234634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T20:29:45.521665Z","iopub.execute_input":"2022-07-04T20:29:45.521921Z","iopub.status.idle":"2022-07-04T20:29:45.533807Z","shell.execute_reply.started":"2022-07-04T20:29:45.521892Z","shell.execute_reply":"2022-07-04T20:29:45.532820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calc spread return sharpe\ncalc_spread_return_sharpe(result, portfolio_size = 200)","metadata":{"papermill":{"duration":1.191834,"end_time":"2022-04-02T13:53:31.971406","exception":false,"start_time":"2022-04-02T13:53:30.779572","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T20:42:33.388774Z","iopub.execute_input":"2022-07-04T20:42:33.389023Z","iopub.status.idle":"2022-07-04T20:42:33.527831Z","shell.execute_reply.started":"2022-07-04T20:42:33.388996Z","shell.execute_reply":"2022-07-04T20:42:33.526909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then, we will show daily spread return of the model.","metadata":{"papermill":{"duration":0.267867,"end_time":"2022-04-02T13:53:32.5075","exception":false,"start_time":"2022-04-02T13:53:32.239633","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _calc_spread_return_per_day(df, portfolio_size, toprank_weight_ratio):\n    \"\"\"\n    Args:\n        df (pd.DataFrame): predicted results\n        portfolio_size (int): # of equities to buy/sell\n        toprank_weight_ratio (float): the relative weight of the most highly ranked stock compared to the least.\n    Returns:\n        (float): spread return\n    \"\"\"\n    assert df['Rank'].min() == 0\n    assert df['Rank'].max() == len(df['Rank']) - 1\n    \n    weights = np.linspace(start=toprank_weight_ratio, stop=1, num=portfolio_size)\n    purchase = (df.sort_values(by='Rank')['Target'][:portfolio_size] * weights).sum() / weights.mean()\n    short = (df.sort_values(by='Rank', ascending=False)['Target'][:portfolio_size] * weights).sum() / weights.mean()\n    \n    return purchase - short\n\ndf_result = result.groupby('Date').apply(_calc_spread_return_per_day, 200, 2)","metadata":{"papermill":{"duration":1.236094,"end_time":"2022-04-02T13:53:34.011016","exception":false,"start_time":"2022-04-02T13:53:32.774922","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T17:53:11.814455Z","iopub.execute_input":"2022-07-04T17:53:11.814713Z","iopub.status.idle":"2022-07-04T17:53:11.954393Z","shell.execute_reply.started":"2022-07-04T17:53:11.814683Z","shell.execute_reply":"2022-07-04T17:53:11.953681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result.plot(figsize=(16, 8))","metadata":{"papermill":{"duration":0.59649,"end_time":"2022-04-02T13:53:34.882518","exception":false,"start_time":"2022-04-02T13:53:34.286028","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-04T17:53:15.083683Z","iopub.execute_input":"2022-07-04T17:53:15.083947Z","iopub.status.idle":"2022-07-04T17:53:15.358160Z","shell.execute_reply.started":"2022-07-04T17:53:15.083917Z","shell.execute_reply":"2022-07-04T17:53:15.357434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving model","metadata":{"papermill":{"duration":0.272106,"end_time":"2022-04-02T13:53:37.417349","exception":false,"start_time":"2022-04-02T13:53:37.145243","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"You need to save your model parameter to use created model for your submission.","metadata":{"papermill":{"duration":0.275818,"end_time":"2022-04-02T13:53:37.970603","exception":false,"start_time":"2022-04-02T13:53:37.694785","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pred_model.booster_.save_model(\"./simple-model.txt\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T18:33:53.162873Z","iopub.execute_input":"2022-06-10T18:33:53.163214Z","iopub.status.idle":"2022-06-10T18:33:53.176789Z","shell.execute_reply.started":"2022-06-10T18:33:53.16318Z","shell.execute_reply":"2022-06-10T18:33:53.176099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have created our simple model and evaluate the output of the model.\n\nAs a next action, let's submit the model into the leaderboard,\n\n[The Submission Demo notebook](https://www.kaggle.com/code/smeitoma/submission-demo) will explain how to submit with your model.","metadata":{}}]}