{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nfrom plotly.subplots import make_subplots\nfrom lightgbm import LGBMRegressor\nfrom lightgbm import plot_tree\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-05T15:03:16.822502Z","iopub.execute_input":"2022-07-05T15:03:16.822979Z","iopub.status.idle":"2022-07-05T15:03:19.736731Z","shell.execute_reply.started":"2022-07-05T15:03:16.822869Z","shell.execute_reply":"2022-07-05T15:03:19.735530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stock_prices_supplemental = pd.read_csv('../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv')\n# df_stock_prices = df_stock_prices_supplemental","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:03:21.830083Z","iopub.execute_input":"2022-07-05T15:03:21.830497Z","iopub.status.idle":"2022-07-05T15:03:22.653315Z","shell.execute_reply.started":"2022-07-05T15:03:21.830463Z","shell.execute_reply":"2022-07-05T15:03:22.651918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stock_prices = pd.read_csv('../input/jpx-tokyo-stock-exchange-prediction/train_files/stock_prices.csv')\ndf_stock_prices = df_stock_prices.groupby('SecuritiesCode').apply(lambda x:x.tail(100)).reset_index(level=0,drop=True)\ndf_stock_prices = pd.concat([df_stock_prices,df_stock_prices_supplemental]).sort_values(['SecuritiesCode','Date']).reset_index(drop=True)\ndf_stock_prices","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:16:04.478911Z","iopub.execute_input":"2022-07-05T15:16:04.479703Z","iopub.status.idle":"2022-07-05T15:16:11.978510Z","shell.execute_reply.started":"2022-07-05T15:16:04.479644Z","shell.execute_reply":"2022-07-05T15:16:11.976942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ma_1_window = 5\nma_2_window = 25\nrsi_window = 14\nstochastic_window = 14\nstochastic_slow_window = 3\npsl_window = 14\n# max_window = ma_2_window","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:03:45.631940Z","iopub.execute_input":"2022-07-05T15:03:45.632338Z","iopub.status.idle":"2022-07-05T15:03:45.637228Z","shell.execute_reply.started":"2022-07-05T15:03:45.632308Z","shell.execute_reply":"2022-07-05T15:03:45.636188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['adjusted_Close',\n            'adjusted_Volume_ratio',\n            'diff_Close_Open',\n            'diff_adjusted_Close_sma_1',\n            'diff_adjusted_Close_sma_2',\n            'rsi',\n            'sma_1_crosses_above_sma_2',\n            'sma_1_crosses_above_sma_2_shifted_1',\n            'sma_1_crosses_above_sma_2_shifted_2',\n            'sma_1_crosses_above_sma_2_shifted_3',\n            'sma_1_crosses_below_sma_2',\n            'sma_1_crosses_below_sma_2_shifted_1',\n            'sma_1_crosses_below_sma_2_shifted_2',\n            'sma_1_crosses_below_sma_2_shifted_3',\n            'first_n_days_of_month',\n            'ExpectedDividend',\n            'ExpectedDividend_shifted_1',\n            'ExpectedDividend_shifted_2',\n            'ExpectedDividend_shifted_3',\n            '%K',\n            '%D',\n            'volatility',\n#             'psl'\n# #             'DMA_sma_2_adjusted_Close_shifted_1',\n# #             'DMA_sma_2_adjusted_Close_shifted_2',\n# #             'DMA_sma_2_adjusted_Close_shifted_3'\n           ]\nfeatures_ = features + ['SecuritiesCode','Target']","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:03:49.618296Z","iopub.execute_input":"2022-07-05T15:03:49.618666Z","iopub.status.idle":"2022-07-05T15:03:49.625163Z","shell.execute_reply.started":"2022-07-05T15:03:49.618637Z","shell.execute_reply":"2022-07-05T15:03:49.623919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make Features\ndef make_features(df):\n    g = df.groupby('SecuritiesCode')\n#     df[['Open','High','Low','Close','Volume']] = g.apply(lambda x:x[['Open','High','Low','Close','Volume']].fillna(method='ffill'))\n    df['ExpectedDividend'] = df['ExpectedDividend'].fillna(0)\n    # Adjust Close values\n    df['cumprod_shifted_AdjustmentFactor'] = g['AdjustmentFactor'].shift(1).fillna(1)\n    df['cumprod_shifted_AdjustmentFactor'] = g['cumprod_shifted_AdjustmentFactor'].cumprod()\n    df['adjusted_Close'] = df['Close']/df['cumprod_shifted_AdjustmentFactor']\n    # Adjust Volume\n    df['adjusted_Volume'] = df['Volume']*df['cumprod_shifted_AdjustmentFactor']\n    # MA for Close\n    df['sma_1_adjusted_Close'] = g['adjusted_Close'].rolling(window=ma_1_window).mean().reset_index(level=0,drop=True)    \n\n    df['sma_2_adjusted_Close'] = g['adjusted_Close'].rolling(window=ma_2_window).mean().reset_index(level=0,drop=True)\n    \n    # MA for volume\n    df['sma_1_adjusted_Volume'] = g['adjusted_Volume'].rolling(window=ma_1_window).mean().reset_index(level=0,drop=True)\n            \n    # Calculate RSI\n    df['diff_consecutive'] = g['adjusted_Close'].transform(lambda x: x-x.shift(1))\n    df['gain'] = df['diff_consecutive']\n    df.loc[df['gain']<0,'gain']=0\n    df['loss'] = df['diff_consecutive']\n    df.loc[df['loss']>=0,'loss'] = 0\n    df['loss'] = -df['loss']\n    df['gain_sma'] = g['gain'].rolling(window=rsi_window, min_periods=rsi_window).mean().reset_index(level=0,drop=True)    \n    df['loss_sma'] = g['loss'].rolling(window=rsi_window, min_periods=rsi_window).mean().reset_index(level=0,drop=True)\n    df['rs'] = df['gain_sma']/df['loss_sma']\n    df['rsi'] = 100-(100/(1+df['rs']))\n    \n    # Calculate other metrics\n    df['adjusted_Volume_ratio'] = df['adjusted_Volume']/df['sma_1_adjusted_Volume']\n    df[\"diff_Close_Open\"] = (df[\"Close\"] - df[\"Open\"]) / df[[\"Close\",\"Open\"]].mean(axis=1)\n    df['diff_adjusted_Close_sma_1'] = (df['adjusted_Close'] - df['sma_1_adjusted_Close'])/df[['sma_1_adjusted_Close','adjusted_Close']].mean(axis=1)\n    df['diff_adjusted_Close_sma_2'] = (df['adjusted_Close'] - df['sma_2_adjusted_Close'])/df[['sma_2_adjusted_Close','adjusted_Close']].mean(axis=1)\n    \n    df['sma_1_adjusted_Close_shifted'] = g['sma_1_adjusted_Close'].transform(lambda x:x.shift(1))\n    \n    df['sma_1_crosses_above_sma_2']=False\n    df['sma_1_crosses_below_sma_2'] = False\n    df.loc[df[(df['sma_1_adjusted_Close_shifted']<df['sma_2_adjusted_Close'])&\n              (df['sma_1_adjusted_Close']>df['sma_2_adjusted_Close'])].index,'sma_1_crosses_above_sma_2'] = True\n    \n    df.loc[df[(df['sma_1_adjusted_Close_shifted']>df['sma_2_adjusted_Close'])&\n              (df['sma_1_adjusted_Close']<df['sma_2_adjusted_Close'])].index,'sma_1_crosses_below_sma_2'] = True\n    df['sma_1_crosses_above_sma_2_shifted_1'] = g['sma_1_crosses_above_sma_2'].transform(lambda x:x.shift(1).fillna(False))\n    df['sma_1_crosses_above_sma_2_shifted_2'] = g['sma_1_crosses_above_sma_2'].transform(lambda x:x.shift(2).fillna(False))\n    df['sma_1_crosses_above_sma_2_shifted_3'] = g['sma_1_crosses_above_sma_2'].transform(lambda x:x.shift(3).fillna(False))\n    \n    df['sma_1_crosses_below_sma_2_shifted_1'] = g['sma_1_crosses_below_sma_2'].transform(lambda x:x.shift(1).fillna(False))\n    df['sma_1_crosses_below_sma_2_shifted_2'] = g['sma_1_crosses_below_sma_2'].transform(lambda x:x.shift(2).fillna(False))\n    df['sma_1_crosses_below_sma_2_shifted_3'] = g['sma_1_crosses_below_sma_2'].transform(lambda x:x.shift(3).fillna(False)) \n    \n    df['first_n_days_of_month'] = False\n    df.loc[df['Date'].str[8:].astype(int) <=3 , \"first_n_days_of_month\"] = True\n    \n    df['ExpectedDividend_shifted_1'] = g['ExpectedDividend'].transform(lambda x:x.shift(1))\n    df['ExpectedDividend_shifted_2'] = g['ExpectedDividend'].transform(lambda x:x.shift(2))\n    df['ExpectedDividend_shifted_3'] = g['ExpectedDividend'].transform(lambda x:x.shift(3))\n    \n    # Calculate stochastic \n    df['rolling_min_adjusted_Close'] = g['adjusted_Close'].rolling(stochastic_window).min().reset_index(level=0,drop=True)\n    df['rolling_max_adjusted_Close'] = g['adjusted_Close'].rolling(stochastic_window).max().reset_index(level=0,drop=True)\n    df['%K'] = ((df['adjusted_Close'] - df['rolling_min_adjusted_Close'])/(df['rolling_max_adjusted_Close']-df['rolling_min_adjusted_Close']))*100\n    df['%D'] = g['%K'].rolling(stochastic_slow_window).mean().reset_index(level=0,drop=True)\n    \n    # Calculate Volatility\n    df['mstd_1_adjusted_Close'] = g['adjusted_Close'].rolling(ma_1_window).std().reset_index(level=0,drop=True) \n    df['volatility'] = (df['adjusted_Close'] - df['sma_1_adjusted_Close'])/df['mstd_1_adjusted_Close']\n    \n    # Calculate PSL\n#     df['psl'] = g['diff_consecutive'].rolling(psl_window).apply(lambda x:(x>=0).mean()).reset_index(level=0,drop=True)\n    # Calculate DMA\n#     df['DMA_sma_2_adjusted_Close_shifted_1'] = g['sma_2_adjusted_Close'].transform(lambda x:x.shift(1))\n#     df['DMA_sma_2_adjusted_Close_shifted_2'] = g['sma_2_adjusted_Close'].transform(lambda x:x.shift(2))\n#     df['DMA_sma_2_adjusted_Close_shifted_3'] = g['sma_2_adjusted_Close'].transform(lambda x:x.shift(3))\n    \n#     df['diff_adjusted_Close_DMA_sma_2_adjusted_Close_shifted_1'] = (df['adjusted_Close'] - df['DMA_sma_2_adjusted_Close_shifted_1'])/df['DMA_sma_2_adjusted_Close_shifted_1']\n#     df['diff_adjusted_Close_DMA_sma_2_adjusted_Close__shifted_2'] = (df['adjusted_Close'] - df['DMA_sma_2_adjusted_Close_shifted_2'])/df['DMA_sma_2_adjusted_Close_shifted_2']\n#     df['diff_adjusted_Close_DMA_sma_2_adjusted_Close_shifted_3'] = (df['adjusted_Close'] - df['DMA_sma_2_adjusted_Close_shifted_3'])/df['DMA_sma_2_adjusted_Close_shifted_3']\n    \n    return df\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:03:54.895036Z","iopub.execute_input":"2022-07-05T15:03:54.895434Z","iopub.status.idle":"2022-07-05T15:03:54.919387Z","shell.execute_reply.started":"2022-07-05T15:03:54.895399Z","shell.execute_reply":"2022-07-05T15:03:54.918340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stock_prices = make_features(df_stock_prices)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:16:16.204465Z","iopub.execute_input":"2022-07-05T15:16:16.204879Z","iopub.status.idle":"2022-07-05T15:16:25.253425Z","shell.execute_reply.started":"2022-07-05T15:16:16.204842Z","shell.execute_reply":"2022-07-05T15:16:25.252189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train LGBM model\n\n# def train_lgbm_model(X,y):\n    \n#     params = {'num_leaves': [31],\n#               'learning_rate': [0.0005],\n#               'max_depth': [12],\n#               'n_estimators': [1000]}\n#     grid = GridSearchCV(LGBMRegressor(random_state=0), params, scoring='r2', cv=2)\n#     grid.fit(X, y)\n#     model = grid.best_estimator_\n#     print(grid.best_params_)\n#     print(grid.best_score_)\n#     print(model.score(X, y))\n#     return model\n\ndef train_lgbm_model(X_train,y_train):\n    model=LGBMRegressor(num_leaves=31,max_depth=12,\n                        learning_rate=0.2, n_estimators=100)\n    model.fit(X_train,y_train)    \n    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:31:12.178836Z","iopub.execute_input":"2022-07-05T15:31:12.179234Z","iopub.status.idle":"2022-07-05T15:31:12.185277Z","shell.execute_reply.started":"2022-07-05T15:31:12.179200Z","shell.execute_reply":"2022-07-05T15:31:12.184105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\n# df_stock_prices = df_stock_prices.dropna(subset=['Target'])\n# df_train,df_test = train_test_split(df_stock_prices,test_size=0.3, random_state=42)\nfor code, d in df_stock_prices.groupby(\"SecuritiesCode\"):\n    d = d.dropna(subset=['Target'])\n    X = d[features]\n    y = d.Target\n    model = train_lgbm_model(X, y)\n    models[code] = model\n#     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n    # Evaluate model according to Sharpe Ratio\n#     df_test_current_stock = df_test[df_test['SecuritiesCode']==code]\n#     X_test = df_test_current_stock[features]\n#     y_test = df_test_current_stock['Target']\n#     print(model.predict(X_test))\n#     sys.exit()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T15:31:22.518614Z","iopub.execute_input":"2022-07-05T15:31:22.519001Z","iopub.status.idle":"2022-07-05T15:31:25.564679Z","shell.execute_reply.started":"2022-07-05T15:31:22.518952Z","shell.execute_reply":"2022-07-05T15:31:25.563134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission\nimport jpx_tokyo_market_prediction\nenv = jpx_tokyo_market_prediction.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test files","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:53:05.692908Z","iopub.status.idle":"2022-07-05T13:53:05.693308Z","shell.execute_reply.started":"2022-07-05T13:53:05.693124Z","shell.execute_reply":"2022-07-05T13:53:05.693142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stock_prices = df_stock_prices_supplemental","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:53:05.695648Z","iopub.status.idle":"2022-07-05T13:53:05.696107Z","shell.execute_reply.started":"2022-07-05T13:53:05.695903Z","shell.execute_reply":"2022-07-05T13:53:05.695923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    sec_codes_in_time_step = prices['SecuritiesCode'].unique()\n    df_stock_prices = pd.concat([df_stock_prices,prices]).reset_index(drop=True)\n    df_stock_prices = make_features(df_stock_prices)\n    prices_with_features = df_stock_prices[df_stock_prices['SecuritiesCode'].isin(sec_codes_in_time_step)].groupby('SecuritiesCode').apply(lambda x:x.iloc[-1]).reset_index(level=0,drop=True)\n    sample_prediction = sample_prediction.reset_index(drop=True)\n    d_merge = pd.merge(sample_prediction,prices_with_features, on=['SecuritiesCode','Date'],how='left').reset_index()\n    for code, d in d_merge.groupby(\"SecuritiesCode\"):\n        X = d[features]\n        d_merge.loc[d.index, \"predicted_Target\"] = models[code].predict(X)\n    d_merge['Rank'] = d_merge['predicted_Target'].rank(na_option='bottom',ascending=False,method='first')\n    d_merge['Rank'] = d_merge['Rank'].astype('int32') - 1\n    sample_prediction['Rank']=d_merge['Rank'].to_list()\n    env.predict(sample_prediction)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:53:05.697917Z","iopub.status.idle":"2022-07-05T13:53:05.69832Z","shell.execute_reply.started":"2022-07-05T13:53:05.698133Z","shell.execute_reply":"2022-07-05T13:53:05.698151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n#     sec_codes_in_time_step = prices['SecuritiesCode'].unique()\n#     df_stock_prices = pd.concat([df_stock_prices,prices]).reset_index(drop=True)\n#     df_stock_prices_temp = df_stock_prices[df_stock_prices['SecuritiesCode'].isin(sec_codes_in_time_step)]\n#     df_stock_prices_temp = df_stock_prices_temp.groupby('SecuritiesCode').apply(lambda x:x.iloc[-max_window:]).reset_index(level=0,drop=True)\n#     df_stock_prices_temp = make_features(df_stock_prices_temp)\n\n#     prices_with_features = df_stock_prices_temp.groupby('SecuritiesCode').apply(lambda x:x.iloc[-1]).reset_index(level=0,drop=True)\n#     sample_prediction = sample_prediction.reset_index(drop=True)\n#     d_merge = pd.merge(sample_prediction,prices_with_features, on=['SecuritiesCode','Date'],how='left').reset_index()\n#     for code, d in d_merge.groupby(\"SecuritiesCode\"):\n#         X = d[features]\n#         d_merge.loc[d.index, \"predicted_Target\"] = models[code].predict(X)\n#     d_merge['Rank'] = d_merge['predicted_Target'].rank(na_option='bottom',ascending=False,method='first')\n#     d_merge['Rank'] = d_merge['Rank'].astype('int32') - 1\n#     sample_prediction['Rank']=d_merge['Rank'].to_list()\n#     env.predict(sample_prediction)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:53:05.699996Z","iopub.status.idle":"2022-07-05T13:53:05.700582Z","shell.execute_reply.started":"2022-07-05T13:53:05.700374Z","shell.execute_reply":"2022-07-05T13:53:05.700395Z"},"trusted":true},"execution_count":null,"outputs":[]}]}