{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install neuralprophet --no-index --find-links=file:///kaggle/input/neuralprophet/neuralprophet","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:21.623624Z","iopub.execute_input":"2022-07-01T16:02:21.624122Z","iopub.status.idle":"2022-07-01T16:02:35.313512Z","shell.execute_reply.started":"2022-07-01T16:02:21.624084Z","shell.execute_reply":"2022-07-01T16:02:35.312492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Dependencies","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nimport pandas as pd\nwarnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)\nimport pandas as pd\nfrom pandas.core.common import SettingWithCopyWarning\nwarnings.simplefilter(action=\"ignore\", category=SettingWithCopyWarning)\nfrom neuralprophet import NeuralProphet\nfrom neuralprophet import NeuralProphet, set_log_level\nset_log_level(\"ERROR\")\nfrom matplotlib import pyplot as plt\nimport plotly.express as px\nimport time\nimport numpy as np\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:35.315474Z","iopub.execute_input":"2022-07-01T16:02:35.315822Z","iopub.status.idle":"2022-07-01T16:02:39.168157Z","shell.execute_reply.started":"2022-07-01T16:02:35.315791Z","shell.execute_reply":"2022-07-01T16:02:39.167041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read price df\ntrain=\"../input/jpx-tokyo-stock-exchange-prediction/train_files/stock_prices.csv\"\ntest=\"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv\"\ndf=pd.read_csv(train)\ndf1=pd.read_csv(test)\ndf.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:39.169899Z","iopub.execute_input":"2022-07-01T16:02:39.170468Z","iopub.status.idle":"2022-07-01T16:02:47.260912Z","shell.execute_reply.started":"2022-07-01T16:02:39.170398Z","shell.execute_reply":"2022-07-01T16:02:47.259916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:47.262967Z","iopub.execute_input":"2022-07-01T16:02:47.26334Z","iopub.status.idle":"2022-07-01T16:02:47.281355Z","shell.execute_reply.started":"2022-07-01T16:02:47.263309Z","shell.execute_reply":"2022-07-01T16:02:47.280442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess Data","metadata":{}},{"cell_type":"code","source":"df_clean = df.drop(columns=['RowId','SupervisionFlag','Target'], axis=1)\ndf_clean.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:47.283376Z","iopub.execute_input":"2022-07-01T16:02:47.284005Z","iopub.status.idle":"2022-07-01T16:02:47.400402Z","shell.execute_reply.started":"2022-07-01T16:02:47.283959Z","shell.execute_reply":"2022-07-01T16:02:47.39937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_clean = df1.drop(columns=['RowId','SupervisionFlag','Target'], axis=1)\ndf1_clean.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:47.401921Z","iopub.execute_input":"2022-07-01T16:02:47.402308Z","iopub.status.idle":"2022-07-01T16:02:47.430075Z","shell.execute_reply.started":"2022-07-01T16:02:47.402274Z","shell.execute_reply":"2022-07-01T16:02:47.428971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_close = df_clean.drop(columns=['Open','High','Low','Volume','AdjustmentFactor','ExpectedDividend'],axis=1)\ndf_close.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:47.43148Z","iopub.execute_input":"2022-07-01T16:02:47.431859Z","iopub.status.idle":"2022-07-01T16:02:47.476816Z","shell.execute_reply.started":"2022-07-01T16:02:47.431826Z","shell.execute_reply":"2022-07-01T16:02:47.475646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_close = df1_clean.drop(columns=['Open','High','Low','Volume','AdjustmentFactor','ExpectedDividend'],axis=1)\ndf1_close.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:47.478102Z","iopub.execute_input":"2022-07-01T16:02:47.478493Z","iopub.status.idle":"2022-07-01T16:02:47.495334Z","shell.execute_reply.started":"2022-07-01T16:02:47.478461Z","shell.execute_reply":"2022-07-01T16:02:47.494473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_grouped = df_clean.groupby(['SecuritiesCode','Date'])['Close'].sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:47.496967Z","iopub.execute_input":"2022-07-01T16:02:47.497359Z","iopub.status.idle":"2022-07-01T16:02:48.886884Z","shell.execute_reply.started":"2022-07-01T16:02:47.497305Z","shell.execute_reply":"2022-07-01T16:02:48.885892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_grouped = df1_clean.groupby(['SecuritiesCode','Date'])['Close'].sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:48.889774Z","iopub.execute_input":"2022-07-01T16:02:48.890133Z","iopub.status.idle":"2022-07-01T16:02:48.997073Z","shell.execute_reply.started":"2022-07-01T16:02:48.890102Z","shell.execute_reply":"2022-07-01T16:02:48.995822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_grouped = df_grouped.to_frame().reset_index()\ndf_grouped.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:48.998368Z","iopub.execute_input":"2022-07-01T16:02:48.998734Z","iopub.status.idle":"2022-07-01T16:02:49.087564Z","shell.execute_reply.started":"2022-07-01T16:02:48.998701Z","shell.execute_reply":"2022-07-01T16:02:49.086227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_grouped = df1_grouped.to_frame().reset_index()\ndf1_grouped.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:49.089062Z","iopub.execute_input":"2022-07-01T16:02:49.089411Z","iopub.status.idle":"2022-07-01T16:02:49.111232Z","shell.execute_reply.started":"2022-07-01T16:02:49.089379Z","shell.execute_reply":"2022-07-01T16:02:49.110225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemlist = df_clean.SecuritiesCode.unique()\nforecast_all = pd.DataFrame()  ","metadata":{"execution":{"iopub.status.busy":"2022-07-01T16:02:49.112527Z","iopub.execute_input":"2022-07-01T16:02:49.113409Z","iopub.status.idle":"2022-07-01T16:02:49.135798Z","shell.execute_reply.started":"2022-07-01T16:02:49.113364Z","shell.execute_reply":"2022-07-01T16:02:49.135008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and Initiate NeuralProphet","metadata":{}},{"cell_type":"code","source":"count=0\nfor i in itemlist:\n    temp = df_grouped[df_grouped.SecuritiesCode == i]\n    temp = temp.drop(columns=['SecuritiesCode'])\n    temp['Date'] = pd.to_datetime(temp['Date'])\n    temp = temp.set_index('Date')\n    d_df = temp.resample('D').sum()\n    d_df = d_df.reset_index().dropna()\n    d_df.columns = ['ds','y']\n    d_df = d_df[d_df['ds'].dt.dayofweek < 5]\n    \n    temp1 = df1_grouped[df1_grouped.SecuritiesCode == i]\n    temp1 = temp1.drop(columns=['SecuritiesCode'])\n    temp1['Date'] = pd.to_datetime(temp1['Date'])\n    temp1 = temp1.set_index('Date')\n    d_df1 = temp1.resample('D').sum()\n    d_df1 = d_df1.reset_index().dropna()\n    d_df1.columns = ['ds','y']\n    d_df1 = d_df1[d_df1['ds'].dt.dayofweek < 5]\n  \n    m = NeuralProphet(epochs=20,learning_rate=1.5)\n\n    \n    # create a test holdout set:\n    df_train_val, df_test = m.split_df(df=d_df1, freq=\"D\", valid_p=0.2)\n    # create a validation holdout set:\n    df_train, df_val = m.split_df(df=d_df, freq=\"D\", valid_p=0.2)\n    \n    # fit a model on training data and evaluate on validation set.\n    metrics_train1 = m.fit(df=df_train, freq=\"D\")\n    metrics_val = m.test(df=df_val)\n    \n    # refit model on training and validation data and evaluate on test set.\n    m = NeuralProphet(epochs=20,learning_rate=1.5)\n\n    metrics_train2 = m.fit(df=df_train_val, freq=\"D\")\n    metrics_test = m.test(df=df_test)\n    \n    m = NeuralProphet(\n            n_changepoints=30,\n            changepoints_range=0.95,\n            yearly_seasonality=True,\n            weekly_seasonality=True,\n            daily_seasonality=False,\n            epochs=50,\n            learning_rate=2.5,\n        )\n\n    metrics = m.fit(d_df,freq='D')\n    \n    future = m.make_future_dataframe(d_df, periods=87, n_historic_predictions=13)\n    forecast = m.predict(future)\n    forecast['SecuritiesCode'] = i\n    forecast_all = pd.concat((forecast_all, forecast))\n   \n    count +=1\n    print(f'Securities Code: {i}, Loop Iteration: {count}')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-01T16:02:49.137036Z","iopub.execute_input":"2022-07-01T16:02:49.13785Z","iopub.status.idle":"2022-07-01T17:05:20.883259Z","shell.execute_reply.started":"2022-07-01T16:02:49.137813Z","shell.execute_reply":"2022-07-01T17:05:20.882235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean Forecast Results","metadata":{}},{"cell_type":"code","source":"# read price df\ndf = forecast_all\ndf = df[df['ds'].dt.dayofweek < 5]\ndf.tail(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:05:20.884407Z","iopub.execute_input":"2022-07-01T17:05:20.884738Z","iopub.status.idle":"2022-07-01T17:05:20.936721Z","shell.execute_reply.started":"2022-07-01T17:05:20.884709Z","shell.execute_reply":"2022-07-01T17:05:20.935939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting close prices for all SecuritiesCodes\nclose_prices = df[['SecuritiesCode','ds','yhat1']]\nclose_prices['Close'] = close_prices['yhat1']\nclose_prices['Date'] = close_prices['ds']\nclose_prices = close_prices.drop(columns=['yhat1','ds'],axis=1)\nclose_prices.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:05:20.937944Z","iopub.execute_input":"2022-07-01T17:05:20.938223Z","iopub.status.idle":"2022-07-01T17:05:20.958543Z","shell.execute_reply.started":"2022-07-01T17:05:20.938195Z","shell.execute_reply":"2022-07-01T17:05:20.957771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trading_dates = np.array(sorted(close_prices['Date'].unique()))\nstock_ids = np.array(sorted(close_prices['SecuritiesCode'].unique()))\n\ntemp_mat = pd.DataFrame(np.nan, index=stock_ids, columns=trading_dates)\n\ndef create_factor(item, temp_mat=temp_mat):\n    output_mat = pd.pivot_table(close_prices,\n                                values=item,\n                                index='SecuritiesCode', columns='Date')\n    \n    output_factor = temp_mat.copy()\n    output_factor.loc[output_mat.index, output_mat.columns] = output_mat.values\n\n    return output_factor\n\nprices_target = create_factor('Close')\nprices_target.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:05:20.959892Z","iopub.execute_input":"2022-07-01T17:05:20.960208Z","iopub.status.idle":"2022-07-01T17:05:21.077094Z","shell.execute_reply.started":"2022-07-01T17:05:20.96018Z","shell.execute_reply":"2022-07-01T17:05:21.076011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stock_return = prices_target.pct_change(1,axis=1)\nstock_return.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:05:21.078523Z","iopub.execute_input":"2022-07-01T17:05:21.079445Z","iopub.status.idle":"2022-07-01T17:05:21.112294Z","shell.execute_reply.started":"2022-07-01T17:05:21.079393Z","shell.execute_reply":"2022-07-01T17:05:21.111461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supp_price_file = \"/kaggle/input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv\"\ndf_supp = pd.read_csv(supp_price_file)\n\n\ncols = pd.date_range(start='2021-12-01', end='2022-12-31').astype(str).tolist()\nidx = df_supp['SecuritiesCode'].unique().tolist()\nprint(len(cols), len(idx))\n\npreds = np.random.rand(len(idx),len(cols))\nprint(preds.shape)\n\nstock_return = pd.DataFrame(index=idx, data=preds, columns=cols)\nprint(stock_return.shape)\ndisplay(stock_return.head(2))\ndisplay(stock_return.tail(2))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:40:22.253653Z","iopub.execute_input":"2022-07-01T17:40:22.254214Z","iopub.status.idle":"2022-07-01T17:40:23.072978Z","shell.execute_reply.started":"2022-07-01T17:40:22.254167Z","shell.execute_reply":"2022-07-01T17:40:23.071943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competition Submission","metadata":{}},{"cell_type":"code","source":"counter=0\n\nimport jpx_tokyo_market_prediction\nenv = jpx_tokyo_market_prediction.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test files\n\nfor (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n\n\n    new_dates = prices['Date'].unique()\n    preds_df = stock_return[new_dates].unstack().reset_index()\n    preds_df.columns = ['Date', 'SecuritiesCode', 'pred']\n\n\n    sample_prediction = sample_prediction[['Date', 'SecuritiesCode']].merge(preds_df[['Date', 'SecuritiesCode', 'pred']], \n                                                                                    on=['Date', 'SecuritiesCode'], how='left')\n\n    sample_prediction['Rank'] = sample_prediction[['Date', 'pred']].groupby('Date').rank(ascending=True, method='first', na_option='bottom')\n\n\n    sample_prediction['Rank'] = np.int64(sample_prediction['Rank'].values-1)\n    env.predict(sample_prediction[['Date', 'SecuritiesCode', 'Rank']])\n    ########################\n\n    counter+=1","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:43:45.810023Z","iopub.execute_input":"2022-07-01T18:43:45.81055Z","iopub.status.idle":"2022-07-01T18:43:45.940717Z","shell.execute_reply.started":"2022-07-01T18:43:45.810496Z","shell.execute_reply":"2022-07-01T18:43:45.939504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:42:33.151069Z","iopub.execute_input":"2022-07-01T17:42:33.15211Z","iopub.status.idle":"2022-07-01T17:42:33.169403Z","shell.execute_reply.started":"2022-07-01T17:42:33.152072Z","shell.execute_reply":"2022-07-01T17:42:33.168185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-07-01T17:42:38.701979Z","iopub.execute_input":"2022-07-01T17:42:38.702494Z","iopub.status.idle":"2022-07-01T17:42:39.478972Z","shell.execute_reply.started":"2022-07-01T17:42:38.702453Z","shell.execute_reply":"2022-07-01T17:42:39.47756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}