{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9807474,"sourceType":"datasetVersion","datasetId":6011754},{"sourceId":9836967,"sourceType":"datasetVersion","datasetId":6034154},{"sourceId":9889976,"sourceType":"datasetVersion","datasetId":6073762}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\nfrom statsmodels.tsa.seasonal import seasonal_decompose\nimport statsmodels.tsa.api as smt\nimport statsmodels.api as sm\nfrom scipy.stats import *\nimport seaborn as sns\nimport datetime\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-13T03:05:03.872538Z","iopub.execute_input":"2024-11-13T03:05:03.872940Z","iopub.status.idle":"2024-11-13T03:05:07.570234Z","shell.execute_reply.started":"2024-11-13T03:05:03.872899Z","shell.execute_reply":"2024-11-13T03:05:07.568978Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data exploration","metadata":{}},{"cell_type":"code","source":"\n# read Parquet\n# df = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\n\n# show data\n# print(df.head(10))\n# print(df.describe())\n# print(sorted(df['symbol_id'].unique()))\n# print(df[df['symbol_id']==0].head(10))","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:20.559056Z","iopub.execute_input":"2024-11-07T04:38:20.559556Z","iopub.status.idle":"2024-11-07T04:38:23.679438Z","shell.execute_reply.started":"2024-11-07T04:38:20.559512Z","shell.execute_reply":"2024-11-07T04:38:23.678214Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Returns**\nThe features are asset's returns","metadata":{}},{"cell_type":"code","source":"# plot\n# for i in range(5,9):\n#     plt.plot(df[df['symbol_id']==0]['feature_' + str(i).zfill(2)])\n#     plt.title('Symbol_0 Feature_{0}'.format(str(i).zfill(2)))\n#     plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:23.681408Z","iopub.execute_input":"2024-11-07T04:38:23.681959Z","iopub.status.idle":"2024-11-07T04:38:25.552738Z","shell.execute_reply.started":"2024-11-07T04:38:23.681899Z","shell.execute_reply":"2024-11-07T04:38:25.551370Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Price\nThe cumulative sum shows the price behavior","metadata":{}},{"cell_type":"code","source":"# # plot\n# for i in range(1,9):\n#     plt.plot(df[df['symbol_id']==0]['feature_' + str(i).zfill(2)].cumsum(),)\n# plt.show()    ","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:25.556414Z","iopub.execute_input":"2024-11-07T04:38:25.557064Z","iopub.status.idle":"2024-11-07T04:38:27.869119Z","shell.execute_reply.started":"2024-11-07T04:38:25.556971Z","shell.execute_reply":"2024-11-07T04:38:27.867876Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Symbols\n\nThe symbols look like different portfolios that have the same asset. Feature_05 shows the same behavior in all portfolios, although not exactly the same. This may be because each portfolio is trading at high frequncy and with different amounts to trade.","metadata":{}},{"cell_type":"code","source":"# symbols = sorted(df['symbol_id'].unique())\n# for s in symbols:\n#     plt.plot(df[df['symbol_id']==s]['feature_05'].cumsum(),)\n# plt.show()\n\n# plot\n# for i in range(5,6):\n#     for j in range(0,10):\n#         plt.plot(df[df['symbol_id']==j]['feature_' + str(i).zfill(2)].cumsum(),)\n# plt.show() ","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:27.870771Z","iopub.execute_input":"2024-11-07T04:38:27.871288Z","iopub.status.idle":"2024-11-07T04:38:29.618801Z","shell.execute_reply.started":"2024-11-07T04:38:27.871233Z","shell.execute_reply":"2024-11-07T04:38:29.617381Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Responders\nResponders seem to be associated with the symbol, associated with the portfolio.","metadata":{}},{"cell_type":"code","source":"# plot\n\n# for j in range(0,1):\n#     plt.plot(df[df['symbol_id']==j]['responder_6'].cumsum(),)\n# plt.show() \n\n# plot\n# for i in range(9):\n#     for j in range(0,5):\n#         plt.plot(df[df['symbol_id']==j]['responder_' + str(i).zfill(1)].cumsum(),)\n# plt.show() ","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:29.620524Z","iopub.execute_input":"2024-11-07T04:38:29.621008Z","iopub.status.idle":"2024-11-07T04:38:30.005639Z","shell.execute_reply.started":"2024-11-07T04:38:29.620958Z","shell.execute_reply":"2024-11-07T04:38:30.004275Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Diversification\nEach portfolio appears to be well diversified.","metadata":{}},{"cell_type":"code","source":"# symbols = sorted(df['symbol_id'].unique())\n\n# for s in symbols[:1]:\n#     cor = df[df['symbol_id']==s].corr()\n#     sns.heatmap(cor, annot=False, cmap='coolwarm', center=0)\n#     plt.title('Correlation Symbol_'+str(s))\n#     plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:30.007634Z","iopub.execute_input":"2024-11-07T04:38:30.008176Z","iopub.status.idle":"2024-11-07T04:38:33.652133Z","shell.execute_reply.started":"2024-11-07T04:38:30.008116Z","shell.execute_reply":"2024-11-07T04:38:33.650916Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Weights\nWeights are changing over time. According to interpretation of the scoring function $w_i$ gives greater inportance to external economic shocks.","metadata":{}},{"cell_type":"code","source":"# symbols = sorted(df['symbol_id'].unique())\n\n# for s in symbols[:1]:\n#     fig, ax1 = plt.subplots()\n#     ax1.plot(df[df['symbol_id']==s]['date_id'], df[df['symbol_id']==s]['responder_6'].cumsum()/100, 'g-')\n#     ax1.set_xlabel('Date')\n#     ax1.set_ylabel('responder_6', color='g')\n#     ax2 = ax1.twinx()\n#     ax2.plot(df[df['symbol_id']==s]['date_id'], df[df['symbol_id']==s]['weight'], 'b-')\n#     ax2.set_ylabel('weight', color='b')\n#     plt.title('Responder_6 and Weight over Symbol_' + str(s))\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:33.653889Z","iopub.execute_input":"2024-11-07T04:38:33.654383Z","iopub.status.idle":"2024-11-07T04:38:34.493406Z","shell.execute_reply.started":"2024-11-07T04:38:33.654331Z","shell.execute_reply":"2024-11-07T04:38:34.491787Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ACF\nACF reveals patterns of autocorrelation.","metadata":{}},{"cell_type":"code","source":"#for f in range(79):\n#    ret  = df[df['symbol_id']==0]['feature_' + str(f).zfill(2)].dropna()\n#    if len(ret)>100:\n#        smt.graphics.plot_acf(ret, lags=50, alpha=0.05)\n#        plt.title('ACF symbol_0 ' + 'feature_' + str(f).zfill(2))\n#        plt.show()\n\n# r = df[(df['symbol_id']==1) & (df['date_id']==0)]['responder_6']\n# smt.graphics.plot_acf(r, lags=50, alpha=0.05)\n# plt.title('ACF symbol_0 date_0 responder_6')\n# plt.show()\n\n# smt.graphics.plot_pacf(r, lags=50, alpha=0.05)\n# plt.title('PACF symbol_0 date_0 responder_6')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:34.495015Z","iopub.execute_input":"2024-11-07T04:38:34.495512Z","iopub.status.idle":"2024-11-07T04:38:35.122802Z","shell.execute_reply.started":"2024-11-07T04:38:34.495467Z","shell.execute_reply":"2024-11-07T04:38:35.121279Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Regression\n","metadata":{}},{"cell_type":"code","source":"#features = ['feature_'+str(f).zfill(2) for f in range(79)]\n#days = sorted(df['date_id'].unique())\n#symbols = sorted(df['symbol_id'].unique())\n#df['fit'] = None\n\n#print('start:',datetime.datetime.now())\n#for d in days:\n#    for s in symbols:\n#        df_day = df[(df['symbol_id']==s) & (df['date_id']==d)]\n#        if not df_day.empty:\n#            model = sm.OLS(df_day['responder_6'], sm.add_constant(df_day[features].fillna(0))).fit()\n#            df.loc[(df['symbol_id'] == s) & (df['date_id'] == d), 'fit'] = np.array(model.fittedvalues)\n\n#print('End regression:',datetime.datetime.now())\n#print(df['fit'].info())\n#print('End info fit column:',datetime.datetime.now())\n\n#def evaluateR2(obs, pron, weight):\n#    numerator = np.sum(weight * ((obs - pron)**2))\n#    denominator = np.sum(weight * (obs**2))\n#    r2 = 1 - (numerator/denominator)\n#    print('weighted zero-mean r2 score: {0:.4f}'.format(r2))\n#    return r2\n\n#result = evaluateR2(df['responder_6'],df['fit'],df['weight'])\n\n#print(result)\n#print('End results:',datetime.datetime.now())","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:35.126759Z","iopub.execute_input":"2024-11-07T04:38:35.127238Z","iopub.status.idle":"2024-11-07T04:38:35.134081Z","shell.execute_reply.started":"2024-11-07T04:38:35.127177Z","shell.execute_reply":"2024-11-07T04:38:35.132766Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# lags_ : pl.DataFrame | None = None\n\n\n# def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \n#     global lags_\n#     if lags is not None:\n#         lags_ = lags\n        \n#     predictions = pd.DataFrame(columns=['row_id','responder_6'])\n\n#     features = ['feature_'+str(f).zfill(2) for f in range(79)]\n#     days = sorted(test['date_id'].unique())\n#     symbols = sorted(test['symbol_id'].unique())\n\n#     predictions['row_id'] = test['row_id']  #range(len(test)) test.index\n\n\n    \n#     for d in days:\n#         for s in symbols:\n#             df_day = test[(test['symbol_id']==s) & (test['date_id']==d)]\n#             if not df_day.empty:\n#                 model = sm.OLS(df_day['responder_6'], sm.add_constant(df_day[features].fillna(0))).fit()\n#                 predictions.loc[df_day['row_id'], 'responder_6'] = np.array(model.fittedvalues)\n                \n#     # The predict function must return a DataFrame\n#     assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n#     # with columns 'row_id', 'responer_6'\n#     assert predictions.columns == ['row_id', 'responder_6']\n#     # and as many rows as the test data.\n#     assert len(predictions) == len(test)\n\n#     return predictions","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:35.135972Z","iopub.execute_input":"2024-11-07T04:38:35.136505Z","iopub.status.idle":"2024-11-07T04:38:35.153316Z","shell.execute_reply.started":"2024-11-07T04:38:35.136446Z","shell.execute_reply":"2024-11-07T04:38:35.151526Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n# if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n#     inference_server.serve()\n# else:\n#     inference_server.run_local_gateway(\n#         (\n#             '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n#             '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n#         )\n#     )","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:35.155703Z","iopub.execute_input":"2024-11-07T04:38:35.156343Z","iopub.status.idle":"2024-11-07T04:38:35.173392Z","shell.execute_reply.started":"2024-11-07T04:38:35.156281Z","shell.execute_reply":"2024-11-07T04:38:35.171876Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# lags_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet'\n# test_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet'\n# coef_path = '/kaggle/input/ar-model/data_ar1_model.csv'\n# df = pd.read_parquet(test_path)\n# df_lags = pd.read_parquet(lags_path)\n# coef_data = pd.read_csv(coef_path)\n\n# dates = sorted(df['date_id'].unique())\n# symbols = sorted(df['symbol_id'].unique())\n\n# pronostics = pd.DataFrame(columns=['row_id','responder_6'])\n# pronostics['row_id'] = df['row_id']\n\n# for s in symbols:\n#     for d in dates:\n#         sub = df[(df['symbol_id'] == s) & (df['date_id'] == d)].copy()\n#         sub_lag = df_lags[(df_lags['symbol_id'] == s) & (df_lags['date_id'] == d)].copy()\n#         times = sorted(sub['time_id'].unique())\n#         for t in times:\n#             row_id = sub[sub['time_id'] == t]['row_id'].values[0]\n\n#             lag = sub_lag[sub_lag['time_id'] == t]['responder_6_lag_1'].values[0]\n\n#             try:\n\n#                 coef = coef_data[coef_data['symbol_id']==s]['ar.L1'].values[0]\n\n#                 pronostics.loc[pronostics['row_id'] == row_id, 'responder_6'] = coef * lag\n\n#             except:\n#                 coef = 0.85\n#                 pronostics.loc[pronostics['row_id'] == row_id, 'responder_6'] = coef * lag\n\n# # print(pronostics)\n# pronostics.to_csv('/kaggle/working/prons.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-07T04:38:35.175107Z","iopub.execute_input":"2024-11-07T04:38:35.175717Z","iopub.status.idle":"2024-11-07T04:38:35.189637Z","shell.execute_reply.started":"2024-11-07T04:38:35.175651Z","shell.execute_reply":"2024-11-07T04:38:35.188443Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# AR(1) Model","metadata":{}},{"cell_type":"code","source":"# for par in range(10):\n#     cols = ['date_id','time_id','weight','symbol_id','responder_6']\n#     df = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=' + str(par) + '/part-0.parquet', columns=cols)\n    \n#     dates = sorted(df['date_id'].unique())\n#     symbols = sorted(df['symbol_id'].unique())\n    \n#     data_ar1 = pd.DataFrame(columns=['symbol_id','const','prob_const', 'ar.L1', 'prob_ar.L1', 'sigma2', 'prob_sigma2'])\n#     cont = 0\n#     for s in symbols:\n#         subset = df[df['symbol_id']==s]\n#         serie = subset['responder_6'] * subset['weight']\n#         model = sm.tsa.ARIMA(serie, order=(1, 0, 0)).fit()\n#         data_ar1.loc[cont] = [s, model.params['const'], model.pvalues['const'], model.params['ar.L1'], model.pvalues['ar.L1'], model.params['sigma2'], model.pvalues['sigma2']]\n#         cont += 1\n#         print('...',cont)\n#     data_ar1['part'] = str(par)\n#     data_ar1.to_csv('/kaggle/working/data_ar1_model_part' + str(par) + '.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-08T01:14:34.074721Z","iopub.execute_input":"2024-11-08T01:14:34.075726Z","iopub.status.idle":"2024-11-08T02:15:38.222793Z","shell.execute_reply.started":"2024-11-08T01:14:34.075660Z","shell.execute_reply":"2024-11-08T02:15:38.218637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# params0 = pd.read_csv('/kaggle/working/data_ar1_model_part0.csv')\n# params1 = pd.read_csv('/kaggle/working/data_ar1_model_part1.csv')\n\n# params = pd.concat([params0, params1])\n# print(params)\n# print(pd.read_csv('/kaggle/working/data_ar1_model_part2.csv'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-07T18:24:55.023886Z","iopub.execute_input":"2024-11-07T18:24:55.024456Z","iopub.status.idle":"2024-11-07T18:24:55.049330Z","shell.execute_reply.started":"2024-11-07T18:24:55.024398Z","shell.execute_reply":"2024-11-07T18:24:55.047967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# lags_ : pl.DataFrame | None = None\n\n\n# def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n#     global lags_\n#     if lags is not None:\n#         lags_ = lags\n        \n#     # coef_path = '/kaggle/input/ar-model2/data_ar1_model_part2.csv'\n#     # coef_data = pd.read_csv(coef_path)\n    \n#     dates = sorted(test['date_id'].unique())\n#     symbols = sorted(test['symbol_id'].unique())\n\n    \n#     pronostics = pd.DataFrame(columns=['row_id','responder_6'])\n#     pronostics['row_id'] = test['row_id']\n    \n#     mask = [True] * test.shape[1]\n#     if isinstance(test, pl.DataFrame): \n#         test = test.to_pandas() \n#     if isinstance(lags_, pl.DataFrame): \n#         lags_ = lags_.to_pandas()\n    \n#     for s in symbols:\n#         for d in dates:\n#             sub = test[(test['symbol_id'] == s) & (test['date_id'] == d)].copy()\n#             sub_lag = lags_[(lags_['symbol_id'] == s) & (lags_['date_id'] == d)].copy()\n#             times = sorted(sub['time_id'].unique())\n#             for t in times:\n#                 row_id = sub[sub['time_id'] == t]['row_id'].values[0]\n\n#                 # try:\n#                 #     cons = coef_data[coef_data['symbol_id']==s]['const'].values[0]\n#                 #     coef = coef_data[coef_data['symbol_id']==s]['ar.L1'].values[0]\n#                 # except:\n#                 #     cons = 0\n#                 #     coef = 0.85\n#                 try:\n#                     lag = sub_lag[sub_lag['time_id'] == t]['responder_6_lag_1'].values[0]\n#                 except:\n#                     lag = 0\n                    \n#                 pronostics.loc[pronostics['row_id'] == row_id, 'responder_6'] =  0.8970*lag #cons + coef * lag \n\n#     predictions = pronostics\n#     print(predictions)\n    \n# #     output_path = '/kaggle/working/prons.csv'\n# #     prons = pd.read_csv(output_path)\n\n    \n# #     predictions = pd.DataFrame(columns=['row_id','responder_6'])\n# #     predictions['row_id'] = test['row_id']\n# #     predictions['responder_6'] = prons['responder_6']\n \n    \n\n#     if isinstance(predictions, pl.DataFrame):\n#         assert predictions.columns == ['row_id', 'responder_6']\n#     elif isinstance(predictions, pd.DataFrame):\n#         assert (predictions.columns == ['row_id', 'responder_6']).all()\n#     else:\n#         raise TypeError('The predict function must return a DataFrame')\n#     # Confirm has as many rows as the test data.\n#     assert len(predictions) == len(test)\n\n#     return predictions","metadata":{"execution":{"iopub.status.busy":"2024-11-08T03:21:35.649861Z","iopub.execute_input":"2024-11-08T03:21:35.650865Z","iopub.status.idle":"2024-11-08T03:21:35.665445Z","shell.execute_reply.started":"2024-11-08T03:21:35.650818Z","shell.execute_reply":"2024-11-08T03:21:35.664091Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# ARMA Model\nthe best model for date=1 and symbol=0 is arma(1,2)\nfor symbol=1 is arma(1,1)","metadata":{}},{"cell_type":"code","source":"# # get data\n# cols = ['date_id','time_id','weight','symbol_id','responder_6']\n# df = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet', columns=cols)\n\n# #individual symbol\n# s = 0\n# d = 1\n\n# # serie\n# serie = df[(df['date_id']==d) & (df['symbol_id']==s)]['responder_6']\n# # print(serie.head(10))\n\n# # model\n# model = sm.tsa.ARIMA(serie, order=([1], 0, [1])).fit()\n# errors = model.resid\n\n# # results\n# print(model.summary())\n\n# # ACF errors\n# smt.graphics.plot_acf(errors.dropna(), lags=20, alpha=0.05)\n# plt.title(\"ACF\")\n# plt.show()\n\n# # PACF errors\n# smt.graphics.plot_pacf(errors.dropna(), lags=20, alpha=0.05)\n# plt.title(\"ACF\")\n# plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-09T22:07:22.016197Z","iopub.execute_input":"2024-11-09T22:07:22.016634Z","iopub.status.idle":"2024-11-09T22:07:22.755468Z","shell.execute_reply.started":"2024-11-09T22:07:22.016596Z","shell.execute_reply":"2024-11-09T22:07:22.754010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # for all symbols\n# for par in range(1):\n#     cols = ['date_id','time_id','weight','symbol_id','responder_6']\n#     df = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=' + str(par) + '/part-0.parquet', columns=cols)\n    \n#     dates = sorted(df['date_id'].unique())\n#     symbols = sorted(df['symbol_id'].unique())\n    \n#     data_arma = pd.DataFrame(columns=['symbol_id','const','prob_const', 'ar.L1', 'prob_ar.L1', 'ma.L1', 'prob_ma.L1', 'sigma2', 'prob_sigma2'])\n#     cont = 0\n#     for s in symbols:\n#         subset = df[df['symbol_id']==s]\n#         serie = subset['responder_6'] \n#         model = sm.tsa.ARIMA(serie, order=(1, 0, 1)).fit()\n#         data_arma.loc[cont] = [s, model.params['const'], model.pvalues['const'], model.params['ar.L1'], model.pvalues['ar.L1'], model.params['ma.L1'], model.pvalues['ma.L1'], model.params['sigma2'], model.pvalues['sigma2']]\n#         cont += 1\n#         print('...',cont)\n#     data_arma['part'] = str(par)\n#     data_arma.to_csv('/kaggle/working/data_arma_model_part' + str(par) + '.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T03:05:23.657924Z","iopub.execute_input":"2024-11-13T03:05:23.659265Z","iopub.status.idle":"2024-11-13T03:11:14.621166Z","shell.execute_reply.started":"2024-11-13T03:05:23.659216Z","shell.execute_reply":"2024-11-13T03:11:14.617435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show data\n# df = pd.read_csv('/kaggle/working/data_arma_model_part0.csv')\n# df = pd.read_csv('/kaggle/input/arma-model/data_arma_model_part0.csv')\n# print(df)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T03:19:26.110359Z","iopub.execute_input":"2024-11-13T03:19:26.110764Z","iopub.status.idle":"2024-11-13T03:19:26.132236Z","shell.execute_reply.started":"2024-11-13T03:19:26.110725Z","shell.execute_reply":"2024-11-13T03:19:26.130889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    global lags_\n    if lags is not None:\n        lags_ = lags\n        \n    coef_path = '/kaggle/input/arma-model/data_arma_model_part0.csv'\n    coef_data = pd.read_csv(coef_path)\n    \n    dates = sorted(test['date_id'].unique())\n    symbols = sorted(test['symbol_id'].unique())\n\n    \n    pronostics = pd.DataFrame(columns=['row_id','responder_6'])\n    pronostics['row_id'] = test['row_id']\n    \n    # mask = [True] * test.shape[1]\n    if isinstance(test, pl.DataFrame): \n        test = test.to_pandas() \n    if isinstance(lags_, pl.DataFrame): \n        lags_ = lags_.to_pandas()\n    \n    for s in symbols:\n        for d in dates:\n            sub = test[(test['symbol_id'] == s) & (test['date_id'] == d)].copy()\n            sub_lag = lags_[(lags_['symbol_id'] == s) & (lags_['date_id'] == d)].copy()\n            times = sorted(sub['time_id'].unique())\n            for t in times:\n                row_id = sub[sub['time_id'] == t]['row_id'].values[0]\n\n                try:\n                    cons = coef_data[coef_data['symbol_id']==s]['const'].values[0]\n                    coefAR = coef_data[coef_data['symbol_id']==s]['ar.L1'].values[0]\n                    coefMA = coef_data[coef_data['symbol_id']==s]['ma.L1'].values[0]\n                except:\n                    cons = 0\n                    coefAR = 0.9292\n                    coefMA = -0.20\n                try:\n                    lag = sub_lag[sub_lag['time_id'] == t]['responder_6_lag_1'].values[0]\n                except:\n                    lag = 0\n                    \n                pronostics.loc[pronostics['row_id'] == row_id, 'responder_6'] =  cons + coefAR*lag #+ coefMA * #cons + coef * lag \n\n    predictions = pronostics\n    print(predictions)\n    \n\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    # Confirm has as many rows as the test data.\n    assert len(predictions) == len(test)\n\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T03:49:00.652775Z","iopub.execute_input":"2024-11-13T03:49:00.653222Z","iopub.status.idle":"2024-11-13T03:49:00.666702Z","shell.execute_reply.started":"2024-11-13T03:49:00.653180Z","shell.execute_reply":"2024-11-13T03:49:00.665497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-11-13T03:50:11.084530Z","iopub.execute_input":"2024-11-13T03:50:11.085589Z","iopub.status.idle":"2024-11-13T03:50:11.262127Z","shell.execute_reply.started":"2024-11-13T03:50:11.085543Z","shell.execute_reply":"2024-11-13T03:50:11.260781Z"},"trusted":true},"outputs":[],"execution_count":null}]}