{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:20:49.950022Z","iopub.execute_input":"2024-12-10T09:20:49.950485Z","iopub.status.idle":"2024-12-10T09:20:50.464773Z","shell.execute_reply.started":"2024-12-10T09:20:49.950442Z","shell.execute_reply":"2024-12-10T09:20:50.463274Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***Download required libraries***","metadata":{}},{"cell_type":"code","source":"import gc\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom keras.models import Sequential\nfrom keras.layers import LSTM\nfrom keras.layers import Dense\nfrom sklearn import preprocessing","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***Download and collect data files***","metadata":{}},{"cell_type":"code","source":"Parts = [] # Initialize a list to hold file parts\nfor i in range(10): # Load each part\n    df_path = f\"/kaggle/input/c/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pd.read_parquet(df_path)\n    Part_chunk = chunk[:500000]  # Take a sample of the data\n    Parts.append(Part_chunk)\ndel chunk\ngc.collect()  # The Interpreter is performing a collection\nMarketData_df = pd.concat(Parts, ignore_index=True)\ndel Parts\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:22:10.943765Z","iopub.execute_input":"2024-12-10T09:22:10.944165Z","iopub.status.idle":"2024-12-10T09:24:19.810635Z","shell.execute_reply.started":"2024-12-10T09:22:10.944130Z","shell.execute_reply":"2024-12-10T09:24:19.808557Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***Data Processing***","metadata":{}},{"cell_type":"code","source":"MarketData_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:57:56.183413Z","iopub.execute_input":"2024-12-09T07:57:56.183874Z","iopub.status.idle":"2024-12-09T07:57:56.872452Z","shell.execute_reply.started":"2024-12-09T07:57:56.183837Z","shell.execute_reply":"2024-12-09T07:57:56.871323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:34.363282Z","iopub.execute_input":"2024-12-09T07:59:34.363773Z","iopub.status.idle":"2024-12-09T07:59:34.370415Z","shell.execute_reply.started":"2024-12-09T07:59:34.363735Z","shell.execute_reply":"2024-12-09T07:59:34.369361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:39.600118Z","iopub.execute_input":"2024-12-09T07:59:39.600526Z","iopub.status.idle":"2024-12-09T07:59:39.632084Z","shell.execute_reply.started":"2024-12-09T07:59:39.600489Z","shell.execute_reply":"2024-12-09T07:59:39.630923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T11:21:53.294332Z","iopub.execute_input":"2024-12-09T11:21:53.295531Z","iopub.status.idle":"2024-12-09T11:22:17.161225Z","shell.execute_reply.started":"2024-12-09T11:21:53.295461Z","shell.execute_reply":"2024-12-09T11:22:17.160072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:01:01.218027Z","iopub.execute_input":"2024-12-09T08:01:01.218424Z","iopub.status.idle":"2024-12-09T08:01:01.541760Z","shell.execute_reply.started":"2024-12-09T08:01:01.218387Z","shell.execute_reply":"2024-12-09T08:01:01.540632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(MarketData_df.duplicated())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T09:47:13.728758Z","iopub.execute_input":"2024-12-09T09:47:13.729238Z","iopub.status.idle":"2024-12-09T09:48:45.228941Z","shell.execute_reply.started":"2024-12-09T09:47:13.729190Z","shell.execute_reply":"2024-12-09T09:48:45.224035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df= MarketData_df.drop(['symbol_id'],axis=1)\nMarketData_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T11:55:26.554472Z","iopub.execute_input":"2024-12-09T11:55:26.554950Z","iopub.status.idle":"2024-12-09T11:55:29.949834Z","shell.execute_reply.started":"2024-12-09T11:55:26.554878Z","shell.execute_reply":"2024-12-09T11:55:29.948661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df['date_id']=pd.to_datetime(MarketData_df['date_id'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:30:16.574588Z","iopub.execute_input":"2024-12-10T09:30:16.575737Z","iopub.status.idle":"2024-12-10T09:30:16.751297Z","shell.execute_reply.started":"2024-12-10T09:30:16.575644Z","shell.execute_reply":"2024-12-10T09:30:16.749518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df['time_id']=pd.to_datetime(MarketData_df['time_id'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:30:19.520463Z","iopub.execute_input":"2024-12-10T09:30:19.521051Z","iopub.status.idle":"2024-12-10T09:30:19.664316Z","shell.execute_reply.started":"2024-12-10T09:30:19.521004Z","shell.execute_reply":"2024-12-10T09:30:19.662476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df.set_index(['date_id','time_id'],drop=False,inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:30:22.692071Z","iopub.execute_input":"2024-12-10T09:30:22.692519Z","iopub.status.idle":"2024-12-10T09:30:23.052035Z","shell.execute_reply.started":"2024-12-10T09:30:22.692478Z","shell.execute_reply":"2024-12-10T09:30:23.050748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MarketData_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:30:28.402903Z","iopub.execute_input":"2024-12-10T09:30:28.403323Z","iopub.status.idle":"2024-12-10T09:30:28.442007Z","shell.execute_reply.started":"2024-12-10T09:30:28.403287Z","shell.execute_reply":"2024-12-10T09:30:28.439628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.displot(MarketData_df['feature_00'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:35:51.411382Z","iopub.execute_input":"2024-12-10T09:35:51.411870Z","iopub.status.idle":"2024-12-10T09:35:59.305995Z","shell.execute_reply.started":"2024-12-10T09:35:51.411831Z","shell.execute_reply":"2024-12-10T09:35:59.303808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.boxplot(MarketData_df['feature_00'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T09:36:53.558760Z","iopub.execute_input":"2024-12-10T09:36:53.560585Z","iopub.status.idle":"2024-12-10T09:36:54.814482Z","shell.execute_reply.started":"2024-12-10T09:36:53.560525Z","shell.execute_reply":"2024-12-10T09:36:54.813024Z"}},"outputs":[],"execution_count":null}]}