{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T05:14:56.449392Z","iopub.execute_input":"2024-12-03T05:14:56.450054Z","iopub.status.idle":"2024-12-03T05:14:56.516819Z","shell.execute_reply.started":"2024-12-03T05:14:56.450003Z","shell.execute_reply":"2024-12-03T05:14:56.515571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# imports\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\n%matplotlib inline\npd.set_option('display.float_format', '{:.2f}'.format)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T05:14:56.51806Z","iopub.execute_input":"2024-12-03T05:14:56.518375Z","iopub.status.idle":"2024-12-03T05:14:56.526952Z","shell.execute_reply.started":"2024-12-03T05:14:56.518343Z","shell.execute_reply":"2024-12-03T05:14:56.525671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!pip install datatable > /dev/null\n\n!pip install datatable\nimport datatable as dt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:08:17.884654Z","iopub.execute_input":"2024-12-02T05:08:17.884969Z","iopub.status.idle":"2024-12-02T05:13:10.101894Z","shell.execute_reply.started":"2024-12-02T05:08:17.884937Z","shell.execute_reply":"2024-12-02T05:13:10.099925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:10.107133Z","iopub.execute_input":"2024-12-02T05:13:10.107641Z","iopub.status.idle":"2024-12-02T05:13:10.149195Z","shell.execute_reply.started":"2024-12-02T05:13:10.107593Z","shell.execute_reply":"2024-12-02T05:13:10.147851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read the files\nfeat = pd.read_csv(\"/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\")\n\nprint(feat.head())\nprint(\"\")\nprint(feat.describe())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:10.150601Z","iopub.execute_input":"2024-12-02T05:13:10.15111Z","iopub.status.idle":"2024-12-02T05:13:10.229719Z","shell.execute_reply.started":"2024-12-02T05:13:10.151065Z","shell.execute_reply":"2024-12-02T05:13:10.228514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"resp = pd.read_csv(\"/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv\")\n\nprint(resp.head())\nprint(\"\")\nprint(resp.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:10.231365Z","iopub.execute_input":"2024-12-02T05:13:10.231873Z","iopub.status.idle":"2024-12-02T05:13:10.256774Z","shell.execute_reply.started":"2024-12-02T05:13:10.23182Z","shell.execute_reply":"2024-12-02T05:13:10.255425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"resp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:10.25846Z","iopub.execute_input":"2024-12-02T05:13:10.259029Z","iopub.status.idle":"2024-12-02T05:13:10.27478Z","shell.execute_reply.started":"2024-12-02T05:13:10.25898Z","shell.execute_reply":"2024-12-02T05:13:10.273535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"samplesub = pd.read_csv(\"/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\")\n\nprint(samplesub.head())\nprint(\"\")\nprint(samplesub.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:10.276476Z","iopub.execute_input":"2024-12-02T05:13:10.27687Z","iopub.status.idle":"2024-12-02T05:13:10.306723Z","shell.execute_reply.started":"2024-12-02T05:13:10.276832Z","shell.execute_reply":"2024-12-02T05:13:10.305072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df1 = pd.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\")\n\nprint(df1.head())\nprint(\"\")\nprint(df1.describe())\nprint(df1.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T05:24:42.894954Z","iopub.execute_input":"2024-12-03T05:24:42.895342Z","iopub.status.idle":"2024-12-03T05:24:50.676385Z","shell.execute_reply.started":"2024-12-03T05:24:42.895301Z","shell.execute_reply":"2024-12-03T05:24:50.675141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Output the df1 to csv to observe the data, saved to output tab on RHS under NOTEBOOK\n## df1.to_csv(\"partition_id0.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:22.643815Z","iopub.execute_input":"2024-12-02T05:13:22.644237Z","iopub.status.idle":"2024-12-02T05:13:22.65295Z","shell.execute_reply.started":"2024-12-02T05:13:22.6442Z","shell.execute_reply":"2024-12-02T05:13:22.650923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# to delete unused files from kaggle's output folder \n## os.remove(\"/kaggle/working/file_id0.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:22.654313Z","iopub.execute_input":"2024-12-02T05:13:22.654692Z","iopub.status.idle":"2024-12-02T05:13:22.673254Z","shell.execute_reply.started":"2024-12-02T05:13:22.654654Z","shell.execute_reply":"2024-12-02T05:13:22.671953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# unique values for symbol_id column\ndf1['symbol_id'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:22.675769Z","iopub.execute_input":"2024-12-02T05:13:22.676321Z","iopub.status.idle":"2024-12-02T05:13:22.709988Z","shell.execute_reply.started":"2024-12-02T05:13:22.676263Z","shell.execute_reply":"2024-12-02T05:13:22.708816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#look at symbol_id 14, create a df for symbol_id 14\nsym14 = df1[df1.symbol_id==14]\n#sym14.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:22.711921Z","iopub.execute_input":"2024-12-02T05:13:22.712426Z","iopub.status.idle":"2024-12-02T05:13:22.764208Z","shell.execute_reply.started":"2024-12-02T05:13:22.712372Z","shell.execute_reply":"2024-12-02T05:13:22.761879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot features\nplt.plot(sym14[\"feature_05\"])\n\n## ranges from -7.5 to 7.5 for symbol 14","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:22.766814Z","iopub.execute_input":"2024-12-02T05:13:22.767294Z","iopub.status.idle":"2024-12-02T05:13:23.1466Z","shell.execute_reply.started":"2024-12-02T05:13:22.767251Z","shell.execute_reply":"2024-12-02T05:13:23.144751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot feature against time_id on x axis for symbol id 14 \nplt.plot( sym14[\"time_id\"] , sym14[\"feature_09\"])\n## feature 9 is a constant value ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:23.148119Z","iopub.execute_input":"2024-12-02T05:13:23.14848Z","iopub.status.idle":"2024-12-02T05:13:23.367203Z","shell.execute_reply.started":"2024-12-02T05:13:23.148413Z","shell.execute_reply":"2024-12-02T05:13:23.364918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot feature against time_id on x axis for date_id 0 for symbol id 14 \n## note time_id repeats across multiple date_ids \nsym14dt = sym14.loc[ ( sym14[\"date_id\"]==0), :]\nplt.plot( sym14dt[\"time_id\"] , sym14dt[\"feature_36\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:23.369Z","iopub.execute_input":"2024-12-02T05:13:23.369366Z","iopub.status.idle":"2024-12-02T05:13:23.666647Z","shell.execute_reply.started":"2024-12-02T05:13:23.369331Z","shell.execute_reply":"2024-12-02T05:13:23.665216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot feature against time_id on x axis for time between 200 and 400 for symbol id 14 \nsym14t = sym14.loc[ (sym14[\"time_id\"] >= 200) & (sym14[\"time_id\"] <= 210) &( sym14[\"date_id\"]==0), :]\nplt.plot( sym14t[\"time_id\"] , sym14t[\"feature_36\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:23.668601Z","iopub.execute_input":"2024-12-02T05:13:23.669035Z","iopub.status.idle":"2024-12-02T05:13:23.936861Z","shell.execute_reply.started":"2024-12-02T05:13:23.668997Z","shell.execute_reply":"2024-12-02T05:13:23.93469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## plot weight of each symbol_id\n#df1.groupby(\"symbol_id\")[\"weight\"].plot(kind=\"bar\")\ngrouped = df1.loc[df1['symbol_id'].isin([14 , 33 ,10])]\ngrouped['period']= df1['date_id'].astype(str)+ df1['time_id'].astype(str)\ngrouped.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T05:50:58.067146Z","iopub.execute_input":"2024-12-03T05:50:58.067685Z","iopub.status.idle":"2024-12-03T05:50:59.727738Z","shell.execute_reply.started":"2024-12-03T05:50:58.067633Z","shell.execute_reply":"2024-12-03T05:50:59.726448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsns.lineplot(x='period', y='feature_05', hue='symbol_id', data=grouped)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T05:51:20.444564Z","iopub.execute_input":"2024-12-03T05:51:20.44552Z","iopub.status.idle":"2024-12-03T06:00:11.925104Z","shell.execute_reply.started":"2024-12-03T05:51:20.445461Z","shell.execute_reply":"2024-12-03T06:00:11.923006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot for each category\nfor name, group in grouped:\n    group.plot(kind='bar', y='weight', title=weight of symbols)\n    plt.show()\n\n###df1.groupby(\"symbol_id\")[\"weight\"].plot(kind=\"bar\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T05:13:24.019711Z","iopub.status.idle":"2024-12-02T05:13:24.020641Z","shell.execute_reply.started":"2024-12-02T05:13:24.020295Z","shell.execute_reply":"2024-12-02T05:13:24.020331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# nulls \n# nulls missing value? \nhas_nulls = set(df.columns[df.isnull().sum()!=0])\nprint(\"There are \"+str(len(has_nulls))+\" many cols with at least one null value\")\nprint(has_nulls)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}