{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-12T18:11:52.564626Z","iopub.execute_input":"2023-02-12T18:11:52.565290Z","iopub.status.idle":"2023-02-12T18:11:53.790531Z","shell.execute_reply.started":"2023-02-12T18:11:52.565226Z","shell.execute_reply":"2023-02-12T18:11:53.789152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's read and see the main train data file:","metadata":{}},{"cell_type":"code","source":"data = pd.read_parquet(\"/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet\")\ndata","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:11:53.799276Z","iopub.execute_input":"2023-02-12T18:11:53.799816Z","iopub.status.idle":"2023-02-12T18:12:36.364312Z","shell.execute_reply.started":"2023-02-12T18:11:53.799766Z","shell.execute_reply":"2023-02-12T18:12:36.362793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Wow. 131 million rows in train dataset.","metadata":{}},{"cell_type":"code","source":"display(data.batch_id.nunique())\nprint()\ndisplay(data.event_id.nunique())\nprint()\ndisplay(data.isna().sum())\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:12:36.366072Z","iopub.execute_input":"2023-02-12T18:12:36.366447Z","iopub.status.idle":"2023-02-12T18:15:10.288890Z","shell.execute_reply.started":"2023-02-12T18:12:36.366406Z","shell.execute_reply":"2023-02-12T18:15:10.287723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We could see 660 batches from 1 to 660, uniques event_id, and no missing values.","metadata":{}},{"cell_type":"code","source":"#display(data.duplicated().sum())","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:15:10.292829Z","iopub.execute_input":"2023-02-12T18:15:10.293802Z","iopub.status.idle":"2023-02-12T18:15:10.298706Z","shell.execute_reply.started":"2023-02-12T18:15:10.293762Z","shell.execute_reply":"2023-02-12T18:15:10.297460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.hist(bins=50, figsize=(15, 15))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:15:10.300005Z","iopub.execute_input":"2023-02-12T18:15:10.300407Z","iopub.status.idle":"2023-02-12T18:16:38.604066Z","shell.execute_reply.started":"2023-02-12T18:15:10.300376Z","shell.execute_reply":"2023-02-12T18:16:38.602741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.zenith.median()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:18:45.967217Z","iopub.execute_input":"2023-02-12T18:18:45.968922Z","iopub.status.idle":"2023-02-12T18:18:48.855128Z","shell.execute_reply.started":"2023-02-12T18:18:45.968835Z","shell.execute_reply":"2023-02-12T18:18:48.853944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see zenith median is near pi/2","metadata":{}},{"cell_type":"code","source":"data_test = pd.read_parquet(\"/kaggle/input/icecube-neutrinos-in-deep-ice/test_meta.parquet\")\ndata_test","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:38.606152Z","iopub.execute_input":"2023-02-12T18:16:38.606628Z","iopub.status.idle":"2023-02-12T18:16:38.632096Z","shell.execute_reply.started":"2023-02-12T18:16:38.606564Z","shell.execute_reply":"2023-02-12T18:16:38.630886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In test data there is only 661 batch","metadata":{}},{"cell_type":"code","source":"data_batch_11 = pd.read_parquet(\"/kaggle/input/icecube-neutrinos-in-deep-ice/train/batch_11.parquet\")\ndata_batch_11","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:38.644967Z","iopub.execute_input":"2023-02-12T18:16:38.645445Z","iopub.status.idle":"2023-02-12T18:16:42.074078Z","shell.execute_reply.started":"2023-02-12T18:16:38.645402Z","shell.execute_reply":"2023-02-12T18:16:42.073158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"32 million rows. Oh","metadata":{}},{"cell_type":"code","source":"display(data_batch_11.index.nunique())\nprint()\ndisplay(data_batch_11.isna().sum())\nprint()\ndisplay(data_batch_11.duplicated().sum())","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:42.075370Z","iopub.execute_input":"2023-02-12T18:16:42.075894Z","iopub.status.idle":"2023-02-12T18:16:52.854303Z","shell.execute_reply.started":"2023-02-12T18:16:42.075854Z","shell.execute_reply":"2023-02-12T18:16:52.853056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We could see here 200k event_ids, no missings and 236k diplicates.","metadata":{}},{"cell_type":"code","source":"data_batch_112 = pd.read_parquet(\"/kaggle/input/icecube-neutrinos-in-deep-ice/train/batch_112.parquet\")\ndata_batch_112","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:52.855879Z","iopub.execute_input":"2023-02-12T18:16:52.856250Z","iopub.status.idle":"2023-02-12T18:16:56.114060Z","shell.execute_reply.started":"2023-02-12T18:16:52.856216Z","shell.execute_reply":"2023-02-12T18:16:56.113188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_batch_112.index.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:56.115306Z","iopub.execute_input":"2023-02-12T18:16:56.115807Z","iopub.status.idle":"2023-02-12T18:16:56.682452Z","shell.execute_reply.started":"2023-02-12T18:16:56.115775Z","shell.execute_reply":"2023-02-12T18:16:56.681129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Batches are similar with event_id","metadata":{}},{"cell_type":"code","source":"data_batch_112.auxiliary.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:56.684271Z","iopub.execute_input":"2023-02-12T18:16:56.685041Z","iopub.status.idle":"2023-02-12T18:16:56.922014Z","shell.execute_reply.started":"2023-02-12T18:16:56.684990Z","shell.execute_reply":"2023-02-12T18:16:56.920795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We could see here 9 mlns true events and 25 mlns false events","metadata":{}},{"cell_type":"code","source":"data_geometry = pd.read_csv(\"/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv\")\ndata_geometry","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:56.926495Z","iopub.execute_input":"2023-02-12T18:16:56.927065Z","iopub.status.idle":"2023-02-12T18:16:56.954946Z","shell.execute_reply.started":"2023-02-12T18:16:56.927030Z","shell.execute_reply":"2023-02-12T18:16:56.954140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ok. We have 5160 sensors","metadata":{}},{"cell_type":"code","source":"geom = px.scatter_3d(data_geometry, x='x', y='y', z='z', color='z', opacity=0.7)\ngeom.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T18:16:56.956174Z","iopub.execute_input":"2023-02-12T18:16:56.956691Z","iopub.status.idle":"2023-02-12T18:16:58.153277Z","shell.execute_reply.started":"2023-02-12T18:16:56.956657Z","shell.execute_reply":"2023-02-12T18:16:58.151851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are interesting additional sensors in the icecube","metadata":{}}]}