{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nmatplotlib.style.set('ggplot')\n\nPATH_DATASET = \"/kaggle/input/icecube-neutrinos-in-deep-ice\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-24T10:33:40.800695Z","iopub.status.idle":"2023-01-24T10:33:40.801121Z","shell.execute_reply.started":"2023-01-24T10:33:40.800927Z","shell.execute_reply":"2023-01-24T10:33:40.800945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Browse meta data\n\n**[train/test]_meta.parquet**\n\n- **batch_id** (int): the ID of the batch the event was placed into.\n- **event_id** (int): the event ID.\n- **[first/last]_pulse_index** (int): index of the first/last row in the features dataframe belonging to this event.\n- **[azimuth/zenith]** (float32): the [azimuth/zenith] angle in radians of the neutrino. A value between 0 and 2*pi for the azimuth and 0 and pi for zenith. The target columns. Not provided for the test set. The direction vector represented by zenith and azimuth points to where the neutrino came from.","metadata":{}},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"meta_test = pd.read_parquet(os.path.join(PATH_DATASET, \"test_meta.parquet\"))\nprint(f\"length: {len(meta_test)}\")\nmeta_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:02:34.287119Z","iopub.execute_input":"2023-01-24T10:02:34.287526Z","iopub.status.idle":"2023-01-24T10:02:34.439526Z","shell.execute_reply.started":"2023-01-24T10:02:34.287494Z","shell.execute_reply":"2023-01-24T10:02:34.438448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train = pd.read_parquet(os.path.join(PATH_DATASET, \"train_meta.parquet\"))\nprint(f\"length: {len(meta_train)}\")\ndisplay(meta_train.head())\ndisplay(meta_train.info())","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:02:45.885202Z","iopub.execute_input":"2023-01-24T10:02:45.885624Z","iopub.status.idle":"2023-01-24T10:03:29.048923Z","shell.execute_reply.started":"2023-01-24T10:02:45.88559Z","shell.execute_reply":"2023-01-24T10:03:29.047979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6, 2))\nmeta_train.groupby(\"batch_id\").size().plot.hist(bins=50)\nplt.xlabel(\"nb. events in batch\"), plt.ylabel(\"nb. cases\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:08:20.792081Z","iopub.execute_input":"2023-01-24T10:08:20.793693Z","iopub.status.idle":"2023-01-24T10:08:24.282022Z","shell.execute_reply.started":"2023-01-24T10:08:20.793642Z","shell.execute_reply":"2023-01-24T10:08:24.280985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"seems that event id is unique per whole dataset as grouping is 1","metadata":{}},{"cell_type":"code","source":"meta_train.groupby(\"event_id\").size().max()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:08:46.824447Z","iopub.execute_input":"2023-01-24T10:08:46.824925Z","iopub.status.idle":"2023-01-24T10:10:14.367195Z","shell.execute_reply.started":"2023-01-24T10:08:46.824884Z","shell.execute_reply":"2023-01-24T10:10:14.365364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hitogram of first and last pulses","metadata":{}},{"cell_type":"code","source":"meta_train[[\"first_pulse_index\", \"last_pulse_index\"]].plot.hist(bins=100, alpha=0.5, figsize=(8, 3))\nplt.xlabel('time')","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:10:14.370074Z","iopub.execute_input":"2023-01-24T10:10:14.370771Z","iopub.status.idle":"2023-01-24T10:10:34.434516Z","shell.execute_reply.started":"2023-01-24T10:10:14.370728Z","shell.execute_reply":"2023-01-24T10:10:34.433326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Delay between first and last pulse","metadata":{}},{"cell_type":"code","source":"meta_train[\"delay_pulse_index\"] = meta_train[\"last_pulse_index\"] - meta_train[\"first_pulse_index\"]\n_ = plt.hist(meta_train[\"delay_pulse_index\"], bins=100, log=True)\nplt.ylabel('count cases'), plt.xlabel('duration')\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:13:14.130922Z","iopub.execute_input":"2023-01-24T10:13:14.131446Z","iopub.status.idle":"2023-01-24T10:13:18.286196Z","shell.execute_reply.started":"2023-01-24T10:13:14.131407Z","shell.execute_reply":"2023-01-24T10:13:18.284923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### azimuth and zenith","metadata":{}},{"cell_type":"code","source":"meta_train[[\"azimuth\", \"zenith\"]].plot.hist(bins=50, alpha=0.5, figsize=(8, 3))\nplt.xlabel('azimuth/zenith')\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:13:19.328498Z","iopub.execute_input":"2023-01-24T10:13:19.329287Z","iopub.status.idle":"2023-01-24T10:14:02.033288Z","shell.execute_reply.started":"2023-01-24T10:13:19.329246Z","shell.execute_reply":"2023-01-24T10:14:02.032044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist2d(meta_train[\"azimuth\"], meta_train[\"zenith\"], bins=(50, 50), cmap=plt.cm.jet)\nplt.xlabel('azimuth'), plt.ylabel('zenith')\nplt.colorbar()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:14:02.035679Z","iopub.execute_input":"2023-01-24T10:14:02.036452Z","iopub.status.idle":"2023-01-24T10:14:19.042987Z","shell.execute_reply.started":"2023-01-24T10:14:02.036401Z","shell.execute_reply":"2023-01-24T10:14:19.0416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Browse geometry\n\nThe x, y, and z positions for each of the 5160 IceCube sensors. The row index corresponds to the sensor_idx feature of pulses. The x, y, and z coordinates are in units of meters, with the origin at the center of the IceCube detector. The coordinate system is right-handed, and the z-axis points upwards when standing at the South Pole.","metadata":{}},{"cell_type":"code","source":"geometry = pd.read_csv(os.path.join(PATH_DATASET, \"sensor_geometry.csv\"))\nprint(f\"length: {len(geometry)}\")\ngeometry.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:16:58.338757Z","iopub.execute_input":"2023-01-24T10:16:58.339398Z","iopub.status.idle":"2023-01-24T10:16:58.369389Z","shell.execute_reply.started":"2023-01-24T10:16:58.339353Z","shell.execute_reply":"2023-01-24T10:16:58.367874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\nfig = px.scatter_3d(geometry, x='x', y='y', z='z', opacity=0.6, color=\"z\")\nfig.update_traces(marker_size=2)\nfig.update_layout(height=600, width=600)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:17:09.916535Z","iopub.execute_input":"2023-01-24T10:17:09.916953Z","iopub.status.idle":"2023-01-24T10:17:12.698651Z","shell.execute_reply.started":"2023-01-24T10:17:09.916919Z","shell.execute_reply":"2023-01-24T10:17:12.697305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Browse training/test data\n\n**[train/test]/batch_[n].parquet** Each batch contains tens of thousands of events. Each event may contain thousands of pulses, each of which is the digitized output from a photomultiplier tube and occupies one row.\n\n- **event_id** (int): the event ID. Saved as the index column in parquet.\n- **time** (int): the time of the pulse in nanoseconds in the current event time window. The absolute time of a pulse has no relevance, and only the relative time with respect to other pulses within an event is of relevance.\n- **sensor_id** (int): the ID of which of the 5160 IceCube photomultiplier sensors recorded this pulse.\n- **charge** (float32): An estimate of the amount of light in the pulse, in units of photoelectrons (p.e.). A physical photon does not exactly result in a measurement of 1 p.e. but rather can take values spread around 1 p.e. As an example, a pulse with charge 2.7 p.e. could quite likely be the result of two or three photons hitting the photomultiplier tube around the same time. This data has float16 precision but is stored as float32 due to limitations of the version of pyarrow the data was prepared with.\n- **auxiliary** (bool): If True, the pulse was not fully digitized, is of lower quality, and was more likely to originate from noise. If False, then this pulse was contributed to the trigger decision and the pulse was fully digitized.","metadata":{}},{"cell_type":"code","source":"test = pd.read_parquet(os.path.join(PATH_DATASET, \"test/batch_661.parquet\"))\nprint(f\"length: {len(test)}\")\nprint(f\"events: {len(test.index.unique())}\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:21:07.460602Z","iopub.execute_input":"2023-01-24T10:21:07.461091Z","iopub.status.idle":"2023-01-24T10:21:07.505179Z","shell.execute_reply.started":"2023-01-24T10:21:07.461056Z","shell.execute_reply":"2023-01-24T10:21:07.504152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(os.path.join(PATH_DATASET, \"train/batch_15.parquet\"))\nprint(f\"length: {len(train)}\")\nprint(f\"events: {len(train.index.unique())}\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:21:20.740137Z","iopub.execute_input":"2023-01-24T10:21:20.741162Z","iopub.status.idle":"2023-01-24T10:21:25.530988Z","shell.execute_reply.started":"2023-01-24T10:21:20.741089Z","shell.execute_reply":"2023-01-24T10:21:25.529755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"seems that not all event have same number of samples","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 4))\n_ = plt.hist(train.groupby(level=0).size(), bins=50, log=True)\nplt.xlabel(\"nb. ALL measument per event\"), plt.ylabel(\"nb. of event\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:28:35.852803Z","iopub.execute_input":"2023-01-24T10:28:35.853277Z","iopub.status.idle":"2023-01-24T10:28:37.400298Z","shell.execute_reply.started":"2023-01-24T10:28:35.853213Z","shell.execute_reply":"2023-01-24T10:28:37.399006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 4))\n_ = plt.hist(train[~train['auxiliary']].groupby(level=0).size(), bins=50, log=True)\nplt.xlabel(\"nb. (aux==False) measument per event\"), plt.ylabel(\"nb. of event\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:33:49.511713Z","iopub.execute_input":"2023-01-24T10:33:49.512254Z","iopub.status.idle":"2023-01-24T10:33:52.246616Z","shell.execute_reply.started":"2023-01-24T10:33:49.512182Z","shell.execute_reply":"2023-01-24T10:33:52.244608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Chages per auxiliary","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\n_ = plt.hist(train[~train['auxiliary']]['charge'], bins=50, log=True, label=\"False\")\n_ = plt.hist(train[train['auxiliary']]['charge'], bins=50, log=True, label=\"True\")\nplt.ylabel('count cases'), plt.xlabel('charge')\nplt.grid(), plt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:33:55.904158Z","iopub.execute_input":"2023-01-24T10:33:55.904714Z","iopub.status.idle":"2023-01-24T10:33:59.048155Z","shell.execute_reply.started":"2023-01-24T10:33:55.90467Z","shell.execute_reply":"2023-01-24T10:33:59.046696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sample event\n\ninspiration from: https://github.com/plotly/plotly.py/blob/master/doc/python/subplots.md","metadata":{}},{"cell_type":"code","source":"! pip download -q --prefer-binary scikit-spatial --dest frozen-packages/\n! ls -lh frozen-packages/","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-24T10:34:23.602627Z","iopub.execute_input":"2023-01-24T10:34:23.603043Z","iopub.status.idle":"2023-01-24T10:34:37.858558Z","shell.execute_reply.started":"2023-01-24T10:34:23.603011Z","shell.execute_reply":"2023-01-24T10:34:37.856868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train[meta_train['event_id'].isin([46528394, 2135637939, 2084362251])]","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:34:52.373136Z","iopub.execute_input":"2023-01-24T10:34:52.3737Z","iopub.status.idle":"2023-01-24T10:34:53.824509Z","shell.execute_reply.started":"2023-01-24T10:34:52.373651Z","shell.execute_reply":"2023-01-24T10:34:53.823306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\ndef show_event(event_id=46528394, data=train, sensors=geometry, metadata=meta_train):\n    event = data[data.index == event_id]\n    event = event.merge(sensors, on=\"sensor_id\")\n    meta = dict(metadata[metadata[\"event_id\"]==event_id].iloc[0])\n    display(meta)\n    # event.head()\n    azimuth = meta[\"azimuth\"]\n    zenith = meta[\"zenith\"]\n    x_ = math.cos(azimuth) * math.sin(zenith)\n    y_ = math.sin(azimuth) * math.sin(zenith)\n    z_ = math.cos(zenith)\n    \n    auxiliaries = [False, True]\n    fig = make_subplots(\n        rows=2, specs=[[{'type': 'scene'}], [{'type': 'scene'}]],\n        subplot_titles=[f\"auxiliary={aux}\" for aux in auxiliaries],\n        vertical_spacing=0.05,\n    )\n    for i, aux in enumerate(auxiliaries):\n        evt_ = event[event['auxiliary'] == aux]\n        # sensors as background\n        fig.add_trace(\n            go.Scatter3d(\n                x=sensors['x'], y=sensors['y'], z=sensors['z'], \n                mode='markers', marker=dict(size=1, color=0), opacity=0.2\n            ), row=(i+1), col=1)\n        # sensors reading\n        fig.add_trace(\n            go.Scatter3d(\n                x=evt_['x'], y=evt_['y'], z=evt_['z'], opacity=0.8,\n                mode='markers', marker=dict(size=evt_['charge'] * 10, color=evt_['time'], colorscale='Viridis')\n            ), row=(i+1), col=1)\n        # direction from metad data\n        fig.add_trace(\n            go.Scatter3d(\n                x=[-x_ * 500, x_ * 500], y=[-y_ * 500, y_ * 500], z=[-z_ * 500, z_ * 500],\n                opacity=0.8, mode='lines', line=dict(color='red', width=3)\n            ), row=(i+1), col=1)\n    fig.update_layout(\n        height=800, width=600, showlegend=False,\n        title_text=f\"Event #{event_id} / azimuth={azimuth:0.3}; zenith={zenith:0.3}\\n\"\n        f\"-> x={x_:0.2}; y={y_:0.2}; z={z_:0.2}\",\n    )\n    # fig = px.scatter_3d(event, x='x', y='y', z='z', size=\"charge\", color=\"auxiliary\",\n    #     opacity=0.8, title=f\"Event #{event_id}\\n -> x={x_}; y={y_}; z={z_}\")\n    return fig\n\nshow_event().show()","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:35:06.355358Z","iopub.execute_input":"2023-01-24T10:35:06.357153Z","iopub.status.idle":"2023-01-24T10:35:06.897614Z","shell.execute_reply.started":"2023-01-24T10:35:06.357099Z","shell.execute_reply":"2023-01-24T10:35:06.896409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**This one is an interactive chart when you can slide over all event in the training batch**\n\nbut to get in you need to ioen the notebook in edior mode (so to copy as your own)","metadata":{}},{"cell_type":"code","source":"from ipywidgets import interact, IntSlider\n\ndef interactive_show(events):\n    interact(\n        lambda i: show_event(events[i]).show(),\n        i=IntSlider(min=0, max=len(events), step=1, value=len(events) // 2),\n    )\n\nevents = train.index.unique().tolist()\ninteractive_show(events)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T10:36:06.851516Z","iopub.execute_input":"2023-01-24T10:36:06.852083Z","iopub.status.idle":"2023-01-24T10:36:07.594953Z","shell.execute_reply.started":"2023-01-24T10:36:06.852042Z","shell.execute_reply":"2023-01-24T10:36:07.593587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Browse submission\n\nAn example submission with the correct columns and properly ordered event IDs. The sample submission is provided in the parquet format so it can be read quickly but your final submission must be a csv.","metadata":{}},{"cell_type":"code","source":"ssub = pd.read_parquet(os.path.join(PATH_DATASET, \"sample_submission.parquet\"))\nprint(f\"length: {len(ssub)}\")\nssub.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-23T10:46:45.108935Z","iopub.execute_input":"2023-01-23T10:46:45.109322Z","iopub.status.idle":"2023-01-23T10:46:45.131075Z","shell.execute_reply.started":"2023-01-23T10:46:45.109287Z","shell.execute_reply":"2023-01-23T10:46:45.129421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssub['zenith'] = meta_train['zenith'].median()\nssub['azimuth'] = meta_train['azimuth'].median()\nssub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T10:46:45.132652Z","iopub.execute_input":"2023-01-23T10:46:45.133031Z","iopub.status.idle":"2023-01-23T10:46:53.077304Z","shell.execute_reply.started":"2023-01-23T10:46:45.132997Z","shell.execute_reply":"2023-01-23T10:46:53.076204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-01-23T10:46:53.080554Z","iopub.execute_input":"2023-01-23T10:46:53.080995Z","iopub.status.idle":"2023-01-23T10:46:54.277724Z","shell.execute_reply.started":"2023-01-23T10:46:53.080948Z","shell.execute_reply":"2023-01-23T10:46:54.276095Z"},"trusted":true},"execution_count":null,"outputs":[]}]}