{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport pyarrow.parquet as pq\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:44.801218Z","iopub.execute_input":"2023-01-18T15:57:44.802357Z","iopub.status.idle":"2023-01-18T15:57:44.810111Z","shell.execute_reply.started":"2023-01-18T15:57:44.802314Z","shell.execute_reply":"2023-01-18T15:57:44.808608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic Data Loading and Exploration\n\n## Geometry\n\nThe file geometry holds the positions in x,y and z of each IceCube sensor","metadata":{}},{"cell_type":"code","source":"geometry = pd.read_csv('/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv')\ngeometry","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:44.811532Z","iopub.execute_input":"2023-01-18T15:57:44.811861Z","iopub.status.idle":"2023-01-18T15:57:44.837695Z","shell.execute_reply.started":"2023-01-18T15:57:44.811833Z","shell.execute_reply":"2023-01-18T15:57:44.836867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following plot shows the 3d position of each IceCube sensor. What can be seen are 86 vertical strings with 60 sensors on each. The strings in the middle, lower part with the denser sensor spacing are known as the \"DeepCore\" region.","metadata":{}},{"cell_type":"code","source":"ax = plt.figure(figsize=(8,8)).add_subplot(projection='3d')\nax.scatter(geometry.x, geometry.y, geometry.z, s=1);","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:44.838870Z","iopub.execute_input":"2023-01-18T15:57:44.839352Z","iopub.status.idle":"2023-01-18T15:57:45.259617Z","shell.execute_reply.started":"2023-01-18T15:57:44.839323Z","shell.execute_reply":"2023-01-18T15:57:45.258499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train metadata\n\nThe metadata file holds information per event, and points to the correct location to load the event's features (variable length series).\nWe load in the tarin metedata in batches to save memory, and only take the first batch here:","metadata":{}},{"cell_type":"code","source":"train_meta = pq.ParquetFile('/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet')\nit = train_meta.iter_batches()\ntrain = next(it).to_pandas()\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:45.261051Z","iopub.execute_input":"2023-01-18T15:57:45.261742Z","iopub.status.idle":"2023-01-18T15:57:45.907454Z","shell.execute_reply.started":"2023-01-18T15:57:45.261702Z","shell.execute_reply":"2023-01-18T15:57:45.906343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The distribution of the numbers of features per event, and the two regression target variables are shown next:","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20,6))\n\nax[0].hist(train.last_pulse_index - train.first_pulse_index, bins=np.logspace(1,4,20))\nax[0].set_xscale('log')\nax[0].set_xlabel('# pulses')\nax[1].hist(train.azimuth, bins=np.linspace(0, 2*np.pi, 20))\nax[1].set_xlabel('Azimuth (rad)')\nax[2].hist(np.cos(train.zenith), bins=np.linspace(0, 1, 20))\nax[2].set_xlabel('cos(zenith)');","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:45.910807Z","iopub.execute_input":"2023-01-18T15:57:45.911137Z","iopub.status.idle":"2023-01-18T15:57:46.783828Z","shell.execute_reply.started":"2023-01-18T15:57:45.911109Z","shell.execute_reply":"2023-01-18T15:57:46.782612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features\n\nNow we load the features, for simplicity here we only load batch number 1, since all the events in our small sample point to that batch:","metadata":{}},{"cell_type":"code","source":"assert np.all(train.batch_id == 1)\ntrain_features = pd.read_parquet('/kaggle/input/icecube-neutrinos-in-deep-ice/train/batch_1.parquet')\ntrain_features","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:46.785208Z","iopub.execute_input":"2023-01-18T15:57:46.785994Z","iopub.status.idle":"2023-01-18T15:57:48.249783Z","shell.execute_reply.started":"2023-01-18T15:57:46.785954Z","shell.execute_reply":"2023-01-18T15:57:48.248909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can take a single event at index `i` and look at it in more detail","metadata":{}},{"cell_type":"code","source":"i = 99\nevent = train.iloc[i]\nfeatures = train_features.iloc[train.first_pulse_index[i] : train.last_pulse_index[i]+1]\npos = geometry.iloc[features.sensor_id]","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:48.251078Z","iopub.execute_input":"2023-01-18T15:57:48.252286Z","iopub.status.idle":"2023-01-18T15:57:48.259844Z","shell.execute_reply.started":"2023-01-18T15:57:48.252247Z","shell.execute_reply":"2023-01-18T15:57:48.258763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's plot the event:","metadata":{}},{"cell_type":"code","source":"from matplotlib.colors import ListedColormap, LinearSegmentedColormap\n\nfig = plt.figure(figsize=(15,8))\nax1 = plt.subplot(1, 2, 1, projection='3d')\nax2 = plt.subplot(1, 2, 2, projection='3d')\n\nplt.tight_layout()\nplt.subplots_adjust(wspace=0.1)\nplt.subplots_adjust(hspace=0.)\ncmap = ListedColormap(np.array(plt.cm.turbo.colors)[np.concatenate((np.zeros(150), np.arange(256), np.ones(200)*255)).astype(int)])\n\nfor ax in [ax1, ax2]:\n    ax.set_xlabel('x')\n    ax.set_ylabel('y')\n    ax.set_zlabel('z')\n    ax.view_init(azim=-30, elev=30)\n    ax.scatter(geometry.x, geometry.y, geometry.z, s=0.3, color='k', alpha=0.2)\n    ax.grid(False)\n\n# Primary pulses\nmask = ~features.auxiliary\nim = ax1.scatter(pos.x.to_numpy()[mask],\n                 pos.y.to_numpy()[mask],\n                 pos.z.to_numpy()[mask],\n                 s=features.charge[mask]*100,\n                 c=features.time[mask],\n                 cmap=cmap, \n                 alpha=0.7,\n                 vmin=np.min(features.time),\n                 vmax=np.max(features.time))\n\n# Auxiliary pulses\nim = ax2.scatter(pos.x.to_numpy()[~mask],\n                 pos.y.to_numpy()[~mask],\n                 pos.z.to_numpy()[~mask],\n                 s=features.charge[~mask]*100,\n                 c=features.time[~mask],\n                 cmap=cmap,\n                 alpha=0.7,\n                 vmin=np.min(features.time),\n                 vmax=np.max(features.time))\n\nfig.colorbar(im, ax=[ax1, ax2], shrink=0.5, label='time',)\nax1.set_title('auxiliary == False', y=1.05)\nax2.set_title('auxiliary == True', y=1.05)\nfig.suptitle('Example event from the dataset:\\n(azimuth = %.2f rad, zenith = %.2f rad)'%(event.azimuth, event.zenith));","metadata":{"execution":{"iopub.status.busy":"2023-01-18T15:57:48.261759Z","iopub.execute_input":"2023-01-18T15:57:48.262533Z","iopub.status.idle":"2023-01-18T15:57:49.192806Z","shell.execute_reply.started":"2023-01-18T15:57:48.262497Z","shell.execute_reply":"2023-01-18T15:57:49.191592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}