{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style= \"border:4px solid #5c5c5c; padding: 6px; background: #A8B6FF; text-align:center\">\n\n# IceCube - Neutrinos in Deep Ice","metadata":{}},{"cell_type":"markdown","source":"#### **import package**","metadata":{"execution":{"iopub.status.busy":"2023-02-10T19:41:47.376227Z","iopub.execute_input":"2023-02-10T19:41:47.376589Z","iopub.status.idle":"2023-02-10T19:41:47.385380Z","shell.execute_reply.started":"2023-02-10T19:41:47.376541Z","shell.execute_reply":"2023-02-10T19:41:47.383165Z"}}},{"cell_type":"code","source":"from IPython.display import HTML\nimport os\nimport math\n\nimport numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-15T15:58:49.806551Z","iopub.execute_input":"2023-02-15T15:58:49.807066Z","iopub.status.idle":"2023-02-15T15:58:51.185442Z","shell.execute_reply.started":"2023-02-15T15:58:49.806947Z","shell.execute_reply":"2023-02-15T15:58:51.184331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **constants**","metadata":{}},{"cell_type":"code","source":"DATA_SAMPLE_PATH = '/kaggle/input/icecube-neutrinos-in-deep-ice/sample_submission.parquet'\nDATA_SENSOR_PATH = '/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv'\nDATA_TEST_META_PATH = '/kaggle/input/icecube-neutrinos-in-deep-ice/test_meta.parquet'\nDATA_TRAIN_META_PATH = '/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet'\nDATA_TRAIN_BATCH_PATH = \"/kaggle/input/icecube-neutrinos-in-deep-ice/train/\"","metadata":{"execution":{"iopub.status.busy":"2023-02-15T15:58:51.187077Z","iopub.execute_input":"2023-02-15T15:58:51.187404Z","iopub.status.idle":"2023-02-15T15:58:51.192037Z","shell.execute_reply.started":"2023-02-15T15:58:51.187375Z","shell.execute_reply":"2023-02-15T15:58:51.191179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **import styles**","metadata":{}},{"cell_type":"code","source":"style = \"\"\"<style>\ndiv.blue_line {height:35px;border:1px solid #A8B6FF; border-left:10px solid #A8B6FF; \n                padding-left:5px;font-size: 160% ; font-family: monospace}\n                \ndiv.red_line {height:35px; border-left:6px solid #DC143C; \n                padding-left:7px;font-size: 130%}\n</style>\"\"\"\n\nHTML(style)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T15:58:51.193392Z","iopub.execute_input":"2023-02-15T15:58:51.193970Z","iopub.status.idle":"2023-02-15T15:58:51.207543Z","shell.execute_reply.started":"2023-02-15T15:58:51.193923Z","shell.execute_reply":"2023-02-15T15:58:51.206703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"blue_line\">\n    \n*Project description*","metadata":{}},{"cell_type":"markdown","source":"Цель этого конкурса — определить, откуда пришли нейтрино \\\nОписание проекта - будет!!!","metadata":{}},{"cell_type":"markdown","source":"---\n`sensor_geometry.csv` - the x, y, and z positions for each of the 5160 IceCube sensors\n - ***sensor_id*** -  corresponds to the sensor_idx feature of pulses\n - ***x, y, z*** -  coordinates are in units of meters\n \n`sample_submission.parquet` - пример события \\\n`data_train_meta` - файл с данными для обучения модели\n - ***batch_id*** - the ID of the batch the event was placed into\n - ***event_id*** - the event ID\n - ***first_pulse_index*** - index of the first row in the features dataframe belonging to this event\n - ***last_pulse_index*** - index of the last row in the features dataframe belonging to this event\n \n`train/batch_[n].parquet` - batch, содержащие показатели разных датчиков\n- ***event_id*** - the event ID. Saved as the index column in parquet\n- ***time*** - the time of the pulse in nanoseconds in the current event time window\n- ***sensor_id*** - the ID of which of the 5160 IceCube photomultiplier sensors recorded this pulse.\n- ***charge*** - an estimate of the amount of light in the pulse, in units of photoelectrons (p.e.)\n- ***auxiliary*** - if True, the pulse was not fully digitized, is of lower quality, and was more likely to originate from noise\n\n\n ---","metadata":{}},{"cell_type":"markdown","source":"<div class=\"blue_line\">\n    \n*Import Data*","metadata":{"execution":{"iopub.status.busy":"2023-02-11T05:43:44.983248Z","iopub.execute_input":"2023-02-11T05:43:44.983634Z","iopub.status.idle":"2023-02-11T05:43:44.991289Z","shell.execute_reply.started":"2023-02-11T05:43:44.983602Z","shell.execute_reply":"2023-02-11T05:43:44.989807Z"}}},{"cell_type":"code","source":"data_sample = pd.read_parquet(DATA_SAMPLE_PATH)\ndata_sensors = pd.read_csv(DATA_SENSOR_PATH)\ndata_train_meta = pd.read_parquet(DATA_TRAIN_META_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T15:58:51.209830Z","iopub.execute_input":"2023-02-15T15:58:51.210769Z","iopub.status.idle":"2023-02-15T15:59:34.546485Z","shell.execute_reply.started":"2023-02-15T15:58:51.210735Z","shell.execute_reply":"2023-02-15T15:59:34.545286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking the number of batch","metadata":{}},{"cell_type":"code","source":"data_train_meta.batch_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T15:59:34.548088Z","iopub.execute_input":"2023-02-15T15:59:34.548536Z","iopub.status.idle":"2023-02-15T15:59:35.437122Z","shell.execute_reply.started":"2023-02-15T15:59:34.548491Z","shell.execute_reply":"2023-02-15T15:59:35.435900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"red_line\">\n    \n*Import batch for EDA*","metadata":{"execution":{"iopub.status.busy":"2023-02-11T08:50:21.373618Z","iopub.execute_input":"2023-02-11T08:50:21.374389Z","iopub.status.idle":"2023-02-11T08:50:21.382540Z","shell.execute_reply.started":"2023-02-11T08:50:21.374332Z","shell.execute_reply":"2023-02-11T08:50:21.380989Z"}}},{"cell_type":"code","source":"def get_batch_data(num_batch):\n    \"\"\"import one batch for analytics\"\"\"\n    _data_train_meta = data_train_meta[data_train_meta['batch_id'] == num_batch]\n    _data_train_batch = pd.read_parquet(\n                        DATA_TRAIN_BATCH_PATH + f\"batch_{num_batch}.parquet\")\n    return _data_train_meta, _data_train_batch","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:14:04.380712Z","iopub.execute_input":"2023-02-15T16:14:04.381268Z","iopub.status.idle":"2023-02-15T16:14:04.388732Z","shell.execute_reply.started":"2023-02-15T16:14:04.381215Z","shell.execute_reply":"2023-02-15T16:14:04.387017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train_meta_batch, data_train_batch = get_batch_data(30)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:22:19.578215Z","iopub.execute_input":"2023-02-15T16:22:19.578643Z","iopub.status.idle":"2023-02-15T16:22:24.196054Z","shell.execute_reply.started":"2023-02-15T16:22:19.578608Z","shell.execute_reply":"2023-02-15T16:22:24.194998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(data_train_meta.info())\nprint(\"-\" *80)\ndisplay(data_train_batch.info())","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:22:25.945575Z","iopub.execute_input":"2023-02-15T16:22:25.946795Z","iopub.status.idle":"2023-02-15T16:22:25.971714Z","shell.execute_reply.started":"2023-02-15T16:22:25.946740Z","shell.execute_reply":"2023-02-15T16:22:25.970799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"blue_line\">\n    \n*Sensors*","metadata":{"execution":{"iopub.status.busy":"2023-02-11T09:01:45.810520Z","iopub.execute_input":"2023-02-11T09:01:45.811033Z","iopub.status.idle":"2023-02-11T09:01:45.819993Z","shell.execute_reply.started":"2023-02-11T09:01:45.810995Z","shell.execute_reply":"2023-02-11T09:01:45.817920Z"}}},{"cell_type":"code","source":"data_sensors","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:22:27.350087Z","iopub.execute_input":"2023-02-15T16:22:27.350796Z","iopub.status.idle":"2023-02-15T16:22:27.366909Z","shell.execute_reply.started":"2023-02-15T16:22:27.350755Z","shell.execute_reply":"2023-02-15T16:22:27.365316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_3d(data_sensors, x='x', y='y', z='z',\n                   size=[3]*len(data_sensors),opacity = 1)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:22:27.970373Z","iopub.execute_input":"2023-02-15T16:22:27.970759Z","iopub.status.idle":"2023-02-15T16:22:28.036896Z","shell.execute_reply.started":"2023-02-15T16:22:27.970728Z","shell.execute_reply":"2023-02-15T16:22:28.035955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Под датчикам, мы видим, что они также распределены неравномерно и в центре их концентрация выше \\\nВозможно в дальнейшем как вариант стоит рассмотреть классификацию датчиков","metadata":{}},{"cell_type":"markdown","source":"<div class=\"blue_line\">\n    \n*Event*","metadata":{"execution":{"iopub.status.busy":"2023-02-11T09:45:59.236280Z","iopub.execute_input":"2023-02-11T09:45:59.236842Z","iopub.status.idle":"2023-02-11T09:45:59.244946Z","shell.execute_reply.started":"2023-02-11T09:45:59.236801Z","shell.execute_reply":"2023-02-11T09:45:59.243043Z"}}},{"cell_type":"code","source":"def get_event_data(data_train_meta_batch, data_train_batch,\n                  num_event):\n    _data_train_meta_batch = data_train_meta_batch[data_train_meta_batch.event_id == num_event]\n    _data_train_batch = data_train_batch[data_train_batch.index == num_event]\n\n    \n    _data = _data_train_batch.merge(right = data_sensors, on = 'sensor_id')\n    return _data , _data_train_meta_batch[['azimuth','zenith']]\n\ndef get_coord(azimuth, zenith):\n    x = math.cos(azimuth) * math.sin(zenith)\n    y = math.sin(azimuth) * math.sin(zenith)\n    z = math.cos(zenith)\n    return x,y,z","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:22:31.062071Z","iopub.execute_input":"2023-02-15T16:22:31.062463Z","iopub.status.idle":"2023-02-15T16:22:31.069567Z","shell.execute_reply.started":"2023-02-15T16:22:31.062430Z","shell.execute_reply":"2023-02-15T16:22:31.068620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train_meta_batch.event_id.unique()[:20]","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:22:31.838643Z","iopub.execute_input":"2023-02-15T16:22:31.839118Z","iopub.status.idle":"2023-02-15T16:22:31.930268Z","shell.execute_reply.started":"2023-02-15T16:22:31.839074Z","shell.execute_reply":"2023-02-15T16:22:31.929097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_num = 94363850\ndata_event = get_event_data(data_train_meta_batch, data_train_batch , event_num)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:35:31.246468Z","iopub.execute_input":"2023-02-15T17:35:31.247264Z","iopub.status.idle":"2023-02-15T17:35:31.306486Z","shell.execute_reply.started":"2023-02-15T17:35:31.247223Z","shell.execute_reply":"2023-02-15T17:35:31.305363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def line_data(data):\n    df_line = pd.DataFrame(data = {\n        'x':0,\n        'y':0,\n        'z':0\n    } , index = [0])\n\n    df_line.loc[1] = get_coord(data['azimuth'],data['zenith'])\n    df_line.loc[2] = df_line.loc[1] * 600\n    df_line.loc[3] = df_line.loc[1] * - 600\n    return df_line","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:35:31.446084Z","iopub.execute_input":"2023-02-15T17:35:31.446736Z","iopub.status.idle":"2023-02-15T17:35:31.452366Z","shell.execute_reply.started":"2023-02-15T17:35:31.446699Z","shell.execute_reply":"2023-02-15T17:35:31.451447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_event[0][\"time_scale\"] = data_event[0].time/data_event[0].time.mean()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:35:31.781666Z","iopub.execute_input":"2023-02-15T17:35:31.782141Z","iopub.status.idle":"2023-02-15T17:35:31.789630Z","shell.execute_reply.started":"2023-02-15T17:35:31.782099Z","shell.execute_reply":"2023-02-15T17:35:31.788251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"red_line\">\n    \n*Event with auxiliary*","metadata":{}},{"cell_type":"code","source":"fig = px.scatter_3d(data_event[0], x=\"x\", y=\"y\", z=\"z\", color=\"time_scale\",size = 'charge',\n           hover_name=\"sensor_id\",range_x=[-600,600], range_y=[-600,600], range_z=[-600,600])\n\n\n\nfig.add_trace(\n             go.Scatter3d(x = line_data(data_event[1])[\"x\"], y=line_data(data_event[1])[\"y\"], z=line_data(data_event[1])[\"z\"],mode='lines', \n                          line=dict(color='red', width=5)))","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:35:32.549621Z","iopub.execute_input":"2023-02-15T17:35:32.550322Z","iopub.status.idle":"2023-02-15T17:35:32.647491Z","shell.execute_reply.started":"2023-02-15T17:35:32.550275Z","shell.execute_reply":"2023-02-15T17:35:32.646365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"red_line\">\n    \n*Event with not auxiliary*","metadata":{"execution":{"iopub.status.busy":"2023-02-11T12:05:42.180762Z","iopub.execute_input":"2023-02-11T12:05:42.181136Z","iopub.status.idle":"2023-02-11T12:05:42.189487Z","shell.execute_reply.started":"2023-02-11T12:05:42.181106Z","shell.execute_reply":"2023-02-11T12:05:42.187503Z"}}},{"cell_type":"code","source":"not_work_sensors = set(data_sensors.sensor_id) -  set(data_event[0].sensor_id)\ndata_sensors_ = data_sensors.query('sensor_id in @not_work_sensors')","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:35:37.865715Z","iopub.execute_input":"2023-02-15T17:35:37.866544Z","iopub.status.idle":"2023-02-15T17:35:37.877558Z","shell.execute_reply.started":"2023-02-15T17:35:37.866501Z","shell.execute_reply":"2023-02-15T17:35:37.876537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_3d(data_event[0], x=\"x\", y=\"y\", z=\"z\", color=\"auxiliary\",size = 'charge',\n           hover_name=\"sensor_id\",range_x=[-600,600], range_y=[-600,600], range_z=[-600,600])\n\nfig.add_trace(go.Scatter3d(x= data_sensors_['x'], \n                           y= data_sensors_['y'], \n                           z= data_sensors_['z'],\n                          opacity = 0.05))\n\nfig.add_trace(\n             go.Scatter3d(x = line_data(data_event[1])[\"x\"], y=line_data(data_event[1])[\"y\"], z=line_data(data_event[1])[\"z\"],mode='lines', \n                          line=dict(color='red', width=5)))","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:35:38.109851Z","iopub.execute_input":"2023-02-15T17:35:38.110552Z","iopub.status.idle":"2023-02-15T17:35:38.209805Z","shell.execute_reply.started":"2023-02-15T17:35:38.110512Z","shell.execute_reply":"2023-02-15T17:35:38.208400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Попробуем посмотреть, какие сенсоры чаще срабатывали без шумов","metadata":{}},{"cell_type":"code","source":"data_train_batch_with_cord = data_train_batch.merge(right = data_sensors, on = 'sensor_id')","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:40:20.487983Z","iopub.execute_input":"2023-02-15T17:40:20.488490Z","iopub.status.idle":"2023-02-15T17:40:26.921304Z","shell.execute_reply.started":"2023-02-15T17:40:20.488448Z","shell.execute_reply":"2023-02-15T17:40:26.919995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train_batch_with_cord = data_train_batch_with_cord[data_train_batch_with_cord.auxiliary == False]","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:40:27.835531Z","iopub.execute_input":"2023-02-15T17:40:27.835941Z","iopub.status.idle":"2023-02-15T17:40:30.048581Z","shell.execute_reply.started":"2023-02-15T17:40:27.835895Z","shell.execute_reply":"2023-02-15T17:40:30.047366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train_batch_with_cord.groupby('sensor_id')['sensor_id'].count().hist(bins = 100)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:40:49.100890Z","iopub.execute_input":"2023-02-15T17:40:49.101337Z","iopub.status.idle":"2023-02-15T17:40:50.024327Z","shell.execute_reply.started":"2023-02-15T17:40:49.101295Z","shell.execute_reply":"2023-02-15T17:40:50.023130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_sensors","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:40:51.988273Z","iopub.execute_input":"2023-02-15T17:40:51.989028Z","iopub.status.idle":"2023-02-15T17:40:52.005978Z","shell.execute_reply.started":"2023-02-15T17:40:51.988983Z","shell.execute_reply":"2023-02-15T17:40:52.004632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_3d(data_sensors.query('sensor_id<1500 or sensor_id> 2000'), x='x', y='y', z='z',\n                   size=[1]*len(data_sensors.query('sensor_id<1500 or sensor_id> 2000')),opacity = 0.2)\n\n\nfig.add_trace(go.Scatter3d(x= data_sensors.query('1500 < sensor_id< 2000')['x'], \n                           y= data_sensors.query('1500 < sensor_id< 2000')['y'], \n                           z= data_sensors.query('1500 < sensor_id< 2000')['z']))\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-02-15T17:40:54.875773Z","iopub.execute_input":"2023-02-15T17:40:54.876226Z","iopub.status.idle":"2023-02-15T17:40:54.959085Z","shell.execute_reply.started":"2023-02-15T17:40:54.876189Z","shell.execute_reply":"2023-02-15T17:40:54.957817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Попробуем рассмотреть под каким углом происходило несколько событий","metadata":{}},{"cell_type":"code","source":"fig = px.scatter_3d(data_sensors, x='x', y='y', z='z',\n                   size=[3]*len(data_sensors),opacity = 0.001)\n\nfor index in data_train_meta_batch[:40].index:\n    fig.add_trace(\n             go.Scatter3d(x = line_data(data_train_meta_batch.loc[index])[\"x\"], \n                          y=line_data(data_train_meta_batch.loc[index])[\"y\"], \n                          z=line_data(data_train_meta_batch.loc[index])[\"z\"],\n                          mode='lines', \n                          line=dict(color='blue', width=5)))\n    \n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T16:36:09.481739Z","iopub.execute_input":"2023-02-15T16:36:09.482488Z","iopub.status.idle":"2023-02-15T16:36:10.345468Z","shell.execute_reply.started":"2023-02-15T16:36:09.482439Z","shell.execute_reply":"2023-02-15T16:36:10.344563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}