{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, we will conduct a small EDA with the plotting of events graphs and sensor geometry, as well as calculate the confidence interval for sensors accuracy.","metadata":{}},{"cell_type":"markdown","source":"Importing the necessary libraries","metadata":{"execution":{"iopub.status.busy":"2023-02-09T08:40:54.670371Z","iopub.execute_input":"2023-02-09T08:40:54.670854Z","iopub.status.idle":"2023-02-09T08:40:54.701726Z","shell.execute_reply.started":"2023-02-09T08:40:54.670752Z","shell.execute_reply":"2023-02-09T08:40:54.700610Z"}}},{"cell_type":"code","source":"import math\nimport os\nimport pandas as pd\nimport numpy as np\nfrom numpy.random import RandomState\n\nfrom scipy.stats import t\n\nimport matplotlib.pyplot as plt\n\nROOT_PATH = '/kaggle/input/icecube-neutrinos-in-deep-ice'\nSTATE = RandomState(12345)\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-09T10:08:26.227756Z","iopub.execute_input":"2023-02-09T10:08:26.228258Z","iopub.status.idle":"2023-02-09T10:08:27.177104Z","shell.execute_reply.started":"2023-02-09T10:08:26.228153Z","shell.execute_reply":"2023-02-09T10:08:27.176147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Meta","metadata":{}},{"cell_type":"code","source":"train_meta = pd.read_parquet(\n    ROOT_PATH + '/train_meta.parquet'\n)\nprint(f'number of rows: {len(train_meta)}\\n\\\n{train_meta.head()}')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:08:34.481104Z","iopub.execute_input":"2023-02-09T10:08:34.481936Z","iopub.status.idle":"2023-02-09T10:09:18.859938Z","shell.execute_reply.started":"2023-02-09T10:08:34.481884Z","shell.execute_reply":"2023-02-09T10:09:18.859121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plotting the last 100 events","metadata":{}},{"cell_type":"code","source":"def events_plotting(azimuth, zenith, title, count=100):\n    fig = plt.figure(figsize=(12, 10))\n    ax = fig.add_subplot(projection='3d')\n    azimuth = azimuth[:count]\n    zenith = zenith[:count]\n    \n    for az, ze in zip(azimuth, zenith):\n        x = math.cos(az) * math.sin(ze)\n        y = math.sin(az) * math.sin(ze)\n        z = math.cos(ze)\n        ax.scatter(x, y, z)\n        \n    plt.title(title)    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:09:18.861602Z","iopub.execute_input":"2023-02-09T10:09:18.862148Z","iopub.status.idle":"2023-02-09T10:09:18.868813Z","shell.execute_reply.started":"2023-02-09T10:09:18.862114Z","shell.execute_reply":"2023-02-09T10:09:18.867611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_plotting(train_meta['azimuth'], train_meta['zenith'],'Last 100 events', count=100)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:09:18.870461Z","iopub.execute_input":"2023-02-09T10:09:18.870799Z","iopub.status.idle":"2023-02-09T10:09:20.601718Z","shell.execute_reply.started":"2023-02-09T10:09:18.870769Z","shell.execute_reply":"2023-02-09T10:09:20.600433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sensor geometry","metadata":{}},{"cell_type":"code","source":"sensor_geometry = pd.read_csv(ROOT_PATH + '/sensor_geometry.csv')\nsensor_geometry.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:09:27.338779Z","iopub.execute_input":"2023-02-09T10:09:27.339248Z","iopub.status.idle":"2023-02-09T10:09:27.370507Z","shell.execute_reply.started":"2023-02-09T10:09:27.339207Z","shell.execute_reply":"2023-02-09T10:09:27.369099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plotting sensor geometry","metadata":{}},{"cell_type":"code","source":"plt.scatter(sensor_geometry.x, sensor_geometry.y)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:09:28.300738Z","iopub.execute_input":"2023-02-09T10:09:28.301240Z","iopub.status.idle":"2023-02-09T10:09:28.452661Z","shell.execute_reply.started":"2023-02-09T10:09:28.301199Z","shell.execute_reply":"2023-02-09T10:09:28.451409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sensor_plotting(x, y, z, title):\n    fig = plt.figure(figsize=(12, 10))\n    ax = fig.add_subplot(projection='3d')\n    \n    for x, y, z in zip(x, y, z):\n        ax.scatter(x, y, z)\n        \n    plt.title(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:09:28.914721Z","iopub.execute_input":"2023-02-09T10:09:28.915171Z","iopub.status.idle":"2023-02-09T10:09:28.921263Z","shell.execute_reply.started":"2023-02-09T10:09:28.915134Z","shell.execute_reply":"2023-02-09T10:09:28.920396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_plotting(sensor_geometry['x'], sensor_geometry['y'], sensor_geometry['z'], 'Sensor Geometry')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:09:29.904441Z","iopub.execute_input":"2023-02-09T10:09:29.905111Z","iopub.status.idle":"2023-02-09T10:10:36.544420Z","shell.execute_reply.started":"2023-02-09T10:09:29.905072Z","shell.execute_reply":"2023-02-09T10:10:36.543379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Batch 100","metadata":{}},{"cell_type":"code","source":"batch_100 = pd.read_parquet(ROOT_PATH + '/train/batch_100.parquet')\n\nevent_ids_100 = batch_100.index.unique()\n\nprint(f'{batch_100.head()}\\n\\n\\\nNum of all events: {len(batch_100)}\\n\\\nNum of unique events: {len(event_ids_100)}\\n')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:11:58.608186Z","iopub.execute_input":"2023-02-09T10:11:58.608726Z","iopub.status.idle":"2023-02-09T10:12:03.285970Z","shell.execute_reply.started":"2023-02-09T10:11:58.608680Z","shell.execute_reply":"2023-02-09T10:12:03.284661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_100['time'].hist(bins=100, figsize=(12, 5), range=(0, 30000))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:23:50.211344Z","iopub.execute_input":"2023-02-09T10:23:50.211793Z","iopub.status.idle":"2023-02-09T10:23:51.350911Z","shell.execute_reply.started":"2023-02-09T10:23:50.211758Z","shell.execute_reply":"2023-02-09T10:23:51.350075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_100['charge'].hist(bins=200, figsize=(12, 5), range=(0, 30))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:25:25.416874Z","iopub.execute_input":"2023-02-09T10:25:25.417321Z","iopub.status.idle":"2023-02-09T10:25:26.895949Z","shell.execute_reply.started":"2023-02-09T10:25:25.417288Z","shell.execute_reply":"2023-02-09T10:25:26.894432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confidence Interval of Mean Sensors Accuracy","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(round(batch_100.isna().mean() * 100)).style.background_gradient('coolwarm')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:12:03.288143Z","iopub.execute_input":"2023-02-09T10:12:03.288595Z","iopub.status.idle":"2023-02-09T10:12:03.596858Z","shell.execute_reply.started":"2023-02-09T10:12:03.288562Z","shell.execute_reply":"2023-02-09T10:12:03.595318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_true = batch_100[batch_100['auxiliary'] == 1]\nevents_false = batch_100[batch_100['auxiliary'] == 0]\nprint(f'Num of Auxiliary == True: {len(events_true)}\\n\\\nNum of Auxiliary == False: {len(events_false)}')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:12:03.598273Z","iopub.execute_input":"2023-02-09T10:12:03.598655Z","iopub.status.idle":"2023-02-09T10:12:05.018137Z","shell.execute_reply.started":"2023-02-09T10:12:03.598621Z","shell.execute_reply":"2023-02-09T10:12:05.017112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we know from the description of the data, the attribute 'auxiliary' - boolean value, that have True if pulse wasn't fully digitized and was more likely to originate from noise, or False if pulse was contributed to the trigger decision and was fully digitized. \n\nTherefore, it can be assumed that the lower the average of this feature, the higher the accuracy of the sensor.\n\nFor better clarity, we will convert this indicator into percentages.","metadata":{}},{"cell_type":"code","source":"sensors_accuracy = (\n    batch_100\n    .pivot_table(index='sensor_id',\n                 values='auxiliary',\n                 aggfunc='mean')\n    .sort_values(by='auxiliary', ascending=False)\n)\nsensors_accuracy = sensors_accuracy.rename(columns={'auxiliary':'accuracy'})\nsensors_accuracy['accuracy'] = abs((sensors_accuracy['accuracy'] * 100) - 100)\nsensors_accuracy\n","metadata":{"execution":{"iopub.status.busy":"2023-02-09T11:42:25.593354Z","iopub.execute_input":"2023-02-09T11:42:25.593746Z","iopub.status.idle":"2023-02-09T11:42:27.249327Z","shell.execute_reply.started":"2023-02-09T11:42:25.593715Z","shell.execute_reply":"2023-02-09T11:42:27.248031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensors_accuracy.hist(bins=100, figsize=(12, 5))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T11:42:38.387130Z","iopub.execute_input":"2023-02-09T11:42:38.388073Z","iopub.status.idle":"2023-02-09T11:42:38.778215Z","shell.execute_reply.started":"2023-02-09T11:42:38.388024Z","shell.execute_reply":"2023-02-09T11:42:38.776988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensors_accuracy.boxplot(figsize=(5, 10))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T11:42:42.520659Z","iopub.execute_input":"2023-02-09T11:42:42.521140Z","iopub.status.idle":"2023-02-09T11:42:42.721154Z","shell.execute_reply.started":"2023-02-09T11:42:42.521099Z","shell.execute_reply":"2023-02-09T11:42:42.719941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see from the graphs above, the median value of the exact sensor response is ~70%, but since we are considering data from only one batch, using the bootstrap technique we will find the confidence interval of the general population for all data batches.","metadata":{}},{"cell_type":"code","source":"values = []\n\nfor i in range(1000):\n    data_sampled = sensors_accuracy.sample(n=250, replace=True, random_state=STATE)\n    values.append(data_sampled['accuracy'].mean())\n\nvalues = pd.Series(values)\n\nconfidence_interval = (\n    t\n    .interval(.9, \n              len(values) - 1, \n              values.mean(), \n              values.sem())\n)\nupper = values.quantile(.95, interpolation='linear')\nlower = values.quantile(.05, interpolation='linear')\nprint(f'Confidence Interval of Sensors Accuracy: {confidence_interval}\\n\\\n95% Quantile: {upper}\\n\\\n5% Quantile: {lower}')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T11:50:22.885976Z","iopub.execute_input":"2023-02-09T11:50:22.887182Z","iopub.status.idle":"2023-02-09T11:50:23.241562Z","shell.execute_reply.started":"2023-02-09T11:50:22.887132Z","shell.execute_reply":"2023-02-09T11:50:23.240217Z"},"trusted":true},"execution_count":null,"outputs":[]}]}