{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA for IceCube\nLet's do an exploratory data analysis.","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-12T11:39:30.886265Z","iopub.execute_input":"2023-02-12T11:39:30.887057Z","iopub.status.idle":"2023-02-12T11:39:32.413138Z","shell.execute_reply.started":"2023-02-12T11:39:30.886626Z","shell.execute_reply":"2023-02-12T11:39:32.411784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating file path variables\ntrain_meta_path = '/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet'\n# sample_submission_path = '/kaggle/input/icecube-neutrinos-in-deep-ice/sample_submission.parquet'\n# test_meta_path = '/kaggle/input/icecube-neutrinos-in-deep-ice/test_meta.parquet'\nsensor_geometry_path = '/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv'\n\ntrain_path = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:39:32.414791Z","iopub.execute_input":"2023-02-12T11:39:32.415104Z","iopub.status.idle":"2023-02-12T11:39:32.420413Z","shell.execute_reply.started":"2023-02-12T11:39:32.415080Z","shell.execute_reply":"2023-02-12T11:39:32.419284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading and writing files to variables\ntrain_meta = pd.read_parquet(train_meta_path)\n# sample_submission = pd.read_parquet(sample_submission_path)\n# test_meta = pd.read_parquet(test_meta_path)\nsensor_geometry = pd.read_csv(sensor_geometry_path)\n\nbatch_1 = pd.read_parquet(f'{train_path}batch_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:39:32.422059Z","iopub.execute_input":"2023-02-12T11:39:32.422399Z","iopub.status.idle":"2023-02-12T11:40:20.163886Z","shell.execute_reply.started":"2023-02-12T11:39:32.422373Z","shell.execute_reply":"2023-02-12T11:40:20.162864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train_meta.parquet","metadata":{}},{"cell_type":"code","source":"train_meta","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:40:20.167423Z","iopub.execute_input":"2023-02-12T11:40:20.167772Z","iopub.status.idle":"2023-02-12T11:40:20.200227Z","shell.execute_reply.started":"2023-02-12T11:40:20.167733Z","shell.execute_reply":"2023-02-12T11:40:20.199293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:40:20.202004Z","iopub.execute_input":"2023-02-12T11:40:20.202377Z","iopub.status.idle":"2023-02-12T11:40:20.221835Z","shell.execute_reply.started":"2023-02-12T11:40:20.202342Z","shell.execute_reply":"2023-02-12T11:40:20.220663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:40:20.223056Z","iopub.execute_input":"2023-02-12T11:40:20.224198Z","iopub.status.idle":"2023-02-12T11:40:21.072300Z","shell.execute_reply.started":"2023-02-12T11:40:20.224149Z","shell.execute_reply":"2023-02-12T11:40:21.070902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.hist(bins=50, figsize=(20, 20))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:40:21.073830Z","iopub.execute_input":"2023-02-12T11:40:21.074188Z","iopub.status.idle":"2023-02-12T11:41:26.916448Z","shell.execute_reply.started":"2023-02-12T11:40:21.074159Z","shell.execute_reply":"2023-02-12T11:41:26.915039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The average value of the zenith angle is: {}'.format(train_meta['zenith'].mean()))","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:41:26.917890Z","iopub.execute_input":"2023-02-12T11:41:26.918264Z","iopub.status.idle":"2023-02-12T11:41:27.175097Z","shell.execute_reply.started":"2023-02-12T11:41:26.918223Z","shell.execute_reply":"2023-02-12T11:41:27.173328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (8,6))\nmatrix = np.triu(train_meta.corr())\nsns.heatmap(train_meta.corr(), cmap='Blues', annot=True, mask=matrix)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:41:27.176765Z","iopub.execute_input":"2023-02-12T11:41:27.177168Z","iopub.status.idle":"2023-02-12T11:41:50.757737Z","shell.execute_reply.started":"2023-02-12T11:41:27.177134Z","shell.execute_reply":"2023-02-12T11:41:50.756786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_sample = train_meta.sample(10**4)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:41:50.760959Z","iopub.execute_input":"2023-02-12T11:41:50.761253Z","iopub.status.idle":"2023-02-12T11:41:57.374741Z","shell.execute_reply.started":"2023-02-12T11:41:50.761229Z","shell.execute_reply":"2023-02-12T11:41:57.373426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_sample.plot(x='event_id', y='azimuth', kind='scatter', \n                       grid=True, figsize=(15,8), alpha=0.3)\ntrain_meta_sample.plot(x='event_id', y='azimuth', kind='hexbin', \n                       gridsize=20, figsize=(15,8))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:41:57.376007Z","iopub.execute_input":"2023-02-12T11:41:57.376316Z","iopub.status.idle":"2023-02-12T11:41:57.895520Z","shell.execute_reply.started":"2023-02-12T11:41:57.376290Z","shell.execute_reply":"2023-02-12T11:41:57.894252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_sample.plot(x='zenith', y='azimuth', kind='scatter', \n                       grid=True, figsize=(15,8), alpha=0.3)\ntrain_meta_sample.plot(x='zenith', y='azimuth', kind='hexbin', \n                       gridsize=20, figsize=(15,8))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:41:57.897169Z","iopub.execute_input":"2023-02-12T11:41:57.897730Z","iopub.status.idle":"2023-02-12T11:41:58.367671Z","shell.execute_reply.started":"2023-02-12T11:41:57.897697Z","shell.execute_reply":"2023-02-12T11:41:58.366530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_train_meta = ['first_pulse_index', 'last_pulse_index', 'azimuth', 'zenith']\n\nfor column in numeric_train_meta:\n    train_meta.boxplot(column=column, figsize=(6, 6))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T11:41:58.368977Z","iopub.execute_input":"2023-02-12T11:41:58.369210Z","iopub.status.idle":"2023-02-12T12:00:06.442804Z","shell.execute_reply.started":"2023-02-12T11:41:58.369188Z","shell.execute_reply":"2023-02-12T12:00:06.441657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Section Conclusion:**\n- no missing values;\n- the zenith angle has a dome-shaped distribution without tails, the symmetry axis is at a value slightly less than pi / 2, namely at 1.5344185980269713;\n- no obvious correlation between dataframe columns was found, except for the obvious linear relationship between event_id and batch_id and between first_pulse_index and last_pulse_index;\n- no outliers detected.","metadata":{}},{"cell_type":"markdown","source":"## sensor_geometry.csv","metadata":{}},{"cell_type":"code","source":"sensor_geometry","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:06.444292Z","iopub.execute_input":"2023-02-12T12:00:06.444760Z","iopub.status.idle":"2023-02-12T12:00:06.461924Z","shell.execute_reply.started":"2023-02-12T12:00:06.444731Z","shell.execute_reply":"2023-02-12T12:00:06.460514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_geometry.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:06.464134Z","iopub.execute_input":"2023-02-12T12:00:06.464569Z","iopub.status.idle":"2023-02-12T12:00:06.487191Z","shell.execute_reply.started":"2023-02-12T12:00:06.464536Z","shell.execute_reply":"2023-02-12T12:00:06.485819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_geometry.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:06.489292Z","iopub.execute_input":"2023-02-12T12:00:06.489723Z","iopub.status.idle":"2023-02-12T12:00:06.507295Z","shell.execute_reply.started":"2023-02-12T12:00:06.489695Z","shell.execute_reply":"2023-02-12T12:00:06.505327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_3d(sensor_geometry, x='x', y='y', z='z', \n                    color='z', opacity=0.5)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:06.508865Z","iopub.execute_input":"2023-02-12T12:00:06.509243Z","iopub.status.idle":"2023-02-12T12:00:08.434179Z","shell.execute_reply.started":"2023-02-12T12:00:06.509212Z","shell.execute_reply":"2023-02-12T12:00:08.432950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Section Conclusion:**\n- a 3D model of the location of the detectors was built, due to which it can be seen that there are additional lines of detectors in the middle of the array of detectors.","metadata":{}},{"cell_type":"markdown","source":"## train/batch_[n].parquet","metadata":{}},{"cell_type":"code","source":"batch_1 = batch_1.reset_index()\ndisplay(batch_1)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:08.436276Z","iopub.execute_input":"2023-02-12T12:00:08.436701Z","iopub.status.idle":"2023-02-12T12:00:09.027339Z","shell.execute_reply.started":"2023-02-12T12:00:08.436668Z","shell.execute_reply":"2023-02-12T12:00:09.026036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_1.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:09.028871Z","iopub.execute_input":"2023-02-12T12:00:09.029591Z","iopub.status.idle":"2023-02-12T12:00:09.040347Z","shell.execute_reply.started":"2023-02-12T12:00:09.029558Z","shell.execute_reply":"2023-02-12T12:00:09.039340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_1.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:09.041987Z","iopub.execute_input":"2023-02-12T12:00:09.042484Z","iopub.status.idle":"2023-02-12T12:00:09.221172Z","shell.execute_reply.started":"2023-02-12T12:00:09.042450Z","shell.execute_reply":"2023-02-12T12:00:09.219665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The number of full duplicates is: {}'.format(batch_1.duplicated().sum()))","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:09.224039Z","iopub.execute_input":"2023-02-12T12:00:09.224479Z","iopub.status.idle":"2023-02-12T12:00:19.479467Z","shell.execute_reply.started":"2023-02-12T12:00:09.224446Z","shell.execute_reply":"2023-02-12T12:00:19.478447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_1.hist(bins=50, figsize=(20, 20))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:19.480892Z","iopub.execute_input":"2023-02-12T12:00:19.481205Z","iopub.status.idle":"2023-02-12T12:00:22.670263Z","shell.execute_reply.started":"2023-02-12T12:00:19.481178Z","shell.execute_reply":"2023-02-12T12:00:22.669169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_1['time'].hist(bins=100, figsize=(8, 8))\nplt.xlim(5000,20000)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:22.671468Z","iopub.execute_input":"2023-02-12T12:00:22.672121Z","iopub.status.idle":"2023-02-12T12:00:23.378132Z","shell.execute_reply.started":"2023-02-12T12:00:22.672093Z","shell.execute_reply":"2023-02-12T12:00:23.377249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_counts = batch_1['sensor_id'].value_counts()\n\nsns.barplot(x=sensor_counts.index, y=sensor_counts.values)\nplt.xlabel('Sensor id')\nplt.ylabel('Frequency of pulses')\nplt.title('Bar plot of the frequency of pulses recorded by each sensor')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:00:23.380007Z","iopub.execute_input":"2023-02-12T12:00:23.380624Z","iopub.status.idle":"2023-02-12T12:01:20.548311Z","shell.execute_reply.started":"2023-02-12T12:00:23.380589Z","shell.execute_reply":"2023-02-12T12:01:20.547003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (8,6))\nmatrix = np.triu(batch_1.corr())\nsns.heatmap(batch_1.corr(), cmap='Blues', annot=True, mask=matrix)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:01:20.550067Z","iopub.execute_input":"2023-02-12T12:01:20.550409Z","iopub.status.idle":"2023-02-12T12:01:25.048692Z","shell.execute_reply.started":"2023-02-12T12:01:20.550380Z","shell.execute_reply":"2023-02-12T12:01:25.047655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_1_sample = batch_1.sample(10**4)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:01:25.050272Z","iopub.execute_input":"2023-02-12T12:01:25.050646Z","iopub.status.idle":"2023-02-12T12:01:26.638477Z","shell.execute_reply.started":"2023-02-12T12:01:25.050614Z","shell.execute_reply":"2023-02-12T12:01:26.637103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ng = sns.pairplot(batch_1_sample, hue='auxiliary', diag_kind=\"kde\", corner=True)\ng.map_lower(sns.kdeplot, levels=4)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:01:26.640071Z","iopub.execute_input":"2023-02-12T12:01:26.640451Z","iopub.status.idle":"2023-02-12T12:02:11.759701Z","shell.execute_reply.started":"2023-02-12T12:01:26.640414Z","shell.execute_reply":"2023-02-12T12:02:11.758618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_1_sample.loc[batch_1_sample['auxiliary'] == True].plot(x='sensor_id', y='charge', kind='scatter', grid=True, figsize=(15,8), alpha=0.3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:02:11.764034Z","iopub.execute_input":"2023-02-12T12:02:11.765169Z","iopub.status.idle":"2023-02-12T12:02:11.954566Z","shell.execute_reply.started":"2023-02-12T12:02:11.765135Z","shell.execute_reply":"2023-02-12T12:02:11.953478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_batch = ['time', 'charge']\n\nfor column in numeric_batch:\n    batch_1.boxplot(column=column, figsize=(6, 6))\n    plt.show()\n    q3 = batch_1[column].quantile(0.75)\n    q1 = batch_1[column].quantile(0.25)\n    max_lim = q3 + 1.5 * (q3 - q1)\n    min_lim = q1 - 1.5 * (q3 - q1)\n    percent = batch_1.loc[(batch_1[column] > max_lim) | (batch_1[column] < min_lim)][column].count()/len(batch_1[column])*100\n    print(f'The maximum sample value is: {max_lim}')\n    print('The percentage of outliers in the column `{}` is: {}%'.format(column, percent))","metadata":{"execution":{"iopub.status.busy":"2023-02-12T12:02:11.956895Z","iopub.execute_input":"2023-02-12T12:02:11.957792Z","iopub.status.idle":"2023-02-12T12:04:16.027960Z","shell.execute_reply.started":"2023-02-12T12:02:11.957756Z","shell.execute_reply":"2023-02-12T12:04:16.026136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Section Conclusion:**\n- no missing values;\n- there is an extremely small number of complete duplicates (0.0009%);\n- the largest number of detected particles falls on sensors with id numbers greater than 4500;\n- the most common pulse time is in the range (9500-17000) ns;\n- the dependence of the quality of the digitization of the pulse on the charge is traced: all poorly digitized pulses of low quality, which probably arose due to noise, correspond to small (0-4 p.e.) charges;\n- the maximum sample value of the 'time' column is 18941 ns, and the percentage of outliers is 8.31242199415865%;\n- the maximum sample value of the 'charge' column is 3.274999976158142, and the percentage of outliers is 15.125402776056513%.","metadata":{}}]}