{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-03T03:07:20.398423Z","iopub.execute_input":"2023-02-03T03:07:20.398754Z","iopub.status.idle":"2023-02-03T03:07:20.871146Z","shell.execute_reply.started":"2023-02-03T03:07:20.398727Z","shell.execute_reply":"2023-02-03T03:07:20.870213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor = pd.read_csv('/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv')\ndf = pd.read_csv(\"/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv\")\ntrain_meta = pq.ParquetFile('/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet')\nit = train_meta.iter_batches()\ntrain_meta = next(it).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:07:20.978698Z","iopub.execute_input":"2023-02-03T03:07:20.979082Z","iopub.status.idle":"2023-02-03T03:07:38.536214Z","shell.execute_reply.started":"2023-02-03T03:07:20.979052Z","shell.execute_reply":"2023-02-03T03:07:38.534866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inspection of sensors with annotated data : Auxiliary.\n- Each signals yielded from sensors provide \"Auxiliary\" information, which represents the quality of the signal.\n\n- This inspection aims to detect sensors with highly interrupted by noise, across all possible train bathces.","metadata":{}},{"cell_type":"code","source":"def get_batch(batchfile):\n    batch1 = pq.ParquetFile(batchfile)\n    it = batch1.iter_batches()\n    batch1 = next(it).to_pandas()\n    return(batch1)","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:07:38.537868Z","iopub.execute_input":"2023-02-03T03:07:38.538177Z","iopub.status.idle":"2023-02-03T03:07:38.543359Z","shell.execute_reply.started":"2023-02-03T03:07:38.538149Z","shell.execute_reply":"2023-02-03T03:07:38.542392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Appending parquet dataset","metadata":{}},{"cell_type":"code","source":"path_batch = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\nsensor_info = [get_batch(path_batch+'batch_'+str(i+1)+'.parquet') for i in tqdm(range(len(os.listdir(path_batch))))]","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:07:38.544547Z","iopub.execute_input":"2023-02-03T03:07:38.544799Z","iopub.status.idle":"2023-02-03T03:30:58.437126Z","shell.execute_reply.started":"2023-02-03T03:07:38.544778Z","shell.execute_reply":"2023-02-03T03:30:58.435680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aux_true = []\nfor s in tqdm(sensor_info):\n    aux_true.append(pd.DataFrame(s[s.auxiliary==True][['sensor_id','auxiliary']].value_counts()).sort_index())","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:32:14.109974Z","iopub.execute_input":"2023-02-03T03:32:14.110811Z","iopub.status.idle":"2023-02-03T03:32:18.898776Z","shell.execute_reply.started":"2023-02-03T03:32:14.110779Z","shell.execute_reply":"2023-02-03T03:32:18.897735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing number of Auxiliary==True signals for each sensors","metadata":{}},{"cell_type":"code","source":"aux_true = pd.concat(aux_true).reset_index().drop('auxiliary',axis=1)\naux_true_result = aux_true.groupby('sensor_id').sum()\naux_true_result.columns = ['Num_Aux_True']","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:32:18.900749Z","iopub.execute_input":"2023-02-03T03:32:18.901130Z","iopub.status.idle":"2023-02-03T03:32:19.554604Z","shell.execute_reply.started":"2023-02-03T03:32:18.901096Z","shell.execute_reply":"2023-02-03T03:32:19.553222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Looks like sensor id around ~=5000 is quite noisy...","metadata":{}},{"cell_type":"code","source":"aux_true_result.sort_values('Num_Aux_True')[::-1].head(3)","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:32:20.713338Z","iopub.execute_input":"2023-02-03T03:32:20.713723Z","iopub.status.idle":"2023-02-03T03:32:20.736826Z","shell.execute_reply.started":"2023-02-03T03:32:20.713694Z","shell.execute_reply":"2023-02-03T03:32:20.735886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of auxiliary==True sensors using standard deviation","metadata":{}},{"cell_type":"code","source":"std_t2_p = aux_true_result.Num_Aux_True.median() + 2*aux_true_result.Num_Aux_True.std()\nstd_t2_m = aux_true_result.Num_Aux_True.median() - 2*aux_true_result.Num_Aux_True.std()\n\n\nsns.set(font_scale = 2,style = 'white')\nplt.figure(figsize = (15,8))\nsns.scatterplot(data = aux_true_result,x = aux_true_result.index.tolist(),y = 'Num_Aux_True',linewidth =0,s = 15,color = 'grey')\nplt.axhline(aux_true_result.Num_Aux_True.median(),color = 'red')\nplt.axhline(std_t2_p,color = 'red',linestyle='--')\nplt.axhline(std_t2_m,color = 'red',linestyle='--')\n\nplt.text(5500,aux_true_result.Num_Aux_True.median(),'Median sum of Aux numbers',color = 'black')\nplt.text(5500,std_t2_p,'STD*2',color = 'black')\nplt.text(5500,std_t2_m,'STD*2',color = 'black')\n\nsns.despine()\nplt.xlabel('Sensor number')\nplt.title('Distribution of cumulative Auxiliary number per sensors')","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:44:51.305943Z","iopub.execute_input":"2023-02-03T03:44:51.306374Z","iopub.status.idle":"2023-02-03T03:44:51.587742Z","shell.execute_reply.started":"2023-02-03T03:44:51.306342Z","shell.execute_reply":"2023-02-03T03:44:51.586303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Annotating outlier sensors","metadata":{}},{"cell_type":"code","source":"sensor_under_std2 = aux_true_result[aux_true_result.Num_Aux_True<std_t2_m].index.tolist()\nsensor_over_std2 = aux_true_result[aux_true_result.Num_Aux_True>std_t2_p].index.tolist()\nsensor['Outlier_sensors'] = 'Normal'\nsensor.loc[sensor.sensor_id.isin(sensor_under_std2), 'Outlier_sensors'] = '< 2*STD'\nsensor.loc[sensor.sensor_id.isin(sensor_over_std2), 'Outlier_sensors'] = '> 2*STD'\n","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:49:48.658985Z","iopub.execute_input":"2023-02-03T03:49:48.659446Z","iopub.status.idle":"2023-02-03T03:49:48.670447Z","shell.execute_reply.started":"2023-02-03T03:49:48.659411Z","shell.execute_reply":"2023-02-03T03:49:48.668683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize outlier sensors\n> From https://www.kaggle.com/code/ridwanultanvir/icecube-extreme-gradient-boosting-supervised","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\nfig = px.scatter_3d(sensor, x='x', y='y', z='z', color='Outlier_sensors', size='sensor_id',\n                   title=\"IceCube Observatory : Visualization of irregular sensors\")\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-03T03:49:50.361977Z","iopub.execute_input":"2023-02-03T03:49:50.362356Z","iopub.status.idle":"2023-02-03T03:49:50.431509Z","shell.execute_reply.started":"2023-02-03T03:49:50.362327Z","shell.execute_reply":"2023-02-03T03:49:50.430358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}