{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I was interested to find out if it is possible to identify some sensors that are most often associated with auxiliary = True.  \nLet's conduct a very simple study.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:44:57.006659Z","iopub.execute_input":"2023-01-25T23:44:57.007053Z","iopub.status.idle":"2023-01-25T23:44:57.012425Z","shell.execute_reply.started":"2023-01-25T23:44:57.007023Z","shell.execute_reply":"2023-01-25T23:44:57.011512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\nfiles = os.listdir(train_dir)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:44:58.701009Z","iopub.execute_input":"2023-01-25T23:44:58.701418Z","iopub.status.idle":"2023-01-25T23:44:58.707227Z","shell.execute_reply.started":"2023-01-25T23:44:58.701384Z","shell.execute_reply":"2023-01-25T23:44:58.706345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Make initial dataframe with all sensors.","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({\n    'sensor_id': list(range(5160)),\n    'count': 0,\n    'sum': 0\n}).set_index('sensor_id')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:44:59.783698Z","iopub.execute_input":"2023-01-25T23:44:59.784089Z","iopub.status.idle":"2023-01-25T23:44:59.792071Z","shell.execute_reply.started":"2023-01-25T23:44:59.784059Z","shell.execute_reply":"2023-01-25T23:44:59.791069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Run over 10% files in train dataset. We have a lot of them!","metadata":{}},{"cell_type":"code","source":"random.shuffle(files)\n\nfiles = files[:int(len(files)*0.1)]\n\nfor i, f in enumerate(files):\n    data = pd.read_parquet(f'{train_dir}/{f}')\n    grouped = data.groupby('sensor_id')['auxiliary'].agg(['count', 'sum'])\n    df.iloc[grouped.index] += grouped\n    \ndf['mean_auxiliary'] = df['sum'] / df['count']","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:45:01.743133Z","iopub.execute_input":"2023-01-25T23:45:01.743566Z","iopub.status.idle":"2023-01-25T23:49:55.442335Z","shell.execute_reply.started":"2023-01-25T23:45:01.743530Z","shell.execute_reply":"2023-01-25T23:49:55.439622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Time to get some pictures.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,7))\ndf['mean_auxiliary'].hist(bins=100)\nplt.title('Distribution of mean number of events with auxiliary = True by sensor')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:52:23.148921Z","iopub.execute_input":"2023-01-25T23:52:23.149344Z","iopub.status.idle":"2023-01-25T23:52:23.535827Z","shell.execute_reply.started":"2023-01-25T23:52:23.149308Z","shell.execute_reply":"2023-01-25T23:52:23.534660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.boxplot(column=['mean_auxiliary'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:49:55.750348Z","iopub.execute_input":"2023-01-25T23:49:55.750707Z","iopub.status.idle":"2023-01-25T23:49:55.934975Z","shell.execute_reply.started":"2023-01-25T23:49:55.750676Z","shell.execute_reply":"2023-01-25T23:49:55.934177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'm going to use IQR to find out the upper limit for outliers.","metadata":{}},{"cell_type":"code","source":"q25, q75 = df['mean_auxiliary'].quantile([0.25, 0.75]).values\nupper_limit = q75 + 1.5 * (q75 - q25)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:49:55.936770Z","iopub.execute_input":"2023-01-25T23:49:55.937729Z","iopub.status.idle":"2023-01-25T23:49:55.946007Z","shell.execute_reply.started":"2023-01-25T23:49:55.937694Z","shell.execute_reply":"2023-01-25T23:49:55.945105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'We might consider all sensors with mean_auxiliary {upper_limit:.4} to be \"Dirty\"')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:49:55.948034Z","iopub.execute_input":"2023-01-25T23:49:55.948507Z","iopub.status.idle":"2023-01-25T23:49:55.955321Z","shell.execute_reply.started":"2023-01-25T23:49:55.948463Z","shell.execute_reply":"2023-01-25T23:49:55.954124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.mean_auxiliary > upper_limit]","metadata":{"execution":{"iopub.status.busy":"2023-01-25T23:49:55.956879Z","iopub.execute_input":"2023-01-25T23:49:55.957648Z","iopub.status.idle":"2023-01-25T23:49:55.983453Z","shell.execute_reply.started":"2023-01-25T23:49:55.957603Z","shell.execute_reply":"2023-01-25T23:49:55.982400Z"},"trusted":true},"execution_count":null,"outputs":[]}]}