{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install fasteda","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-27T01:36:30.502492Z","iopub.execute_input":"2023-01-27T01:36:30.504535Z","iopub.status.idle":"2023-01-27T01:36:44.919728Z","shell.execute_reply.started":"2023-01-27T01:36:30.504476Z","shell.execute_reply":"2023-01-27T01:36:44.918226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fasteda import fast_eda\nimport pyarrow.parquet as pq\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt #graphing\nimport plotly.express as px #graphing\nimport seaborn as sns #graphing\nimport missingno as msno #describe data\nimport os\nfrom sklearn.decomposition import PCA\nimport plotly.graph_objects as go\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:44.922712Z","iopub.execute_input":"2023-01-27T01:36:44.923114Z","iopub.status.idle":"2023-01-27T01:36:45.361440Z","shell.execute_reply.started":"2023-01-27T01:36:44.923072Z","shell.execute_reply":"2023-01-27T01:36:45.359953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv\")\ntrain_meta = pq.ParquetFile('/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet')\nit = train_meta.iter_batches()\ntrain_meta = next(it).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:45.363125Z","iopub.execute_input":"2023-01-27T01:36:45.363565Z","iopub.status.idle":"2023-01-27T01:36:54.669370Z","shell.execute_reply.started":"2023-01-27T01:36:45.363524Z","shell.execute_reply":"2023-01-27T01:36:54.668098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv\", delimiter=',', encoding='ISO-8859-2')\npd.set_option('display.max_columns', None)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:54.670814Z","iopub.execute_input":"2023-01-27T01:36:54.671205Z","iopub.status.idle":"2023-01-27T01:36:54.698053Z","shell.execute_reply.started":"2023-01-27T01:36:54.671169Z","shell.execute_reply":"2023-01-27T01:36:54.696762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_DATASET = \"/kaggle/input/icecube-neutrinos-in-deep-ice\"","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:54.701179Z","iopub.execute_input":"2023-01-27T01:36:54.701706Z","iopub.status.idle":"2023-01-27T01:36:54.706650Z","shell.execute_reply.started":"2023-01-27T01:36:54.701666Z","shell.execute_reply":"2023-01-27T01:36:54.705593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_geometry = pd.read_csv('/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv')\nsensor_geometry.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:54.707880Z","iopub.execute_input":"2023-01-27T01:36:54.708829Z","iopub.status.idle":"2023-01-27T01:36:54.734579Z","shell.execute_reply.started":"2023-01-27T01:36:54.708793Z","shell.execute_reply":"2023-01-27T01:36:54.733548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(os.path.join(PATH_DATASET, \"train/batch_15.parquet\"))\nprint(f\"length: {len(train)}\")\nprint(f\"events: {len(train.index.unique())}\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:54.735987Z","iopub.execute_input":"2023-01-27T01:36:54.736341Z","iopub.status.idle":"2023-01-27T01:36:58.820429Z","shell.execute_reply.started":"2023-01-27T01:36:54.736309Z","shell.execute_reply":"2023-01-27T01:36:58.819144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 4))\n_ = plt.hist(train.groupby(level=0).size(), bins=50, log=True)\nplt.xlabel(\"nb. ALL measument per event\"), plt.ylabel(\"nb. of event\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:36:58.822211Z","iopub.execute_input":"2023-01-27T01:36:58.823474Z","iopub.status.idle":"2023-01-27T01:37:00.774166Z","shell.execute_reply.started":"2023-01-27T01:36:58.823426Z","shell.execute_reply":"2023-01-27T01:37:00.772737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 4))\n_ = plt.hist(train[~train['auxiliary']].groupby(level=0).size(), bins=50, log=True)\nplt.xlabel(\"nb. (aux==False) measument per event\"), plt.ylabel(\"nb. of event\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:00.775874Z","iopub.execute_input":"2023-01-27T01:37:00.776310Z","iopub.status.idle":"2023-01-27T01:37:03.293534Z","shell.execute_reply.started":"2023-01-27T01:37:00.776275Z","shell.execute_reply":"2023-01-27T01:37:03.292188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\n_ = plt.hist(train[~train['auxiliary']]['charge'], bins=50, log=True, label=\"False\")\n_ = plt.hist(train[train['auxiliary']]['charge'], bins=50, log=True, label=\"True\")\nplt.ylabel('count cases'), plt.xlabel('charge')\nplt.grid(), plt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:03.294857Z","iopub.execute_input":"2023-01-27T01:37:03.295223Z","iopub.status.idle":"2023-01-27T01:37:05.719598Z","shell.execute_reply.started":"2023-01-27T01:37:03.295182Z","shell.execute_reply":"2023-01-27T01:37:05.718415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train = pd.read_parquet(os.path.join(PATH_DATASET, \"train_meta.parquet\"))\nprint(f\"length: {len(meta_train)}\")\ndisplay(meta_train.head())\ndisplay(meta_train.info())","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:05.721202Z","iopub.execute_input":"2023-01-27T01:37:05.721702Z","iopub.status.idle":"2023-01-27T01:37:29.274759Z","shell.execute_reply.started":"2023-01-27T01:37:05.721640Z","shell.execute_reply":"2023-01-27T01:37:29.273458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train[meta_train['event_id'].isin([46528394, 2135637939, 2084362251])]","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:29.276717Z","iopub.execute_input":"2023-01-27T01:37:29.277093Z","iopub.status.idle":"2023-01-27T01:37:30.517538Z","shell.execute_reply.started":"2023-01-27T01:37:29.277059Z","shell.execute_reply":"2023-01-27T01:37:30.516248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(sensor_geometry.x, sensor_geometry.y)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:30.519172Z","iopub.execute_input":"2023-01-27T01:37:30.519637Z","iopub.status.idle":"2023-01-27T01:37:30.705792Z","shell.execute_reply.started":"2023-01-27T01:37:30.519593Z","shell.execute_reply":"2023-01-27T01:37:30.704272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\n# Loading 1st train parquet file\nfiles = [file for idx, file in enumerate(os.listdir(path)) if file.endswith('.parquet') and idx < 1]\nparquets = pd.concat([pd.read_parquet(path+file) for file in files])","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:30.710228Z","iopub.execute_input":"2023-01-27T01:37:30.711302Z","iopub.status.idle":"2023-01-27T01:37:35.430938Z","shell.execute_reply.started":"2023-01-27T01:37:30.711257Z","shell.execute_reply":"2023-01-27T01:37:35.427783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquets.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:35.434976Z","iopub.execute_input":"2023-01-27T01:37:35.435824Z","iopub.status.idle":"2023-01-27T01:37:35.460550Z","shell.execute_reply.started":"2023-01-27T01:37:35.435788Z","shell.execute_reply":"2023-01-27T01:37:35.455083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eda = parquets.iloc[:1000].reset_index()\neda[\"auxiliary\"] = eda[\"auxiliary\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:35.464936Z","iopub.execute_input":"2023-01-27T01:37:35.466004Z","iopub.status.idle":"2023-01-27T01:37:35.493699Z","shell.execute_reply.started":"2023-01-27T01:37:35.465955Z","shell.execute_reply":"2023-01-27T01:37:35.489476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fasteda on batch of train_meta\nfast_eda(train_meta, correlation = False, pairplot = False)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T01:37:35.497805Z","iopub.execute_input":"2023-01-27T01:37:35.499255Z"},"trusted":true},"execution_count":null,"outputs":[]}]}