{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install fasteda","metadata":{"execution":{"iopub.status.busy":"2023-01-19T18:35:26.627486Z","iopub.execute_input":"2023-01-19T18:35:26.627927Z","iopub.status.idle":"2023-01-19T18:35:40.073536Z","shell.execute_reply.started":"2023-01-19T18:35:26.627892Z","shell.execute_reply":"2023-01-19T18:35:40.072165Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fasteda import fast_eda\nimport pyarrow.parquet as pq\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt #graphing\nimport plotly.express as px #graphing\nimport seaborn as sns #graphing\nimport missingno as msno #describe data\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:11:11.965789Z","iopub.execute_input":"2023-01-19T20:11:11.966312Z","iopub.status.idle":"2023-01-19T20:11:11.973379Z","shell.execute_reply.started":"2023-01-19T20:11:11.966271Z","shell.execute_reply":"2023-01-19T20:11:11.972106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv\")\ntrain_meta = pq.ParquetFile('/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet')\nit = train_meta.iter_batches()\ntrain_meta = next(it).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:12:36.215481Z","iopub.execute_input":"2023-01-19T20:12:36.215921Z","iopub.status.idle":"2023-01-19T20:12:36.987688Z","shell.execute_reply.started":"2023-01-19T20:12:36.215889Z","shell.execute_reply":"2023-01-19T20:12:36.986548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_3d(df, x = \"x\", y = \"y\", z = \"z\", opacity = 0.8, color = \"z\")\n\nfig.update_traces(marker = dict(size = 2, symbol = \"diamond-open\"))\nfig.update_coloraxes(showscale = False)\nfig.update_layout(template = \"plotly_dark\", font = dict(family = \"PT Sans\", size = 12, color = \"#97FFFF\"))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:09:41.859935Z","iopub.execute_input":"2023-01-19T20:09:41.860921Z","iopub.status.idle":"2023-01-19T20:09:43.802515Z","shell.execute_reply.started":"2023-01-19T20:09:41.860878Z","shell.execute_reply":"2023-01-19T20:09:43.801465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\n# Loading 1st train parquet file\nfiles = [file for idx, file in enumerate(os.listdir(path)) if file.endswith('.parquet') and idx < 1]\nparquets = pd.concat([pd.read_parquet(path+file) for file in files])","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:09:43.804997Z","iopub.execute_input":"2023-01-19T20:09:43.805773Z","iopub.status.idle":"2023-01-19T20:09:46.890501Z","shell.execute_reply.started":"2023-01-19T20:09:43.805710Z","shell.execute_reply":"2023-01-19T20:09:46.888193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquets.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:09:46.892526Z","iopub.execute_input":"2023-01-19T20:09:46.893085Z","iopub.status.idle":"2023-01-19T20:09:46.906990Z","shell.execute_reply.started":"2023-01-19T20:09:46.893034Z","shell.execute_reply":"2023-01-19T20:09:46.905190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eda = parquets.iloc[:1000].reset_index()\neda[\"auxiliary\"] = eda[\"auxiliary\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:09:46.909970Z","iopub.execute_input":"2023-01-19T20:09:46.911141Z","iopub.status.idle":"2023-01-19T20:09:46.931838Z","shell.execute_reply.started":"2023-01-19T20:09:46.911081Z","shell.execute_reply":"2023-01-19T20:09:46.929843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fasteda on batch of train_meta\nfast_eda(train_meta, correlation = False, pairplot = False)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:13:54.334288Z","iopub.execute_input":"2023-01-19T20:13:54.334814Z","iopub.status.idle":"2023-01-19T20:13:58.113292Z","shell.execute_reply.started":"2023-01-19T20:13:54.334774Z","shell.execute_reply":"2023-01-19T20:13:58.111653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fasteda on the first 1000 rows of train parquet file\nfast_eda(eda, target = \"auxiliary\")","metadata":{"execution":{"iopub.status.busy":"2023-01-19T19:38:31.499482Z","iopub.execute_input":"2023-01-19T19:38:31.499939Z","iopub.status.idle":"2023-01-19T19:38:39.506319Z","shell.execute_reply.started":"2023-01-19T19:38:31.499902Z","shell.execute_reply":"2023-01-19T19:38:39.504834Z"},"trusted":true},"execution_count":null,"outputs":[]}]}