{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pyarrow.parquet as pq","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:33:35.010369Z","iopub.execute_input":"2023-01-20T14:33:35.010832Z","iopub.status.idle":"2023-01-20T14:33:35.017166Z","shell.execute_reply.started":"2023-01-20T14:33:35.010783Z","shell.execute_reply":"2023-01-20T14:33:35.016021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the meta data\ntrain_meta = pq.read_pandas('/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet').to_pandas()\ndata = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\n# Read the batch data in chunks\nbatch_files = [data+'batch_1.parquet', data+'batch_2.parquet',data+'batch_10.parquet']\nbatch_data = pd.concat([pq.read_pandas(file, columns=['event_id', 'time', 'sensor_id', 'charge', 'auxiliary']).to_pandas() for file in batch_files])","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:33:35.026919Z","iopub.execute_input":"2023-01-20T14:33:35.027684Z","iopub.status.idle":"2023-01-20T14:34:08.006368Z","shell.execute_reply.started":"2023-01-20T14:33:35.027641Z","shell.execute_reply":"2023-01-20T14:34:08.005330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read sensor_geometry.csv using pandas\nsensor_geometry = pd.read_csv('/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv')\n\n# examine the data\nsensor_geometry.info()\nsensor_geometry.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:08.008735Z","iopub.execute_input":"2023-01-20T14:34:08.009513Z","iopub.status.idle":"2023-01-20T14:34:08.040220Z","shell.execute_reply.started":"2023-01-20T14:34:08.009464Z","shell.execute_reply":"2023-01-20T14:34:08.038792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select a subset of the data to train the model on\nsubset = train_meta.sample(frac=0.00002, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:08.043893Z","iopub.execute_input":"2023-01-20T14:34:08.044308Z","iopub.status.idle":"2023-01-20T14:34:17.684559Z","shell.execute_reply.started":"2023-01-20T14:34:08.044274Z","shell.execute_reply":"2023-01-20T14:34:17.683342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:17.688042Z","iopub.execute_input":"2023-01-20T14:34:17.688945Z","iopub.status.idle":"2023-01-20T14:34:17.704735Z","shell.execute_reply.started":"2023-01-20T14:34:17.688889Z","shell.execute_reply":"2023-01-20T14:34:17.703322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\n# Create a 3D scatter plot\nfig = px.scatter_3d(sensor_geometry, x='x', y='y', z='z', color='sensor_id', size='sensor_id',\n                   title=\"Sensor Positions in the IceCube Observatory\")\n\n# Show the plot\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:17.706171Z","iopub.execute_input":"2023-01-20T14:34:17.706524Z","iopub.status.idle":"2023-01-20T14:34:17.785508Z","shell.execute_reply.started":"2023-01-20T14:34:17.706491Z","shell.execute_reply":"2023-01-20T14:34:17.784173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# examine the distribution of azimuth and zenith angles\nsubset['azimuth'].hist()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:17.787254Z","iopub.execute_input":"2023-01-20T14:34:17.787739Z","iopub.status.idle":"2023-01-20T14:34:18.069104Z","shell.execute_reply.started":"2023-01-20T14:34:17.787690Z","shell.execute_reply":"2023-01-20T14:34:18.067983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset['zenith'].hist()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:18.070279Z","iopub.execute_input":"2023-01-20T14:34:18.070593Z","iopub.status.idle":"2023-01-20T14:34:18.364774Z","shell.execute_reply.started":"2023-01-20T14:34:18.070563Z","shell.execute_reply":"2023-01-20T14:34:18.363523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:18.366085Z","iopub.execute_input":"2023-01-20T14:34:18.366813Z","iopub.status.idle":"2023-01-20T14:34:18.374448Z","shell.execute_reply.started":"2023-01-20T14:34:18.366778Z","shell.execute_reply":"2023-01-20T14:34:18.372952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:18.375948Z","iopub.execute_input":"2023-01-20T14:34:18.377185Z","iopub.status.idle":"2023-01-20T14:34:18.393208Z","shell.execute_reply.started":"2023-01-20T14:34:18.377148Z","shell.execute_reply":"2023-01-20T14:34:18.391565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select a subset of the data to train the model on\nsubset_data = batch_data.sample(frac=0.00002, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:18.398767Z","iopub.execute_input":"2023-01-20T14:34:18.399218Z","iopub.status.idle":"2023-01-20T14:34:24.614785Z","shell.execute_reply.started":"2023-01-20T14:34:18.399177Z","shell.execute_reply":"2023-01-20T14:34:24.613522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_data","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:24.619087Z","iopub.execute_input":"2023-01-20T14:34:24.619473Z","iopub.status.idle":"2023-01-20T14:34:24.643724Z","shell.execute_reply.started":"2023-01-20T14:34:24.619437Z","shell.execute_reply":"2023-01-20T14:34:24.642501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:24.645874Z","iopub.execute_input":"2023-01-20T14:34:24.646792Z","iopub.status.idle":"2023-01-20T14:34:24.655061Z","shell.execute_reply.started":"2023-01-20T14:34:24.646722Z","shell.execute_reply":"2023-01-20T14:34:24.653841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation matrix\ncorr = subset.corr()\n#print(corr)\nplt.figure(figsize=(13,13))\n\n# Heatmap of correlation matrix\nsns.heatmap(corr,annot=True,fmt='.2f', annot_kws={\"size\": 10}, xticklabels=corr.columns, yticklabels=corr.columns)\nplt.title('Heatmap of Correlation Matrix')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:24.656825Z","iopub.execute_input":"2023-01-20T14:34:24.657660Z","iopub.status.idle":"2023-01-20T14:34:25.194239Z","shell.execute_reply.started":"2023-01-20T14:34:24.657615Z","shell.execute_reply":"2023-01-20T14:34:25.192663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation matrix\ncorr = subset_data.corr()\n#print(corr)\nplt.figure(figsize=(13,13))\n\n# Heatmap of correlation matrix\nsns.heatmap(corr,annot=True,fmt='.2f', annot_kws={\"size\": 10}, xticklabels=corr.columns, yticklabels=corr.columns)\nplt.title('Heatmap of Correlation Matrix')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:25.196252Z","iopub.execute_input":"2023-01-20T14:34:25.197078Z","iopub.status.idle":"2023-01-20T14:34:25.566840Z","shell.execute_reply.started":"2023-01-20T14:34:25.197025Z","shell.execute_reply":"2023-01-20T14:34:25.565611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:25.568545Z","iopub.execute_input":"2023-01-20T14:34:25.569045Z","iopub.status.idle":"2023-01-20T14:34:25.585806Z","shell.execute_reply.started":"2023-01-20T14:34:25.568996Z","shell.execute_reply":"2023-01-20T14:34:25.584384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(subset_data['time'], subset_data['charge'])\nplt.xlabel('Time')\nplt.ylabel('Charge')\nplt.title('Scatter plot of Time and Charge values')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:25.587941Z","iopub.execute_input":"2023-01-20T14:34:25.588559Z","iopub.status.idle":"2023-01-20T14:34:25.871172Z","shell.execute_reply.started":"2023-01-20T14:34:25.588503Z","shell.execute_reply":"2023-01-20T14:34:25.869509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_data['charge'].hist(bins=50)\nplt.xlabel('Charge')\nplt.ylabel('Frequency')\nplt.title('Histogram of Charge values')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:25.872659Z","iopub.execute_input":"2023-01-20T14:34:25.873583Z","iopub.status.idle":"2023-01-20T14:34:26.238626Z","shell.execute_reply.started":"2023-01-20T14:34:25.873543Z","shell.execute_reply":"2023-01-20T14:34:26.237249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"heat_data = subset_data.pivot_table(values='charge', index='sensor_id', columns='auxiliary', aggfunc='mean')\nplt.figure(figsize=(13,13))\nsns.heatmap(heat_data, cmap='YlGnBu')\nplt.title('Heatmap of Mean Charge by Sensor ID and Auxiliary')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:26.240018Z","iopub.execute_input":"2023-01-20T14:34:26.240881Z","iopub.status.idle":"2023-01-20T14:34:27.378003Z","shell.execute_reply.started":"2023-01-20T14:34:26.240843Z","shell.execute_reply":"2023-01-20T14:34:27.376960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_subplot(111, projection='3d')\n\n#grouping data by event_id\ngrouped_data=subset.groupby(['event_id'])[['azimuth','zenith']].mean()\n\nax.plot(grouped_data['azimuth'],grouped_data['zenith'])\nax.set_xlabel('Azimuth')\nax.set_ylabel('Zenith')\nax.set_title(\"Line plot of Azimuth and Zenith over time\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:27.379482Z","iopub.execute_input":"2023-01-20T14:34:27.380518Z","iopub.status.idle":"2023-01-20T14:34:27.742599Z","shell.execute_reply.started":"2023-01-20T14:34:27.380477Z","shell.execute_reply":"2023-01-20T14:34:27.741666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a bar plot\n\n#grouping data by sensor_id\ngrouped_data=subset_data.groupby(['sensor_id'])['charge'].mean()\nplt.bar(grouped_data.index, grouped_data.values)\n\n# Label the axes\nplt.xlabel('Sensor ID')\nplt.ylabel('Mean Charge')\nplt.title(\"Bar plot of Mean Charge by Sensor ID\")\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:27.743832Z","iopub.execute_input":"2023-01-20T14:34:27.744684Z","iopub.status.idle":"2023-01-20T14:34:31.099554Z","shell.execute_reply.started":"2023-01-20T14:34:27.744647Z","shell.execute_reply":"2023-01-20T14:34:31.098323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='sensor_id', y='charge', data=subset_data)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:34:31.101447Z","iopub.execute_input":"2023-01-20T14:34:31.101819Z","iopub.status.idle":"2023-01-20T14:35:32.336538Z","shell.execute_reply.started":"2023-01-20T14:34:31.101784Z","shell.execute_reply":"2023-01-20T14:35:32.335102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.pairplot(subset)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:36:05.665478Z","iopub.execute_input":"2023-01-20T14:36:05.665940Z","iopub.status.idle":"2023-01-20T14:36:13.709534Z","shell.execute_reply.started":"2023-01-20T14:36:05.665901Z","shell.execute_reply":"2023-01-20T14:36:13.707766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_data.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:40:48.856962Z","iopub.execute_input":"2023-01-20T14:40:48.857410Z","iopub.status.idle":"2023-01-20T14:40:48.867295Z","shell.execute_reply.started":"2023-01-20T14:40:48.857375Z","shell.execute_reply":"2023-01-20T14:40:48.865604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:41:03.337556Z","iopub.execute_input":"2023-01-20T14:41:03.338174Z","iopub.status.idle":"2023-01-20T14:41:03.348774Z","shell.execute_reply.started":"2023-01-20T14:41:03.338116Z","shell.execute_reply":"2023-01-20T14:41:03.347436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(subset['azimuth'], shade=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:41:59.465289Z","iopub.execute_input":"2023-01-20T14:41:59.465727Z","iopub.status.idle":"2023-01-20T14:41:59.732622Z","shell.execute_reply.started":"2023-01-20T14:41:59.465690Z","shell.execute_reply":"2023-01-20T14:41:59.731307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(subset['last_pulse_index'], shade=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:41:45.389631Z","iopub.execute_input":"2023-01-20T14:41:45.390105Z","iopub.status.idle":"2023-01-20T14:41:45.667459Z","shell.execute_reply.started":"2023-01-20T14:41:45.390060Z","shell.execute_reply":"2023-01-20T14:41:45.666043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.swarmplot(x='sensor_id', y='charge', data=subset_data)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:42:16.483868Z","iopub.execute_input":"2023-01-20T14:42:16.484780Z","iopub.status.idle":"2023-01-20T14:44:04.438050Z","shell.execute_reply.started":"2023-01-20T14:42:16.484725Z","shell.execute_reply":"2023-01-20T14:44:04.436153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='sensor_id', data=subset_data)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:44:10.208078Z","iopub.execute_input":"2023-01-20T14:44:10.208641Z","iopub.status.idle":"2023-01-20T14:45:08.815176Z","shell.execute_reply.started":"2023-01-20T14:44:10.208592Z","shell.execute_reply":"2023-01-20T14:45:08.813849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(subset_data['time'], subset_data['charge'])\nplt.xlabel('Time')\nplt.ylabel('Charge')","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:45:08.817405Z","iopub.execute_input":"2023-01-20T14:45:08.817794Z","iopub.status.idle":"2023-01-20T14:45:09.078715Z","shell.execute_reply.started":"2023-01-20T14:45:08.817757Z","shell.execute_reply":"2023-01-20T14:45:09.077621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.fill_between(subset_data['time'], subset_data['charge'])\nplt.xlabel('Time')\nplt.ylabel('Charge')","metadata":{"execution":{"iopub.status.busy":"2023-01-20T14:46:43.937528Z","iopub.execute_input":"2023-01-20T14:46:43.937944Z","iopub.status.idle":"2023-01-20T14:46:44.532039Z","shell.execute_reply.started":"2023-01-20T14:46:43.937911Z","shell.execute_reply":"2023-01-20T14:46:44.530642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import pi\nimport matplotlib.pyplot as plt\n\n# select the columns to use in the plot\ndata = subset_data[['charge','time','auxiliary']]\n\n# calculate the mean of each column\nmean_data = data.mean()\n\n# number of variable\ncategories=list(data.columns)\nN = len(categories)\n\n# We are going to plot the first line of the data frame.\n# But we need to repeat the first value to close the circular graph:\nvalues=mean_data.values.flatten().tolist()\nvalues += values[:1]\n\n# What will be the angle of each axis in the plot? (we divide the plot / number of variable)\nangles = [n / float(N) * 2 * pi for n in range(N)]\nangles += angles[:1]\n\n# Initialise the spider plot\nax = plt.subplot(111, polar=True)\n\n# Draw one axe per variable + add labels labels yet\nplt.xticks(angles[:-1], categories, color='blue', size=8)\n\n# Draw ylabels\nax.set_rlabel_position(0)\nplt.yticks([10,20,30], [\"10\",\"20\",\"30\"], color=\"green\", size=10)\nplt.ylim(0,40)\n\n\nax.plot(angles, values, linewidth=3, linestyle='solid', color='blue', alpha=0.7)\n#ax.plot(angles, values, linewidth=3, linestyle='solid')\n\n# Plot data\n#ax.plot(angles, values, linewidth=1, linestyle='solid')\nplt.style.use('dark_background')\n\n# Show plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T15:30:17.680548Z","iopub.execute_input":"2023-01-20T15:30:17.681816Z","iopub.status.idle":"2023-01-20T15:30:18.241113Z","shell.execute_reply.started":"2023-01-20T15:30:17.681765Z","shell.execute_reply":"2023-01-20T15:30:18.239965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above polar plot represents the mean values of the selected columns ('charge','time','auxiliary') of the subset_data DataFrame, where each variable is represented by a spoke on the plot. The length of each spoke corresponds to the mean value of the variable.\n\nThe plot is useful for comparing the relative magnitude of the mean values of the variables. For example, if the spoke for 'charge' is longer than the spoke for 'time', it indicates that the mean value of 'charge' is greater than the mean value of 'time'.\n\nIt is also useful for identifying patterns or trends in the data. For example, if the spoke for 'auxiliary' is much shorter than the other two spokes, it may indicate that the majority of the pulses in the subset_data DataFrame are of high quality, and not likely to originate from noise.","metadata":{}}]}