{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this inference notebook I show that with the correct data pre-processing, model architecture and training setup an LSTM model can achieve the same performance as the current top performing Graph based models.\n\nThis notebook is based on the notebook originally published with LSTM training and inference: [3 LSTMs; with Data Picking and Shifting](https://www.kaggle.com/code/seungmoklee/3-lstms-with-data-picking-and-shifting). So if you like my notebooks don't forget the work that it is based!\n\nThis Inference notebook is for the largest part the same. It doesn't use the shifting as applied in the original work, it is based on only using 96 pulses and only 6 features.\nIt does ensemble 3 models but they are from the same training run (just different epochs).\n\nCombining models from different model trainings with slightly different hyperparameters will very likely further increase the score. Increasing the number of bins and using more data for training is another way to increase the score of the models. Consider the Inference and Training notebooks a starting point to explore that yourself!\n\nThe training notebook for these models can be found [here](https://www.kaggle.com/code/rsmits/tensorflow-lstm-model-training-tpu). Note that I performed training on my local laptop (32GB RAM / NVidia 3070). For local training I loaded the files for 70 batches into RAM. The training notebook is equal to my local setup with the change that it loads multiple rounds of training data.\n\nIn the training notebook I will further explain what the differences are and how I improved the achieved score.\n\nI hope you enjoy this notebook and if you do please give it an upvote :-)\n\n!! Update in Latest Version: In the comments it was mentioned that using model files from a TPU training caused an error when trying to load the model. I've updated the load_model() with compile = False. This solves the error. Nothing else has changed in the notebook.","metadata":{}},{"cell_type":"code","source":"# Import Modules\nimport gc\nimport os\nimport multiprocessing\nimport time\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport tensorflow as tf\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-14T20:48:14.908749Z","iopub.execute_input":"2023-03-14T20:48:14.909605Z","iopub.status.idle":"2023-03-14T20:48:14.919958Z","shell.execute_reply.started":"2023-03-14T20:48:14.909565Z","shell.execute_reply":"2023-03-14T20:48:14.918963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directories and constants\nhome_dir = \"/kaggle/input/icecube-neutrinos-in-deep-ice/\"\ntest_format = home_dir + 'test/batch_{batch_id:d}.parquet'\nmodel_home = \"/kaggle/input/lstmicecubesdata/\"\n\n# Model(s)\nmodel_names = [\"4347_MAE_1-02076_bin24_pp96_n6_batch2048_epoch29.h5\",\n               \"4347_MAE_1-02039_bin24_pp96_n6_batch2048_epoch25.h5\", \n               \"4346_MAE_1-02020_bin24_pp96_n6_batch2048_epoch27.h5\"]\nmodel_weights = np.array([0.30, \n                          0.30,\n                          0.40])","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:14.921817Z","iopub.execute_input":"2023-03-14T20:48:14.922369Z","iopub.status.idle":"2023-03-14T20:48:14.928207Z","shell.execute_reply.started":"2023-03-14T20:48:14.922342Z","shell.execute_reply":"2023-03-14T20:48:14.926961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Model(s)","metadata":{}},{"cell_type":"code","source":"# Load Models\nmodels = []\nfor model_name in model_names:\n    print(f'\\n========== Model File: {model_name}')\n    \n    # Load Model\n    model_path = model_home + model_name\n    model = tf.keras.models.load_model(model_path, compile = False)\n    models.append(model)      \n    \n    # Model summary\n    model.summary()\n    \n# Get Model Parameters\npulse_count = model.inputs[0].shape[1]\nfeature_count = model.inputs[0].shape[2]\noutput_bins = model.layers[-1].weights[0].shape[-1]\nbin_num = int(np.sqrt(output_bins))\n\n# Model Parameter Summary\nprint(\"\\n==== Model Parameters\")\nprint(f\"Bin Numbers: {bin_num}\")\nprint(f\"Maximum Pulse Count: {pulse_count}\")\nprint(f\"Features Count: {feature_count}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:14.929836Z","iopub.execute_input":"2023-03-14T20:48:14.930481Z","iopub.status.idle":"2023-03-14T20:48:25.481716Z","shell.execute_reply.started":"2023-03-14T20:48:14.930445Z","shell.execute_reply":"2023-03-14T20:48:25.480585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detector Information","metadata":{}},{"cell_type":"code","source":"# Load sensor_geometry\nsensor_geometry_df = pd.read_csv(home_dir + \"sensor_geometry.csv\")\n\n# Get Sensor Information\nsensor_x = sensor_geometry_df.x\nsensor_y = sensor_geometry_df.y\nsensor_z = sensor_geometry_df.z\n\n# Detector constants\nc_const = 0.299792458  # speed of light [m/ns]\n\n# Sensor Min / Max Coordinates\nx_min = sensor_x.min()\nx_max = sensor_x.max()\ny_min = sensor_y.min()\ny_max = sensor_y.max()\nz_min = sensor_z.min()\nz_max = sensor_z.max()\n\ndetector_length = np.sqrt((x_max - x_min)**2 + (y_max - y_min)**2 + (z_max - z_min)**2)\nt_valid_length = detector_length / c_const\n\nprint(f\"time valid length: {t_valid_length} ns\")","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.485018Z","iopub.execute_input":"2023-03-14T20:48:25.485319Z","iopub.status.idle":"2023-03-14T20:48:25.499753Z","shell.execute_reply.started":"2023-03-14T20:48:25.485291Z","shell.execute_reply":"2023-03-14T20:48:25.498358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Angle encoding edges\n\n- It is efficient to train the model by classification task, initially.\n- azimuth and zenith are independent\n- azimuth distribution is flat and zenith distribution is sinusoidal.\n  - Flat on the spherical surface\n  - $\\phi > \\pi$ events are a little bit rarer than $\\phi < \\pi$ events, (maybe) because of the neutrino attenuation by earth.\n- So, the uniform bin is used for azimuth, and $\\left| \\cos \\right|$ bin is used for zenith","metadata":{}},{"cell_type":"code","source":"# Create Azimuth Edges\nazimuth_edges = np.linspace(0, 2 * np.pi, bin_num + 1)\nprint(azimuth_edges)\n\n# Create Zenith Edges\nzenith_edges = []\nzenith_edges.append(0)\nfor bin_idx in range(1, bin_num):\n    zenith_edges.append(np.arccos(np.cos(zenith_edges[-1]) - 2 / (bin_num)))\nzenith_edges.append(np.pi)\nzenith_edges = np.array(zenith_edges)\nprint(zenith_edges)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.503204Z","iopub.execute_input":"2023-03-14T20:48:25.503485Z","iopub.status.idle":"2023-03-14T20:48:25.510604Z","shell.execute_reply.started":"2023-03-14T20:48:25.503459Z","shell.execute_reply":"2023-03-14T20:48:25.509476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define a function converts from prediction to angles\n\n- Calculation of the mean-vector in a bin $\\theta \\in ( \\theta_0, \\theta_1 )$ and $\\phi \\in ( \\phi_0, \\phi_1 )$\n  - $\\vec{r} \\left( \\theta, ~ \\phi \\right) = \\left< \\sin \\theta \\cos \\phi, ~ \\sin \\theta \\sin \\phi, ~ \\cos \\theta \\right>$\n  - $\\bar{\\vec{r}} = \\frac{ \\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} \\vec{r} \\left( \\theta, ~ \\phi \\right) \\sin \\theta \\,d\\phi \\,d\\theta }{ \\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} 1 \\sin \\theta \\,d\\phi \\,d\\theta }$\n  - $ \\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} 1 \\sin \\theta \\,d\\phi \\,d\\theta = \\left( \\phi_1 - \\phi_0 \\right) \\left( \\cos \\theta_0 - \\cos \\theta_1 \\right)$\n  - $\n\\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} {r}_{x} \\left( \\theta, ~ \\phi \\right) \\sin \\theta \\,d\\phi \\,d\\theta = \n\\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} \\sin^2 \\theta \\cos \\phi \\,d\\phi \\,d\\theta = \n\\left( \\sin \\phi_1 - \\sin \\phi_0 \\right) \\left( \\frac{\\theta_1 - \\theta_0}{2} - \\frac{\\sin 2 \\theta_1 - \\sin 2 \\theta_0}{4} \\right)\n$\n  - $\n\\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} {r}_{y} \\left( \\theta, ~ \\phi \\right) \\sin \\theta \\,d\\phi \\,d\\theta = \n\\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} \\sin^2 \\theta \\sin \\phi \\,d\\phi \\,d\\theta = \n\\left( \\cos \\phi_0 - \\cos \\phi_1 \\right) \\left( \\frac{\\theta_1 - \\theta_0}{2} - \\frac{\\sin 2 \\theta_1 - \\sin 2 \\theta_0}{4} \\right)\n$\n  - $\n\\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} {r}_{z} \\left( \\theta, ~ \\phi \\right) \\sin \\theta \\,d\\phi \\,d\\theta = \n\\int_{\\theta_{0}}^{\\theta_{1}} \\int_{\\phi_0}^{\\phi_1} \\sin \\theta \\cos \\theta \\,d\\phi \\,d\\theta = \n\\left( \\phi_1 - \\phi_0 \\right) \\left( \\frac{\\cos 2 \\theta_0 - \\cos 2 \\theta_1}{4} \\right)\n$","metadata":{}},{"cell_type":"code","source":"angle_bin_zenith0 = np.tile(zenith_edges[:-1], bin_num)\nangle_bin_zenith1 = np.tile(zenith_edges[1:], bin_num)\nangle_bin_azimuth0 = np.repeat(azimuth_edges[:-1], bin_num)\nangle_bin_azimuth1 = np.repeat(azimuth_edges[1:], bin_num)\n\nangle_bin_area = (angle_bin_azimuth1 - angle_bin_azimuth0) * (np.cos(angle_bin_zenith0) - np.cos(angle_bin_zenith1))\nangle_bin_vector_sum_x = (np.sin(angle_bin_azimuth1) - np.sin(angle_bin_azimuth0)) * ((angle_bin_zenith1 - angle_bin_zenith0) / 2 - (np.sin(2 * angle_bin_zenith1) - np.sin(2 * angle_bin_zenith0)) / 4)\nangle_bin_vector_sum_y = (np.cos(angle_bin_azimuth0) - np.cos(angle_bin_azimuth1)) * ((angle_bin_zenith1 - angle_bin_zenith0) / 2 - (np.sin(2 * angle_bin_zenith1) - np.sin(2 * angle_bin_zenith0)) / 4)\nangle_bin_vector_sum_z = (angle_bin_azimuth1 - angle_bin_azimuth0) * ((np.cos(2 * angle_bin_zenith0) - np.cos(2 * angle_bin_zenith1)) / 4)\n\nangle_bin_vector_mean_x = angle_bin_vector_sum_x / angle_bin_area\nangle_bin_vector_mean_y = angle_bin_vector_sum_y / angle_bin_area\nangle_bin_vector_mean_z = angle_bin_vector_sum_z / angle_bin_area\n\nangle_bin_vector = np.zeros((1, bin_num * bin_num, 3))\nangle_bin_vector[:, :, 0] = angle_bin_vector_mean_x\nangle_bin_vector[:, :, 1] = angle_bin_vector_mean_y\nangle_bin_vector[:, :, 2] = angle_bin_vector_mean_z\n\nangle_bin_vector_unit = angle_bin_vector[0].copy()\nangle_bin_vector_unit /= np.sqrt((angle_bin_vector_unit**2).sum(axis=1).reshape((-1, 1)))","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.512298Z","iopub.execute_input":"2023-03-14T20:48:25.513123Z","iopub.status.idle":"2023-03-14T20:48:25.526486Z","shell.execute_reply.started":"2023-03-14T20:48:25.513085Z","shell.execute_reply":"2023-03-14T20:48:25.525521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pred_to_angle(pred, epsilon = 1e-8):\n    # Convert prediction\n    pred_vector = (pred.reshape((-1, bin_num**2, 1)) * angle_bin_vector).sum(axis = 1)\n    \n    # Normalize\n    pred_vector_norm = np.sqrt((pred_vector**2).sum(axis = 1))\n    mask = pred_vector_norm < epsilon\n    pred_vector_norm[mask] = 1\n    \n    # Assign <1, 0, 0> to very small vectors (badly predicted)\n    pred_vector /= pred_vector_norm.reshape((-1, 1))\n    pred_vector[mask] = np.array([1., 0., 0.])\n    \n    # Convert to angle\n    azimuth = np.arctan2(pred_vector[:, 1], pred_vector[:, 0])\n    azimuth[azimuth < 0] += 2 * np.pi\n    zenith = np.arccos(pred_vector[:, 2])\n    \n    # Mask bad norm predictions as 0, 0\n    azimuth[mask] = 0.\n    zenith[mask] = 0.\n    \n    return azimuth, zenith","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.528201Z","iopub.execute_input":"2023-03-14T20:48:25.528549Z","iopub.status.idle":"2023-03-14T20:48:25.538902Z","shell.execute_reply.started":"2023-03-14T20:48:25.528515Z","shell.execute_reply":"2023-03-14T20:48:25.537795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Weighted-Vector Ensemble","metadata":{}},{"cell_type":"code","source":"def weighted_vector_ensemble(angles, weight):\n    # Convert angle to vector\n    vec_models = list()\n    for angle in angles:\n        az, zen = angle\n        sa = np.sin(az)\n        ca = np.cos(az)\n        sz = np.sin(zen)\n        cz = np.cos(zen)\n        vec = np.stack([sz * ca, sz * sa, cz], axis=1)\n        vec_models.append(vec)\n    vec_models = np.array(vec_models)\n\n    # Weighted-mean\n    vec_mean = (weight.reshape((-1, 1, 1)) * vec_models).sum(axis=0) / weight.sum()\n    vec_mean /= np.sqrt((vec_mean**2).sum(axis=1)).reshape((-1, 1))\n\n    # Convert vector to angle\n    zenith = np.arccos(vec_mean[:, 2])\n    azimuth = np.arctan2(vec_mean[:, 1], vec_mean[:, 0])\n    azimuth[azimuth < 0] += 2 * np.pi\n    \n    return azimuth, zenith","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.540901Z","iopub.execute_input":"2023-03-14T20:48:25.541682Z","iopub.status.idle":"2023-03-14T20:48:25.552980Z","shell.execute_reply.started":"2023-03-14T20:48:25.541646Z","shell.execute_reply":"2023-03-14T20:48:25.552028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Single event reader function\n\n- Pick-up important data points first\n    - Rank 3 (First)\n        - not aux, in valid time window\n    - Rank 2\n        - not aux, out of valid time window\n    - Rank 1\n        - aux, in valid time window\n    - Rank 0 (Last)\n        - aux, out of valid time window\n    - In each ranks, take pulses from highest charge","metadata":{}},{"cell_type":"code","source":"# Placeholder\nopen_batch_dict = dict()\n\n# Read single event from batch_meta_df\ndef read_event(event_idx, batch_meta_df, pulse_count):\n    # Read metadata\n    batch_id, first_pulse_index, last_pulse_index = batch_meta_df.iloc[event_idx][[\"batch_id\", \"first_pulse_index\", \"last_pulse_index\"]].astype(\"int\")\n\n    # close past batch df\n    if batch_id - 1 in open_batch_dict.keys():\n        del open_batch_dict[batch_id - 1]\n\n    # open current batch df\n    if batch_id not in open_batch_dict.keys():\n        open_batch_dict.update({batch_id: pd.read_parquet(test_format.format(batch_id=batch_id))})\n    \n    batch_df = open_batch_dict[batch_id]\n    \n    # Read event\n    event_feature = batch_df[first_pulse_index:last_pulse_index + 1]\n    sensor_id = event_feature.sensor_id\n    \n    # Merge features into single structured array\n    dtype = [(\"time\", \"float16\"),\n             (\"charge\", \"float16\"),\n             (\"auxiliary\", \"float16\"),\n             (\"x\", \"float16\"),\n             (\"y\", \"float16\"),\n             (\"z\", \"float16\"),\n             (\"rank\", \"short\")]    \n    \n    # Create event_x\n    event_x = np.zeros(last_pulse_index - first_pulse_index + 1, dtype)\n    event_x[\"time\"] = event_feature.time.values - event_feature.time.min()\n    event_x[\"charge\"] = event_feature.charge.values\n    event_x[\"auxiliary\"] = event_feature.auxiliary.values\n    event_x[\"x\"] = sensor_geometry_df.x[sensor_id].values\n    event_x[\"y\"] = sensor_geometry_df.y[sensor_id].values\n    event_x[\"z\"] = sensor_geometry_df.z[sensor_id].values\n\n    # For long event, pick-up\n    if len(event_x) > pulse_count:\n        # Find valid time window\n        t_peak = event_x[\"time\"][event_x[\"charge\"].argmax()]\n        t_valid_min = t_peak - t_valid_length\n        t_valid_max = t_peak + t_valid_length\n        t_valid = (event_x[\"time\"] > t_valid_min) * (event_x[\"time\"] < t_valid_max)\n\n        # Rank\n        event_x[\"rank\"] = 2 * (1 - event_x[\"auxiliary\"]) + (t_valid)\n\n        # Sort by Rank and Charge (important goes to backward)\n        event_x = np.sort(event_x, order = [\"rank\", \"charge\"])\n\n        # pick-up from backward\n        event_x = event_x[-pulse_count:]\n\n        # Sort events by time \n        event_x = np.sort(event_x, order = \"time\")\n\n    return event_idx, len(event_x), event_x","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.554326Z","iopub.execute_input":"2023-03-14T20:48:25.554848Z","iopub.status.idle":"2023-03-14T20:48:25.568103Z","shell.execute_reply.started":"2023-03-14T20:48:25.554812Z","shell.execute_reply":"2023-03-14T20:48:25.567170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test metadata","metadata":{}},{"cell_type":"code","source":"# Read Test Meta data\ntest_meta_df = pq.read_table(home_dir + 'test_meta.parquet').to_pandas()\nbatch_counts = test_meta_df.batch_id.value_counts().sort_index()\n\nbatch_max_index = batch_counts.cumsum()\nbatch_max_index[test_meta_df.batch_id.min() - 1] = 0\nbatch_max_index = batch_max_index.sort_index()\n\n# Support Function\ndef test_meta_df_spliter(batch_id):\n    return test_meta_df.loc[batch_max_index[batch_id - 1]:batch_max_index[batch_id] - 1]","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.570686Z","iopub.execute_input":"2023-03-14T20:48:25.570938Z","iopub.status.idle":"2023-03-14T20:48:25.590554Z","shell.execute_reply.started":"2023-03-14T20:48:25.570915Z","shell.execute_reply":"2023-03-14T20:48:25.589534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read test data and predict batchwise","metadata":{}},{"cell_type":"code","source":"# Get Batch IDs\ntest_batch_ids = test_meta_df.batch_id.unique()\n\n# Submission Placeholders\ntest_event_id = []\ntest_azimuth = []\ntest_zenith = []\n\n# Batch Loop\nfor batch_id in test_batch_ids:\n    # Batch Meta DF\n    batch_meta_df = test_meta_df_spliter(batch_id)\n\n    # Set Pulses\n    test_x = np.zeros((len(batch_meta_df), pulse_count, feature_count), dtype = \"float16\")    \n    test_x[:, :, 2] = -1    \n\n    # Read Event Data\n    def read_event_local(event_idx):\n        return read_event(event_idx, batch_meta_df, pulse_count)\n    \n    # Multiprocess Events\n    iterator = range(len(batch_meta_df))\n    with multiprocessing.Pool() as pool:\n        for event_idx, pulsecount, event_x in pool.map(read_event_local, iterator):\n            # Features\n            test_x[event_idx, :pulsecount, 0] = event_x[\"time\"]\n            test_x[event_idx, :pulsecount, 1] = event_x[\"charge\"]\n            test_x[event_idx, :pulsecount, 2] = event_x[\"auxiliary\"]\n            test_x[event_idx, :pulsecount, 3] = event_x[\"x\"]\n            test_x[event_idx, :pulsecount, 4] = event_x[\"y\"]\n            test_x[event_idx, :pulsecount, 5] = event_x[\"z\"]\n    \n    del batch_meta_df\n    \n    # Normalize\n    test_x[:, :, 0] /= 1000  # time\n    test_x[:, :, 1] /= 300  # charge\n    test_x[:, :, 3:] /= 600  # space\n        \n    # Predict\n    pred_angles = []\n    for model in models:\n        pred_model = model.predict(test_x, verbose=0)\n        az_model, zen_model = pred_to_angle(pred_model)\n        pred_angles.append((az_model, zen_model))\n    \n    # Get Predicted Azimuth and Zenith\n    pred_azimuth, pred_zenith = weighted_vector_ensemble(pred_angles, model_weights)\n    \n    # Get Event IDs\n    event_ids = test_meta_df.event_id[test_meta_df.batch_id == batch_id].values\n    \n    # Finalize \n    for event_id, azimuth, zenith in zip(event_ids, pred_azimuth, pred_zenith):\n        if np.isfinite(azimuth) and np.isfinite(zenith):\n            test_event_id.append(int(event_id))\n            test_azimuth.append(azimuth)\n            test_zenith.append(zenith)\n        else:\n            test_event_id.append(int(event_id))\n            test_azimuth.append(0.)\n            test_zenith.append(0.)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:25.592254Z","iopub.execute_input":"2023-03-14T20:48:25.592776Z","iopub.status.idle":"2023-03-14T20:48:37.286988Z","shell.execute_reply.started":"2023-03-14T20:48:25.592740Z","shell.execute_reply":"2023-03-14T20:48:37.285861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Submission","metadata":{}},{"cell_type":"code","source":"# Create and Save Submission.csv\nsubmission_df = pd.DataFrame({\"event_id\": test_event_id,\n                              \"azimuth\": test_azimuth,\n                              \"zenith\": test_zenith})\nsubmission_df = submission_df.sort_values(by = ['event_id'])\nsubmission_df.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:37.289383Z","iopub.execute_input":"2023-03-14T20:48:37.289980Z","iopub.status.idle":"2023-03-14T20:48:37.300744Z","shell.execute_reply.started":"2023-03-14T20:48:37.289940Z","shell.execute_reply":"2023-03-14T20:48:37.299747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Summary\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T20:48:37.302347Z","iopub.execute_input":"2023-03-14T20:48:37.303069Z","iopub.status.idle":"2023-03-14T20:48:37.320387Z","shell.execute_reply.started":"2023-03-14T20:48:37.303031Z","shell.execute_reply":"2023-03-14T20:48:37.319560Z"},"trusted":true},"execution_count":null,"outputs":[]}]}