{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"# Code explanation (junseonglee11)\njunseonglee11: I just copied this notebook from Robert Smith's  \nhttps://www.kaggle.com/code/rsmits/tensorflow-lstm-model-data-preprocessor/notebook   \nand added line fit code results of Robert Hatch's    \nhttps://www.kaggle.com/code/roberthatch/lb-1-183-lightning-fast-baseline-with-polars  \n\nand converted it to TFRecords Dataset.","metadata":{}},{"cell_type":"markdown","source":"The code in this notebook is based on the code as was originally written in [LSTM Preprocessing Point Picker](https://www.kaggle.com/code/seungmoklee/lstm-preprocessing-point-picker). The author of this notebook did a great job of setting a clear baseline.\n\nI modified the code in the following part:\n* Maximum pulse count is set to 96.\n* Remove the features r_err and z_err.\n* Remove all non-essential code and graphics. \n\nWith these few changes the output files only contain the features for the events as I use them in my [Tensorflow LSTM Model Training TPU](https://www.kaggle.com/code/rsmits/tensorflow-lstm-model-training-tpu) notebook and [Tensorflow LSTM Model Inference](https://www.kaggle.com/code/rsmits/tensorflow-lstm-model-inference) notebook.","metadata":{}},{"cell_type":"code","source":"# Data I/O and preprocessing\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\n\n# System\nimport time\nimport os\nimport gc\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf\nfrom contextlib import ExitStack\n# multiprocessing\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:38.248070Z","iopub.execute_input":"2023-04-07T01:47:38.248441Z","iopub.status.idle":"2023-04-07T01:47:45.198703Z","shell.execute_reply.started":"2023-04-07T01:47:38.248406Z","shell.execute_reply":"2023-04-07T01:47:45.197564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data setting\ntrain_batch_id_first = 118\ntrain_batch_id_last = 156\ntrain_batch_ids = range(train_batch_id_first, train_batch_id_last + 1)\n\n# Feature Settings\nmax_pulse_count = 96\nn_features = 6  # time, charge, aux, x, y, z, rank \n\n# Directories\nhome_dir = \"/kaggle/input/icecube-neutrinos-in-deep-ice/\"\ntrain_format = home_dir + 'train/batch_{batch_id:d}.parquet'\npoint_picker_format = 'pp_mpc96_n7_batch_{batch_id:d}.npz'","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:45.200642Z","iopub.execute_input":"2023-04-07T01:47:45.201200Z","iopub.status.idle":"2023-04-07T01:47:45.206756Z","shell.execute_reply.started":"2023-04-07T01:47:45.201170Z","shell.execute_reply":"2023-04-07T01:47:45.205738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# counts\ndoms_per_string = 60\nstring_num = 86\n\n# index\nouter_long_strings = np.concatenate([np.arange(0, 25), np.arange(27, 34), np.arange(37, 44), np.arange(46, 78)])\ninner_long_strings = np.array([25, 26, 34, 35, 36, 44, 45])\ninner_short_strings = np.array([78, 79, 80, 81, 82, 83, 84, 85])\n# known specs\nouter_xy_resolution = 125. / 2\ninner_xy_resolution = 70. / 2\nlong_z_resolution = 17. / 2\nshort_z_resolution = 7. / 2\n\n\n# Sensor Geometry Data\nsensor_geometry_df = pd.read_csv(home_dir + \"sensor_geometry.csv\")\n\n# X, Y, Z coordinates\nsensor_x = sensor_geometry_df.x\nsensor_y = sensor_geometry_df.y\nsensor_z = sensor_geometry_df.z\nsensor_r_err = np.ones(doms_per_string * string_num)\nsensor_z_err = np.ones(doms_per_string * string_num)\nfor string_id in outer_long_strings:\n    sensor_r_err[string_id * doms_per_string:(string_id + 1) * doms_per_string] *= outer_xy_resolution\nfor string_id in np.concatenate([inner_long_strings, inner_short_strings]):\n    sensor_r_err[string_id * doms_per_string:(string_id + 1) * doms_per_string] *= inner_xy_resolution\n\nfor string_id in outer_long_strings:\n    sensor_z_err[string_id * doms_per_string:(string_id + 1) * doms_per_string] *= long_z_resolution\nfor string_id in np.concatenate([inner_long_strings, inner_short_strings]):\n    for dom_id in range(doms_per_string):\n        z = sensor_z[string_id * doms_per_string + dom_id]\n        if (z < -156.) or (z > 95.5 and z < 191.5):\n            sensor_z_err[string_id * doms_per_string + dom_id] *= short_z_resolution\n# register\nsensor_geometry_df[\"r_err\"] = sensor_r_err\nsensor_geometry_df[\"z_err\"] = sensor_z_err\n\n\n# Detector constants\nc_const = 0.299792458  # speed of light [m/ns]\n\n# Min / Max information\nx_min = sensor_x.min()\nx_max = sensor_x.max()\ny_min = sensor_y.min()\ny_max = sensor_y.max()\nz_min = sensor_z.min()\nz_max = sensor_z.max()\n\n# Detector Valid Length\ndetector_length = np.sqrt((x_max - x_min)**2 + (y_max - y_min)**2 + (z_max - z_min)**2)\nt_valid_length = detector_length / c_const\n\nprint(f\"time valid length: {t_valid_length} ns\")","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:45.207885Z","iopub.execute_input":"2023-04-07T01:47:45.208177Z","iopub.status.idle":"2023-04-07T01:47:45.245216Z","shell.execute_reply.started":"2023-04-07T01:47:45.208141Z","shell.execute_reply":"2023-04-07T01:47:45.244083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n\n## Single event reader function\n\n- Pick-up important data points first\n    - Rank 3 (First)\n        - not aux, in valid time window\n    - Rank 2\n        - not aux, out of valid time window\n    - Rank 1\n        - aux, in valid time window\n    - Rank 0 (Last)\n        - aux, out of valid time window\n    - In each ranks, take pulses from highest charge\n\n\"\"\"\n\n# read single event from batch_meta_df\ndef read_event(event_idx, batch_meta_df, max_pulse_count, batch_df, train=True):\n    # read metadata\n    batch_id, first_pulse_index, last_pulse_index = batch_meta_df.iloc[event_idx][[\"batch_id\", \"first_pulse_index\", \"last_pulse_index\"]].astype(\"int\")\n\n    # read event\n    event_feature = batch_df[first_pulse_index:last_pulse_index + 1]\n    sensor_id = event_feature.sensor_id\n    \n    # merge features into single structured array\n    dtype = [(\"time\", \"float16\"),\n             (\"charge\", \"float16\"),\n             (\"auxiliary\", \"float16\"),\n             (\"x\", \"float16\"),\n             (\"y\", \"float16\"),\n             (\"z\", \"float16\"),\n             (\"rank\", \"short\"),\n            (\"r_err\", \"float16\"),\n            (\"z_err\", \"float16\")]\n    event_x = np.zeros(last_pulse_index - first_pulse_index + 1, dtype)\n\n    event_x[\"time\"] = event_feature.time.values - event_feature.time.min()\n    event_x[\"charge\"] = event_feature.charge.values\n    event_x[\"auxiliary\"] = event_feature.auxiliary.values\n\n    event_x[\"x\"] = sensor_geometry_df.x[sensor_id].values\n    event_x[\"y\"] = sensor_geometry_df.y[sensor_id].values\n    event_x[\"z\"] = sensor_geometry_df.z[sensor_id].values\n\n    event_x[\"r_err\"] = sensor_geometry_df.r_err[sensor_id].values\n    event_x[\"z_err\"] = sensor_geometry_df.z_err[sensor_id].values\n    \n    # For long event, pick-up\n    if len(event_x) > max_pulse_count:\n        # Find valid time window\n        t_peak = event_x[\"time\"][event_x[\"charge\"].argmax()]\n        t_valid_min = t_peak - t_valid_length\n        t_valid_max = t_peak + t_valid_length\n\n        t_valid = (event_x[\"time\"] > t_valid_min) * (event_x[\"time\"] < t_valid_max)\n\n        # rank\n        event_x[\"rank\"] = 2 * (1 - event_x[\"auxiliary\"]) + (t_valid)\n\n        # sort by rank and charge (important goes to backward)\n        event_x = np.sort(event_x, order=[\"rank\", \"charge\"])\n\n        # pick-up from backward\n        event_x = event_x[-max_pulse_count:]\n\n        # resort by time\n        event_x = np.sort(event_x, order=\"time\")\n\n    # resort by time\n    event_x = np.sort(event_x, order=\"time\")\n        \n    # for train data, give angles together\n    azimuth, zenith = batch_meta_df.iloc[event_idx][[\"azimuth\", \"zenith\"]].astype(\"float32\")\n    event_y = np.array([azimuth, zenith], dtype=\"float32\")\n        \n    return event_idx, len(event_x), event_x, event_y","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:45.247846Z","iopub.execute_input":"2023-04-07T01:47:45.248170Z","iopub.status.idle":"2023-04-07T01:47:45.261402Z","shell.execute_reply.started":"2023-04-07T01:47:45.248140Z","shell.execute_reply":"2023-04-07T01:47:45.260207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#230316 junseonglee11 implementation of normalization and inverse function \n#                     for data preprocessing\ndef normalize_data(x_data, y_data):\n    #최소, 배율, time은 놔두는게 좋을듯\n    norm_coeff_x = np.zeros((9, 2), np.float32)\n    norm_coeff_x[0] = 0, 1000\n    norm_coeff_x[1] = 0, 300\n    norm_coeff_x[2] = 0, 1\n    norm_coeff_x[3] = 0, 600\n    norm_coeff_x[4] = 0, 600\n    norm_coeff_x[5] = 0, 600\n    #norm_coeff_x[6] = 0, 62.5\n    #norm_coeff_x[7] = 0, 8.5\n    #norm_coeff_x[8] = 0, 3.0\n\n    #categorical\n\n    for i in range(0, 6):\n        x_data[:,:,i] = (x_data[:,:,i]-norm_coeff_x[i,0])/norm_coeff_x[i,1]\n        \n    #time to diff_time\n    x_data[:,:-1,0] = x_data[:,1:,0] - x_data[:,:-1,0]\n    x_data[:,-1,0] = 0            \n    \n    \n    #pseudo momentum (next time position - current position)\n    pseudo_momentum = x_data[:, :, 3:].copy()\n    pseudo_momentum[:,:-1,:] = pseudo_momentum[:,1:,:] - x_data[:,:-1, 3:]\n    pseudo_momentum[:,-1,:] = 0\n    \n    for i in range(0, 3):\n        pseudo_momentum[:,:-1,i][x_data[:,:-1,0]<0]=0\n\n    \n    x_data[:,:-1,0][x_data[:,:-1,0]<0] = 0\n    \n    x_data = np.append(x_data, pseudo_momentum, axis = 2)\n\n    return x_data, y_data","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:45.262714Z","iopub.execute_input":"2023-04-07T01:47:45.263023Z","iopub.status.idle":"2023-04-07T01:47:45.277344Z","shell.execute_reply.started":"2023-04-07T01:47:45.262997Z","shell.execute_reply":"2023-04-07T01:47:45.276302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_example_protobuff(event_pulses, line_fit_angles, origin_azimuth, origin_zenith):\n    # convert to binary string format for Example protobuf\n    event_data = tf.io.serialize_tensor(tf.cast(event_pulses, tf.float16))\n    return tf.train.Example(\n        features=tf.train.Features(\n            feature={\n                'event_pulses': tf.train.Feature(bytes_list=tf.train.BytesList(value=[event_data.numpy()])),\n                #'azimuth': tf.train.Feature(bytes_list=tf.train.BytesList(value=[azimuth.numpy()])),\n                #'zenith': tf.train.Feature(bytes_list=tf.train.BytesList(value=[zenith.numpy()])),\n                'fitted_azimuth': tf.train.Feature(float_list=tf.train.FloatList(value=[line_fit_angles[0]])),\n                'fitted_zenith': tf.train.Feature(float_list=tf.train.FloatList(value=[line_fit_angles[1]])),\n                'origin_azimuth': tf.train.Feature(float_list=tf.train.FloatList(value=[origin_azimuth])),\n                'origin_zenith' : tf.train.Feature(float_list=tf.train.FloatList(value=[origin_zenith])),\n            }\n        )\n    )\n\ndef write_tfrecords(filename, batch_x, line_fit_angles, batch_origin_y):\n    options = tf.io.TFRecordOptions(compression_type='GZIP', compression_level=9)\n    with ExitStack() as stack:\n        writer = stack.enter_context(tf.io.TFRecordWriter(filename, options = options))\n\n        # create example protobuffs from instances\n        for i in tqdm(np.arange(0, len(batch_x))):\n            event = tf.Variable(batch_x[i])\n            #azimuth, zenith = dataset[i,1152:]            \n            #azimuth, zenith = batch_y[i]       \n            origin_azimuth, origin_zenith = batch_origin_y[i]\n            example = create_example_protobuff(event, line_fit_angles[i], origin_azimuth, origin_zenith)\n            writer.write(example.SerializeToString())\n            \ndef save_to_tfrecord(batch_x, line_fit_angles, batch_origin_y, name):   \n    # write Dataset to files\n    write_tfrecords(f\"{name}\", batch_x, line_fit_angles, batch_origin_y)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:55:23.565722Z","iopub.execute_input":"2023-04-07T01:55:23.566798Z","iopub.status.idle":"2023-04-07T01:55:23.584317Z","shell.execute_reply.started":"2023-04-07T01:55:23.566743Z","shell.execute_reply":"2023-04-07T01:55:23.582952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚡🧊⚡[LB 1.183] Polar Lightning\nI took this from the invaluable works of roberthatch  \nhttps://www.kaggle.com/code/roberthatch/lb-1-183-lightning-fast-baseline-with-polars/comments","metadata":{}},{"cell_type":"code","source":"## Configuration parameters\nMODE = 'train'\n#MODE = 'test'\n\n# USE_POLARS = False\nUSE_POLARS = True\n\n# TRAIN_MAX_EVENTS = 20000\nTRAIN_MAX_EVENTS = None\nTRAIN_BATCH_START = 1\nTRAIN_N_BATCHES = 1\n\n## I pulled in one piece of older code to demonstrate \"before and after\".\n## Set to True (and USE_POLARS=False) if interested in seeing the difference.\nUSE_UNOPTIMIZED = False\n\n\n#### HYPERPARAMETERS ####\n\n## For setting auxiliary = False\n## Hand-tuned and hand-validated, I mostly used batches 100-105, and probably early on also touched batch 1.\n## TODO: revisit the deep core logic now that I've learned about the deep veto layer: https://www.kaggle.com/competitions/icecube-neutrinos-in-deep-ice/discussion/381702\nFIND_BEST_POINTS = True\nMIN_PRIMARY_DATAPOINTS = 2\nMAX_Z = 3\nMAX_DEEP_Z = 1\nMAX_T = 350\nMAX_DEEP_T = 180\nif USE_POLARS:\n    ## Only implmented in polars version.\n    AUX_FALSE_WEIGHT = 0.01\n\n## Ensemble and algorithm selection parameters.\n## First weight is center of charge algorithm introduced in this notebook.\n## Second weight is the unweighted version. Which turns out to be the least-squares algorithm: https://www.kaggle.com/competitions/icecube-neutrinos-in-deep-ice/discussion/381747\nUSE_ENSEMBLE = True\nWEIGHTS = [0.58, 0.42]\nif not USE_ENSEMBLE and not USE_POLARS:\n    ALGORITHM = 'center of charge'\n#     ALGORITHM = 'least squares'\n    if ALGORITHM == 'least squares':\n        USE_WEIGHTED_LEAST_SQUARES = False\n\n\n## Constants\nINPUT_DIR = '/kaggle/input/icecube-neutrinos-in-deep-ice'\n\n## Basic configuration override logic\nif MODE == 'test':\n    TRAIN_MAX_EVENTS = None\nif USE_ENSEMBLE:\n    USE_WEIGHTED_LEAST_SQUARES = False\n    \nif USE_POLARS:\n    try:\n        import polars as pl\n    except:\n        print('Installing polars, please wait about 35 seconds...')\n        !pip install /kaggle/input/polars01516/polars-0.15.16-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n        import polars as pl\n        \nimport numpy as np\nimport pandas as pd\nimport math\nimport time\nimport gc\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\n## Condensed for space. See here for expanded original version: https://www.kaggle.com/code/sohier/mean-angular-error\ndef angular_dist_score(az_true, zen_true, az_pred, zen_pred):\n    if not (np.all(np.isfinite(az_true)) and\n            np.all(np.isfinite(zen_true)) and\n            np.all(np.isfinite(az_pred)) and\n            np.all(np.isfinite(zen_pred))):\n        raise ValueError(\"All arguments must be finite\")\n    sa1 = np.sin(az_true)\n    ca1 = np.cos(az_true)\n    sz1 = np.sin(zen_true)\n    cz1 = np.cos(zen_true)\n    sa2 = np.sin(az_pred)\n    ca2 = np.cos(az_pred)\n    sz2 = np.sin(zen_pred)\n    cz2 = np.cos(zen_pred)\n    scalar_prod = sz1*sz2*(ca1*ca2 + sa1*sa2) + (cz1*cz2)\n    scalar_prod =  np.clip(scalar_prod, -1, 1)\n    return np.average(np.abs(np.arccos(scalar_prod)))\n\n## TODO: It would be good to benchmark versus other implementations, like arctan2 used here: https://www.kaggle.com/code/shlomoron/icecube-eda-pca-baseline-cv-1-28-lb-1-274 \n\n## This version has a small optimization trick, calculating azimuth without regard for z or zenith\n## This version is suboptimal if the vectors are already unit vectors, or if you need 3d unit vectors again later for some other step.\ndef angles_from_vectors(vectors):\n    v_squared = np.square(vectors)\n    \n    ## Shortcut optimization for azimuth: calculate 2d unit vectors for x and y independent of z\n    xy_sq = np.sum(v_squared[:, 0:2], axis=1)\n    xy_d = np.sqrt(xy_sq)[:, None]\n    np.seterr(divide='ignore', invalid='ignore') ## Turn off the warning temporarily\n    vectors[:, 0:2] = np.where(xy_d == 0, xy_d, vectors[:, 0:2]/xy_d)\n\n    ## For z, use full 3d unit vector\n    d = np.sqrt(xy_sq + v_squared[:, 2])\n    vectors[:, 2] = np.where(d == 0, d, vectors[:, 2]/d)\n    np.seterr(divide='warn', invalid='warn') ## Turn back on\n\n    ## As mentioned by others, clip solely to avoid floating point errors, the unit vectors should already be within this range.\n    vectors =  np.clip(vectors, -1, 1)\n\n    azimuth = np.arccos(vectors[:, 0])\n    ## if y < 0, convert from quadrants 1 and 2 to quadrants 3 and 4\n    azimuth = np.where(vectors[:, 1] >= 0, azimuth, 2*math.pi - azimuth)\n    azimuth = np.where(np.isfinite(azimuth), azimuth, 0.0)\n\n    zenith = np.arccos(vectors[:, 2])\n    ## IMPORTANT: zenith angles are not evenly distributed, so set the error case to pi/2!\n    ## (even though x, y, z might be. It would be a fun exercise to check if random values\n    ##  for x, y, z converted to zenith angles would match the observed distribution of zenith angles in the train labels)\n    zenith = np.where(np.isfinite(zenith), zenith, math.pi/2)\n\n    return np.stack([azimuth, zenith], axis=1)\n\n## Takes a list of azimuth np arrays, a list of zenith np arrays, and an optional list of numerical weights,\n## and ensembles into a final direction.\n##\n## It's not really optimal in terms of lines of code nor performance,\n## since in most or all cases you are converting a unit vector to an angle,\n## converting back to a unit vector, averaging, then converting to the final angle.\n## However, it is quite convenient, because you can always use this at the end\n## to ensemble the results originating from any number of notebooks or sources.\ndef average_angles(az_list, zen_list, weights=None):\n    assert(len(az_list) == len(zen_list))\n    total = az_list[0].shape[0]\n    x = np.zeros(total)\n    y = np.zeros(total)\n    z = np.zeros(total)\n    for i in range(len(az_list)):\n        w = 1\n        if weights is not None:\n            w = weights[i]\n        az = az_list[i]\n        zen = zen_list[i]\n        assert(az.shape[0] == total)\n        assert(zen.shape[0] == total)\n        if not (np.all(np.isfinite(az)) and\n                np.all(np.isfinite(zen))):\n            raise ValueError(\"All arguments must be finite\")\n        sz = np.sin(zen)\n        x += w*np.cos(az)*sz\n        y += w*np.sin(az)*sz\n        z += w*np.cos(zen)\n    tot_w = len(az_list)\n    if weights is not None:\n        tot_w = sum(weights)\n    x = x / tot_w\n    y = y / tot_w\n    z = z / tot_w\n    d = np.sqrt(np.square(x) + np.square(y) + np.square(z))\n    x = x / d\n    y = y / d\n    z = z / d\n    return angles_from_vectors(np.stack([x, y, z], axis=1))\n\ndef center_of_charge(batch):\n    ## Groupby -> transform is the key trick to avoiding the dreaded 'for each event' loop,\n    ## and thus getting ~30-60x speed boost improvement!\n    ## If you need any min, max, mean or other simple value from the event group,\n    ## you can precalculate it for each group and broadcast it\n    batch['ev_t_min'] = batch.groupby('event_id')['time'].transform('min')\n    batch['ev_t_max'] = batch.groupby('event_id')['time'].transform('max')\n    \n    ## Now we can just implement our formula! w0 and w1 are the time-weighted charge cases.\n    ## Gather the values we need\n    batch['w1'] = batch.charge * (batch.time - batch.ev_t_min) / (batch.ev_t_max - batch.ev_t_min)\n    batch['w0'] = batch.charge - batch.w1\n    batch['wx0'] = batch.x * batch.w0\n    batch['wy0'] = batch.y * batch.w0\n    batch['wz0'] = batch.z * batch.w0\n    batch['wx1'] = batch.x * batch.w1\n    batch['wy1'] = batch.y * batch.w1\n    batch['wz1'] = batch.z * batch.w1\n    df = batch[['w0', 'w1', 'wx0', 'wy0', 'wz0', 'wx1', 'wy1', 'wz1']]\n\n    ## Calculate all the sums!\n    df = df.groupby('event_id').sum()\n    \n    ## Now do the final divide of the weighted center by the sum of the weights.\n    df[['wx0', 'wy0', 'wz0']] = df[['wx0', 'wy0', 'wz0']].div(df.w0, axis=0)\n    df[['wx1', 'wy1', 'wz1']] = df[['wx1', 'wy1', 'wz1']].div(df.w1, axis=0)\n    \n    ## The direction the neutrino is traveling FROM is point0 - point1, instead of point1 - point0.\n    ## Counter-intuitive to me, but fortunately, easy to notice and correct if your score is > 1.57 instead of less.\n    df[['x', 'y', 'z']] = df[['wx0', 'wy0', 'wz0']].values - df[['wx1', 'wy1', 'wz1']].values\n\n    df = df[['x', 'y', 'z']]\n    df[['azimuth', 'zenith']] = angles_from_vectors(df.values)\n\n    return(df[['azimuth', 'zenith']])\n\ndef least_squares(batch, weighted=False):\n    batch['xt'] = batch.x * batch.time\n    batch['yt'] = batch.y * batch.time\n    batch['zt'] = batch.z * batch.time\n    batch['tt'] = batch.time * batch.time\n    if weighted:\n        df = batch[['x', 'y', 'z', 'time', 'xt', 'yt', 'zt', 'tt']] * batch.charge.values[:, None]\n        df['charge'] = batch.charge\n        df = df.groupby('event_id').sum()\n        df = df.div(df.charge, axis=0)\n    else:\n        df = batch[['x', 'y', 'z', 'time', 'xt', 'yt', 'zt', 'tt']]\n        df = df.groupby('event_id').mean()\n    df[['x', 'y', 'z']] = (\n                              (df[['xt', 'yt', 'zt']].values - (df[['x', 'y', 'z']].values * df['time'].values[:, None]))\n                            / (df['tt'].values - (df.time.values * df.time.values))[:, None]\n                          )\n    ## Reverse it\n    df = -df[['x', 'y', 'z']]\n    df[['azimuth', 'zenith']] = angles_from_vectors(df.values)\n    return df[['azimuth', 'zenith']]\n\n\ndef process_batch(batch_id, sensor, max_events=None):\n    print('load batch...')\n    batch = pd.read_parquet(f'{INPUT_DIR}/{MODE}/batch_{batch_id}.parquet')\n\n    ## Limit to max_events\n    if max_events is not None:\n        batch_i = batch.reset_index()\n        event_ids = batch_i.event_id.drop_duplicates()\n        end_index = event_ids.index[max_events]\n        batch = batch_i[:end_index].set_index('event_id')\n    print(batch.shape)\n\n    ## Merge in sensor x,y,z data\n    batch = batch.reset_index().merge(sensor, how='left', on='sensor_id', left_index=False).set_index('event_id')\n\n    ## The logic for auxiliary = False is very basic, we can improve on it. Initial discussion here: \n    if FIND_BEST_POINTS:\n        batch = find_best_points_pandas(batch)\n\n    ## Limit to primary (aux=False) datapoints if there's enough of them.\n    ## For event_ids with too few, set all rows to aux=False. This handles these cases without any for loop logic.\n    ## MIN_PRIMARY_DATAPOINTS is a tuned value, based on optimizing the score on batches 101-110.\n    ## But the result was 'as low as possible'.\n    batch['primary_count'] = batch.groupby('event_id')['auxiliary'].transform('count') - batch.groupby('event_id')['auxiliary'].transform('sum')\n    batch.loc[batch.primary_count < MIN_PRIMARY_DATAPOINTS, 'auxiliary'] = False\n    batch = batch[batch.auxiliary == False]\n    print(batch.shape)\n\n    if USE_ENSEMBLE or ALGORITHM == 'center of charge':\n        df = center_of_charge(batch)\n        if USE_ENSEMBLE:\n            df1 = df\n    if USE_ENSEMBLE or ALGORITHM == 'least squares':\n        df = least_squares(batch, weighted=USE_WEIGHTED_LEAST_SQUARES)\n    if USE_ENSEMBLE:\n        df[['azimuth', 'zenith']] = average_angles([df1.azimuth.values, df.azimuth.values],\n                                                   [df1.zenith.values, df.zenith.values], \n                                                   weights=WEIGHTS)\n    return df\n\ndef proximity(e, col):\n    ## Since we explode the data to an NxN array, if N is too high it will take a long time and anyways we'll run out of memory.\n    ## TODO: see how high we can go before we run out of memory\n    ## TODO: consider subsampling instead of simply returning the baseline auxiliary setting?\n    ##       And/or splitting on string_id to get much smaller groups\n    if e.shape[0] > 2000:\n        return e[:, col['auxiliary']]\n\n    ## The magic None in the line below is a new thing I learned while working on this notebook.\n    ## It is shorthand for np.newaxis, and broadcasts the array to another dimension.\n    ## This allows us to create an NxN array for each event, so that we can\n    ## check if *any* other row in the event meets our proximity in time and height requirements.\n    ## For any matching pairs, then we set both rows as auxiliary = False. Rows without matches are auxiliary = True.\n    deltas = np.abs(e[:, [col['string_id'], col['depth_id'], col['time']]] - e[:, None, [col['string_id'], col['depth_id'], col['time']]])\n    dz = deltas[:, :, 1]\n\n    ## if same depth or different string id, ignore by setting dz > the max threshold used later.\n    dz[(dz == 0) | (deltas[:, :, 0] != 0)] = MAX_Z + MAX_DEEP_Z + 1\n\n    ## if sensor is not a deep ice sensor, and time > MAX_T, ignore\n    mask = (e[:, col['sensor_id']] < 4680)\n    mask = np.broadcast_to(mask, (mask.shape[0], mask.shape[0])).T\n    dz[mask & (deltas[:, :, 2] > MAX_T)] = MAX_Z + MAX_DEEP_Z + 1\n    ## if sensor IS a deep ice sensor, and time > MAX_DEEP_T, ignore\n    mask = (e[:, col['sensor_id']] >= 4680)\n    mask = np.broadcast_to(mask, (mask.shape[0], mask.shape[0])).T\n    dz[mask & (deltas[:, :, 2] > MAX_DEEP_T)] = MAX_Z + MAX_DEEP_Z + 1\n\n    ## Now take the min (best) result for each row com\n    dz = dz.min(axis=1)\n    ## If no matches, the default, then everything is aux=True\n    e[:, col['auxiliary']] = True\n    ## If not deep ice and distance less than threshold, or deep ice and distance less than other threshold, then we have a match!\n    e[((e[:, col['sensor_id']] < 4680) & (dz <= MAX_Z)) | ((e[:, col['sensor_id']] >= 4680) & (dz <= MAX_DEEP_Z)), col['auxiliary']] = False\n    ## Return only the data needed to speed up the np.concatenate called next.\n    return e[:, col['auxiliary']]\n\n## You can ignore this one unless interested in a deep dive on performance optimization\n## It is provided as a way of comparing the changes versus the pure numpy version.\n## Comments removed from this copy to conserve vertical space.\ndef proximity_unoptimized(df):\n    if df.shape[0] > 2000:\n        return df\n    deltas = np.abs(df[['string_id', 'depth_id', 'time']].values - df[['string_id', 'depth_id', 'time']].values[:, None, :])\n    mask = (df.sensor_id < 4680)\n    dz = deltas[:, :, 1]\n\n    dz[(dz == 0) | (deltas[:, :, 0] != 0)] = MAX_Z + MAX_DEEP_Z + 1\n\n    mask = (df.sensor_id < 4680)\n    mask = np.broadcast_to(mask, (mask.shape[0], mask.shape[0])).T\n    dz[mask & (deltas[:, :, 2] > MAX_T)] = MAX_Z + MAX_DEEP_Z + 1\n    mask = (df.sensor_id >= 4680)\n    mask = np.broadcast_to(mask, (mask.shape[0], mask.shape[0])).T\n    dz[mask & (deltas[:, :, 2] > MAX_DEEP_T)] = MAX_Z + MAX_DEEP_Z + 1\n\n    dz = dz.min(axis=1)\n    df.auxiliary = True\n    df.loc[((df.sensor_id < 4680) & (dz <= MAX_Z)) | ((df.sensor_id >= 4680) & (dz <= MAX_DEEP_Z)), 'auxiliary'] = False\n    return df\n\n\ndef find_best_points_pandas(batch):\n    if USE_UNOPTIMIZED:\n        cols = ['sensor_id', 'time', 'auxiliary', 'string_id', 'depth_id']\n        batch[cols] = batch[cols].groupby('event_id').progress_apply(proximity_unoptimized)\n        return batch\n\n    ## np.split used as a pure numpy equivalent of groupby\n    ## Note this version didn't minimize the size of the inputs, but does make sure all dtypes are the same for an efficient np array.\n    column_to_index = { k:v for v,k in enumerate(batch.columns)}\n    events = np.split(batch.values.astype('float32'), np.unique(batch.index.values, return_index=True)[1][1:])\n\n    ## Run each event sequentially in a list comprehension, then join back together with np.concatenate.\n    ## So far tried and failed to find a reasonable solution to avoid this groupby > apply > join loop.\n    ## Note that this line overrides the dtype of 'auxiliary' column to float32\n    batch.auxiliary = np.concatenate([proximity(e, column_to_index) for e in tqdm(events)])\n    return batch\n\ndef time_weighted_centering(batch, charge_weighted=True):\n    ## Polars equivalent to groupby->transform is called 'over'. We again use this to get min and max without a for loop.\n    batch = batch.with_columns([pl.col('time').min().over('event_id').alias('ev_t_min'),\n                                pl.col('time').max().over('event_id').alias('ev_t_max')])\n    if charge_weighted:\n        batch = batch.with_columns((pl.col('charge') * (pl.col('time') - pl.col('ev_t_min'))\n                                    / (pl.col('ev_t_max') - pl.col('ev_t_min'))).alias('w1'))\n        batch = batch.with_columns((pl.col('charge') - pl.col('w1')).alias('w0'))\n    else:\n        batch = batch.with_columns(((pl.col('time') - pl.col('ev_t_min'))\n                                    / (pl.col('ev_t_max') - pl.col('ev_t_min'))).alias('w1'))\n        batch = batch.with_columns((pl.lit(1) - pl.col('w1')).alias('w0'))\n\n    batch = batch.select(\n        [\n            pl.col('event_id'),\n            pl.col('w0'),\n            pl.col('w1'),\n            (pl.col('x') * pl.col('w0')).alias('wx0'),\n            (pl.col('y') * pl.col('w0')).alias('wy0'),\n            (pl.col('z') * pl.col('w0')).alias('wz0'),\n            (pl.col('x') * pl.col('w1')).alias('wx1'),\n            (pl.col('y') * pl.col('w1')).alias('wy1'),\n            (pl.col('z') * pl.col('w1')).alias('wz1'),\n        ]\n    ).collect().groupby('event_id', maintain_order=True).sum()\n\n    ## The direction the neutrino is traveling FROM is point0 - point1, instead of point1 - point0.\n    ## Counter-intuitive to me, but fortunately, easy to notice and correct if your score is > 1.57 instead of less.\n    batch_values = batch.select(\n        [\n            ((pl.col('wx0') / pl.col('w0')) - (pl.col('wx1') / pl.col('w1'))).alias('x'),\n            ((pl.col('wy0') / pl.col('w0')) - (pl.col('wy1') / pl.col('w1'))).alias('y'),\n            ((pl.col('wz0') / pl.col('w0')) - (pl.col('wz1') / pl.col('w1'))).alias('z'),\n        ]\n    ).to_numpy()\n    return angles_from_vectors(batch_values), batch\n\n\n## We use the same numpy proximity function, the only difference is we convert from and to a Polars df instead of a Pandas df.\ndef find_best_points_polars(batch):\n    ## Minimize the size of the inputs, and make sure all data types are the same for an efficient np array\n    df = batch.select([pl.col('sensor_id'), pl.col('time'), pl.col('auxiliary'), pl.col('string_id'), pl.col('depth_id')])\n    column_to_index = { k:v for v,k in enumerate(df.columns)}\n    batch_values = df.to_numpy().astype('float32')\n\n    ## Pure numpy equivalent of groupby\n    events = np.split(batch_values, np.unique(batch.select(pl.col('event_id')), return_index=True)[1][1:])\n\n    ## Run each event sequentially in a list comprehension, then join back together with np.concatenate. So far tried and failed to find a reasonable solution to avoid this groupby > apply > join loop.\n    ## Rename 'auxiliary' -> 'best', and flip the boolean value\n    batch = batch.with_columns(pl.Series(np.concatenate([proximity(e, column_to_index) for e in tqdm(events)]).astype('bool')).alias('best')).with_columns(pl.col('best').is_not())\n\n    return batch\n\n\n\nif USE_POLARS:\n    sub = []\n    #for batch_id in range(batch_id_start,batch_id_end):\ndef get_line_fit_angles(batch_id):\n    print(MODE)\n    ## Scan parquet is part of Polars lazy evaluation, so no cost yet,\n    ## and it can figure out optimizations when we finally 'collect' it later.\n    meta = pl.scan_parquet(f'{INPUT_DIR}/{MODE}_meta.parquet')\n\n    print('load sensor data...')\n    sensor = (pl.scan_csv(f'{INPUT_DIR}/sensor_geometry.csv')\n                .with_columns([\n                    pl.col('sensor_id').cast(pl.Int16),\n                    (pl.col('sensor_id') // 60).alias('string_id'),\n                    (pl.col('sensor_id') % 60).alias('depth_id'),                \n                ])\n             )\n\n    print(sensor)\n\n    if MODE == 'train':\n        batch_id_start = TRAIN_BATCH_START\n        batch_id_end = batch_id_start + TRAIN_N_BATCHES\n    else:\n        batch_id_start = meta.select(pl.col('batch_id')).collect()[0, 0]\n        batch_id_end = meta.select(pl.col('batch_id')).collect()[-1, 0] + 1\n\n    print(batch_id_start, batch_id_end)\n    \n    print(batch_id)\n    t = time.time()\n    max_events=TRAIN_MAX_EVENTS\n    print('load batch...')\n    batch = pl.scan_parquet(f'{INPUT_DIR}/{MODE}/batch_{batch_id}.parquet')\n\n    ## Limit to max_events\n    if max_events is not None:\n        batch = batch.collect()\n        last_event_id = batch.select(pl.col('event_id')).unique()[TRAIN_MAX_EVENTS-1, 0]\n        batch = batch.lazy().filter(pl.col('event_id') <= last_event_id)\n\n    ## Merge in sensor x,y,z data\n    batch = batch.join(sensor, on='sensor_id', how='left').collect()\n\n    ## The logic for auxiliary = False is very basic, we can improve on it. Initial discussion here: \n    if FIND_BEST_POINTS:\n        batch = find_best_points_polars(batch)\n\n    ## Use data point weights instead of filtering. Still set most points to 0 weight, unless they are the only data points available.\n    ## MIN_PRIMARY_DATAPOINTS and AUX_FALSE_WEIGHT are tuned values, based on optimizing the score on batches 101-110.\n    batch = batch.lazy().with_columns((pl.col('best').count().over('event_id') - pl.col('best').sum().over('event_id')).alias('best_count'))\n    batch = batch.lazy().with_columns((pl.col('auxiliary').count().over('event_id') - pl.col('auxiliary').sum().over('event_id')).alias('non_aux_count'))\n    batch = batch.with_columns(( pl.when( pl.col('best') | ((pl.col('best_count') < MIN_PRIMARY_DATAPOINTS) & (pl.col('non_aux_count') < MIN_PRIMARY_DATAPOINTS)) )\n                                            .then(1.0)\n                                            .otherwise(pl.when(pl.col('auxiliary').is_not())\n                                                         .then(AUX_FALSE_WEIGHT)\n                                                         .otherwise(0.0)\n                                                    ) \n                                      ).alias('trust'))\n    batch = batch.lazy().with_columns((pl.col('charge') * pl.col('trust')).alias('charge'))\n\n    preds, events = time_weighted_centering(batch)\n    if USE_ENSEMBLE:\n        preds1 = preds\n#             preds, events = time_weighted_centering(batch, charge_weighted=False)\n        ## Instead of charge_weighted=False, set charge*aux to just equal aux.\n        batch = batch.lazy().with_columns(pl.col('trust').alias('charge'))\n        preds, events = time_weighted_centering(batch)\n        preds = average_angles([preds1[:, 0], preds[:, 0]], [preds1[:, 1], preds[:, 1]], weights=WEIGHTS)\n\n    '''\n    if MODE == 'test':\n        sub.append(events.select([pl.col('event_id'), pl.Series(preds[:, 0]).alias('azimuth'),\n                                                     pl.Series(preds[:, 1]).alias('zenith')]))\n    else:\n        meta = meta.filter((pl.col('batch_id') >= batch_id_start) & (pl.col('batch_id') < batch_id_end))\n        if isinstance(meta, pl.LazyFrame):\n            meta = meta.collect()\n        meta_values = meta.filter(pl.col('batch_id') == batch_id).select([pl.col('azimuth'), pl.col('zenith')]).to_numpy()\n        if TRAIN_MAX_EVENTS is not None:\n            print(angular_dist_score(meta_values[:TRAIN_MAX_EVENTS, 0], meta_values[:TRAIN_MAX_EVENTS, 1], preds[:, 0], preds[:, 1]))\n        else:\n            print(angular_dist_score(meta_values[:, 0], meta_values[:, 1], preds[:, 0], preds[:, 1]))\n    '''\n    print(f'Time: {time.time() - t:0.2f}s')\n    return preds\n\n#fitted_angle = get_line_fit_angles(1)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:45.295575Z","iopub.execute_input":"2023-04-07T01:47:45.295986Z","iopub.status.idle":"2023-04-07T01:47:45.646195Z","shell.execute_reply.started":"2023-04-07T01:47:45.295942Z","shell.execute_reply":"2023-04-07T01:47:45.644808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Read Train Meta Data\ntrain_meta_df = pd.read_parquet(home_dir + 'train_meta.parquet')\n\nbatch_counts = train_meta_df.batch_id.value_counts().sort_index()\n\nbatch_max_index = batch_counts.cumsum()\nbatch_max_index[train_meta_df.batch_id.min() - 1] = 0\nbatch_max_index = batch_max_index.sort_index()\n\ndef train_meta_df_spliter(batch_id):\n    return train_meta_df.loc[batch_max_index[batch_id - 1]:batch_max_index[batch_id] - 1]\n\nfor batch_id in train_batch_ids:\n    #line fit\n    line_fit_angles = get_line_fit_angles(batch_id)\n    \n    print(\"Reading batch \", batch_id, end=\"\")\n    # get batch meta data and data\n    batch_meta_df = train_meta_df_spliter(batch_id)\n    batch_df = pd.read_parquet(train_format.format(batch_id=batch_id))\n\n    # register pulses\n    batch_x = np.zeros((len(batch_meta_df), max_pulse_count, n_features), dtype=\"float16\")\n    batch_y = np.zeros((len(batch_meta_df), 2), dtype=\"float32\")\n    \n    batch_x[:, :, 2] = -1\n\n    def read_event_local(event_idx):\n        return read_event(event_idx, batch_meta_df, max_pulse_count, batch_df, train=True)\n\n    # Proces Events\n    iterator = range(len(batch_meta_df))\n    with multiprocessing.Pool() as pool:\n        for event_idx, pulse_count, event_x, event_y in pool.map(read_event_local, iterator):\n            batch_x[event_idx, :pulse_count, 0] = event_x[\"time\"]\n            batch_x[event_idx, :pulse_count, 1] = event_x[\"charge\"]\n            batch_x[event_idx, :pulse_count, 2] = event_x[\"auxiliary\"]\n            batch_x[event_idx, :pulse_count, 3] = event_x[\"x\"]\n            batch_x[event_idx, :pulse_count, 4] = event_x[\"y\"]\n            batch_x[event_idx, :pulse_count, 5] = event_x[\"z\"]\n\n            batch_y[event_idx] = event_y\n                \n    del batch_meta_df, batch_df\n\n    batch_x, batch_y = normalize_data(batch_x, batch_y)\n    \n\n    \n\n    batch_x = batch_x.reshape(np.shape(batch_x)[0], max_pulse_count*(n_features+3))\n    \n\n    # Save    \n    print(\" DONE! Saving...\")\n    save_to_tfrecord(batch_x, line_fit_angles, batch_y, f'{batch_id}.tfrecord')\n    del batch_x, line_fit_angles, batch_y\n    gc.collect()\n    #np.savez(point_picker_format.format(batch_id=batch_id), x=batch_x, y=batch_y)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:47:45.650962Z","iopub.execute_input":"2023-04-07T01:47:45.651463Z","iopub.status.idle":"2023-04-07T01:54:00.549211Z","shell.execute_reply.started":"2023-04-07T01:47:45.651425Z","shell.execute_reply":"2023-04-07T01:54:00.548196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"batch_x = np.zeros((200000, max_pulse_count, n_features), dtype=\"float16\")\nbatch_y = np.zeros((200000, 2), dtype=\"float16\")\nbatch_x, batch_y, batch_origin_y = normalize_data(batch_x, batch_y)\nbatch_x = batch_x.reshape(np.shape(batch_x)[0], max_pulse_count*(n_features+6))\n\n# Save    \nprint(\" DONE! Saving...\")\nsave_to_tfrecord(batch_x, batch_origin_y, f'{batch_id}.tfrecord')","metadata":{"execution":{"iopub.status.busy":"2023-04-02T08:16:30.382900Z","iopub.execute_input":"2023-04-02T08:16:30.384319Z","iopub.status.idle":"2023-04-02T08:16:44.985069Z","shell.execute_reply.started":"2023-04-02T08:16:30.384261Z","shell.execute_reply":"2023-04-02T08:16:44.982452Z"}}},{"cell_type":"code","source":"def decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'event_pulses': tf.io.FixedLenFeature([], tf.string),\n        'fitted_azimuth': tf.io.FixedLenFeature([], tf.float32),               \n        'fitted_zenith' : tf.io.FixedLenFeature([], tf.float32),\n        'origin_azimuth': tf.io.FixedLenFeature([], tf.float32),               \n        'origin_zenith' : tf.io.FixedLenFeature([], tf.float32),\n    })        \n\n    event_pulses = tf.io.parse_tensor(features['event_pulses'], out_type=tf.float16)\n    event_pulses = tf.cast(event_pulses, tf.float16)\n    event_pulses = tf.reshape(event_pulses, [max_pulse_count, 9])\n    \n    fitted_azimuth = features['fitted_azimuth']\n    fitted_zenith  = features['fitted_zenith']\n    fitted_targets = tf.stack([fitted_azimuth, fitted_zenith])\n    \n    origin_azimuth = features['origin_azimuth']    \n    origin_zenith =  features['origin_zenith']\n    origin_targets = tf.stack([origin_azimuth, origin_zenith])\n    \n    return event_pulses, fitted_targets, origin_targets","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:58:33.406490Z","iopub.execute_input":"2023-04-07T01:58:33.407794Z","iopub.status.idle":"2023-04-07T01:58:33.415694Z","shell.execute_reply.started":"2023-04-07T01:58:33.407737Z","shell.execute_reply":"2023-04-07T01:58:33.414512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample TFRecord Dataset\ndef get_train_dataset():\n    # Read all TFRecord file paths\n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob('/kaggle/working/1.tfrecord')\n    # initialize TFRecord dataset\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=1, compression_type='GZIP')\n    # Decode samples by mapping with decode function\n    train_dataset = train_dataset.map(decode_tfrecord)\n    # Batch samples\n    train_dataset = train_dataset.batch(1000)\n    return train_dataset","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:58:33.580555Z","iopub.execute_input":"2023-04-07T01:58:33.581610Z","iopub.status.idle":"2023-04-07T01:58:33.587265Z","shell.execute_reply.started":"2023-04-07T01:58:33.581548Z","shell.execute_reply":"2023-04-07T01:58:33.586040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show Example Batch\ntrain_dataset = get_train_dataset()\niteration = iter(train_dataset)\npulses=[]\ntargetss=[]\nfitted_targetss=[]\norigin_targetss=[]\nfor i in tqdm(range(0, 200)):\n    event_pulses,fitted_targets, origin_targets = next(iteration)    \n    pulses.append(event_pulses.numpy())\n    fitted_targetss.append(fitted_targets.numpy())\n    origin_targetss.append(origin_targets.numpy())\n    \npulses = np.vstack(pulses)\nfitted_targetss = np.vstack(fitted_targetss)\norigin_targetss = np.vstack(origin_targetss)\n\nprint(np.shape(pulses))\nprint(np.shape(fitted_targetss))\nprint(np.shape(origin_targetss))\n'''\nsample_batch_x = batch_x[:1000]\nsample_batch_y = batch_y[:1000]\nsample_batch_origin_y = batch_origin_y[:1000]\n\nsample_batch_x = sample_batch_x.reshape(1000, max_pulse_count, 9)\n\nprint(np.max(sample_batch_x - pulses))\nprint(np.max(sample_batch_y - targetss))\nprint(np.max(sample_batch_origin_y - origin_targetss))\n'''","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:58:33.743436Z","iopub.execute_input":"2023-04-07T01:58:33.743832Z","iopub.status.idle":"2023-04-07T01:58:42.519196Z","shell.execute_reply.started":"2023-04-07T01:58:33.743798Z","shell.execute_reply":"2023-04-07T01:58:42.518023Z"}}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}