{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-01T03:21:56.930812Z","iopub.execute_input":"2023-02-01T03:21:56.931268Z","iopub.status.idle":"2023-02-01T03:22:08.833788Z","shell.execute_reply.started":"2023-02-01T03:21:56.931232Z","shell.execute_reply":"2023-02-01T03:22:08.831974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport os\n\nimport numpy as np\nimport polars as pl\nimport matplotlib.pyplot as plt\n\nOUTPUT_DIR = '/kaggle/working'\nDATA_ROOT = '/kaggle/input/icecube-neutrinos-in-deep-ice'\nDATA_TEST = f'{DATA_ROOT}/test'\nDATA_TRAIN = f'{DATA_ROOT}/train'\n\nSAMPLE_SUBMISSION = f'{DATA_ROOT}/sample_submission.parquet'\nSENSOR_GEOMETRY = f'{DATA_ROOT}/sensor_geometry.csv'\nTEST_META = f'{DATA_ROOT}/test_meta.parquet'\nTRAIN_META = f'{DATA_ROOT}/train_meta.parquet'","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.836572Z","iopub.execute_input":"2023-02-01T03:22:08.836981Z","iopub.status.idle":"2023-02-01T03:22:08.845762Z","shell.execute_reply.started":"2023-02-01T03:22:08.836942Z","shell.execute_reply":"2023-02-01T03:22:08.844239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub_df = pl.read_parquet(SAMPLE_SUBMISSION)\nsample_sub_df","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.847409Z","iopub.execute_input":"2023-02-01T03:22:08.847814Z","iopub.status.idle":"2023-02-01T03:22:08.870246Z","shell.execute_reply.started":"2023-02-01T03:22:08.847768Z","shell.execute_reply":"2023-02-01T03:22:08.868515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_geo_df = pl.read_csv(SENSOR_GEOMETRY)\\\n    .with_columns(pl.col('sensor_id').cast(pl.Int16))\nsensor_geo_df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.872045Z","iopub.execute_input":"2023-02-01T03:22:08.872474Z","iopub.status.idle":"2023-02-01T03:22:08.886251Z","shell.execute_reply.started":"2023-02-01T03:22:08.872435Z","shell.execute_reply":"2023-02-01T03:22:08.885004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_meta_df = pl.read_parquet(TEST_META)\ntest_meta_df","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.890308Z","iopub.execute_input":"2023-02-01T03:22:08.890763Z","iopub.status.idle":"2023-02-01T03:22:08.902201Z","shell.execute_reply.started":"2023-02-01T03:22:08.890725Z","shell.execute_reply":"2023-02-01T03:22:08.900680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile(f'{OUTPUT_DIR}/train_meta_top100.parquet'):\n    train_meta_top100_df = pl.read_parquet(f'{OUTPUT_DIR}/train_meta_top100.parquet')\nelse:\n    train_meta_top100_df = pl.read_parquet(TRAIN_META, n_rows=100)\n    train_meta_top100_df.write_parquet(f'{OUTPUT_DIR}/train_meta_top100.parquet')\ntrain_meta_top100_df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.904901Z","iopub.execute_input":"2023-02-01T03:22:08.905749Z","iopub.status.idle":"2023-02-01T03:22:08.920762Z","shell.execute_reply.started":"2023-02-01T03:22:08.905701Z","shell.execute_reply":"2023-02-01T03:22:08.919306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile(f'{OUTPUT_DIR}/batch1_100events.parquet'):\n    batch1_100events_df = pl.read_parquet(f'{OUTPUT_DIR}/batch1_100events.parquet')\nelse:\n    batch1_100events_df = pl.read_parquet(f'{DATA_TRAIN}/batch_1.parquet')\n    batch1_100events_df = batch1_100events_df\\\n        .filter(pl.col('event_id').is_in(train_meta_top100_df['event_id'].unique()))\n    batch1_100events_df.write_parquet(f'{OUTPUT_DIR}/batch1_100events.parquet')\nbatch1_100events_df.filter((pl.col('event_id') == 24) & ~pl.col('auxiliary'))","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.922379Z","iopub.execute_input":"2023-02-01T03:22:08.922800Z","iopub.status.idle":"2023-02-01T03:22:08.939640Z","shell.execute_reply.started":"2023-02-01T03:22:08.922763Z","shell.execute_reply":"2023-02-01T03:22:08.938571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_agg = batch1_100events_df\\\n    .groupby('event_id')\\\n    .agg([\n        pl.col('time').min().alias('min_time'),\n        pl.col('time').max().alias('max_time'),\n        pl.col('charge').mean().alias('mean_charge'),\n        pl.col('charge').std().alias('std_charge')\n    ])\n\nprint(event_agg.sample(10), end='\\n\\n')\n\nevents = batch1_100events_df\\\n    .join(event_agg, left_on='event_id', right_on='event_id')\\\n    .with_columns([\n        ((pl.col('time') - pl.col('min_time')) / (pl.col('max_time') - pl.col('min_time'))).alias('rel_time'),\n        ((pl.col('charge') - pl.col('mean_charge')) / pl.col('std_charge')).alias('norm_charge')\n    ])\\\n    .select([\n        'event_id',\n        'sensor_id',\n        'auxiliary',\n        'rel_time',\n        'norm_charge'\n    ])\n\nevents.filter(pl.col('event_id') == 24)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:22:08.941165Z","iopub.execute_input":"2023-02-01T03:22:08.941822Z","iopub.status.idle":"2023-02-01T03:22:08.968500Z","shell.execute_reply.started":"2023-02-01T03:22:08.941784Z","shell.execute_reply":"2023-02-01T03:22:08.966969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib widget\n\nevent24 = events\\\n    .filter(pl.col('event_id') == 24)\\\n    .join(sensor_geo_df, left_on='sensor_id', right_on='sensor_id')\n\nevent24_no_aux = event24.filter(~pl.col('auxiliary'))\n\nfig = plt.figure()\nax0 = fig.add_subplot(1, 2, 1, projection='3d')\nax1 = fig.add_subplot(1, 2, 2, projection='3d')\n\nax0.set_xlabel('X')\nax0.set_ylabel('Y')\nax0.set_zlabel('Z')\nax1.set_xlabel('X')\nax1.set_ylabel('Y')\nax1.set_zlabel('Z')\n\nax0.scatter(\n    event24['x'],\n    event24['y'],\n    event24['z'],\n    s=event24['norm_charge'] - event24['norm_charge'].min(),\n    c=event24['rel_time']\n)\n\nax1.scatter(\n    event24_no_aux['x'],\n    event24_no_aux['y'],\n    event24_no_aux['z'],\n    s=event24_no_aux['norm_charge'] - event24_no_aux['norm_charge'].min(),\n    c=event24_no_aux['rel_time']\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-01T03:29:42.761931Z","iopub.execute_input":"2023-02-01T03:29:42.762342Z","iopub.status.idle":"2023-02-01T03:29:42.868823Z","shell.execute_reply.started":"2023-02-01T03:29:42.762310Z","shell.execute_reply":"2023-02-01T03:29:42.867339Z"},"trusted":true},"execution_count":null,"outputs":[]}]}