{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport math\nfrom pathlib import Path\nfrom collections import Counter\nimport datetime\nimport gc\nimport time\n\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\n\nfrom sklearn.model_selection import train_test_split, KFold\n# from sklearn.preprocessing import StandardScaler, scale, MinMaxScaler\n# from sklearn.decomposition import TruncatedSVD","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-28T13:39:02.815401Z","iopub.execute_input":"2023-01-28T13:39:02.816276Z","iopub.status.idle":"2023-01-28T13:39:05.731265Z","shell.execute_reply.started":"2023-01-28T13:39:02.816139Z","shell.execute_reply":"2023-01-28T13:39:05.729222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-01-28T13:39:05.739622Z","iopub.execute_input":"2023-01-28T13:39:05.744186Z","iopub.status.idle":"2023-01-28T13:39:05.754356Z","shell.execute_reply.started":"2023-01-28T13:39:05.744104Z","shell.execute_reply":"2023-01-28T13:39:05.753070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 333\ndef seedBasic(seed=SEED):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    \n    \ndef seedTorch(seed=SEED):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n      \n# basic + torch \ndef seedEverything(seed=SEED):\n    seedBasic(seed)\n    seedTorch(seed)\n\nseedEverything()","metadata":{"execution":{"iopub.status.busy":"2023-01-28T13:39:05.759447Z","iopub.execute_input":"2023-01-28T13:39:05.760820Z","iopub.status.idle":"2023-01-28T13:39:05.781604Z","shell.execute_reply.started":"2023-01-28T13:39:05.760734Z","shell.execute_reply":"2023-01-28T13:39:05.779788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = Path(\"/kaggle/input/icecube-neutrinos-in-deep-ice/\")","metadata":{"execution":{"iopub.status.busy":"2023-01-28T13:39:05.789963Z","iopub.execute_input":"2023-01-28T13:39:05.790606Z","iopub.status.idle":"2023-01-28T13:39:05.795800Z","shell.execute_reply.started":"2023-01-28T13:39:05.790568Z","shell.execute_reply":"2023-01-28T13:39:05.794629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# parquet_file = pq.ParquetFile(\"/kaggle/input/icecube-data/train_meta.parquet\")\n# for p_batch in parquet_file.iter_batches(batch_size=1000000):\n#     chunk = p_batch.to_pandas()\n#     for name, group in chunk.groupby('batch_id'):\n#         group.to_csv(f'train_meta_batch_{name}.csv', mode='a', index=False, header=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-28T13:39:05.797755Z","iopub.execute_input":"2023-01-28T13:39:05.798617Z","iopub.status.idle":"2023-01-28T13:39:05.835325Z","shell.execute_reply.started":"2023-01-28T13:39:05.798573Z","shell.execute_reply":"2023-01-28T13:39:05.834192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta = pd.read_parquet(data_path / \"train_meta.parquet\")\ntrain_meta['pulses'] = train_meta['last_pulse_index'] - train_meta['first_pulse_index']\ntrain_meta = train_meta.drop(['last_pulse_index', 'first_pulse_index'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-28T13:16:32.396364Z","iopub.execute_input":"2023-01-28T13:16:32.397501Z","iopub.status.idle":"2023-01-28T13:17:23.347283Z","shell.execute_reply.started":"2023-01-28T13:16:32.397453Z","shell.execute_reply":"2023-01-28T13:17:23.343163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta = train_meta.sort_values(by=['pulses']).groupby('batch_id')\nfor name, group in train_meta:\n    group.to_parquet(f'train_meta_batch_{name}.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-28T13:30:06.726968Z","iopub.execute_input":"2023-01-28T13:30:06.727384Z","iopub.status.idle":"2023-01-28T13:30:07.477317Z","shell.execute_reply.started":"2023-01-28T13:30:06.727350Z","shell.execute_reply":"2023-01-28T13:30:07.476158Z"},"trusted":true},"execution_count":null,"outputs":[]}]}