{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38257,"databundleVersionId":4319132,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport glob\nimport time\n\n# Đường dẫn file\n#file_pattern = \"/kaggle/input/icecube-neutrinos-in-deep-ice/train/batch_*.parquet\"\nfile_pattern = '/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet'\nfiles = glob.glob(file_pattern)\n\nprint(f\"Tìm thấy {len(files)} file. Đang đếm dòng bằng Metadata...\")\nstart_time = time.time()\ntotal_rows = 0\n\n# Chỉ đọc Header, không đọc nội dung\nfor f in files:\n    # pq.read_metadata chỉ đọc vài KB thông tin mô tả file\n    total_rows += pq.read_metadata(f).num_rows\n\nend_time = time.time()\n\nprint(\"-\" * 30)\nprint(f\"TỔNG SỐ DÒNG: {total_rows:,}\")\nprint(f\"Thời gian: {end_time - start_time:.2f} giây\")\nprint(\"-\" * 30)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:38:28.569962Z","iopub.execute_input":"2026-02-02T04:38:28.571010Z","iopub.status.idle":"2026-02-02T04:38:28.702722Z","shell.execute_reply.started":"2026-02-02T04:38:28.570960Z","shell.execute_reply":"2026-02-02T04:38:28.701935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ND3\nimport polars as pl\nlf = pl.scan_parquet(file_pattern)\n\nprint(lf.explain())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:38:28.704995Z","iopub.execute_input":"2026-02-02T04:38:28.705499Z","iopub.status.idle":"2026-02-02T04:38:29.361169Z","shell.execute_reply.started":"2026-02-02T04:38:28.705453Z","shell.execute_reply":"2026-02-02T04:38:29.360414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Đọc dữ liệu lazy","metadata":{}},{"cell_type":"code","source":"type(lf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:38:29.362131Z","iopub.execute_input":"2026-02-02T04:38:29.362510Z","iopub.status.idle":"2026-02-02T04:38:29.368610Z","shell.execute_reply.started":"2026-02-02T04:38:29.362486Z","shell.execute_reply":"2026-02-02T04:38:29.367729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lf.limit(5).collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:38:29.370045Z","iopub.execute_input":"2026-02-02T04:38:29.370508Z","iopub.status.idle":"2026-02-02T04:38:29.498769Z","shell.execute_reply.started":"2026-02-02T04:38:29.370480Z","shell.execute_reply":"2026-02-02T04:38:29.498022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lf.collect_schema()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:38:29.500495Z","iopub.execute_input":"2026-02-02T04:38:29.500821Z","iopub.status.idle":"2026-02-02T04:38:29.506885Z","shell.execute_reply.started":"2026-02-02T04:38:29.500797Z","shell.execute_reply":"2026-02-02T04:38:29.506009Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 6 cột với kiểu kiểu dữ liệu tương ứng với cột đó","metadata":{}},{"cell_type":"code","source":"cols = [\"azimuth\", \"zenith\"]\n\nplans = []\n\nfor cur_col in cols:\n    # Với mỗi cột, ta tạo một lệnh select trả về 1 dòng và 6 cột thống kê\n    p = lf.select([\n        pl.lit(cur_col).alias(\"feature\"),\n        pl.col(cur_col).count().alias(\"count\"),\n        pl.col(cur_col).null_count().alias(\"null_count\"),\n        pl.col(cur_col).mean().alias(\"mean\"),\n        pl.col(cur_col).std().alias(\"std\"),\n        pl.col(cur_col).min().alias(\"min\"),\n        pl.col(cur_col).max().alias(\"max\"),\n    ])\n    plans.append(p)\n\n# Dùng pl.concat để ghép dọc các kế hoạch lại và collect 1 lần duy nhất\n# Polars sẽ tối ưu hóa để quét dữ liệu hiệu quả nhất\nresult = pl.concat(plans).collect()\n\nprint(f\"Shape: {result.shape}\")\nprint(result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:42:15.494400Z","iopub.execute_input":"2026-02-02T04:42:15.494976Z","iopub.status.idle":"2026-02-02T04:42:19.762370Z","shell.execute_reply.started":"2026-02-02T04:42:15.494947Z","shell.execute_reply":"2026-02-02T04:42:19.761444Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Ghi dữ liệu","metadata":{}},{"cell_type":"code","source":"lf.sink_parquet(\"/kaggle/working/output.parquet\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:38:47.735600Z","iopub.execute_input":"2026-02-02T04:38:47.736028Z","iopub.status.idle":"2026-02-02T04:39:15.148493Z","shell.execute_reply.started":"2026-02-02T04:38:47.736003Z","shell.execute_reply":"2026-02-02T04:39:15.147513Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Quan sát dữ liệu\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lưu ý: Vì dữ liệu thực tế rất lớn, ta không nên collect() toàn bộ.\n# Thay vào đó, dùng fetch(n) để lấy mẫu thống kê nhanh.\n\n# Lấy mẫu 50,000 dòng để xem thống kê\nsample_stats = lf.fetch(50000).describe()\nprint(sample_stats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:41:47.470074Z","iopub.execute_input":"2026-02-02T04:41:47.471216Z","iopub.status.idle":"2026-02-02T04:41:47.503485Z","shell.execute_reply.started":"2026-02-02T04:41:47.471149Z","shell.execute_reply":"2026-02-02T04:41:47.502424Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Kết hợp .select() và .filter()","metadata":{}},{"cell_type":"code","source":"# Kịch bản: Chỉ lấy ID sự kiện và góc zenith, \n# nhưng LỌC chỉ lấy các bản ghi có góc azimuth > 3.0 (radian)\n\nfiltered_df = (\n    lf.select([\"event_id\", \"azimuth\", \"zenith\"])  # Chọn cột\n      .filter(pl.col(\"azimuth\") > 3.0)            # Lọc dòng (Đang thiếu phần này)\n      .collect()                                  # Thực thi\n)\n\nprint(f\"Số lượng bản ghi sau khi lọc: {filtered_df.height}\")\nprint(filtered_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T04:39:15.183995Z","iopub.execute_input":"2026-02-02T04:39:15.184690Z","iopub.status.idle":"2026-02-02T04:39:25.458133Z","shell.execute_reply.started":"2026-02-02T04:39:15.184662Z","shell.execute_reply":"2026-02-02T04:39:25.457088Z"}},"outputs":[],"execution_count":null}]}