{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38257,"databundleVersionId":4319132,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport glob\nimport time\n\n# Đường dẫn file\n#file_pattern = \"/kaggle/input/icecube-neutrinos-in-deep-ice/train/batch_*.parquet\"\nfile_pattern = '/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet'\nfiles = glob.glob(file_pattern)\n\nprint(f\"Tìm thấy {len(files)} file. Đang đếm dòng bằng Metadata...\")\nstart_time = time.time()\ntotal_rows = 0\n\n# Vòng lặp siêu tốc: Chỉ đọc Header, không đọc nội dung\nfor f in files:\n    # pq.read_metadata chỉ đọc vài KB thông tin mô tả file\n    total_rows += pq.read_metadata(f).num_rows\n\nend_time = time.time()\n\nprint(\"-\" * 30)\nprint(f\"TỔNG SỐ DÒNG: {total_rows:,}\")\nprint(f\"Thời gian: {end_time - start_time:.2f} giây\")\nprint(\"-\" * 30)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:36.300700Z","iopub.execute_input":"2026-01-31T15:50:36.302032Z","iopub.status.idle":"2026-01-31T15:50:36.314869Z","shell.execute_reply.started":"2026-01-31T15:50:36.301981Z","shell.execute_reply":"2026-01-31T15:50:36.313497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ND3\nimport polars as pl\nlf = pl.scan_parquet(file_pattern)\n\nprint(lf.explain())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:36.316890Z","iopub.execute_input":"2026-01-31T15:50:36.317314Z","iopub.status.idle":"2026-01-31T15:50:36.329983Z","shell.execute_reply.started":"2026-01-31T15:50:36.317275Z","shell.execute_reply":"2026-01-31T15:50:36.328862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### Đọc dữ liệu lazy","metadata":{}},{"cell_type":"code","source":"type(lf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:36.331252Z","iopub.execute_input":"2026-01-31T15:50:36.331588Z","iopub.status.idle":"2026-01-31T15:50:36.346789Z","shell.execute_reply.started":"2026-01-31T15:50:36.331561Z","shell.execute_reply":"2026-01-31T15:50:36.345667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lf.limit(5).collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:36.348376Z","iopub.execute_input":"2026-01-31T15:50:36.348733Z","iopub.status.idle":"2026-01-31T15:50:36.377170Z","shell.execute_reply.started":"2026-01-31T15:50:36.348705Z","shell.execute_reply":"2026-01-31T15:50:36.376082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lf.schema","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:36.379345Z","iopub.execute_input":"2026-01-31T15:50:36.380562Z","iopub.status.idle":"2026-01-31T15:50:36.387840Z","shell.execute_reply.started":"2026-01-31T15:50:36.380526Z","shell.execute_reply":"2026-01-31T15:50:36.386946Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 5 cột với kiểu kiểu dữ liệu tương ứng với cột đó","metadata":{}},{"cell_type":"code","source":"cur_col = \"azimuth\"\n\naggs = {\n    \"count\": pl.col(cur_col).count(),\n    \"null_count\": pl.col(cur_col).null_count(),\n    \"mean\": pl.col(cur_col).mean(),\n    \"std\": pl.col(cur_col).std(),\n    \"min\": pl.col(cur_col).min(),\n    \"max\": pl.col(cur_col).max(),\n}\n\nlf.select([\n    expr.alias(f\"{cur_col}_{name}\")\n    for name, expr in aggs.items()\n]).collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:36.389175Z","iopub.execute_input":"2026-01-31T15:50:36.389496Z","iopub.status.idle":"2026-01-31T15:50:38.064578Z","shell.execute_reply.started":"2026-01-31T15:50:36.389463Z","shell.execute_reply":"2026-01-31T15:50:38.063347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lf.sink_parquet(\"/kaggle/working/output.parquet\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T15:50:38.065910Z","iopub.execute_input":"2026-01-31T15:50:38.066212Z","iopub.status.idle":"2026-01-31T15:51:03.158498Z","shell.execute_reply.started":"2026-01-31T15:50:38.066186Z","shell.execute_reply":"2026-01-31T15:51:03.157103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Ghi dữ liệu","metadata":{}}]}