{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I am publishing this notebook in case anyone else is bothered by inconsistent schemas of parquet files.\nThe dtypes of some columns in the test samples are wrong.\nI guess the parquet files were converted from the csv files\nso the schemas were inferred from the values in the csv files.\nBut because the test samples only contain 10 cases,\nsome columns may contain only null values\nin which case the dtypes cannot be inferred.","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport glob","metadata":{"execution":{"iopub.status.busy":"2024-02-16T00:57:37.087082Z","iopub.execute_input":"2024-02-16T00:57:37.087543Z","iopub.status.idle":"2024-02-16T00:57:37.093264Z","shell.execute_reply.started":"2024-02-16T00:57:37.087509Z","shell.execute_reply":"2024-02-16T00:57:37.091834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following function returns `{column_name: {dtype: [file, ...], ...}, ...}`.","metadata":{}},{"cell_type":"code","source":"def collect_dtypes(pattern):\n    dtypes = dict()\n    for f in glob.iglob(pattern):\n        s = pl.read_parquet_schema(f)\n        for k, v in s.items():\n            a = dtypes.get(k, dict())\n            a = {**a, v: a.get(v, list()) + [f]}\n            dtypes[k] = a\n    return dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:09:26.130695Z","iopub.execute_input":"2024-02-16T01:09:26.131096Z","iopub.status.idle":"2024-02-16T01:09:26.136755Z","shell.execute_reply.started":"2024-02-16T01:09:26.131060Z","shell.execute_reply":"2024-02-16T01:09:26.135916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So if the length of `{dtype: [file, ...], ...}` is greater than `1`, then that column has inconsistent dtypes.","metadata":{}},{"cell_type":"code","source":"def collect_inconsistent_dtypes(pattern):\n    dtypes = collect_dtypes(pattern)\n    return {k: v for k, v in dtypes.items() if len(v) > 1}","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:09:29.072324Z","iopub.execute_input":"2024-02-16T01:09:29.072770Z","iopub.status.idle":"2024-02-16T01:09:29.078123Z","shell.execute_reply.started":"2024-02-16T01:09:29.072737Z","shell.execute_reply":"2024-02-16T01:09:29.076955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_applprev_1_*.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:09:30.224702Z","iopub.execute_input":"2024-02-16T01:09:30.225095Z","iopub.status.idle":"2024-02-16T01:09:30.242230Z","shell.execute_reply.started":"2024-02-16T01:09:30.225063Z","shell.execute_reply":"2024-02-16T01:09:30.241093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_applprev_2.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:10:17.731848Z","iopub.execute_input":"2024-02-16T01:10:17.732283Z","iopub.status.idle":"2024-02-16T01:10:17.744065Z","shell.execute_reply.started":"2024-02-16T01:10:17.732249Z","shell.execute_reply":"2024-02-16T01:10:17.743169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_base.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:10:28.818571Z","iopub.execute_input":"2024-02-16T01:10:28.819043Z","iopub.status.idle":"2024-02-16T01:10:28.833298Z","shell.execute_reply.started":"2024-02-16T01:10:28.819002Z","shell.execute_reply":"2024-02-16T01:10:28.831999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_credit_bureau_a_1_*.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:10:29.433622Z","iopub.execute_input":"2024-02-16T01:10:29.434540Z","iopub.status.idle":"2024-02-16T01:10:29.465231Z","shell.execute_reply.started":"2024-02-16T01:10:29.434503Z","shell.execute_reply":"2024-02-16T01:10:29.464057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`num_group1 : Float64` is wrong.","metadata":{}},{"cell_type":"code","source":"display(pl.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_a_1_3.parquet').select('num_group1'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:20:08.074885Z","iopub.execute_input":"2024-02-16T01:20:08.075309Z","iopub.status.idle":"2024-02-16T01:20:08.086759Z","shell.execute_reply.started":"2024-02-16T01:20:08.075280Z","shell.execute_reply":"2024-02-16T01:20:08.085869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"But this seems to safely be casted to `Int64`.","metadata":{}},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_credit_bureau_a_2_*.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:12:59.220472Z","iopub.execute_input":"2024-02-16T01:12:59.220939Z","iopub.status.idle":"2024-02-16T01:12:59.388492Z","shell.execute_reply.started":"2024-02-16T01:12:59.220907Z","shell.execute_reply":"2024-02-16T01:12:59.387211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_credit_bureau_b_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:13:25.885924Z","iopub.execute_input":"2024-02-16T01:13:25.886383Z","iopub.status.idle":"2024-02-16T01:13:25.913966Z","shell.execute_reply.started":"2024-02-16T01:13:25.886350Z","shell.execute_reply":"2024-02-16T01:13:25.912901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_2.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:13:43.894127Z","iopub.execute_input":"2024-02-16T01:13:43.894481Z","iopub.status.idle":"2024-02-16T01:13:43.917903Z","shell.execute_reply.started":"2024-02-16T01:13:43.894456Z","shell.execute_reply":"2024-02-16T01:13:43.917040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_debitcard_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:14:01.350095Z","iopub.execute_input":"2024-02-16T01:14:01.350481Z","iopub.status.idle":"2024-02-16T01:14:01.380024Z","shell.execute_reply.started":"2024-02-16T01:14:01.350455Z","shell.execute_reply":"2024-02-16T01:14:01.378841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_deposit_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:14:16.118484Z","iopub.execute_input":"2024-02-16T01:14:16.118927Z","iopub.status.idle":"2024-02-16T01:14:16.142710Z","shell.execute_reply.started":"2024-02-16T01:14:16.118898Z","shell.execute_reply":"2024-02-16T01:14:16.141670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_other_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:14:30.080530Z","iopub.execute_input":"2024-02-16T01:14:30.081014Z","iopub.status.idle":"2024-02-16T01:14:30.104316Z","shell.execute_reply.started":"2024-02-16T01:14:30.080981Z","shell.execute_reply":"2024-02-16T01:14:30.103081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_other_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:14:44.800216Z","iopub.execute_input":"2024-02-16T01:14:44.800660Z","iopub.status.idle":"2024-02-16T01:14:44.813138Z","shell.execute_reply.started":"2024-02-16T01:14:44.800630Z","shell.execute_reply":"2024-02-16T01:14:44.811508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_person_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:15:01.912225Z","iopub.execute_input":"2024-02-16T01:15:01.912602Z","iopub.status.idle":"2024-02-16T01:15:01.937174Z","shell.execute_reply.started":"2024-02-16T01:15:01.912575Z","shell.execute_reply":"2024-02-16T01:15:01.936362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_person_2.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:15:52.345449Z","iopub.execute_input":"2024-02-16T01:15:52.345852Z","iopub.status.idle":"2024-02-16T01:15:52.373814Z","shell.execute_reply.started":"2024-02-16T01:15:52.345825Z","shell.execute_reply":"2024-02-16T01:15:52.372881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_static_0_*.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:16:13.489147Z","iopub.execute_input":"2024-02-16T01:16:13.489551Z","iopub.status.idle":"2024-02-16T01:16:13.589069Z","shell.execute_reply.started":"2024-02-16T01:16:13.489522Z","shell.execute_reply":"2024-02-16T01:16:13.587728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_static_cb_0.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:18:11.284817Z","iopub.execute_input":"2024-02-16T01:18:11.285252Z","iopub.status.idle":"2024-02-16T01:18:11.311858Z","shell.execute_reply.started":"2024-02-16T01:18:11.285224Z","shell.execute_reply":"2024-02-16T01:18:11.311027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`*D : Float64` is wrong.","metadata":{}},{"cell_type":"code","source":"display(pl.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_cb_0.parquet').select(\n    'assignmentdate_238D', 'birthdate_574D', 'responsedate_1012D',\n))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:19:34.989225Z","iopub.execute_input":"2024-02-16T01:19:34.989636Z","iopub.status.idle":"2024-02-16T01:19:35.009244Z","shell.execute_reply.started":"2024-02-16T01:19:34.989606Z","shell.execute_reply":"2024-02-16T01:19:35.008307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"But all values are `null`.","metadata":{}},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_tax_registry_a_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:21:32.892536Z","iopub.execute_input":"2024-02-16T01:21:32.893055Z","iopub.status.idle":"2024-02-16T01:21:32.918810Z","shell.execute_reply.started":"2024-02-16T01:21:32.893022Z","shell.execute_reply":"2024-02-16T01:21:32.917947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_tax_registry_b_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:21:39.819934Z","iopub.execute_input":"2024-02-16T01:21:39.820388Z","iopub.status.idle":"2024-02-16T01:21:39.848071Z","shell.execute_reply.started":"2024-02-16T01:21:39.820358Z","shell.execute_reply":"2024-02-16T01:21:39.847134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(collect_inconsistent_dtypes('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/*/*_tax_registry_c_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-02-16T01:21:46.249365Z","iopub.execute_input":"2024-02-16T01:21:46.249736Z","iopub.status.idle":"2024-02-16T01:21:46.303620Z","shell.execute_reply.started":"2024-02-16T01:21:46.249708Z","shell.execute_reply":"2024-02-16T01:21:46.302387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A problem","metadata":{}},{"cell_type":"markdown","source":"The inconsistency of schemas can cause an error. I don't know the exact reason.","metadata":{}},{"cell_type":"code","source":"try:\n    _ = (\n        pl.scan_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_0_*.parquet')\n        .select('cardtype_51L')\n        .collect()\n    )\nexcept Exception as e:\n    print('Something happend.')\n    display(e)","metadata":{"execution":{"iopub.status.busy":"2024-02-16T02:18:21.266966Z","iopub.execute_input":"2024-02-16T02:18:21.267423Z","iopub.status.idle":"2024-02-16T02:18:21.288475Z","shell.execute_reply.started":"2024-02-16T02:18:21.267393Z","shell.execute_reply":"2024-02-16T02:18:21.287154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Separate scan and concatenation work.","metadata":{}},{"cell_type":"code","source":"try:\n    display(pl.concat([\n        pl.scan_parquet(f)\n        for f in glob.iglob('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_0_*.parquet')\n    ]).select('cardtype_51L').collect())\nexcept Exception as e:\n    print('Something happend.')\n    display(e)","metadata":{"execution":{"iopub.status.busy":"2024-02-16T02:20:53.743395Z","iopub.execute_input":"2024-02-16T02:20:53.743879Z","iopub.status.idle":"2024-02-16T02:20:53.762903Z","shell.execute_reply.started":"2024-02-16T02:20:53.743829Z","shell.execute_reply":"2024-02-16T02:20:53.761892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"markdown","source":"- Schemas are consistent accross the training dataset.\n- Some schemas in the test samples are wrong due to short samples.\n- It is likely the case that schemas in the real test dataset are correct, but not sure.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}