{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"学習データをサンプリングして容量を減らす\n- target=1 はすべて残す　47944 case\n- target=0 は50000件ランダムサンプリング","metadata":{}},{"cell_type":"code","source":"import polars as pl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-16T12:46:01.537668Z","iopub.execute_input":"2024-03-16T12:46:01.538733Z","iopub.status.idle":"2024-03-16T12:46:01.849278Z","shell.execute_reply.started":"2024-03-16T12:46:01.538688Z","shell.execute_reply":"2024-03-16T12:46:01.848159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/'","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:46:01.851393Z","iopub.execute_input":"2024-03-16T12:46:01.851774Z","iopub.status.idle":"2024-03-16T12:46:01.856409Z","shell.execute_reply.started":"2024-03-16T12:46:01.851742Z","shell.execute_reply":"2024-03-16T12:46:01.855321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files = [\n    'train_applprev_1_0.parquet',\n    'train_applprev_1_1.parquet',\n    'train_applprev_2.parquet',\n    'train_base.parquet',\n    'train_credit_bureau_a_1_0.parquet',\n    'train_credit_bureau_a_1_1.parquet',\n    'train_credit_bureau_a_1_2.parquet',\n    'train_credit_bureau_a_1_3.parquet',\n    'train_credit_bureau_a_2_0.parquet',\n    'train_credit_bureau_a_2_1.parquet',\n    'train_credit_bureau_a_2_10.parquet',\n    'train_credit_bureau_a_2_2.parquet',\n    'train_credit_bureau_a_2_3.parquet',\n    'train_credit_bureau_a_2_4.parquet',\n    'train_credit_bureau_a_2_5.parquet',\n    'train_credit_bureau_a_2_6.parquet',\n    'train_credit_bureau_a_2_7.parquet',\n    'train_credit_bureau_a_2_8.parquet',\n    'train_credit_bureau_a_2_9.parquet',\n    'train_credit_bureau_b_1.parquet',\n    'train_credit_bureau_b_2.parquet',\n    'train_debitcard_1.parquet',\n    'train_deposit_1.parquet',\n    'train_other_1.parquet',\n    'train_person_1.parquet',\n    'train_person_2.parquet',\n    'train_static_0_0.parquet',\n    'train_static_0_1.parquet',\n    'train_static_cb_0.parquet',\n    'train_tax_registry_a_1.parquet',\n    'train_tax_registry_b_1.parquet',\n    'train_tax_registry_c_1.parquet',\n]","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:46:01.857936Z","iopub.execute_input":"2024-03-16T12:46:01.858351Z","iopub.status.idle":"2024-03-16T12:46:01.869177Z","shell.execute_reply.started":"2024-03-16T12:46:01.858312Z","shell.execute_reply":"2024-03-16T12:46:01.868027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pl.read_parquet(f'{train_path}train_base.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:46:01.871796Z","iopub.execute_input":"2024-03-16T12:46:01.872255Z","iopub.status.idle":"2024-03-16T12:46:02.237891Z","shell.execute_reply.started":"2024-03-16T12:46:01.872214Z","shell.execute_reply":"2024-03-16T12:46:02.236851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0 = df.filter(pl.col('target')==0).sample(50000)\ndf_1 = df.filter(pl.col('target')==1)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:47:05.754939Z","iopub.execute_input":"2024-03-16T12:47:05.755387Z","iopub.status.idle":"2024-03-16T12:47:05.849477Z","shell.execute_reply.started":"2024-03-16T12:47:05.755353Z","shell.execute_reply":"2024-03-16T12:47:05.848345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = set(pl.concat([df_0,df_1])['case_id'])","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:48:12.092434Z","iopub.execute_input":"2024-03-16T12:48:12.092833Z","iopub.status.idle":"2024-03-16T12:48:12.125401Z","shell.execute_reply.started":"2024-03-16T12:48:12.092804Z","shell.execute_reply":"2024-03-16T12:48:12.124141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.mkdir('train')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:48:31.842025Z","iopub.execute_input":"2024-03-16T12:48:31.842446Z","iopub.status.idle":"2024-03-16T12:48:31.848623Z","shell.execute_reply.started":"2024-03-16T12:48:31.842404Z","shell.execute_reply":"2024-03-16T12:48:31.847309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in files:\n    df = pl.read_parquet(f'{train_path}{file}')\n    df = df.filter(pl.col('case_id').is_in(case_ids))\n    print(len(df),end=' : ')\n    print(file)\n    df.write_parquet(f'train/{file}')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:48:36.063811Z","iopub.execute_input":"2024-03-16T12:48:36.064181Z","iopub.status.idle":"2024-03-16T12:50:41.948125Z","shell.execute_reply.started":"2024-03-16T12:48:36.064153Z","shell.execute_reply":"2024-03-16T12:50:41.946332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# shutil.make_archive('train', format='zip', root_dir='/kaggle/working/train')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T12:50:41.951717Z","iopub.execute_input":"2024-03-16T12:50:41.952973Z","iopub.status.idle":"2024-03-16T12:50:48.348026Z","shell.execute_reply.started":"2024-03-16T12:50:41.952910Z","shell.execute_reply":"2024-03-16T12:50:48.346846Z"},"trusted":true},"execution_count":null,"outputs":[]}]}