{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom tqdm.auto import tqdm\nfrom IPython.display import display\n\nfrom sklearn.model_selection import train_test_split\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-02T18:54:49.698061Z","iopub.execute_input":"2024-05-02T18:54:49.698834Z","iopub.status.idle":"2024-05-02T18:54:53.077805Z","shell.execute_reply.started":"2024-05-02T18:54:49.698790Z","shell.execute_reply":"2024-05-02T18:54:53.076473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_0.parquet\")\ndf2 = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_1.parquet\")\ntrain_base = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet\")\n\ndisplay(df1, df2, train_base)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T18:54:53.080234Z","iopub.execute_input":"2024-05-02T18:54:53.082505Z","iopub.status.idle":"2024-05-02T18:55:00.676437Z","shell.execute_reply.started":"2024-05-02T18:54:53.082456Z","shell.execute_reply":"2024-05-02T18:55:00.675136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_case_ids = list(set(df1[\"case_id\"])|set(df2[\"case_id\"]))\nunique_case_ids.sort()\n\nprint(f\"Unique case_id in df1: {len(df1['case_id'].unique())}\")\nprint(f\"Unique case_id in df2: {len(df2['case_id'].unique())}\")\nprint(f\"Unique case_id in both datasets: {len(unique_case_ids)}\")\n\nprint(f\"Unique case_id in train_base: {len(train_base['case_id'].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-02T18:55:00.678078Z","iopub.execute_input":"2024-05-02T18:55:00.678424Z","iopub.status.idle":"2024-05-02T18:55:01.328709Z","shell.execute_reply.started":"2024-05-02T18:55:00.678396Z","shell.execute_reply":"2024-05-02T18:55:01.327244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids, test_ids = train_test_split(train_base[\"case_id\"], test_size=0.3, stratify=train_base[\"target\"], shuffle=True, random_state=42)\nprint(f\"Train: {len(train_ids)}\")\nprint(f\"Test: {len(test_ids)}\")\n\ntrain_df = train_base[train_base[\"case_id\"].isin(train_ids)].reset_index(drop=True)\ntest_df = train_base[train_base[\"case_id\"].isin(test_ids)].reset_index(drop=True)\n\ndisplay(train_df, test_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T18:55:01.331782Z","iopub.execute_input":"2024-05-02T18:55:01.332119Z","iopub.status.idle":"2024-05-02T18:55:02.340099Z","shell.execute_reply.started":"2024-05-02T18:55:01.332092Z","shell.execute_reply":"2024-05-02T18:55:02.338709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\n\nplt.subplot(1, 2, 1)\nsns.histplot(train_df[\"target\"]).set(title = \"Train df target distribution\")\n\nplt.subplot(1, 2, 2)\nsns.histplot(test_df[\"target\"]).set(title = \"Test df target distribution\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T18:55:02.341645Z","iopub.execute_input":"2024-05-02T18:55:02.342065Z","iopub.status.idle":"2024-05-02T18:55:04.702057Z","shell.execute_reply.started":"2024-05-02T18:55:02.342027Z","shell.execute_reply":"2024-05-02T18:55:04.700835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\n\nplt.subplot(1, 2, 1)\nsns.histplot(train_df[\"WEEK_NUM\"]).set(title = \"Train df WEEK_NUM distribution\")\n\nplt.subplot(1, 2, 2)\nsns.histplot(test_df[\"WEEK_NUM\"]).set(title = \"Test df WEEK_NUM distribution\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T18:55:04.703375Z","iopub.execute_input":"2024-05-02T18:55:04.703707Z","iopub.status.idle":"2024-05-02T18:55:07.256525Z","shell.execute_reply.started":"2024-05-02T18:55:04.703679Z","shell.execute_reply":"2024-05-02T18:55:07.255255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train\"\noutput_dir = \"/kaggle/working/csv_files\"\noutput_train_dir = os.path.join(output_dir, \"train\")\noutput_test_dir = os.path.join(output_dir, \"test\")\n\nos.makedirs(output_train_dir, exist_ok=True)\nos.makedirs(output_test_dir, exist_ok=True)\n\nfor file in tqdm(os.listdir(train_dir)[15:]):\n    df_to_split = pd.read_csv(os.path.join(train_dir, file))\n    \n    train_part = df_to_split[df_to_split[\"case_id\"].isin(train_ids)].reset_index(drop=True)\n    test_part = df_to_split[df_to_split[\"case_id\"].isin(test_ids)].reset_index(drop=True)\n    \n    print(len(train_part), len(test_part))\n    \n    train_part.to_csv(os.path.join(output_train_dir, file), index=False)\n    test_part.to_csv(os.path.join(output_test_dir, \"test\"+file[5:]), index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T18:55:07.258175Z","iopub.execute_input":"2024-05-02T18:55:07.258644Z"},"trusted":true},"execution_count":null,"outputs":[]}]}