{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom tqdm.auto import tqdm\nfrom IPython.display import display\n\nfrom sklearn.model_selection import train_test_split\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-29T15:55:42.796600Z","iopub.execute_input":"2024-04-29T15:55:42.796991Z","iopub.status.idle":"2024-04-29T15:55:46.655912Z","shell.execute_reply.started":"2024-04-29T15:55:42.796946Z","shell.execute_reply":"2024-04-29T15:55:46.654688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_0.parquet\")\ndf2 = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_1.parquet\")\ntrain_base = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet\")\n\ndisplay(df1, df2, train_base)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T15:55:49.850874Z","iopub.execute_input":"2024-04-29T15:55:49.851737Z","iopub.status.idle":"2024-04-29T15:55:57.439417Z","shell.execute_reply.started":"2024-04-29T15:55:49.851687Z","shell.execute_reply":"2024-04-29T15:55:57.438056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_case_ids = list(set(df1[\"case_id\"])|set(df2[\"case_id\"]))\nunique_case_ids.sort()\n\nprint(f\"Unique case_id in df1: {len(df1['case_id'].unique())}\")\nprint(f\"Unique case_id in df2: {len(df2['case_id'].unique())}\")\nprint(f\"Unique case_id in both datasets: {len(unique_case_ids)}\")\n\nprint(f\"Unique case_id in train_base: {len(train_base['case_id'].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-29T15:55:59.505296Z","iopub.execute_input":"2024-04-29T15:55:59.506349Z","iopub.status.idle":"2024-04-29T15:56:00.114443Z","shell.execute_reply.started":"2024-04-29T15:55:59.506310Z","shell.execute_reply":"2024-04-29T15:56:00.113269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids, test_ids = train_test_split(train_base[\"case_id\"], test_size=0.3, stratify=train_base[\"target\"], shuffle=True, random_state=42)\nprint(f\"Train: {len(train_ids)}\")\nprint(f\"Test: {len(test_ids)}\")\n\ntrain_df = train_base[train_base[\"case_id\"].isin(train_ids)].reset_index(drop=True)\ntest_df = train_base[train_base[\"case_id\"].isin(test_ids)].reset_index(drop=True)\n\ndisplay(train_df, test_df)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T15:56:02.484869Z","iopub.execute_input":"2024-04-29T15:56:02.485314Z","iopub.status.idle":"2024-04-29T15:56:03.461937Z","shell.execute_reply.started":"2024-04-29T15:56:02.485279Z","shell.execute_reply":"2024-04-29T15:56:03.460732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\n\nplt.subplot(1, 2, 1)\nsns.histplot(train_df[\"target\"]).set(title = \"Train df target distribution\")\n\nplt.subplot(1, 2, 2)\nsns.histplot(test_df[\"target\"]).set(title = \"Test df target distribution\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T15:56:11.048889Z","iopub.execute_input":"2024-04-29T15:56:11.049330Z","iopub.status.idle":"2024-04-29T15:56:13.398281Z","shell.execute_reply.started":"2024-04-29T15:56:11.049299Z","shell.execute_reply":"2024-04-29T15:56:13.397036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\n\nplt.subplot(1, 2, 1)\nsns.histplot(train_df[\"WEEK_NUM\"]).set(title = \"Train df WEEK_NUM distribution\")\n\nplt.subplot(1, 2, 2)\nsns.histplot(test_df[\"WEEK_NUM\"]).set(title = \"Test df WEEK_NUM distribution\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T15:56:31.970263Z","iopub.execute_input":"2024-04-29T15:56:31.971114Z","iopub.status.idle":"2024-04-29T15:56:34.575036Z","shell.execute_reply.started":"2024-04-29T15:56:31.971077Z","shell.execute_reply":"2024-04-29T15:56:34.574002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train\"\noutput_dir = \"/kaggle/working/parquet_files\"\noutput_train_dir = os.path.join(output_dir, \"train\")\noutput_test_dir = os.path.join(output_dir, \"test\")\n\nos.makedirs(output_train_dir, exist_ok=True)\nos.makedirs(output_test_dir, exist_ok=True)\n\nfor file in tqdm(os.listdir(train_dir)):\n    df_to_split = pd.read_parquet(os.path.join(train_dir, file))\n    \n    train_part = df_to_split[df_to_split[\"case_id\"].isin(train_ids)].reset_index(drop=True)\n    test_part = df_to_split[df_to_split[\"case_id\"].isin(test_ids)].reset_index(drop=True)\n    \n    print(len(train_part), len(test_part))\n    \n    train_part.to_parquet(os.path.join(output_train_dir, file), index=False)\n    test_part.to_parquet(os.path.join(output_test_dir, \"test\"+file[5:]), index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T14:50:31.913033Z","iopub.execute_input":"2024-04-29T14:50:31.913363Z"},"trusted":true},"execution_count":null,"outputs":[]}]}