{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# credit_bureau_a_2\n# Properties: depth=2, external data source, Credit bureau provider A\n# Train Files:\n\n# train_credit_bureau_a_2_0.csv\n# train_credit_bureau_a_2_1.csv\n# train_credit_bureau_a_2_2.csv\n# train_credit_bureau_a_2_3.csv\n# train_credit_bureau_a_2_4.csv\n# train_credit_bureau_a_2_5.csv\n# train_credit_bureau_a_2_6.csv\n# train_credit_bureau_a_2_7.csv\n# train_credit_bureau_a_2_8.csv\n# train_credit_bureau_a_2_9.csv\n# train_credit_bureau_a_2_10.csv","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.770214Z","iopub.execute_input":"2024-09-26T15:54:05.770854Z","iopub.status.idle":"2024-09-26T15:54:05.776473Z","shell.execute_reply.started":"2024-09-26T15:54:05.770808Z","shell.execute_reply":"2024-09-26T15:54:05.775114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install dask","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.786107Z","iopub.execute_input":"2024-09-26T15:54:05.786547Z","iopub.status.idle":"2024-09-26T15:54:05.794052Z","shell.execute_reply.started":"2024-09-26T15:54:05.786511Z","shell.execute_reply":"2024-09-26T15:54:05.792866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# columns_to_drop = ['collater_typofvalofguarant_298M','collater_typofvalofguarant_407M','collaterals_typeofguarante_359M','collaterals_typeofguarante_669M','subjectroles_name_541M','subjectroles_name_838M','collater_typofvalofguarant_298M','collater_typofvalofguarant_407M','subjectroles_name_541M','subjectroles_name_838M','collaterals_typeofguarante_359M','collaterals_typeofguarante_669M','collater_typofvalofguarant_298M','collater_typofvalofguarant_407M','collaterals_typeofguarante_359M','collaterals_typeofguarante_669M','subjectroles_name_541M','subjectroles_name_838M']\n# columns_to_drop=list(set(columns_to_drop))\n# columns_to_drop","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.800259Z","iopub.execute_input":"2024-09-26T15:54:05.800663Z","iopub.status.idle":"2024-09-26T15:54:05.806749Z","shell.execute_reply.started":"2024-09-26T15:54:05.800617Z","shell.execute_reply":"2024-09-26T15:54:05.805661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nimport numpy as np\nimport dask.dataframe as dd","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.808784Z","iopub.execute_input":"2024-09-26T15:54:05.809186Z","iopub.status.idle":"2024-09-26T15:54:05.817153Z","shell.execute_reply.started":"2024-09-26T15:54:05.809139Z","shell.execute_reply":"2024-09-26T15:54:05.816114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_0.csv')\ndf2 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_1.csv')\ndf3 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_2.csv')\ndf4 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_3.csv')\ndf5 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_4.csv')\ndf6 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_5.csv')\ndf7 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_6.csv')\ndf8 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_7.csv')\ndf9 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_8.csv')\ndf10 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_9.csv')\ndf11 = dd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_10.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.819169Z","iopub.execute_input":"2024-09-26T15:54:05.820142Z","iopub.status.idle":"2024-09-26T15:54:05.831828Z","shell.execute_reply.started":"2024-09-26T15:54:05.820099Z","shell.execute_reply":"2024-09-26T15:54:05.830668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_df = dd.concat([df1, df2, df3,df4,df5,df6])\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.834025Z","iopub.execute_input":"2024-09-26T15:54:05.834520Z","iopub.status.idle":"2024-09-26T15:54:05.847330Z","shell.execute_reply.started":"2024-09-26T15:54:05.834465Z","shell.execute_reply":"2024-09-26T15:54:05.845982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_drop=['collaterals_typeofguarante_669M',\n 'collaterals_typeofguarante_359M',\n 'collater_typofvalofguarant_298M',\n 'subjectroles_name_541M',\n 'subjectroles_name_838M',\n 'collater_typofvalofguarant_407M']\n\ncombined_df = combined_df.drop(columns=columns_to_drop)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.849739Z","iopub.execute_input":"2024-09-26T15:54:05.850158Z","iopub.status.idle":"2024-09-26T15:54:05.861218Z","shell.execute_reply.started":"2024-09-26T15:54:05.850117Z","shell.execute_reply":"2024-09-26T15:54:05.859903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_df = combined_df.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.862928Z","iopub.execute_input":"2024-09-26T15:54:05.863332Z","iopub.status.idle":"2024-09-26T15:54:05.874128Z","shell.execute_reply.started":"2024-09-26T15:54:05.863291Z","shell.execute_reply":"2024-09-26T15:54:05.872919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = combined_df.compute()","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:54:05.979833Z","iopub.execute_input":"2024-09-26T15:54:05.980976Z","iopub.status.idle":"2024-09-26T15:55:18.369851Z","shell.execute_reply.started":"2024-09-26T15:54:05.980919Z","shell.execute_reply":"2024-09-26T15:55:18.367662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:18.373363Z","iopub.execute_input":"2024-09-26T15:55:18.373792Z","iopub.status.idle":"2024-09-26T15:55:18.404293Z","shell.execute_reply.started":"2024-09-26T15:55:18.373751Z","shell.execute_reply":"2024-09-26T15:55:18.402857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df.drop_duplicates()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:18.405809Z","iopub.execute_input":"2024-09-26T15:55:18.406292Z","iopub.status.idle":"2024-09-26T15:55:18.419384Z","shell.execute_reply.started":"2024-09-26T15:55:18.406242Z","shell.execute_reply":"2024-09-26T15:55:18.418276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:18.421195Z","iopub.execute_input":"2024-09-26T15:55:18.421659Z","iopub.status.idle":"2024-09-26T15:55:18.460646Z","shell.execute_reply.started":"2024-09-26T15:55:18.421599Z","shell.execute_reply":"2024-09-26T15:55:18.459468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:18.463954Z","iopub.execute_input":"2024-09-26T15:55:18.464471Z","iopub.status.idle":"2024-09-26T15:55:20.418646Z","shell.execute_reply.started":"2024-09-26T15:55:18.464417Z","shell.execute_reply":"2024-09-26T15:55:20.417184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cn=df.isnull().mean()*100\ndf.drop(cn[cn>=60].index,axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:20.420123Z","iopub.execute_input":"2024-09-26T15:55:20.420570Z","iopub.status.idle":"2024-09-26T15:55:23.599133Z","shell.execute_reply.started":"2024-09-26T15:55:20.420528Z","shell.execute_reply":"2024-09-26T15:55:23.597613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### df=df.fillna(df.median())\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\nfor col in df:\n    Q1 = df[col].quantile(0.25)\n    Q3 = df[col].quantile(0.75)\n    IQR = Q3 - Q1\n    lower_bound = Q1 - 1.5 * IQR\n    upper_bound = Q3 + 1.5 * IQR\n\n    df[col] = np.clip(df[col], lower_bound, upper_bound)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:23.600985Z","iopub.execute_input":"2024-09-26T15:55:23.601360Z","iopub.status.idle":"2024-09-26T15:55:42.523245Z","shell.execute_reply.started":"2024-09-26T15:55:23.601321Z","shell.execute_reply":"2024-09-26T15:55:42.521974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaler = MinMaxScaler()\n# df = scaler.fit_transform(df)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:42.524900Z","iopub.execute_input":"2024-09-26T15:55:42.525371Z","iopub.status.idle":"2024-09-26T15:55:42.530347Z","shell.execute_reply.started":"2024-09-26T15:55:42.525321Z","shell.execute_reply":"2024-09-26T15:55:42.529163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\nscaler = MinMaxScaler()\ndf = pd.DataFrame(scaler.fit_transform(df), columns=df.columns)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T15:55:42.531837Z","iopub.execute_input":"2024-09-26T15:55:42.532292Z","iopub.status.idle":"2024-09-26T15:55:48.702006Z","shell.execute_reply.started":"2024-09-26T15:55:42.532253Z","shell.execute_reply":"2024-09-26T15:55:48.700648Z"},"trusted":true},"execution_count":null,"outputs":[]}]}