{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7600559,"sourceType":"datasetVersion","datasetId":4424545}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-08T12:14:42.828147Z","iopub.execute_input":"2024-04-08T12:14:42.828568Z","iopub.status.idle":"2024-04-08T12:14:42.848236Z","shell.execute_reply.started":"2024-04-08T12:14:42.828535Z","shell.execute_reply":"2024-04-08T12:14:42.846855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\n# Load the DataFrame\ndf= pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_deposit_1.parquet')\ntrain =pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet')\n# Set seed for reproducibility\nnp.random.seed(0)\n\n# Calculate percent missing values of each column\npercent_missing = df.isnull().sum() * 100 / len(df)\nmissing_applprev = pd.DataFrame({'percent_missing': percent_missing})\n\n# Identify columns with percent missing value > 80\nmorethan_80 = missing_applprev[missing_applprev['percent_missing'] > 80].index\n\n# Drop columns with percent missing value > 80\ndrop_deposit = df.drop(columns=morethan_80)\ndrop_deposit\n# Fill NaN values in the column \"actualdpd_943P\" with 0\n#drop_applprev_1_0[\"actualdpd_943P\"].fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:14:47.14229Z","iopub.execute_input":"2024-04-08T12:14:47.143709Z","iopub.status.idle":"2024-04-08T12:14:47.366749Z","shell.execute_reply.started":"2024-04-08T12:14:47.143657Z","shell.execute_reply":"2024-04-08T12:14:47.365712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.swarmplot(x=insurance_data['smoker'],\n              y=insurance_data['charges'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature engineering function demo and testing\ndef feature_eng(data,fealist,agglist):\n    result = pd.DataFrame({'case_id':list(set(data.case_id))})\n    for f in fealist:\n        print(f,'......')\n        df_table = data.loc[:,['case_id',f,'num_group1']].\\\n            sort_values(['case_id','num_group1']).\\\n            groupby(['case_id'])[f].\\\n            agg(agglist).\\\n            add_suffix('_'+ f).\\\n            reset_index()\n        result = result.merge(df_table,on=['case_id'],how='left')\n    return result","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:14:52.834343Z","iopub.execute_input":"2024-04-08T12:14:52.834816Z","iopub.status.idle":"2024-04-08T12:14:52.843166Z","shell.execute_reply.started":"2024-04-08T12:14:52.83478Z","shell.execute_reply":"2024-04-08T12:14:52.841568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ndef suggest_fill_method(dataFrame, col):\n\n    # Check if the column contains numerical or categorical data\n    if pd.api.types.is_numeric_dtype(dataFrame[col]):\n        # If numerical, check if the data is skewed\n        skewness = dataFrame[col].skew()\n        if abs(skewness) > 1:\n            fill_method = 'median'  # If highly skewed, suggest filling with median\n        else:\n            fill_method = 'mean'  # If not highly skewed, suggest filling with mean\n    else:\n        fill_method = 'mode'  # If categorical, suggest filling with mode\n    \n    # Fill missing values with the suggested method\n    if fill_method == 'mode':\n        dataFrame[col].fillna(dataFrame[col].mode()[0], inplace=True)\n    elif fill_method == 'median':\n        dataFrame[col].fillna(dataFrame[col].median(), inplace=True)\n\n    return fill_method","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:14:55.270252Z","iopub.execute_input":"2024-04-08T12:14:55.270669Z","iopub.status.idle":"2024-04-08T12:14:55.279394Z","shell.execute_reply.started":"2024-04-08T12:14:55.270638Z","shell.execute_reply":"2024-04-08T12:14:55.278093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Drop columns from applprev_1_0\ndrop_deposit = df.drop(columns=morethan_80)\n\n# Iterate through columns and drop columns ending with 'D'\nfor col in drop_deposit.columns:\n    if col.endswith(\"D\"):\n        drop_deposit.drop(col, axis=1, inplace=True)\n\n#Print the DataFrame\ndrop_deposit","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:14:58.495197Z","iopub.execute_input":"2024-04-08T12:14:58.495653Z","iopub.status.idle":"2024-04-08T12:14:58.527876Z","shell.execute_reply.started":"2024-04-08T12:14:58.495612Z","shell.execute_reply":"2024-04-08T12:14:58.526537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nA_fea = ['amount_416A']\nA_fea_table = feature_eng(data = drop_deposit, fealist = A_fea,agglist=['mean'])\nA_fea_table\n","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:15:01.221179Z","iopub.execute_input":"2024-04-08T12:15:01.221642Z","iopub.status.idle":"2024-04-08T12:15:01.408078Z","shell.execute_reply.started":"2024-04-08T12:15:01.221606Z","shell.execute_reply":"2024-04-08T12:15:01.406779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in A_fea_table.columns:\n    # Call the function for each column and update the original DataFrame\n    fill_method = suggest_fill_method(A_fea_table, col)\n    print(f\"For column '{col}', suggested fill method:\", fill_method)\n\n# Now cleaned_P has been updated with the filled missing values\nprint(\"Updated DataFrame:\")\nA_fea_table","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:15:04.112399Z","iopub.execute_input":"2024-04-08T12:15:04.11381Z","iopub.status.idle":"2024-04-08T12:15:04.134558Z","shell.execute_reply.started":"2024-04-08T12:15:04.11377Z","shell.execute_reply":"2024-04-08T12:15:04.13329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata = train.\\\n    merge(A_fea_table,on=['case_id'],how = 'left')\ndata","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:15:06.935397Z","iopub.execute_input":"2024-04-08T12:15:06.936252Z","iopub.status.idle":"2024-04-08T12:15:07.265933Z","shell.execute_reply.started":"2024-04-08T12:15:06.936211Z","shell.execute_reply":"2024-04-08T12:15:07.264726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in data.columns:\n    # Call the function for each column and update the original DataFrame\n    fill_method = suggest_fill_method(data, col)\n    print(f\"For column '{col}', suggested fill method:\", fill_method)\n\n# Now cleaned_P has been updated with the filled missing values\nprint(\"Updated DataFrame:\")\n\n\ndata","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:15:10.356887Z","iopub.execute_input":"2024-04-08T12:15:10.357347Z","iopub.status.idle":"2024-04-08T12:15:10.914147Z","shell.execute_reply.started":"2024-04-08T12:15:10.35731Z","shell.execute_reply":"2024-04-08T12:15:10.912839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import StandardScaler\n\n\n# Columns to exclude from normalization\nexclude_cols = ['case_id', 'date_decision', 'MONTH', 'WEEK_NUM', 'target']\n\n# Columns to normalize\ncols_to_normalize = [col for col in data.columns if col not in exclude_cols]\n\n# Apply z-score normalization to each column except excluded ones\nnormalized_data = data.copy()\nfor col in cols_to_normalize:\n    normalized_data[col] = (data[col] - data[col].mean()) / data[col].std()\n\nprint(\"\\nNormalized Data (Standardization):\")\nnormalized_data","metadata":{"execution":{"iopub.status.busy":"2024-04-08T12:15:14.587866Z","iopub.execute_input":"2024-04-08T12:15:14.588271Z","iopub.status.idle":"2024-04-08T12:15:14.694595Z","shell.execute_reply.started":"2024-04-08T12:15:14.588241Z","shell.execute_reply":"2024-04-08T12:15:14.693666Z"},"trusted":true},"execution_count":null,"outputs":[]}]}