{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":170267939,"sourceType":"kernelVersion"},{"sourceId":170735078,"sourceType":"kernelVersion"}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Procedure for planning imputation\n## Check NULL \n\n1. Probability of target for NULL and NOT NULL\n2. Check data type of columns. factor, date, numeric\n\n## Preprocessing\n\nFactor: Remove Minorities --> \"OTHERS\"\n\nDate: Convert to Integers\n\nNumeric: Nothing to do\n\n## Inputation\n\nFactor\n\n1. NULL --> \"Z\", if case_id does not exist in the file, then \"NA\"\n\nDate and Numeric\n\n1. Logistic regression by each X. \n2. Take inverse of probability of target in NULL, replace NULL with this value\n3. if case_id does not exist in the file, fill with 0.5","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time\nimport re\nimport json\n#from sklearn.ensemble import RandomForestClassifier\n#from sklearn import metrics\nfrom sklearn.linear_model import LogisticRegression","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-28T16:59:49.174367Z","iopub.execute_input":"2024-03-28T16:59:49.174782Z","iopub.status.idle":"2024-03-28T16:59:52.739649Z","shell.execute_reply.started":"2024-03-28T16:59:49.174747Z","shell.execute_reply":"2024-03-28T16:59:52.737996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_json(file1):\n    with open(file1, 'r',  encoding='utf-8') as f:\n        data = json.load(f)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.742028Z","iopub.execute_input":"2024-03-28T16:59:52.742704Z","iopub.status.idle":"2024-03-28T16:59:52.751467Z","shell.execute_reply.started":"2024-03-28T16:59:52.742662Z","shell.execute_reply":"2024-03-28T16:59:52.748757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/\"","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.752740Z","iopub.execute_input":"2024-03-28T16:59:52.753225Z","iopub.status.idle":"2024-03-28T16:59:52.765514Z","shell.execute_reply.started":"2024-03-28T16:59:52.753194Z","shell.execute_reply":"2024-03-28T16:59:52.764590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fpath = \"/kaggle/input/homecredit-pipeline-imputationplan/imputation_plan.json\"\nimputation_plan = read_json(fpath)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.767532Z","iopub.execute_input":"2024-03-28T16:59:52.768021Z","iopub.status.idle":"2024-03-28T16:59:52.784579Z","shell.execute_reply.started":"2024-03-28T16:59:52.767988Z","shell.execute_reply":"2024-03-28T16:59:52.783143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fpath = \"/kaggle/input/homecredit-filesbycategory/train_files_subcategory.json\"\nsubcategory_files_train = read_json(fpath)\n\nfpath = \"/kaggle/input/homecredit-filesbycategory/files_depth.json\"\nfile_depth = read_json(fpath)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.786322Z","iopub.execute_input":"2024-03-28T16:59:52.786654Z","iopub.status.idle":"2024-03-28T16:59:52.800882Z","shell.execute_reply.started":"2024-03-28T16:59:52.786621Z","shell.execute_reply":"2024-03-28T16:59:52.799473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readFiles(files, merge_label = False):\n    count = 0\n    for file in files:\n        if count == 0:\n            data = pd.read_parquet(train_dir + file)\n            \n        else:\n            tmp = pd.read_parquet(train_dir + file)\n            data = pd.concat([data, tmp])\n        \n        count += 1\n    \n    if merge_label:\n        data = data.merge(train_base[[\"case_id\", \"target\"]], on = \"case_id\", how = \"inner\")\n    \n    return data\n\n\ndef readFilesAggL1(files, merge_label = False):\n    count = 0\n    \n    \n    for file in files:\n        if count == 0:\n            data = pd.read_parquet(train_dir + file)\n            filter1 = data[\"num_group1\"] == 0\n            data = data.loc[filter1]\n        \n            \n        else:\n            tmp = pd.read_parquet(train_dir + file)\n            filter1 = tmp[\"num_group1\"] == 0\n            tmp = tmp.loc[filter1]\n            \n            data = pd.concat([data, tmp])\n        \n        count += 1\n    \n    if merge_label:\n        data = data.merge(train_base[[\"case_id\", \"target\"]], on = \"case_id\", how = \"inner\")\n    \n    return data\n\n\ndef readFilesAggL2(files, merge_label = False):\n    count = 0\n    \n    for file in files:\n        if count == 0:\n            data = pd.read_parquet(train_dir + file)\n            filter1 = (data[\"num_group1\"] == 0) & (data[\"num_group2\"] == 0)\n            data = data.loc[filter1]\n        \n            \n        else:\n            tmp = pd.read_parquet(train_dir + file)\n            filter1 = (tmp[\"num_group1\"] == 0) & (tmp[\"num_group2\"] == 0)\n            tmp = tmp.loc[filter1]\n            \n            data = pd.concat([data, tmp])\n        \n        count += 1\n    \n    if merge_label:\n        data = data.merge(train_base[[\"case_id\", \"target\"]], on = \"case_id\", how = \"inner\")\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.802264Z","iopub.execute_input":"2024-03-28T16:59:52.803862Z","iopub.status.idle":"2024-03-28T16:59:52.819642Z","shell.execute_reply.started":"2024-03-28T16:59:52.803705Z","shell.execute_reply":"2024-03-28T16:59:52.815760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def impute_factor(df, c, values):\n    \n    filter_na = df[c].isna()\n    \n    set_values = set(values)\n    \n    filter1 =  df[c].apply(lambda x: x not in set_values)\n    \n    df.loc[filter1, c] = \"OTHERS\"\n    \n    df.loc[filter_na, c] = \"Z\"\n    \n    return df[c]\n\ndef impute_numeric(df, c, value):\n    filter_na = df[c].isna()\n    df.loc[filter_na, c] = value\n    return df[c]    \n\ndef impute_date(df, c, value):\n    \n    filter_na = df[c].isna()\n    \n    df.loc[:,c] = pd.to_datetime(df[c]).astype(int)\n    df.loc[filter_na, c] = int(value)\n    \n    return df[c]    ","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.822326Z","iopub.execute_input":"2024-03-28T16:59:52.822894Z","iopub.status.idle":"2024-03-28T16:59:52.839554Z","shell.execute_reply.started":"2024-03-28T16:59:52.822850Z","shell.execute_reply":"2024-03-28T16:59:52.837292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imputation_execute(df, imputation_plan, key):\n\n    selected_cols = list(imputation_plan[key].keys())\n\n    df = df[[\"case_id\"] + selected_cols]\n\n    for c in selected_cols:\n        vals = imputation_plan[key][c]\n        #print(vals[0], c)\n        if vals[0] == \"factor\":\n            df.loc[:,c] = impute_factor(df, c, vals[1])\n        elif vals[0] == \"date\":\n            df.loc[:,c] = impute_date(df, c, vals[1])\n        else:\n            df.loc[:,c] = impute_numeric(df, c, vals[1])\n            \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.840873Z","iopub.execute_input":"2024-03-28T16:59:52.841424Z","iopub.status.idle":"2024-03-28T16:59:52.859355Z","shell.execute_reply.started":"2024-03-28T16:59:52.841391Z","shell.execute_reply":"2024-03-28T16:59:52.857186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in subcategory_files_train.keys():\n    files1 = subcategory_files_train[key]\n    depth1 = file_depth[key]\n    print(depth1, files1)\n    \n    if depth1 == 0:\n        data_df = readFiles(files1, False)\n    elif depth1 == 1:\n        data_df = readFilesAggL1(files1, False)\n    else:\n        data_df = readFilesAggL2(files1, False)\n        \n    data_df = imputation_execute(data_df, imputation_plan, key)\n    \n    data_df.to_csv(key + \".csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T16:59:52.861026Z","iopub.execute_input":"2024-03-28T16:59:52.861580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}