{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30618,"isInternetEnabled":true,"language":"r","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"install.packages(c(\"fastverse\", \"arrow\"))","metadata":{"execution":{"iopub.status.busy":"2024-05-08T08:04:55.065688Z","iopub.execute_input":"2024-05-08T08:04:55.067810Z","iopub.status.idle":"2024-05-08T08:07:28.973794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"packageVersion(\"fastverse\")\npackageVersion(\"collapse\")\npackageVersion(\"arrow\")\npackageVersion(\"data.table\")","metadata":{"execution":{"iopub.status.busy":"2024-05-08T08:07:47.750104Z","iopub.execute_input":"2024-05-08T08:07:47.752982Z","iopub.status.idle":"2024-05-08T08:07:47.794207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Packages Load\nlibrary(tidyverse)\n# library(mlr3verse)\n# library(mlr3mbo)\n# library(mlr3tuningspaces)\n# library(mlr3tuning)\n# library(bbotk)\nlibrary(data.table)\n# library(formattable)\n# library(patchwork)\n# library(ggpubr)\n# library(cowplot)\n# library(rcompanion)\n# library(cvms)\nlibrary(tictoc)\n# library(DiagrammeR)\n# library(iml)\n# library(DALEX)\n# library(DALEXtra)\nlibrary(arrow)\nlibrary(fastverse)\n# library(profvis)\nlibrary(R6)\n# library(lobstr)\n# library(microbenchmark)\nlibrary(cli)\nlibrary(crayon)","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2024-05-08T08:08:16.430032Z","iopub.execute_input":"2024-05-08T08:08:16.431742Z","iopub.status.idle":"2024-05-08T08:08:17.610321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG <- R6Class(\"CFG\",\n  public = list(\n    root_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files\",\n    train_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train\",\n    test_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test\"\n  )\n)\ncfg = CFG$new()\n\nPipeline <- R6Class(\"Pipeline\",\n  public = list(\n    handle_dates = function(dt) {\n      dt = dt %>% \n        fmutate(across(grep(\"D$\", names(.)), function(x) x - .$date_decision)) %>% \n        fselect(-c(date_decision, MONTH))\n      dt\n    },\n\n    filter_cols = function(dt) {\n      miss_95 = names(dt)[dapply(dt, function(x) fmean(is.na(x))) > 0.9]\n      cardinality = dapply(get_vars(dt, is.factor), fndistinct)\n      cardi_cols = names(cardinality)[cardinality < 2 | cardinality > 200]\n      \n      \n      cat(\"    # of cols removed with high missing rate:\", length(miss_95), \"\\n\")\n      cat(\"    # of cols removed with low or high cardinality:\", length(cardi_cols), \"\\n\")\n      \n      dt = dt %>% get_vars(!(names(.) %in% c(miss_95, cardi_cols)))\n      dt\n    }\n  )\n)\npipe = Pipeline$new()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T08:09:02.977456Z","iopub.execute_input":"2024-05-08T08:09:02.980685Z","iopub.status.idle":"2024-05-08T08:09:03.003558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_file_collapse <- function(path) {\n  mode <- strsplit(path, \"_\")[[1]][1]\n  dt <- fread(file.path(cfg$root_dir, mode, path), na.strings = \"\", key = \"case_id\", showProgress = FALSE)\n  vars = names(dt)\n  dt = dt %>% \n    mutate(\n      across(grep(\"case_id|WEEK_NUM|num_group1|num_group2\", vars), as.integer),\n      across(grep(\"date_decision|D$\", vars), as.IDate),\n      across(grep(\"P$|A$\", vars), as.numeric),\n      across(where(is.character), as.factor)\n    )\n  \n  dt\n}\n\nread_file2 = function(path, mode, depth = NA) {\n  files = list.files(paste(cfg$root_dir, mode, sep = \"/\"), path)\n  dt = list()\n  i = 1\n  for (f in files) {\n    cat(sprintf(\"        chunk %d:\", i), f)\n    s = Sys.time()\n    if (depth %in% c(1, 2)) {\n      dt_collapse = read_file_collapse(f) \n      if (nrow(dt_collapse) == 0) {\n        PA_cols = grep(\"P$|A$\", names(dt_collapse), value = T)\n        D_cols = grep(\"D$\", names(dt_collapse), value = T)\n        M_cols = grep(\"M$\", names(dt_collapse), value = T)\n        ng_cols = grep(\"num_group\", names(dt_collapse), value = T)\n        \n        \n        dt_collapse = data.table(matrix(nrow = 10, ncol = 15))\n        \n        names(dt_collapse) = c(\"case_id\", \n                       paste(c(\"max\", \"min\", \"first\", \"last\", \"mean\"), PA_cols, sep = \"_\"),\n                       paste(c(\"max\", \"min\", \"first\", \"last\"), D_cols, sep = \"_\"),\n                       paste(c(\"first\", \"last\", \"mode\", \"nuniq\"), M_cols, sep = \"_\"),\n                       paste(\"nuniq\", ng_cols, sep = \"_\"))\n        \n        dt_collapse = dt_collapse %>% \n          mutate(\n            across(grep(\"case_id|WEEK_NUM|num_group1|num_group2\", names(.)), as.integer),\n            across(grep(\"date_decision|D$\", names(.)), as.IDate),\n            across(grep(\"P$|A$\", names(.)), as.numeric),\n            across(grep(\"M$\", names(.)), as.factor)\n          )\n        dt[[i]] = dt_collapse\n        next\n      }\n      \n      dt_collapse = dt_collapse %>% fgroup_by(case_id)\n      \n      vars = names(dt_collapse)\n      t = dt_collapse %>% fsummarise(across(\n        grep(\"num_group\", vars), list(uniq = fndistinct), .names = \"flip\"\n      ))\n      if (length(grep(\"P$|A$\", vars))) {\n        t = join(t, dt_collapse %>% fsummarise(across(\n          grep(\"P$|A$\", vars), list(max = fmax, min = fmin, first = ffirst, last = flast, mean = fmean), .names = \"flip\"\n        )), on = \"case_id\", verbose = F)\n      }\n      if (length(grep(\"D$\", vars))) {\n        t = join(t, dt_collapse %>% fsummarise(across(\n          grep(\"D$\", vars), list(max = fmax, min = fmin, first = ffirst, last = flast), .names = \"flip\"\n        )), on = \"case_id\", verbose = F)\n      }\n      if (length(grep(\"M$\", vars))) {\n        t = join(t, dt_collapse %>% fsummarise(across(\n          grep(\"M$\", vars), list(first = ffirst, last = flast, mode = fmode, nuniq = fndistinct), .names = \"flip\"\n        )), on = \"case_id\", verbose = F)\n      }\n      if (length(grep(\"T$|L$\", vars))) {\n        t = join(t, dt_collapse %>% fsummarise(across(\n          grep(\"T$|L$\", vars), list(max = fmax, min = fmin, first = ffirst, last = flast), .names = \"flip\"\n        )), on = \"case_id\", verbose = F)\n      }\n      dt[[i]] = t\n      rm(dt_collapse)\n      rm(t)\n      rm(vars)\n    } else {\n      dt[[i]] = read_file_collapse(f)\n    }\n    e = Sys.time()\n    cat(\" \", round(e-s, 4), attr(e-s, \"units\"), \"\\n\")\n    i = i + 1\n    gc()\n  }\n  cat(\"\\n\")\n  dt = rbindlist(dt, use.names = T)\n  gc()\n  \n  dt\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-08T09:16:49.571339Z","iopub.execute_input":"2024-05-08T09:16:49.573976Z","iopub.status.idle":"2024-05-08T09:16:49.604165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collect = function(files_list, mode = 'train', train_cols = NULL, cat_cols = NULL) {\n  s0 = Sys.time()\n  cli_alert_info(blue$underline$bold(\"Collecting Data...\\n\"))\n  # s1 = Sys.time()\n  base_files = sprintf(\"%s_base.csv\", mode)\n  cat(\"    data:\", bold(base_files), \"\\n\")\n  base_df = read_file2(base_files, mode)\n  if (mode == 'train') {\n    base_df = base_df %>% \n      fmutate(month_decision = month(date_decision)) %>% \n      fmutate(weekday_decision = wday(date_decision))\n  } else {\n    base_df = base_df %>% \n      fmutate(month_decision = month(date_decision)) %>% \n      fmutate(weekday_decision = wday(date_decision))\n  }\n  gc()\n  # e1 = Sys.time()\n  # cat(\"        \", base_files, round(e1-s1, 4), attr(e1-s1, \"units\"), \"\\n\")\n  \n  for (i in 1:length(files_list)) {\n    depth = as.numeric(str_match(files_list[[i]], \"\\\\w_(\\\\d)\")[, 2])\n    cat(\"    data:\", bold(files_list[[i]]), red$bold(sprintf(\" (depth: %d)\", depth)), \"\\n\")\n    # s1 = Sys.time()\n    files_df = read_file2(files_list[[i]], mode, depth)\n    # e1 = Sys.time()\n    # cat(\"        \", files_list[[i]], round(e1-s1, 4), attr(e1-s1, \"units\"), \"\\n\")\n    if (i > 1) {\n      base_df = join(base_df, files_df, on = \"case_id\", verbose = F, \n                     suffix = paste0(\"_\", str_extract(c(files_list[[i-1]], files_list[[i]]), \"(?<=_).*(?=_)\")))\n    } else {\n      base_df = join(base_df, files_df, on = \"case_id\", verbose = F)\n    }\n    rm(files_df)\n    gc()\n  }\n  cat(green$bold(sprintf(\"%s_data shape:\", mode)), bold(sprintf(\"(%d, %d)\", nrow(base_df), ncol(base_df))), \"\\n\")\n  cat(bold(\"-------------------------------------------------------------\"), \"\\n\\n\")\n  \n  \n  cli_alert_info(blue$underline$bold(\"Feature Engineering...\\n\"))\n  base_df = pipe$handle_dates(base_df) # D$ variables preprocessing\n  \n  \n  if (mode == \"train\") {\n    base_df = pipe$filter_cols(base_df)  # feature engineering\n  } else {\n    # train_cols = ifelse(length(train_cols) > 0, train_cols, names(base_df))\n    base_df = base_df %>% get_vars(train_cols)\n  }\n  gc()\n  names(base_df)[!names(base_df) %in%  train_cols]\n  train_cols[!names(base_df) %in%  train_cols]\n  if (is.null(cat_cols)) cat_cols = names(base_df %>% get_vars(is.factor))\n  # base_df = base_df %>% fmutate(across(cat_cols, as.factor))\n  gc()\n  cat(\"\\n\")\n  cat(green$bold(sprintf(\"%s_data shape:\", mode)), bold(sprintf(\"(%d, %d)\", nrow(base_df), ncol(base_df))), \"\\n\")\n  e0 = Sys.time()\n  cat(bold(\"-------------------------------------------------------------\"), \"\\n\")\n  cat(green(sprintf(\"Total time: %.4f %s\", e0-s0, attr(e0-s0, \"units\"))), \"\\n\")\n  # cat(\"Total time: \", round(e0-s0, 4), attr(e0-s0, \"units\"), \"\\n\")\n  return(base_df)\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-08T09:16:53.712656Z","iopub.execute_input":"2024-05-08T09:16:53.715787Z","iopub.status.idle":"2024-05-08T09:16:53.740256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_list = list(\n  # depth = 0:\n  static_0_files = \"_static_0_*\",\n  static_cb_0_files = \"_static_cb_0_*\",\n  # depth = 1:\n  applprev_1_files = \"_applprev_1_*\",\n  other_1_files = \"_other_1_*\",\n  deposit_1_files = \"_deposit_1_*\",\n  person_1_files = \"_person_1_*\",\n  debitcard_1_files = \"_debitcard_1_*\",\n  tax_registry_a_1_files = \"_tax_registry_a_1_*\",\n  tax_registry_b_1_files = \"_tax_registry_b_1_*\",\n  tax_registry_c_1_files = \"_tax_registry_c_1_*\",\n  credit_bureau_a_1_files = \"_credit_bureau_a_1_*\",\n  credit_bureau_b_1_files = \"_credit_bureau_b_1_*\",\n  # depth = 2:\n  applprev_2_files = \"_applprev_2_*\",\n  person_2_files = \"_person_2_*\",\n  credit_bureau_a_2_files = \"_credit_bureau_a_2_*\",\n  credit_bureau_b_2_files = \"_credit_bureau_b_2_*\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T08:09:49.799571Z","iopub.execute_input":"2024-05-08T08:09:49.801173Z","iopub.status.idle":"2024-05-08T08:09:49.814015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = data_collect(f_list, \"train\")","metadata":{"execution":{"iopub.status.busy":"2024-05-08T08:10:01.224424Z","iopub.execute_input":"2024-05-08T08:10:01.226042Z","iopub.status.idle":"2024-05-08T08:18:46.501619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols_train = names(train_data %>% get_vars(is.factor))\ntest_data = data_collect(f_list, mode = \"test\",\n                         train_cols = names(train_data %>% fselect(-target)),\n                         cat_cols = cat_cols_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T09:17:29.510245Z","iopub.execute_input":"2024-05-08T09:17:29.514609Z","iopub.status.idle":"2024-05-08T09:17:43.386826Z"},"trusted":true},"execution_count":null,"outputs":[]}]}