{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feature Engineering with dplyr\n\nMy R version of feature engineering based on @cdeotte [notebook](https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793).","metadata":{}},{"cell_type":"code","source":"library(tidyverse)","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-07-04T00:27:13.926223Z","iopub.execute_input":"2022-07-04T00:27:13.928256Z","iopub.status.idle":"2022-07-04T00:27:15.317457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data <- arrow::read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:27:15.319936Z","iopub.execute_input":"2022-07-04T00:27:15.355890Z","iopub.status.idle":"2022-07-04T00:27:32.517444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process and Feature Engineer Train Data\n\nWe need a little hack to process the whole data with 16GB RAM Kaggle Notebook.","metadata":{}},{"cell_type":"markdown","source":"### Bad: simple but consumes too much memory","metadata":{}},{"cell_type":"markdown","source":"See [Programming with dplyr](https://dplyr.tidyverse.org/articles/programming.html) about doubled braces like `{{ cat_features }}`.","metadata":{}},{"cell_type":"code","source":"process_and_feature_engineer <- function(df) {\n  cat_features <- c(\"B_30\", \"B_38\", \"D_114\", \"D_116\", \"D_117\", \"D_120\", \"D_126\", \"D_63\", \"D_64\", \"D_66\", \"D_68\")\n  num_features <- setdiff(colnames(df), c(cat_features, \"customer_ID\", \"S_2\"))\n\n  df %>%\n    group_by(customer_ID) %>%\n    summarise(n = n(),\n              across({{ cat_features }}, list(last = last, nd = n_distinct)),\n              across({{ num_features }}, list(mean = mean, sd = sd, min = min, max = max, last = last)))\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:27:32.521367Z","iopub.execute_input":"2022-07-04T00:27:32.523373Z","iopub.status.idle":"2022-07-04T00:27:32.537074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Not bad: complex but consumes less memory","metadata":{}},{"cell_type":"code","source":"chunk <- function(v, length) split(v, ceiling(seq_along(v) / length))\n\nprocess_and_feature_engineer_with_gc <- function(df) {\n  cat_features <- c(\"B_30\", \"B_38\", \"D_114\", \"D_116\", \"D_117\", \"D_120\", \"D_126\", \"D_63\", \"D_64\", \"D_66\", \"D_68\")\n  num_features <- setdiff(colnames(df), c(cat_features, \"customer_ID\", \"S_2\"))\n\n  cat_df <- df %>%\n    group_by(customer_ID) %>%\n    summarise(n = n(),\n              across({{ cat_features }}, list(last = last, nd = n_distinct)))\n\n  num_features %>%\n    chunk(10) %>%\n    reduce(function(result, features) {\n      gc()\n      bind_cols(result,\n                df %>%\n                group_by(customer_ID) %>%\n                summarise(across({{ features }}, list(mean = mean, sd = sd, min = min, max = max, last = last))) %>%\n                select(-customer_ID))\n    }, .init = cat_df)\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:28:51.204383Z","iopub.execute_input":"2022-07-04T00:28:51.205996Z","iopub.status.idle":"2022-07-04T00:28:51.220491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`chunk` works like this:","metadata":{}},{"cell_type":"code","source":"letters %>% chunk(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:28:55.142410Z","iopub.execute_input":"2022-07-04T00:28:55.144064Z","iopub.status.idle":"2022-07-04T00:28:55.164953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try it.","metadata":{}},{"cell_type":"code","source":"identical(train_data %>% head(100) %>% process_and_feature_engineer,\n          train_data %>% head(100) %>% process_and_feature_engineer_with_gc)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:28:57.732598Z","iopub.execute_input":"2022-07-04T00:28:57.734164Z","iopub.status.idle":"2022-07-04T00:29:04.719560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X <-\n  train_data %>%\n  process_and_feature_engineer_with_gc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X %>% head","metadata":{},"execution_count":null,"outputs":[]}]}