{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"},"papermill":{"default_parameters":{},"duration":487.037682,"end_time":"2022-12-30T11:49:49.308689","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2022-12-30T11:41:42.271007","version":"2.4.0"},"vscode":{"interpreter":{"hash":"3ad933181bd8a04b432d3370b9dc3b0662ad032c4dfaa4e4f1596c548f763858"}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"library(data.table)\nlibrary(ggplot2)\nlibrary(magrittr)\nlibrary(ggrepel)\nlibrary(corrplot)\nlibrary(lares)\nlibrary(funModeling)\nlibrary(dplyr)\nlibrary(caret)","metadata":{"papermill":{"duration":4.650141,"end_time":"2022-12-30T11:41:51.306749","exception":false,"start_time":"2022-12-30T11:41:46.656608","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T02:33:02.396915Z","iopub.execute_input":"2023-01-08T02:33:02.398477Z","iopub.status.idle":"2023-01-08T02:33:07.431637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load train data and train label","metadata":{"papermill":{"duration":0.008902,"end_time":"2022-12-30T11:41:51.324747","exception":false,"start_time":"2022-12-30T11:41:51.315845","status":"completed"},"tags":[]}},{"cell_type":"code","source":"DAT_DIR <- '../input/amex-default-prediction'\nlist.files(DAT_DIR)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:32:16.956652Z","iopub.status.busy":"2023-01-03T02:32:16.954110Z","iopub.status.idle":"2023-01-03T02:32:16.989826Z"},"papermill":{"duration":0.085857,"end_time":"2022-12-30T11:41:51.419119","exception":false,"start_time":"2022-12-30T11:41:51.333262","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_dt_dim <- function(dt) {\n  cat(paste0(deparse(substitute(dt)), \" dimension: (\", paste0(dim(dt), collapse=\", \"), \")\\n\"))  \n}","metadata":{"papermill":{"duration":0.030806,"end_time":"2022-12-30T11:41:51.458224","exception":false,"start_time":"2022-12-30T11:41:51.427418","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T02:34:16.920888Z","iopub.execute_input":"2023-01-08T02:34:16.943952Z","iopub.status.idle":"2023-01-08T02:34:16.952286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dt <-\n  fread(file.path(DAT_DIR, \"train_data.csv\"))\n\nprint_dt_dim(train_dt)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:32:17.017038Z","iopub.status.busy":"2023-01-03T02:32:17.015111Z","iopub.status.idle":"2023-01-03T02:35:00.448744Z"},"papermill":{"duration":174.73014,"end_time":"2022-12-30T11:44:46.197281","exception":false,"start_time":"2022-12-30T11:41:51.467141","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_layout <- train_dt[, lapply(.SD, function(i) class(i)[[1]])] %>% \n  as.character()\n\ntable(train_layout)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:00.454437Z","iopub.status.busy":"2023-01-03T02:35:00.452508Z","iopub.status.idle":"2023-01-03T02:35:00.518671Z"},"papermill":{"duration":0.092616,"end_time":"2022-12-30T11:44:46.302493","exception":false,"start_time":"2022-12-30T11:44:46.209877","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cols <- names(train_dt)\nnon_numeric_cols <- names(train_dt)[unlist(train_dt[, lapply(.SD, function(i) {! is.numeric(i)})])]\nnumeric_cols <- setdiff(train_cols, non_numeric_cols)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:00.524027Z","iopub.status.busy":"2023-01-03T02:35:00.522403Z","iopub.status.idle":"2023-01-03T02:35:00.553682Z"},"papermill":{"duration":0.040808,"end_time":"2022-12-30T11:44:46.352328","exception":false,"start_time":"2022-12-30T11:44:46.311520","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat(paste0(\"There are \", length(unique(train_dt$customer_ID)), \" unique customers in the training\"))","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:00.560838Z","iopub.status.busy":"2023-01-03T02:35:00.558917Z","iopub.status.idle":"2023-01-03T02:35:00.892388Z"},"papermill":{"duration":0.292948,"end_time":"2022-12-30T11:44:46.653765","exception":false,"start_time":"2022-12-30T11:44:46.360817","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"head(train_dt)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:00.898223Z","iopub.status.busy":"2023-01-03T02:35:00.896114Z","iopub.status.idle":"2023-01-03T02:35:00.971915Z"},"papermill":{"duration":0.08205,"end_time":"2022-12-30T11:44:46.744184","exception":false,"start_time":"2022-12-30T11:44:46.662134","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load labels as well\ntrain_labels_dt <- read.csv('../input/amex-default-prediction/train_labels.csv')\nprint_dt_dim(train_labels_dt)","metadata":{"papermill":{"duration":1.203307,"end_time":"2022-12-30T11:44:47.956341","exception":false,"start_time":"2022-12-30T11:44:46.753034","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T02:45:20.623200Z","iopub.execute_input":"2023-01-08T02:45:20.624363Z","iopub.status.idle":"2023-01-08T02:45:21.396136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"head(train_labels_dt)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:02.158032Z","iopub.status.busy":"2023-01-03T02:35:02.156103Z","iopub.status.idle":"2023-01-03T02:35:02.186427Z"},"papermill":{"duration":0.047236,"end_time":"2022-12-30T11:44:48.013304","exception":false,"start_time":"2022-12-30T11:44:47.966068","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = table(train_labels_dt$target)\npie_labels <- paste0(labels, \" = \", round(100 * labels/sum(labels), 2), \"%\")\npie(labels, labels = pie_labels,radius=1.05)\nlegend(\"topleft\", legend = c(\"Default\", \"Paid\"),\n       fill =  c(\"1\", \"0\"), cex=1.4)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:02.191516Z","iopub.status.busy":"2023-01-03T02:35:02.189630Z","iopub.status.idle":"2023-01-03T02:35:02.670833Z"},"papermill":{"duration":0.509221,"end_time":"2022-12-30T11:44:48.533150","exception":false,"start_time":"2022-12-30T11:44:48.023929","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert ID to factor\ntrain_dt$customer_ID <- as.factor(train_dt$customer_ID)\n# convert date\ntrain_dt$S_2 <- as.Date(train_dt$S_2)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:02.675307Z","iopub.status.busy":"2023-01-03T02:35:02.673651Z","iopub.status.idle":"2023-01-03T02:35:29.456550Z"},"papermill":{"duration":32.53815,"end_time":"2022-12-30T11:45:21.081349","exception":false,"start_time":"2022-12-30T11:44:48.543199","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat(paste0(\"Training data from \", min(train_dt$S_2), \" to \",  max(train_dt$S_2)))","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:29.462936Z","iopub.status.busy":"2023-01-03T02:35:29.461333Z","iopub.status.idle":"2023-01-03T02:35:29.672358Z"},"papermill":{"duration":0.250135,"end_time":"2022-12-30T11:45:21.343790","exception":false,"start_time":"2022-12-30T11:45:21.093655","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transform datatype\nfeatures_cat = c('B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68')\nfor (f in features_cat) {\n    train_dt$f <- as.factor(train_dt$f)\n}\n","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:35:29.678178Z","iopub.status.busy":"2023-01-03T02:35:29.676573Z","iopub.status.idle":"2023-01-03T02:37:11.011233Z"},"papermill":{"duration":125.515811,"end_time":"2022-12-30T11:47:26.870882","exception":false,"start_time":"2022-12-30T11:45:21.355071","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory Data Analysis","metadata":{"papermill":{"duration":0.010179,"end_time":"2022-12-30T11:47:26.894717","exception":false,"start_time":"2022-12-30T11:47:26.884538","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_whole <- merge(train_dt, train_labels_dt, by = \"customer_ID\",\n                  all.x = TRUE)\nhead(train_whole)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:37:11.017503Z","iopub.status.busy":"2023-01-03T02:37:11.015707Z","iopub.status.idle":"2023-01-03T02:37:21.908439Z"},"papermill":{"duration":11.203981,"end_time":"2022-12-30T11:47:38.108423","exception":false,"start_time":"2022-12-30T11:47:26.904442","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## release memory\n\nrm(train_dt)\nrm(train_labels_dt)\ngc()","metadata":{"execution":{"iopub.execute_input":"2023-01-03T02:37:21.912204Z","iopub.status.busy":"2023-01-03T02:37:21.910861Z","iopub.status.idle":"2023-01-03T02:37:22.988351Z"},"papermill":{"duration":1.098437,"end_time":"2022-12-30T11:47:39.217871","exception":false,"start_time":"2022-12-30T11:47:38.119434","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categorical data","metadata":{"papermill":{"duration":0.011376,"end_time":"2022-12-30T11:47:39.239982","exception":false,"start_time":"2022-12-30T11:47:39.228606","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# plot distributions\ndata_length = nrow(train_whole)\nfor (f in features_cat) {\n    perc_miss <- round(100*sum(is.na(train_whole[[f]]))/data_length, 4)\n    my_title <- paste0(f, ' - missing: ', perc_miss,'%')\n#     print(my_title)\n    print(ggplot(data = train_whole, aes(x = .data[[f]], fill = factor(.data[['target']]))) + geom_bar()+ggtitle(my_title))\n}","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:47:39.266246Z","iopub.status.busy":"2022-12-30T11:47:39.264480Z","iopub.status.idle":"2022-12-30T11:49:11.706298Z","shell.execute_reply":"2022-12-30T11:49:11.704273Z"},"papermill":{"duration":92.458566,"end_time":"2022-12-30T11:49:11.709385","exception":false,"start_time":"2022-12-30T11:47:39.250819","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Numerical data","metadata":{"papermill":{"duration":0.019031,"end_time":"2022-12-30T11:49:11.748712","exception":false,"start_time":"2022-12-30T11:49:11.729681","status":"completed"},"tags":[]}},{"cell_type":"code","source":"corr_simple <- function(data=df,sig=0.5){\n  #convert data to numeric in order to run correlations\n  #convert to factor first to keep the integrity of the data - each value will become a number rather than turn into NA\n  df_cor <- data %>% dplyr::mutate_if(is.character, as.factor)\n  df_cor <- df_cor %>% dplyr::mutate_if(is.factor, as.numeric)\n  #run a correlation and drop the insignificant ones\n  corr <- cor(df_cor)\n  #prepare to drop duplicates and correlations of 1     \n  corr[lower.tri(corr,diag=TRUE)] <- NA \n  #drop perfect correlations\n  corr[corr == 1] <- NA \n  #turn into a 3-column table\n  corr <- as.data.frame(as.table(corr))\n  #remove the NA values from above \n  corr <- na.omit(corr) \n  #select significant values  \n  corr <- subset(corr, abs(Freq) > sig) \n  #sort by highest correlation\n  corr <- corr[order(-abs(corr$Freq)),] \n  #print table\n  print(corr)\n  #turn corr back into matrix in order to plot with corrplot\n  mtx_corr <- reshape2::acast(corr, Var1~Var2, value.var=\"Freq\")\n  \n  #plot correlations visually\n  corrplot(mtx_corr, is.corr=FALSE, tl.col=\"black\", addCoef.col = 'black', na.label=\" \")\n}","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:11.791784Z","iopub.status.busy":"2022-12-30T11:49:11.790081Z","iopub.status.idle":"2022-12-30T11:49:11.807343Z","shell.execute_reply":"2022-12-30T11:49:11.804973Z"},"papermill":{"duration":0.042588,"end_time":"2022-12-30T11:49:11.810347","exception":false,"start_time":"2022-12-30T11:49:11.767759","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" #### EDA of Delinquency Variables","metadata":{"papermill":{"duration":0.017478,"end_time":"2022-12-30T11:49:11.845355","exception":false,"start_time":"2022-12-30T11:49:11.827877","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# numerical_d <- list()\n# for (n in names(train_whole)){\n#     if (grepl('D_', n)){\n#         numerical_d <- append(numerical_d, n)\n#     }\n# }","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:11.883936Z","iopub.status.busy":"2022-12-30T11:49:11.882222Z","iopub.status.idle":"2022-12-30T11:49:11.896674Z","shell.execute_reply":"2022-12-30T11:49:11.894782Z"},"papermill":{"duration":0.036379,"end_time":"2022-12-30T11:49:11.899158","exception":false,"start_time":"2022-12-30T11:49:11.862779","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_dt <- dplyr::select(train_whole, starts_with('D_'))","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:11.937023Z","iopub.status.busy":"2022-12-30T11:49:11.935213Z","iopub.status.idle":"2022-12-30T11:49:11.967480Z","shell.execute_reply":"2022-12-30T11:49:11.965342Z"},"papermill":{"duration":0.054603,"end_time":"2022-12-30T11:49:11.970411","exception":false,"start_time":"2022-12-30T11:49:11.915808","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_simple(d_dt)","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:12.010201Z","iopub.status.busy":"2022-12-30T11:49:12.008353Z","iopub.status.idle":"2022-12-30T11:49:20.692700Z","shell.execute_reply":"2022-12-30T11:49:20.690456Z"},"papermill":{"duration":8.707848,"end_time":"2022-12-30T11:49:20.695695","exception":false,"start_time":"2022-12-30T11:49:11.987847","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EDA of Spend Variables","metadata":{"papermill":{"duration":0.019006,"end_time":"2022-12-30T11:49:20.734354","exception":false,"start_time":"2022-12-30T11:49:20.715348","status":"completed"},"tags":[]}},{"cell_type":"code","source":"S_dt <- dplyr::select(train_whole, starts_with('S_'))\ncorr_simple(subset(S_dt, select = -c(S_2)))","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:20.774843Z","iopub.status.busy":"2022-12-30T11:49:20.772934Z","iopub.status.idle":"2022-12-30T11:49:22.994536Z","shell.execute_reply":"2022-12-30T11:49:22.991156Z"},"papermill":{"duration":2.245923,"end_time":"2022-12-30T11:49:22.997974","exception":false,"start_time":"2022-12-30T11:49:20.752051","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EDA of Payment Variables","metadata":{"papermill":{"duration":0.019259,"end_time":"2022-12-30T11:49:23.037211","exception":false,"start_time":"2022-12-30T11:49:23.017952","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# There are not any obvious coreletion, so comment out to avoid report error\n# P_dt <- dplyr::select(train_whole, starts_with('P_'))\n# tryCatch(corr_simple(P_dt))","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:23.079742Z","iopub.status.busy":"2022-12-30T11:49:23.077726Z","iopub.status.idle":"2022-12-30T11:49:23.095544Z","shell.execute_reply":"2022-12-30T11:49:23.092708Z"},"papermill":{"duration":0.042882,"end_time":"2022-12-30T11:49:23.099088","exception":false,"start_time":"2022-12-30T11:49:23.056206","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EDA of Balance Variables","metadata":{"papermill":{"duration":0.018962,"end_time":"2022-12-30T11:49:23.136880","exception":false,"start_time":"2022-12-30T11:49:23.117918","status":"completed"},"tags":[]}},{"cell_type":"code","source":"B_dt <- dplyr::select(train_whole, starts_with('B_'))\ncorr_simple(B_dt)","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:23.179728Z","iopub.status.busy":"2022-12-30T11:49:23.177731Z","iopub.status.idle":"2022-12-30T11:49:26.686136Z","shell.execute_reply":"2022-12-30T11:49:26.683999Z"},"papermill":{"duration":3.533127,"end_time":"2022-12-30T11:49:26.689367","exception":false,"start_time":"2022-12-30T11:49:23.156240","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EDA of Risk Variables","metadata":{"papermill":{"duration":0.021642,"end_time":"2022-12-30T11:49:26.732755","exception":false,"start_time":"2022-12-30T11:49:26.711113","status":"completed"},"tags":[]}},{"cell_type":"code","source":"R_dt <- dplyr::select(train_whole, starts_with('R_'))\ncorr_simple(R_dt)","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:26.779762Z","iopub.status.busy":"2022-12-30T11:49:26.777534Z","iopub.status.idle":"2022-12-30T11:49:31.503797Z","shell.execute_reply":"2022-12-30T11:49:31.501639Z"},"papermill":{"duration":4.753942,"end_time":"2022-12-30T11:49:31.507200","exception":false,"start_time":"2022-12-30T11:49:26.753258","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## release unused memory\ngc()","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:31.556325Z","iopub.status.busy":"2022-12-30T11:49:31.554479Z","iopub.status.idle":"2022-12-30T11:49:32.264721Z","shell.execute_reply":"2022-12-30T11:49:32.262812Z"},"papermill":{"duration":0.737453,"end_time":"2022-12-30T11:49:32.267290","exception":false,"start_time":"2022-12-30T11:49:31.529837","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation with target\n","metadata":{"papermill":{"duration":0.023041,"end_time":"2022-12-30T11:49:32.312739","exception":false,"start_time":"2022-12-30T11:49:32.289698","status":"completed"},"tags":[]}},{"cell_type":"code","source":"tmp <- subset(train_whole, select = train_whole[,unlist(lapply(train_whole, is.numeric))])\nnumeric_col = names(tmp)\nrm(tmp)\ngc()","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:32.360739Z","iopub.status.busy":"2022-12-30T11:49:32.358971Z","iopub.status.idle":"2022-12-30T11:49:35.505906Z","shell.execute_reply":"2022-12-30T11:49:35.503893Z"},"papermill":{"duration":3.173767,"end_time":"2022-12-30T11:49:35.508437","exception":false,"start_time":"2022-12-30T11:49:32.334670","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cor_with_target <- list()\nvar_names <- list()\n\nfor (n_f in numeric_col){\n    cor_value <- cor(as.numeric(train_whole$target), train_whole[[n_f]])\n    if (!is.na(cor_value)){\n        cor_with_target <- append(cor_with_target, cor_value)\n        var_names <- append(var_names, n_f)\n    }\n}\n","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:35.558186Z","iopub.status.busy":"2022-12-30T11:49:35.556071Z","iopub.status.idle":"2022-12-30T11:49:47.774711Z","shell.execute_reply":"2022-12-30T11:49:47.772493Z"},"papermill":{"duration":12.246654,"end_time":"2022-12-30T11:49:47.777812","exception":false,"start_time":"2022-12-30T11:49:35.531158","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (idx in order(unlist(cor_with_target), decreasing = TRUE)){\n    cat(paste0(\"Varname: \", var_names[idx], \" correlation with target: \", cor_with_target[idx], '\\n'))\n}","metadata":{"execution":{"iopub.execute_input":"2022-12-30T11:49:47.825856Z","iopub.status.busy":"2022-12-30T11:49:47.824045Z","iopub.status.idle":"2022-12-30T11:49:47.850701Z","shell.execute_reply":"2022-12-30T11:49:47.847920Z"},"papermill":{"duration":0.05478,"end_time":"2022-12-30T11:49:47.854246","exception":false,"start_time":"2022-12-30T11:49:47.799466","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features vs target","metadata":{}},{"cell_type":"markdown","source":"#### Boxplot\nThe cutoffs are **0.4** and **-0.4** because of the scale of correlation coefficient.\n* 0.4 $\\leq$ r $\\leq$ 0.59  moderate correlation\n* 0.6 $\\leq$ r $\\leq$ 0.79  high correlation\n* 0.8 $\\leq$ r $\\leq$ 1  very high correlation","metadata":{}},{"cell_type":"code","source":"# boxplot\npar(mfrow = c(3, 3))\nboxplot(B_9~target, data=train_whole)\nboxplot(D_75~target, data=train_whole)\nboxplot(D_58~target, data=train_whole)\nboxplot(B_7~target, data=train_whole)\nboxplot(B_23~target, data=train_whole)\nboxplot(B_4~target, data=train_whole)\nboxplot(B_18~target, data=train_whole)","metadata":{"execution":{"iopub.execute_input":"2022-12-30T17:50:20.860762Z","iopub.status.busy":"2022-12-30T17:50:20.858426Z","iopub.status.idle":"2022-12-30T17:51:56.067183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Scatter plot\nPick the top two most correlated of each category of features (Based on the absolute value of correlation coefficient).","metadata":{}},{"cell_type":"code","source":"# D_58 vs D_75\nggplot(train_whole, aes(x = D_58, y = D_75)) +\n    geom_point(aes(color = factor(target)), size = 1)\n\n# D_51 vs D_92\nggplot(train_whole, aes(x = D_51, y = D_92)) +\n    geom_point(aes(color = factor(target)), size = 1)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T03:08:31.562443Z","iopub.status.busy":"2023-01-03T03:08:31.558645Z","iopub.status.idle":"2023-01-03T03:17:34.033635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# S_8 vs S_15\nggplot(train_whole, aes(x = S_8, y = S_15)) +\n    geom_point(aes(color = factor(target)), size = 1)\n\n# S_8 vs S_13\nggplot(train_whole, aes(x = S_8, y = S_13)) +\n    geom_point(aes(color = factor(target)), size = 1)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T03:18:36.484807Z","iopub.status.busy":"2023-01-03T03:18:36.480731Z","iopub.status.idle":"2023-01-03T03:26:51.430068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# B_1 vs B_11\nggplot(train_whole, aes(x = B_1, y = B_11)) +\n    geom_point(aes(color = factor(target)), size = 1)\n\n# B_7 vs B_23\nggplot(train_whole, aes(x = B_7, y = B_23)) +\n    geom_point(aes(color = factor(target)), size = 1)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T03:27:55.697543Z","iopub.status.busy":"2023-01-03T03:27:55.695656Z","iopub.status.idle":"2023-01-03T03:36:02.466734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# R_5 vs R_8\nggplot(train_whole, aes(x = R_5, y = R_8)) +\n    geom_point(aes(color = factor(target)), size = 1)\n\n# R_2 vs R_4\nggplot(train_whole, aes(x = R_2, y = R_4)) +\n    geom_point(aes(color = factor(target)), size = 1)","metadata":{"execution":{"iopub.execute_input":"2023-01-03T03:39:11.522637Z","iopub.status.busy":"2023-01-03T03:39:11.520573Z","iopub.status.idle":"2023-01-03T03:47:06.165435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## release unused memory\ngc()","metadata":{"execution":{"iopub.execute_input":"2023-01-03T03:47:06.170224Z","iopub.status.busy":"2023-01-03T03:47:06.168529Z","iopub.status.idle":"2023-01-03T03:47:07.316715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Missing data","metadata":{}},{"cell_type":"code","source":"missing <- data.frame(feature = colnames(train_whole), na_sum = colSums(is.na(train_whole)))\npercentage_of_missing <- round(100*colSums(is.na(train_whole))/nrow(train_whole), 2)\nmissing <- cbind(missing, percentage_of_missing)\nmissing <- missing[ , -1]\n\nmiss1 <- missing[order(missing$percentage_of_missing, decreasing = TRUE), ]\nhead(miss1, n=50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## release memory\nrm(missing)\nrm(percentage_of_missing)\ngc()","metadata":{"execution":{"iopub.execute_input":"2022-12-31T06:50:05.460317Z","iopub.status.busy":"2022-12-31T06:50:05.458639Z","iopub.status.idle":"2022-12-31T06:50:06.775964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parquet Training Data ","metadata":{}},{"cell_type":"code","source":"PARQUET_DAT_DIR <- '../input/amex-data-integer-dtypes-parquet-format'","metadata":{"execution":{"iopub.status.busy":"2023-01-08T02:34:33.897771Z","iopub.execute_input":"2023-01-08T02:34:33.898970Z","iopub.status.idle":"2023-01-08T02:34:33.907470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_par <- \n    arrow::read_parquet(file.path(PARQUET_DAT_DIR, \"train.parquet\"), col_select = 1:190) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T02:36:46.846045Z","iopub.execute_input":"2023-01-08T02:36:46.847240Z","iopub.status.idle":"2023-01-08T02:39:05.295563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#attach training labels to training data\ntrain_label <- as_tibble(train_labels_dt)\ntrain_par <- left_join(train_label,train_par, by=c(\"customer_ID\"))\n\nprint_dt_dim(train_par)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T02:45:42.686455Z","iopub.execute_input":"2023-01-08T02:45:42.687698Z","iopub.status.idle":"2023-01-08T02:45:43.012315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We drop a few columns with more than 50% missing data and remove S_2 (date) and customer_ID.","metadata":{}},{"cell_type":"code","source":"train_par <- train_par %>% select(-D_42, -D_49, -D_50, -D_53, -D_56, \n                                  -D_66, -D_73, -D_76, -D_82, -D_87, \n                                  -D_88, -D_105, -D_106, -D_108, -D_110, \n                                  -D_111, -D_132, -D_134, -D_135, -D_136, \n                                  -D_137, -D_138, -D_142, \n                                  -S_9, \n                                  -B_17, -B_29, -B_39, -B_42, \n                                  -R_9, -R_26)\ntrain_par <- select(train_par, -customer_ID, -S_2)\n\nprint_dt_dim(train_par)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T03:59:01.218475Z","iopub.execute_input":"2023-01-06T03:59:01.219919Z","iopub.status.idle":"2023-01-06T03:59:01.276303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X <- as.matrix(subset(train_par, select = -c(target)))\ntrain_Y <- train_par$target","metadata":{"execution":{"iopub.status.busy":"2023-01-06T04:00:26.478384Z","iopub.execute_input":"2023-01-06T04:00:26.479949Z","iopub.status.idle":"2023-01-06T04:00:27.577404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T05:35:35.395299Z","iopub.execute_input":"2023-01-06T05:35:35.397469Z","iopub.status.idle":"2023-01-06T05:35:36.041510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Recursive feature elimination with cross-validation","metadata":{}},{"cell_type":"code","source":"ctrl <- rfeControl(functions = lmFuncs, method = \"cv\", number = 5)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T05:00:56.459612Z","iopub.execute_input":"2023-01-06T05:00:56.461220Z","iopub.status.idle":"2023-01-06T05:00:56.475521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set.seed(111)\nrfe_result <- rfe(train_X, train_Y,\n                 sizes = c(25, 50, 75, 100),\n                 rfeControl = ctrl)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T05:44:14.214256Z","iopub.execute_input":"2023-01-06T05:44:14.215862Z","iopub.status.idle":"2023-01-06T05:45:03.540259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfe_result","metadata":{"execution":{"iopub.status.busy":"2023-01-06T05:45:18.719712Z","iopub.execute_input":"2023-01-06T05:45:18.721324Z","iopub.status.idle":"2023-01-06T05:45:18.742763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"75個變數和100個的表現差不多，但75 SD比較小，且模型複雜度較低，相對general","metadata":{}},{"cell_type":"code","source":"# The top 75 features\npredictors(rfe_result)[1:75]","metadata":{"execution":{"iopub.status.busy":"2023-01-06T05:52:45.503279Z","iopub.execute_input":"2023-01-06T05:52:45.508757Z","iopub.status.idle":"2023-01-06T05:52:45.542020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"markdown","source":"Boxplots of the top five important features","metadata":{}},{"cell_type":"code","source":"dat_par <- \n    arrow::read_parquet(file.path(PARQUET_DAT_DIR, \"train.parquet\"), col_select = 1:190) %>% \n    mutate(S_2 = lubridate::ymd(S_2))\n\n#attach training labels to training data\ndat_par <- left_join(train_label,dat_par, by=c(\"customer_ID\"))\nprint_dt_dim(dat_par)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T03:30:33.458440Z","iopub.execute_input":"2023-01-08T03:30:33.459746Z","iopub.status.idle":"2023-01-08T03:30:45.290956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_mean <- \n    dat_par %>%\n    group_by(customer_ID) %>% \n    summarise(mean.P_2 = mean(P_2))\n\ntrain_mean <- left_join(train_label, train_mean, by=c(\"customer_ID\"))\n\nprint_dt_dim(train_mean)\nhead(train_mean)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T03:59:38.604332Z","iopub.execute_input":"2023-01-08T03:59:38.605608Z","iopub.status.idle":"2023-01-08T03:59:42.319038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"par(mfrow = c(2, 3))\nboxplot(P_2~target, data=train_par)\nboxplot(mean.P_2~target, data=train_mean)\nboxplot(B_9~target, data=train_par)\nboxplot(D_48~target, data=train_par)\nboxplot(D_44~target, data=train_par)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T04:02:30.846142Z","iopub.execute_input":"2023-01-08T04:02:30.847537Z","iopub.status.idle":"2023-01-08T04:02:33.822219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T04:02:27.144334Z","iopub.execute_input":"2023-01-08T04:02:27.145635Z","iopub.status.idle":"2023-01-08T04:02:27.593757Z"},"trusted":true},"execution_count":null,"outputs":[]}]}