{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"AMEX data can be stored in .rds files in order to use more features in machine learning models.","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-07-11T21:13:28.486813Z","iopub.execute_input":"2022-07-11T21:13:28.488793Z","iopub.status.idle":"2022-07-11T21:13:28.543867Z"}}},{"cell_type":"code","source":"# Basic Logit Submission\n# Working with feather data, but want to see how far basic tools can get me\n# Still have to be very careful with memory, can't really handle test data in memory (even as feather)\n# Started with: https://www.kaggle.com/code/stautxie/amex-default-prediction-model-in-r-part-1\n\nsuppressPackageStartupMessages(library(data.table)) \nsuppressPackageStartupMessages(library(tidyverse))\nsuppressPackageStartupMessages(library(dtplyr)) #data.table with tidy syntax\nsuppressPackageStartupMessages(library(arrow))\nsuppressPackageStartupMessages(library(readr))\nsuppressPackageStartupMessages(library(dplyr))\ndir(\"..\")\nprint('available files...')\nlist.files(path = \"../input/amex-default-prediction\") %>% print()\n\npqt_dir <- '../input/amex-data-integer-dtypes-parquet-format'\ncsv_dir <- '../input/amex-default-prediction'\ndt_threads <- getDTthreads()\ncat(paste(\"Number of threads for data.table: \", dt_threads, \"\\n\", sep=\"\"))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pull in training labels\nt1 <- Sys.time()\ntrain_Y <- fread(\"../input/amex-default-prediction/train_labels.csv\") %>% as_tibble()\nt2 <- Sys.time()\n\nprint('Time to load training labels..')\ndifftime(t2,t1, units=\"secs\")\nprint('Number of rows')\ntrain_Y %>% nrow()\nprint('Number of IDs')\ntrain_Y %>% distinct(customer_ID) %>% nrow()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:13:32.143996Z","iopub.execute_input":"2022-07-11T21:13:32.145662Z","iopub.status.idle":"2022-07-11T21:13:33.022678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load subset of the efficiently-stored training data, collapse to 1 row customer on import\nt1 <- Sys.time()\n\ntrain_df <- \n    arrow::read_parquet(file.path(pqt_dir, \"train.parquet\"), col_select = 1:190) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()\n\nt2 <- Sys.time()\n\nprint('Time to load training parquet file with arrow and grab latest obs per group')\ndifftime(t2,t1, units=\"secs\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:13:37.476149Z","iopub.execute_input":"2022-07-11T21:13:37.477858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_rds(train_df,\"train_df.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:21:02.532658Z","iopub.execute_input":"2022-07-11T20:21:02.534242Z","iopub.status.idle":"2022-07-11T20:21:04.017722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_rds(\"train_df.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:52:40.215844Z","iopub.execute_input":"2022-07-11T20:52:40.217548Z","iopub.status.idle":"2022-07-11T20:52:43.278075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names_train_df<-names(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:03:03.449981Z","iopub.execute_input":"2022-07-11T21:03:03.451909Z","iopub.status.idle":"2022-07-11T21:03:03.466052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X1 <- \n    arrow::read_parquet(file.path(pqt_dir, \"test.parquet\"), col_select = 1:60) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:13:29.322655Z","iopub.execute_input":"2022-07-11T20:13:29.324411Z","iopub.status.idle":"2022-07-11T20:20:48.976532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_rds(test_X1,\"test_X1.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:21:14.278768Z","iopub.execute_input":"2022-07-11T20:21:14.280415Z","iopub.status.idle":"2022-07-11T20:21:15.449369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_rds(\"test_X1.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:49:09.900219Z","iopub.execute_input":"2022-07-11T20:49:09.902317Z","iopub.status.idle":"2022-07-11T20:49:13.004049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(test_X1)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:21:40.152738Z","iopub.execute_input":"2022-07-11T20:21:40.154474Z","iopub.status.idle":"2022-07-11T20:21:40.169553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:26:59.936339Z","iopub.execute_input":"2022-07-11T20:26:59.937934Z","iopub.status.idle":"2022-07-11T20:27:00.880454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X2 <- \n    arrow::read_parquet(file.path(pqt_dir, \"test.parquet\"), col_select =c('customer_ID','S_2',names(train_df[,61:120]))) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:28:01.636650Z","iopub.execute_input":"2022-07-11T20:28:01.638542Z","iopub.status.idle":"2022-07-11T20:35:22.187878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_rds(test_X2,\"test_X2.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:06:13.235382Z","iopub.execute_input":"2022-07-11T21:06:13.236932Z","iopub.status.idle":"2022-07-11T21:06:15.084011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X2<-read_rds(\"test_X2.rds\")\ntest_X2<-subset(test_X2,select=-c(customer_ID,S_2))\nwrite_rds(test_X2,\"test_X2.rds\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(test_X2)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:38:09.530769Z","iopub.execute_input":"2022-07-11T20:38:09.532503Z","iopub.status.idle":"2022-07-11T20:38:09.547976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X3 <- \n    arrow::read_parquet(file.path(pqt_dir, \"test.parquet\"), col_select =c('customer_ID','S_2',names(train_df[,121:190]))) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:39:18.915963Z","iopub.execute_input":"2022-07-11T20:39:18.917753Z","iopub.status.idle":"2022-07-11T20:46:57.902491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest_X3<-subset(test_X3,select=-c(customer_ID,S_2))\nwrite_rds(test_X3,\"test_X3.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:48:21.685099Z","iopub.execute_input":"2022-07-11T20:48:21.687226Z","iopub.status.idle":"2022-07-11T20:48:23.119309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X1<-read_rds(\"test_X1.rds\")\ntest_X2<-read_rds(\"test_X2.rds\")\ntest_X3<-read_rds(\"test_X3.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:50:36.946468Z","iopub.execute_input":"2022-07-11T20:50:36.948367Z","iopub.status.idle":"2022-07-11T20:50:39.682862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X<-cbind(test_X1,test_X2,test_X3)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T20:51:11.778303Z","iopub.execute_input":"2022-07-11T20:51:11.780314Z","iopub.status.idle":"2022-07-11T20:51:11.798857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:07:47.108693Z","iopub.execute_input":"2022-07-11T21:07:47.110567Z","iopub.status.idle":"2022-07-11T21:07:47.352685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df<-read_rds(\"train_df.rds\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:02:14.528010Z","iopub.execute_input":"2022-07-11T21:02:14.529603Z","iopub.status.idle":"2022-07-11T21:02:16.828748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(train_df)\ngc()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:00:41.686818Z","iopub.execute_input":"2022-07-11T21:00:41.691341Z","iopub.status.idle":"2022-07-11T21:00:43.589605Z"},"trusted":true},"execution_count":null,"outputs":[]}]}