{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The idea is from https://www.kaggle.com/code/igjit1/amex-basic-logit-with-feather-data.\n\nCaret R package's train function is used for classification.","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-07-10T19:19:22.044727Z","iopub.execute_input":"2022-07-10T19:19:22.050814Z","iopub.status.idle":"2022-07-10T19:19:28.908611Z"}}},{"cell_type":"code","source":"suppressPackageStartupMessages(library(data.table)) \nsuppressPackageStartupMessages(library(tidyverse))\nsuppressPackageStartupMessages(library(dtplyr)) #data.table with tidy syntax\nsuppressPackageStartupMessages(library(arrow))\nsuppressPackageStartupMessages(library(caret))\nsuppressPackageStartupMessages(library(corrplot))\nsuppressPackageStartupMessages(library(ROSE))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:43:05.140298Z","iopub.execute_input":"2022-08-14T07:43:05.145291Z","iopub.status.idle":"2022-08-14T07:43:08.449306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_Y <- fread(\"../input/amex-default-prediction/train_labels.csv\") %>% as_tibble()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:43:08.451109Z","iopub.execute_input":"2022-08-14T07:43:08.475781Z","iopub.status.idle":"2022-08-14T07:43:10.155303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df <- \n    arrow::read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\", col_select = 1:190) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:43:10.157882Z","iopub.execute_input":"2022-08-14T07:43:10.159133Z","iopub.status.idle":"2022-08-14T07:45:58.261441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#attach training labels to training data\ntrain_df <- left_join(train_Y,train_df,by=c(\"customer_ID\"))\n\nnrow(train_df)\ncolSums(is.na(train_df))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:45:58.263266Z","iopub.execute_input":"2022-08-14T07:45:58.264357Z","iopub.status.idle":"2022-08-14T07:45:59.080577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df<-train_df%>%select(-S_2,-customer_ID)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:45:59.082319Z","iopub.execute_input":"2022-08-14T07:45:59.083383Z","iopub.status.idle":"2022-08-14T07:45:59.120208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:45:59.121939Z","iopub.execute_input":"2022-08-14T07:45:59.123016Z","iopub.status.idle":"2022-08-14T07:45:59.912991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df<-train_df %>% mutate_all(~ifelse(is.na(.), median(., na.rm = TRUE), .))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:45:59.914717Z","iopub.execute_input":"2022-08-14T07:45:59.915790Z","iopub.status.idle":"2022-08-14T07:46:02.157442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols <- c(\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\")\n\nnum_cols <- setdiff(names(train_df), c(\"target\", cat_cols))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:02.159276Z","iopub.execute_input":"2022-08-14T07:46:02.160361Z","iopub.status.idle":"2022-08-14T07:46:02.170804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Delete correlated variables except the target\ncorrelations_notarget<-cor(subset(train_df,select=num_cols))\n\ntmp <- cor(correlations_notarget)\ntmp[upper.tri(tmp)] <- 0\ndiag(tmp) <- 0\n\n\ntrain_df<-train_df%>%select(!names(which(apply(tmp, 2, function(x) any(abs(x) > 0.98, na.rm = TRUE)))))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:02.172559Z","iopub.execute_input":"2022-08-14T07:46:02.173619Z","iopub.status.idle":"2022-08-14T07:46:15.433048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:15.434829Z","iopub.execute_input":"2022-08-14T07:46:15.435887Z","iopub.status.idle":"2022-08-14T07:46:15.807963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_num<-train_df%>%select(-c(cat_cols))\ncorrelations<-cor(train_num)\ncorrelations_target <- as.matrix(sort(correlations[,'target'], decreasing = TRUE))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:15.809655Z","iopub.execute_input":"2022-08-14T07:46:15.810671Z","iopub.status.idle":"2022-08-14T07:46:25.381912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlations_target","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.383618Z","iopub.execute_input":"2022-08-14T07:46:25.384657Z","iopub.status.idle":"2022-08-14T07:46:25.425440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlated<- names(which(apply(correlations_target, 1, function(x) ((x > 0.1) | (x < -0.1)))))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.427110Z","iopub.execute_input":"2022-08-14T07:46:25.428115Z","iopub.status.idle":"2022-08-14T07:46:25.441203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlated","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.442864Z","iopub.execute_input":"2022-08-14T07:46:25.443877Z","iopub.status.idle":"2022-08-14T07:46:25.455842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df<-train_df[,c(correlated,cat_cols)]","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.457506Z","iopub.execute_input":"2022-08-14T07:46:25.458520Z","iopub.status.idle":"2022-08-14T07:46:25.466961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df$target<-factor(make.names(train_df$target), labels = c(\"non_default\", \"default\"))\ntrain_df$target<-relevel(train_df$target,\"default\")","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.468628Z","iopub.execute_input":"2022-08-14T07:46:25.469639Z","iopub.status.idle":"2022-08-14T07:46:25.871582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"levels(train_df$target)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.873286Z","iopub.execute_input":"2022-08-14T07:46:25.874316Z","iopub.status.idle":"2022-08-14T07:46:25.885389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set.seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.887097Z","iopub.execute_input":"2022-08-14T07:46:25.888189Z","iopub.status.idle":"2022-08-14T07:46:25.896888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:25.898652Z","iopub.execute_input":"2022-08-14T07:46:25.899720Z","iopub.status.idle":"2022-08-14T07:46:26.248180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctrl <- trainControl(method=\"cv\",\n                     number=5,\n                     classProbs = TRUE,\n                     summaryFunction = twoClassSummary)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:26.249818Z","iopub.execute_input":"2022-08-14T07:46:26.250862Z","iopub.status.idle":"2022-08-14T07:46:26.259958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model<-train(target~.,data=train_df,method=\"glm\",trControl=ctrl)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:46:26.261724Z","iopub.execute_input":"2022-08-14T07:46:26.262733Z","iopub.status.idle":"2022-08-14T07:49:12.571504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:49:12.574464Z","iopub.execute_input":"2022-08-14T07:49:12.575778Z","iopub.status.idle":"2022-08-14T07:49:12.708209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:49:12.711340Z","iopub.execute_input":"2022-08-14T07:49:12.713573Z","iopub.status.idle":"2022-08-14T07:49:13.133604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict\npredictions = predict(model, newdata=train_df, type=\"prob\")\nhead(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:49:13.135506Z","iopub.execute_input":"2022-08-14T07:49:13.136625Z","iopub.status.idle":"2022-08-14T07:49:31.128494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remotes::install_github(\"igjit/amexmetric\", quiet = TRUE)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:49:31.130267Z","iopub.execute_input":"2022-08-14T07:49:31.131345Z","iopub.status.idle":"2022-08-14T07:50:12.796186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amexmetric::amex_metric(-1*(as.numeric(train_df$target)-2), (predictions$default))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:12.798341Z","iopub.execute_input":"2022-08-14T07:50:12.799567Z","iopub.status.idle":"2022-08-14T07:50:13.118097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names_train_df<-names(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:13.120021Z","iopub.execute_input":"2022-08-14T07:50:13.121233Z","iopub.status.idle":"2022-08-14T07:50:13.130799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names_train_df","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:13.132734Z","iopub.execute_input":"2022-08-14T07:50:13.133894Z","iopub.status.idle":"2022-08-14T07:50:13.147558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"length(names_train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:13.149491Z","iopub.execute_input":"2022-08-14T07:50:13.150690Z","iopub.status.idle":"2022-08-14T07:50:13.163189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(predictions,correlations,correlations_target,correlations_notarget)\ngc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:13.165122Z","iopub.execute_input":"2022-08-14T07:50:13.166300Z","iopub.status.idle":"2022-08-14T07:50:13.742199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(correlated,train_num,train_Y)\ngc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:13.743853Z","iopub.execute_input":"2022-08-14T07:50:13.744856Z","iopub.status.idle":"2022-08-14T07:50:14.184210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols<-c(\"customer_ID\",\"S_2\",names_train_df[!names_train_df %in% c('target')])","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:14.185938Z","iopub.execute_input":"2022-08-14T07:50:14.187011Z","iopub.status.idle":"2022-08-14T07:50:14.195657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:14.197375Z","iopub.execute_input":"2022-08-14T07:50:14.198414Z","iopub.status.idle":"2022-08-14T07:50:14.210217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"length(cols)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:14.211862Z","iopub.execute_input":"2022-08-14T07:50:14.212879Z","iopub.status.idle":"2022-08-14T07:50:14.223502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X1 <- \n    arrow::read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\", col_select = cols[1:51]) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:50:14.225204Z","iopub.execute_input":"2022-08-14T07:50:14.226218Z","iopub.status.idle":"2022-08-14T07:55:45.596778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_X1<-subset(test_X1,select=-c(customer_ID,S_2))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:55:45.599014Z","iopub.execute_input":"2022-08-14T07:55:45.600211Z","iopub.status.idle":"2022-08-14T07:55:45.609007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arrow::write_parquet(test_X1, \"test_X1.parquet\")\nrm(test_X1)\ngc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:55:45.610966Z","iopub.execute_input":"2022-08-14T07:55:45.612167Z","iopub.status.idle":"2022-08-14T07:55:48.372783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X2 <- \n    arrow::read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\", col_select = cols[c(1:2,52:103)]) %>% \n    mutate(S_2 = lubridate::ymd(S_2)) %>%\n    group_by(customer_ID) %>% \n    slice_max(S_2) %>% \n    ungroup()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T07:55:48.374629Z","iopub.execute_input":"2022-08-14T07:55:48.375709Z","iopub.status.idle":"2022-08-14T08:01:18.927181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X2<-subset(test_X2,select=-c(customer_ID,S_2))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:01:18.929080Z","iopub.execute_input":"2022-08-14T08:01:18.930222Z","iopub.status.idle":"2022-08-14T08:01:19.116541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arrow::write_parquet(test_X2, \"test_X2.parquet\")\nrm(test_X2)\ngc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:01:19.118471Z","iopub.execute_input":"2022-08-14T08:01:19.119620Z","iopub.status.idle":"2022-08-14T08:01:21.880728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X1<-read_parquet(\"test_X1.parquet\")\ntest_X2<-read_parquet(\"test_X2.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:01:21.882603Z","iopub.execute_input":"2022-08-14T08:01:21.883676Z","iopub.status.idle":"2022-08-14T08:01:22.392367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X<-cbind(test_X1,test_X2)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:01:22.394187Z","iopub.execute_input":"2022-08-14T08:01:22.395279Z","iopub.status.idle":"2022-08-14T08:01:22.405553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X<-test_X %>% mutate_all(~ifelse(is.na(.), median(., na.rm = TRUE), .))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:01:22.407353Z","iopub.execute_input":"2022-08-14T08:01:22.408435Z","iopub.status.idle":"2022-08-14T08:01:52.944170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make predictions on test data\n\n#predict\npredictions = predict(model, newdata=test_X, type=\"prob\") \nhead(predictions)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:01:52.946589Z","iopub.execute_input":"2022-08-14T08:01:52.947749Z","iopub.status.idle":"2022-08-14T08:02:34.720187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:02:34.722409Z","iopub.execute_input":"2022-08-14T08:02:34.723547Z","iopub.status.idle":"2022-08-14T08:02:35.697274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bind test_IDs and predictions\ntest_df <- test_X %>% mutate(prediction=predictions$default)%>% select(customer_ID,prediction)\nnrow(test_df)\nhead(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:02:35.699589Z","iopub.execute_input":"2022-08-14T08:02:35.700809Z","iopub.status.idle":"2022-08-14T08:02:35.733210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#write to submission.csv \nfwrite(x=test_df, file=\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:02:35.735058Z","iopub.execute_input":"2022-08-14T08:02:35.736668Z","iopub.status.idle":"2022-08-14T08:02:35.923283Z"},"trusted":true},"execution_count":null,"outputs":[]}]}