{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Exploration of a subset of the huge training data in R.\n## Table of Contents\n* [Import](#import)\n* [Data Preparation and First Glance](#data_prep)\n* [EDA](#EDA)\n    * [Target](#target)\n    * [Categorial Features](#cat_feat)\n    * [Numerical Features](#num_feat)\n* [Look at specific Customer](#customer)","metadata":{}},{"cell_type":"code","source":"# packages\nlibrary(tidyverse)\nlibrary(lubridate)\nlibrary(corrplot)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-11T09:34:56.111031Z","iopub.execute_input":"2022-08-11T09:34:56.112650Z","iopub.status.idle":"2022-08-11T09:34:56.129797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configure plot size\nplot_w <- 8\nplot_h <- 5\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)\n\n# configure data frame display\noptions(repr.matrix.max.cols=500)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:34:56.990323Z","iopub.execute_input":"2022-08-11T09:34:56.991997Z","iopub.status.idle":"2022-08-11T09:34:57.011875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show files\nlist.files(path = \"../input/amex-default-prediction/\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:34:58.099763Z","iopub.execute_input":"2022-08-11T09:34:58.101334Z","iopub.status.idle":"2022-08-11T09:34:58.121441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='import'></a>\n# Import - Load only subset of huge training data","metadata":{}},{"cell_type":"code","source":"# import subset\nn_subset <- 130000\nt1 <- Sys.time()\ndf_train_sub <- read.csv('../input/amex-default-prediction/train_data.csv', nrows=n_subset)\nt2 <- Sys.time()\nprint(t2-t1)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:29:17.413074Z","iopub.execute_input":"2022-08-11T09:29:17.414989Z","iopub.status.idle":"2022-08-11T09:30:30.748756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load labels as well\ndf_train_labels <- read.csv('../input/amex-default-prediction/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:30.752291Z","iopub.execute_input":"2022-08-11T09:30:30.754001Z","iopub.status.idle":"2022-08-11T09:30:31.959163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='data_prep'></a>\n# Data Preparation and First Glance","metadata":{}},{"cell_type":"code","source":"# preview\nhead(df_train_sub)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:31.961699Z","iopub.execute_input":"2022-08-11T09:30:31.963143Z","iopub.status.idle":"2022-08-11T09:30:32.067984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert ID to factor\ndf_train_sub$customer_ID <- as.factor(df_train_sub$customer_ID)\n\n# convert date\ndf_train_sub$S_2 <- as.Date(df_train_sub$S_2)\nsummary(df_train_sub$S_2)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:32.072128Z","iopub.execute_input":"2022-08-11T09:30:32.073764Z","iopub.status.idle":"2022-08-11T09:30:32.216700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# year/month distribution\ndf_train_sub$year_month <- paste0(lubridate::year(df_train_sub$S_2),'_',\n                                  lubridate::month(df_train_sub$S_2))\n\nplot(as.factor(df_train_sub$year_month),\n     col='blue', las=2,\n     main='Year/Month - Counts'); grid()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:32.219393Z","iopub.execute_input":"2022-08-11T09:30:32.220938Z","iopub.status.idle":"2022-08-11T09:30:39.607051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorical features (see data description)\nfeatures_cat = c('B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68')\n\nfor (f in features_cat) {\n    df_train_sub[,f] <- as.factor(df_train_sub[,f])\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:39.609413Z","iopub.execute_input":"2022-08-11T09:30:39.610825Z","iopub.status.idle":"2022-08-11T09:30:40.397149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# numerical features\nfeatures_num <- setdiff(colnames(df_train_sub),c('customer_ID','S_2','year_month',features_cat))","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:35:09.708857Z","iopub.execute_input":"2022-08-11T09:35:09.710553Z","iopub.status.idle":"2022-08-11T09:35:09.726414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats - categorical features\nsummary(df_train_sub[features_cat])","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:35:12.583974Z","iopub.execute_input":"2022-08-11T09:35:12.585671Z","iopub.status.idle":"2022-08-11T09:35:12.643803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We see a lot of missing values here.","metadata":{}},{"cell_type":"code","source":"# basic stats - numerical features\nsummary(df_train_sub[features_num])","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:35:14.813841Z","iopub.execute_input":"2022-08-11T09:35:14.815453Z","iopub.status.idle":"2022-08-11T09:35:16.305576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Again a lot of missing values.","metadata":{}},{"cell_type":"code","source":"# convert labels data\ndf_train_labels$customer_ID <- as.factor(df_train_labels$customer_ID)\ndf_train_labels$target <- as.factor(df_train_labels$target)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:42.357638Z","iopub.execute_input":"2022-08-11T09:30:42.359118Z","iopub.status.idle":"2022-08-11T09:30:43.654123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='EDA'></a>\n# EDA","metadata":{}},{"cell_type":"markdown","source":"<a id='target'></a>\n### Target:","metadata":{}},{"cell_type":"code","source":"# target\nplot(df_train_labels$target, col='darkorange', main='Target')\ngrid()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:43.656695Z","iopub.execute_input":"2022-08-11T09:30:43.658199Z","iopub.status.idle":"2022-08-11T09:30:43.758332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats\nsummary(df_train_labels$target)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:43.760797Z","iopub.execute_input":"2022-08-11T09:30:43.762233Z","iopub.status.idle":"2022-08-11T09:30:43.791391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Note: Target has already been artificially subsampled here.","metadata":{}},{"cell_type":"markdown","source":"<a id='cat_feat'></a>\n### Categorical features:","metadata":{}},{"cell_type":"code","source":"# plot distributions\nfor (f in features_cat) {\n    perc_miss <- round(100*sum(is.na(df_train_sub[f])) / n_subset, 4)\n    my_title <- paste0(f, ' - missing: ', perc_miss,'%')\n    plot(df_train_sub[f], col='blue', main=my_title)\n    grid()\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:30:43.793796Z","iopub.execute_input":"2022-08-11T09:30:43.795240Z","iopub.status.idle":"2022-08-11T09:30:44.662155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='num_feat'></a>\n### Numerical features:","metadata":{}},{"cell_type":"code","source":"# plot distributions\nfor (f in features_num) {\n    perc_miss <- round(100*sum(is.na(df_train_sub[f])) / n_subset, 4)\n    my_title <- paste0('histogram of ', f, ' - missing: ', perc_miss, '%')\n    df_temp <- df_train_sub[,f]\n    hist(df_temp, col='blue', main=my_title)\n    grid()\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-11T09:34:07.302975Z","iopub.execute_input":"2022-08-11T09:34:07.304801Z","iopub.status.idle":"2022-08-11T09:34:21.455782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation:","metadata":{}},{"cell_type":"markdown","source":"#### Correlation calc is a little bit tricky here due to the lots of missings...","metadata":{}},{"cell_type":"code","source":"features_num_clean <- setdiff(features_num, c('D_87')) # D_87 = 1 or NA => remove\ncorrelation <- cor(df_train_sub[features_num_clean], method='pearson', use = 'pairwise.complete.obs')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot correlation matrix\noptions(repr.plot.width = 18, repr.plot.height = 18)\ncorrplot::corrplot(correlation)\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export for further analyses\nwrite.csv(correlation, file='correlation_for_subset.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='customer'></a>\n# Look at a specific Customer","metadata":{}},{"cell_type":"code","source":"# select customer\ni_customer <- '0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract table for this customer\ndf_example <- df_train_sub[df_train_sub$customer_ID==i_customer,]\ndf_example","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot time series for all numerical features:","metadata":{}},{"cell_type":"code","source":"# plot time series\nempty_features = c() # collect features having only NAs\n\nfor (f in features_num) {\n    y <- df_example[,f]\n    if (sum(is.na(y)) != length(y)) {\n        plot(df_example$S_2, df_example[,f], type='b',\n             xlab='date', ylab=f,\n             main=f)\n        grid()        \n    } else {\n        cat(f, 'has only NAs.\\n')\n        empty_features <- c(empty_features,f)\n    }\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# features having only NAs:\nempty_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# constant features\nsummary(df_example$B_31)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calc correlation\nfeatures_num_clean <- setdiff(features_num, empty_features)\nfeatures_num_clean <- setdiff(features_num_clean, 'B_31') # B_31 is constant for this customer\ncorrelation_ex <- cor(df_example[features_num_clean], method='pearson', use = 'pairwise.complete.obs')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot correlation matrix for this customer\noptions(repr.plot.width = 18, repr.plot.height = 18)\ncorrplot::corrplot(correlation_ex)\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# D_77 shows strange correlations, let's look closer:\nplot(df_example$S_2, df_example$D_77, type='b',\n     xlab='date',\n     main='D_77'); grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### D_77 has only two vales therefore correlations are meaningless here...","metadata":{}},{"cell_type":"code","source":"# export for further analyses\nwrite.csv(correlation_ex, file='correlation_for_example.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check categorical features as well:","metadata":{}},{"cell_type":"code","source":"df_example[c('S_2',features_cat)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Categorical features seem to be fixed over time...","metadata":{}}]}