{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nlibrary(gbm)\nlibrary(lightgbm)\nlibrary(lubridate)\nlibrary(randomForest)\nlibrary(caret)\nlibrary(dplyr)\nlibrary(data.table)\n# Any results you write to the current directory are saved as output.\ntrain <-fread('../input/train_sample.csv', stringsAsFactors = FALSE, data.table = FALSE)\ntest <-fread('../input/test.csv', stringsAsFactors = FALSE, data.table = FALSE)\ntrainfull<-fread('../input/train.csv', stringsAsFactors = FALSE, data.table = FALSE)","execution_count":39,"outputs":[]},{"metadata":{"_uuid":"8a7475699864e396d0f348a723520c475e551f5f","_cell_guid":"b577e031-c698-491c-af92-e9b1c80db9c5"},"cell_type":"markdown","source":"So first we will try to understand the table from the first look of it. We have the ip address, the app no, device, OS, Channel, Click_time, attribted_time and is_attributed or not"},{"metadata":{"_uuid":"6e7064e65d1e9da5f5c3a61440fa70b3dbb5815b","trusted":true,"_cell_guid":"9779228b-9d6a-4e0d-8357-f4c46e405394"},"cell_type":"code","source":"glimpse(trainfull)","execution_count":14,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0"},"cell_type":"code","source":"glimpse(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0127ac4effc9c386cf723986b9fc869b70da161","trusted":false,"_cell_guid":"4b5ea117-8d00-4226-a765-92c4668dfa57"},"cell_type":"code","source":"glimpse(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f4c54e3e4d7234c0c57d02981e00918a72d5ed0","_cell_guid":"cd5566f1-341d-47c2-9288-d6098d12d8c7"},"cell_type":"markdown","source":"Now lets check for any missing values"},{"metadata":{"_uuid":"2f3e229f029d5e60db387ecb2b18036ca5cf3f7f","trusted":false,"_cell_guid":"e39c6351-7e03-46d9-92ed-6c2045db000e"},"cell_type":"code","source":"colSums(is.na(train))\ncolSums(is.na(test))\ncolSums(train==\"\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66c2660cc3d972efc7ee2fe1bca9c66aea242d68","_cell_guid":"b315f93d-955b-4ec1-9861-2b09b2702a87"},"cell_type":"markdown","source":"Now Lets do a Variable wise analyis and lets start with the target Variable "},{"metadata":{"_uuid":"e2c9a3a30260fef7dada8a7c19d01aa116230efa","trusted":false,"_cell_guid":"d6bef19b-a4e5-477e-b343-9950c7579bc0"},"cell_type":"code","source":"table(train$is_attributed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"656189a060a5f4cf8761243c5267517aac1f5fad","_cell_guid":"7f4d3eae-39d5-4ac7-8f2a-a9cf5a88baff"},"cell_type":"markdown","source":"The data seems to be skewed. Do we need to upsample or downsample lets see?one way is to redo the sampling from the Original dataset and we will keep the proportion as 50 : 50 as inherently there seems to be a skew "},{"metadata":{"_uuid":"16b2b72f110640b4399da3ba34b906934a1da414","trusted":true,"_cell_guid":"119cb0c2-d030-4d20-a074-f60f696748da"},"cell_type":"code","source":"#table(trainfull$is_attributed)\ntrainfull_1<-trainfull[trainfull$is_attributed==1,]\nsample<- trainfull_1[sample(nrow(trainfull_1), 100000), ]\nNewtrain<-rbind(train,sample)\n","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"aefc42e12b7d1eeb77c90c48c13da9354519e1d6","trusted":true,"_cell_guid":"db53a6a3-831e-4cbd-95a0-aecb37700ae5"},"cell_type":"code","source":"Newtrain$is_attributed<-as.factor(Newtrain$is_attributed)\ndown_train<-downSample(x=Newtrain[,-ncol(Newtrain)], y=Newtrain$is_attributed)\ntable(Newtrain$is_attributed)\ntable(down_train$Class)\ncolSums(is.na(down_train))","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"d0c44bc34a810eacd21a71034a3d1264b04a4791","_cell_guid":"f9f8599f-06c8-4176-9c2f-17a95035ec90"},"cell_type":"markdown","source":"Now lets split the trainset and testset"},{"metadata":{"_uuid":"fd568cfdb25180ba4c162a94a5edbaf1c91f4611","trusted":true,"_cell_guid":"ae1bebc1-0119-4053-a771-edf11cdcc73e"},"cell_type":"code","source":"train_rows<-createDataPartition(y = down_train$Class, \n                                 p = 0.94, list = F)\ntrain_set = down_train[ train_rows,]\ntest_set  = down_train[!train_rows,]\ntable(train_set$Class)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d89bb61fe06907d31dad92539b2ca2198baee085"},"cell_type":"code","source":"rm(Newtrain)\nrm(down_train)\nrm(trainfull_1)\nrm(trainfull)","execution_count":5,"outputs":[]},{"metadata":{"_uuid":"8f5fdbcd131c73529daaf09a67c9b686a0b1be29","_cell_guid":"7fbc366b-336f-4b5c-ab71-0bf9fed11905"},"cell_type":"markdown","source":"Lets start with the data visualization"},{"metadata":{"_uuid":"ae1bf8222bcd5d96f01fba62d654a494153ccf48","_cell_guid":"ce86cfe1-7b7a-481c-b8b1-b6895dd74bad"},"cell_type":"markdown","source":"Variable 1: app id "},{"metadata":{"_uuid":"74ae7ea8ba3b381b198ed454cf93d0fe2faf3e85","trusted":false,"_cell_guid":"f06cfca2-e0a1-416e-8061-f65672acf9b4"},"cell_type":"code","source":"train_set%>%mutate(app=as.factor(app))%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(app=factor(app,levels=app))%>%head(20)%>%ggplot(aes(x=app, y=count ))+geom_bar(stat=\"identity\")+labs(y=\"count\",x=\"App id\",title=\"Top 20 apps in train set\")+theme_bw()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ee15b0b00b650b70de2d37cac70d0946284ccc3","trusted":false,"_cell_guid":"1016ca7b-72bb-425c-bb40-3b83443f545f"},"cell_type":"code","source":"top20<- train_set%>%mutate(app=as.factor(app))%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(app=factor(app,levels=app))%>%head(20)\ntrain_set%>%mutate(app=as.factor(app))%>%group_by(app,Class)%>%summarize(count=n())%>%filter(app%in%top20$app)%>%ungroup()%>%mutate(app=factor(app, levels=top20$app))%>%ggplot(aes(x=app, y=count, fill=Class))+geom_bar(stat=\"identity\")+labs(y=\"count\",x=\"App id\",title=\"Top 20 apps in Train set\")+theme_bw()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4ad3eb7eabc3754281b1b7d4463145367727b8a","trusted":false,"_cell_guid":"7ea061be-ce03-478e-9251-41d8328f5af6"},"cell_type":"code","source":"top20<- train_set%>%mutate(app=as.factor(app))%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(app=factor(app,levels=app))%>%head(20)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e0604eea7009c64a65917c402d5fc6860eb35a5","_cell_guid":"7dd666fb-54f1-4d51-ab8b-f6428c6fc54f"},"cell_type":"markdown","source":"From the Vizuaulization we can see the top 20 apps and their Class.  You can get definitely a lots of information from the graph.\nFor exaple the top app 19 has higher probablity of getting downloaded after the click and app 15 has less chances of getting downloaded after the click.\nOverall the app id is a good predictor if the app will get downloaded or not hence we must retain the variable\n"},{"metadata":{"_uuid":"48eef29ecab1c2e901075ed0e16778ee3d3ffce4","_cell_guid":"d7dcfd56-b7f8-4370-8d40-c27ac07746a8"},"cell_type":"markdown","source":"Now lets Analyze the app id from test set "},{"metadata":{"_uuid":"585309b2b2587eee075ef1f7e4fa823fa8c4d3ef","trusted":false,"_cell_guid":"a63f05c8-573a-477c-a5a8-5968578ce84e"},"cell_type":"code","source":"test%>%mutate(app=as.factor(app))%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(app=factor(app,levels=app))%>%head(20)%>%ggplot(aes(x=app, y=count ))+geom_bar(stat=\"identity\")+labs(y=\"count\",x=\"App id\",title=\"top apps in test set\")+theme_bw()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8892be8211f534f8c3e2132fac65e8567c7b773c","_cell_guid":"d9ebb03a-caa1-4bbf-8e4d-85d4448bf68d"},"cell_type":"markdown","source":"Now lets analyze the device id and its importance"},{"metadata":{"_uuid":"840552ffff5e93629f233246d9cf58bec060f68e","trusted":false,"_cell_guid":"40063cdf-a849-4ce7-a187-2516d398a3cc"},"cell_type":"code","source":"top_20_devices<-train_set%>%mutate(device=as.factor(device))%>%group_by(device)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>% mutate(device=factor(device, levels=device))%>%head(20)\ntrain_set%>%mutate(device=as.factor(device))%>%group_by(device, Class)%>%summarize(count=n())%>%filter(device%in%top_20_devices$device)%>%ungroup()%>% mutate(device=factor(device, levels=top_20_devices$device))%>%ggplot(aes(x=device, y=count, fill=Class))+geom_bar(stat=\"identity\")+theme_bw()+labs(x=\"Device\", y=\"Count\", title=\"Top 20 devices in train set \")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"554d01df60442ab0a51004ad5edd75ad43e3b3c5","_cell_guid":"cbc18f77-9fdc-46f3-a195-7b78ed6610fa"},"cell_type":"markdown","source":"Now lets see the top devices in the test set"},{"metadata":{"_uuid":"0d1eb9424df4346c18d2fa890982009f606611c5","trusted":false,"_cell_guid":"ad818784-0d77-48f7-92e6-9510368099d4"},"cell_type":"code","source":"test%>%mutate(device=as.factor(device))%>%group_by(device)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(device=factor(device,levels=device))%>%head(20)%>%ggplot(aes(x=device, y=count ))+geom_bar(stat=\"identity\")+labs(y=\"count\",x=\"Device id\",title=\"top devices in test set\")+theme_bw()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57198c7e66c9c2fd963de93e6459c2fbff6a5510","_cell_guid":"a11f6280-e80f-4e26-9db1-4e406463e4ac"},"cell_type":"markdown","source":"Now lets Analyze the next Variable OS"},{"metadata":{"_uuid":"3d7321c3376b5a468a63487139b2675b04bc1a76","trusted":false,"_cell_guid":"e910a650-bd06-40ba-b4d9-2970b25173d0"},"cell_type":"code","source":"top_20_OS<- train_set%>%mutate(os=as.factor(os))%>%group_by(os)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(os=factor(os, levels=os))%>%head(20)\ntrain_set%>%mutate(os=as.factor(os))%>%group_by(os,Class)%>%summarize(count=n())%>%filter(os%in%top_20_OS$os) %>%ungroup()%>%mutate(os=factor(os, levels=top_20_OS$os))%>%ggplot(aes(x=os, y=count, fill=Class))+geom_bar(stat=\"identity\")+theme_bw()+labs(x=\"OS\", y=\"Count\", title=\"top 20 OS in train_set\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fee1c53b92046745c2ef38afe42a07b8a6396468","_cell_guid":"4ce40c99-f35f-43fb-a0d0-ac5d58f5d3b5"},"cell_type":"markdown","source":"OS in the test set"},{"metadata":{"_uuid":"0f8f740b440cdb51afa4aff4ea53d0509a104da9","trusted":false,"_cell_guid":"94c505b1-631e-4b5b-a027-01f746e8a4fb"},"cell_type":"code","source":"test%>%mutate(os=as.factor(os))%>%group_by(os)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(os=factor(os,levels=os))%>%head(20)%>%ggplot(aes(x=os, y=count ))+geom_bar(stat=\"identity\")+labs(y=\"count\",x=\"OS type\",title=\"top OS in test set\")+theme_bw()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65ccfe9c2e8d5987020c1a335e6c58ce11ce086a","trusted":false,"_cell_guid":"7046c760-dc3b-46c5-99ca-cb9bdb7cf0b2"},"cell_type":"code","source":"names(train_set)\ntop_20_Channel<- train_set%>%mutate(channel=as.factor(channel))%>%group_by(channel)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(cahnnel=factor(channel, levels=channel))%>%head(20)\ntrain_set%>%mutate(channel=as.factor(channel))%>%group_by(channel,Class)%>%summarize(count=n())%>%filter(channel%in%top_20_Channel$channel)%>%ungroup()%>%mutate(channel=factor(channel, levels=top_20_Channel$channel))%>%ggplot(aes(x=channel, y=count, fill=Class))+geom_bar(stat=\"identity\")+theme_bw()+labs(x=\"Channel\", y=\"Count\", title=\"top 20 Channel in train_set\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d41b4831f16282c3bc1c0ff150076dd691821fa5","_cell_guid":"206a59de-ff23-4b5d-be2f-37abf5578a9d"},"cell_type":"markdown","source":"Channle seems to be pretty good predictor"},{"metadata":{"_uuid":"a4c1bacf91b321d78b199f335e7e77daf82d696a","trusted":false,"_cell_guid":"ee35d3bf-973f-413f-baf1-68dcee7b3eea"},"cell_type":"code","source":"test%>%mutate(channel=as.factor(channel))%>%group_by(channel)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup%>%mutate(channel=factor(channel,levels=channel))%>%head(20)%>%ggplot(aes(x=channel, y=count ))+geom_bar(stat=\"identity\")+labs(y=\"count\",x=\"Channel type\",title=\"top Channel in test set\")+theme_bw()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"81055b5e7c16d1330c10c2631ee98bcc0dd84cdf","_cell_guid":"a86777c3-30a8-48a9-9cc7-6f8af1692531"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"0a4d0c8cf623f57e35e9132fd322b39d29fbf1ed","trusted":true,"_cell_guid":"1ecace56-1b5a-4120-8899-a31d9ba9901b"},"cell_type":"code","source":"train_set<-train_set%>%mutate(year= year(ymd_hms(click_time)), month=month(ymd_hms(click_time)), day=day(ymd_hms(click_time)),hour_of_day=hour(ymd_hms(click_time)), week=week(ymd_hms(click_time)) )\nprint(table(train_set$Class))","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"7777c64e3c41785d0513b3c3132975d3e14ab83a","trusted":false,"_cell_guid":"3ca743dd-857a-402d-a6e5-1efd69fd764d"},"cell_type":"code","source":"train_set%>%mutate(hour_of_day=as.factor(hour_of_day))%>%group_by(hour_of_day, Class)%>%summarize(count=n())%>%ggplot(aes(x=hour_of_day, y=count, fill=Class))+geom_bar(stat=\"identity\")+theme_bw()+labs(x=\"Hour\", y=\"Count\", title=\"Hours of the Day \")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d6cefecda239d6d402ca3601c6e4acc21823eda","trusted":false,"_cell_guid":"71c0573e-915f-48f0-82ee-48d3bde2ce3b"},"cell_type":"code","source":"library(reshape2)\ndta<- train_set%>%mutate(hour_of_day=as.factor(hour_of_day))%>%group_by(hour_of_day, Class)%>%summarize(count=n())\ndataa<-dcast(dta, hour_of_day ~ Class, mean,value = 'count')\ndataa$diff<-dataa$'1'-dataa$'0'\nggplot(dataa, aes(x=hour_of_day, y=diff, fill=diff))+geom_bar(stat=\"identity\")+labs(Y=\"Difference in download\" , X=\"Hours of the Day\", titile=\"Difference in downloads\" )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a6382e68138490a666b132fbf5b58006c61dd2f","_cell_guid":"b82e30f4-799b-48b3-9a64-8c54f466cc8c"},"cell_type":"raw","source":"Hours of the day has good predictive power and there seems to be a pattern"},{"metadata":{"_uuid":"7f1b44c3bfe459cc0daa24dcd8ca2a638d16f3be","_cell_guid":"0a12f304-8de1-4149-9506-ce38a3339d1b"},"cell_type":"raw","source":"Channel, Hours of the day and "},{"metadata":{"_uuid":"4a44c5ccc0d1d8e58a02498e4be870b3b0f1c8f2","trusted":false,"_cell_guid":"952d7f55-cc86-4c5b-888d-ef8013739156"},"cell_type":"code","source":"\nprint(table(train_set$Class))\n#train_set<-train_set%>%mutate(hour=ifelse(hour_of_day>12,\"M\",\"E\"))\nprint(table(train_set$Class))\n#train_set<-train_set%>%mutate(app=as.factor(app),device=as.factor(device),os=as.factor(os),channel=as.factor(channel))\nprint(table(train_set$Class))\n#train_set<-train_set%>%mutate(app=as.numeric(app),device=as.numeric(device),os=as.numeric(os),channel=as.numeric(channel))\nprint(table(train_set$Class))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7c9f6438f7d95e72c1570124cf83e34d2c3e632","trusted":false,"_cell_guid":"08b14421-9fff-4e7e-bb2a-3176da4e9d66"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f761f703b2b74a60100f0e9b024a960662d18cf2","_cell_guid":"f7f81107-ba04-4c7f-9790-427325aeaa83"},"cell_type":"markdown","source":"Random Forest Model"},{"metadata":{"_uuid":"0f1719212a9f977f6b88e411b9b25c5a8962648c","trusted":false,"_cell_guid":"279d7190-8c20-4316-9028-11995d3a1453"},"cell_type":"code","source":"#train_set$app<-as.factor(train_set$app)\n#train_set$device<-as.factor(train_set$device)\n#train_set$Class<-as.factor(train_set$Class)\n#train_set$os<-as.factor(train_set$os)\n#train_set$channel<-as.factor(train_set$channel)\n#train_set$hour_of_day<-as.factor(train_set$hour_of_day)\n#top_predictors<-c('app','channel','os', 'device','hour_of_day', 'Class')\n#train_set<-train_set[,top_predictors]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d4114c507cfc2a40aeadb23471f09666ae0b536"},"cell_type":"code","source":"trainset%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b75728116bbc946cc49803eaecd572dde24a00ff"},"cell_type":"code","source":"top_predictors<-c('app','channel','os', 'device','hour_of_day', 'Class')\ntrain_set<-train_set[,top_predictors]","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"e6981a3bf7c8453690b7b462c601f11f3aab76ad","trusted":true,"_cell_guid":"e3fdf72c-dc3f-4037-8ca4-243a069703f3"},"cell_type":"code","source":"#best.iter = gbm.perf(gbm.model, method = \"test\")","execution_count":34,"outputs":[]},{"metadata":{"_uuid":"7690db1b4981e1e795364c0ad4bb70b243d65a42","trusted":false,"_cell_guid":"2c511f61-17de-415f-ad51-c2f3318626fc"},"cell_type":"code","source":"#fitControl <- trainControl(method=\"none\",classProbs = TRUE)\n\n#xgbGrid <- expand.grid(nrounds = 100,\n#                       max_depth = 3,\n#                       eta = .05,\n#                       gamma = 0,\n#                       colsample_bytree = .8,\n#                       min_child_weight = 1,\n#                       subsample = 1)\n\n#set.seed(13)\n\n#ProjectXGB = train(Class~., data = dtrain,\n#                        method = \"xgbTree\",trControl = fitControl,\n#                        tuneGrid = xgbGrid,na.action = na.pass)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef64d8ca886cd84762193678c9c14389c8ce8748","trusted":true,"_cell_guid":"d81415ba-c5f3-4708-82fc-61bf94265ce6"},"cell_type":"code","source":"top_50_OS<- train_set%>%mutate(os=as.factor(os))%>%group_by(os)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(os=factor(os, levels=os))%>%head(50)\nprint(top_50_OS)\ntrain_set<-train_set%>%mutate(OSsub= ifelse(os%in%top_50_OS$os,os,\"other\") )\nunique(train_set$OSsub)","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6597a35620b616fe3723e03a469c6a5e4b1b824a"},"cell_type":"code","source":"top_50_app<- train_set%>%mutate(app=as.factor(app))%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(app=factor(app, levels=app))%>%head(50)\nprint(top_50_OS)\ntrain_set<-train_set%>%mutate(appsub= ifelse(app%in%top_50_app$app,app,\"other\") )\nunique(train_set$appsub)","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7c37c2a209f87a504351fa421eb14fe861f1546"},"cell_type":"code","source":"top_50_devicep<- train_set%>%mutate(device=as.factor(device))%>%group_by(device)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(device=factor(device, levels=device))%>%head(50)\nprint(top_50_OS)\ntrain_set<-train_set%>%mutate(devsub= ifelse(device%in%top_50_devicep$device,device,\"other\") )\nunique(train_set$devsub)","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49f633ff74435402d390a8cf1a4444459f330a27"},"cell_type":"code","source":"top_50_channel<- train_set%>%mutate(channel=as.factor(channel))%>%group_by(channel)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(channel=factor(channel, levels=channel))%>%head(50)\nprint(top_50_channel)\ntrain_set<-train_set%>%mutate(Channelsub= ifelse(channel%in%top_50_channel$channel,channel,\"other\") )\nunique(train_set$Channelsub)","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7460d83003b3013b86ee82bc7e2b9bb1e1d3b9a7"},"cell_type":"code","source":"top_50_app<- train_set%>%mutate(app=as.factor(app))%>%group_by(app)%>%summarize(count=n())%>%arrange(desc(count))%>%ungroup()%>%mutate(app=factor(app, levels=app))%>%head(50)\nprint(top_50_OS)\ntrain_set<-train_set%>%mutate(appsub= ifelse(app%in%top_50_app$app,app,\"other\") )\nunique(train_set$appsub)","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2d05f2170255c37802148d27f422de000fc1c24"},"cell_type":"code","source":"top_predictors<-c('appsub','Channelsub','devsub', 'OSsub','Class')\ntrain_set<-train_set[,top_predictors]\n\ntrain_set$appsub<-as.factor(train_set$appsub)\ntrain_set$Channelsub<-as.factor(train_set$Channelsub)\ntrain_set$devsub<-as.factor(train_set$devsub)\ntrain_set$OSsub<-as.factor(train_set$OSsub)\ntrain_set$Class<-as.factor(train_set$Class)\n","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39745de26172506e491a87ef951ffb20b9b4620f"},"cell_type":"code","source":"model_rand<-randomForest(Class~., data=train_set)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e124dde741be68e64a4d657ba0b5f8ca00d1b97b"},"cell_type":"code","source":"print(model_rand)","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4720d007b2f893ad5e0a95957f89a6aaae667ca1"},"cell_type":"code","source":"test<-test%>%mutate(Channelsub= ifelse(channel%in%top_50_channel$channel,channel,\"other\") )\nunique(train_set$Channelsub)\ntest<-test%>%mutate(devsub= ifelse(device%in%top_50_devicep$device,device,\"other\") )\ntest<-test%>%mutate(appsub= ifelse(app%in%top_50_app$app,app,\"other\") )\ntest<-test%>%mutate(OSsub= ifelse(os%in%top_50_OS$os,os,\"other\") )\n\n\n","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b66fe2e73414bc948c04ebd016cbd77121192d2"},"cell_type":"code","source":"test_predictors<-c('appsub','Channelsub','devsub', 'OSsub')\ntest<-test[,test_predictors]\ntest$appsub<-as.factor(test$appsub)\ntest$Channelsub<-as.factor(test$Channelsub)\ntest$devsub<-as.factor(test$devsub)\ntest$OSsub<-as.factor(test$OSsub)\n#test$Class<-as.factor(test$Class)","execution_count":19,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4b79591bb8d9284528ebbe9ea07b5670192f7d0"},"cell_type":"code","source":"levels(test$devsub)<-levels(train_set$devsub)\nlevels(test$OSsub)<-levels(train_set$OSsub)\ncolSums(is.na(test))\nstr(test)","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f91f7afa67863a2f43e50f06765a59cf75b3569f"},"cell_type":"code","source":"preds = predict(model_rand,test)\npreds = as.data.frame(preds)","execution_count":34,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"b713ca47ee0ea57fbaabf58d2ce66405d7a193f4"},"cell_type":"code","source":"\ntest1 <-fread('../input/test.csv', stringsAsFactors = FALSE, data.table = FALSE)\nsub <- data.table(click_id = test1$click_id, is_attributed = NA) \nsub$is_attributed = preds\nfwrite(sub, \"newmenji.csv\")","execution_count":35,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"646fe1428da5b19c88897480859091252c3d8487"},"cell_type":"code","source":"table(preds$preds)\nvarImpPlot(model_rand,type=2)","execution_count":38,"outputs":[]},{"metadata":{"_uuid":"9f8f15d3a170b83fc63c20ead559cdf3e64ba33a","trusted":false,"_cell_guid":"adc98b67-7acb-4de8-9ab8-0ffb80154bc1"},"cell_type":"code","source":"#model<-gbm(formula =Class~.,distribution=\"bernoulli\",data=train_set,n.trees = 1000,\n #            interaction.depth = 5,\n  #           shrinkage = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d8c97366992dadc52df959f13be7fe4bed39c2c","trusted":false,"_cell_guid":"08836572-23fd-4936-8d75-89942f5ab6ce"},"cell_type":"code","source":"#print(varImp(model,numTrees = 720))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d360641261bbf5c6a5e20a9fa165e6fd0785a35a","trusted":true,"_cell_guid":"e899a140-ed9c-4be8-982e-343ed44871cd"},"cell_type":"code","source":"\n#train_rows<-createDataPartition(y = train_set$Class, \n#                                 p = 0.94, list = F)\n\n#dtrain = train_set[ train_rows,]\n#valid  = train_set[-train_rows,]\n#rm(trainfull)\n#cat(\"train size : \", dim(dtrain), \" | valid size : \", dim(valid), \" | test  size : \", dim(test))\n#names(dtrain)","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"d1879b64fc87e1646cc2646cbcd1a6cb66524150","trusted":false,"_cell_guid":"950304b5-916d-4fde-9f07-de7e310d1099"},"cell_type":"code","source":"#tr_index <- nrow(train)\n#dtrain <- train_set %>% head(0.95 * tr_index) # 95% data for training\n#valid <- train_set %>% tail(0.05 * tr_index) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"96e952d5cf923152f2d2471f24848a8603899541","trusted":false,"_cell_guid":"162ef0c1-66e2-421d-904d-3cb0318e0192"},"cell_type":"code","source":"#train_set<-lapply( train_set, function(x) as.numeric(x))\n#print(table(dtrain$Class))\n#print(table(valid$Class))\n                  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc4da2d0c1a6ce3d50644ecde9b369606199c7ff","trusted":false,"_cell_guid":"8c1d7184-07d7-4b1a-9fd4-85daea220a47"},"cell_type":"code","source":"#head(train_set)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2d01a93239d129d7891e9072c034f093918bf3","trusted":false,"_cell_guid":"be5c2354-09ab-442e-8791-5fbee01293a3"},"cell_type":"code","source":"#top_predictors<-c('app','channel','os', 'device','hour_of_day', 'Class')\n#dtrain<-dtrain[,top_predictors]\n#valid<-valid[,top_predictors]\n#dtrain$Class<-as.integer(dtrain$Class)\n#valid$Class<-as.integer(valid$Class)\n#library(lightgbm)\n#categorical_features = c('app','channel','os', 'device','hour_of_day')\n\n#dtrain = lgb.Dataset(data = as.matrix(dtrain[, colnames(dtrain) != \"Class\"]), \n                     label = dtrain$Class, categorical_feature = categorical_features)\n#valid = lgb.Dataset(data = as.matrix(valid[, colnames(valid) != \"Class\"]), \n                     label = valid$Class, categorical_feature = categorical_features)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0635c442a7307e43bba55e5883219ba07992565e","trusted":false,"_cell_guid":"56508e44-1896-469b-ae23-5e6eb1a1565c"},"cell_type":"code","source":"#glimpse(valid)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e7b64999e8cd1974d90b0bc475465cc4719df17","trusted":false,"_cell_guid":"e59d7d79-ae6c-4c43-a2c5-c96f0ec89788"},"cell_type":"code","source":"params = list(objective = \"binary\", \n              metric = \"auc\", \n              learning_rate= 0.1, \n              num_leaves= 1000,\n              max_depth= 3,\n              min_child_samples= 50000,\n              #max_bin= 100, # RAM dependent as per LightGBM documentation\n              subsample= 0.7,\n              subsample_freq= 1,\n              colsample_bytree= 0.7,\n              min_child_weight= 0,\n              min_split_gain= 0,\n              scale_pos_weight=99) ","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"4fd690400623e202271df6159f5bfa124b06480a","trusted":false,"_cell_guid":"3d140f56-d4d6-4610-be7f-e2df997a99f1"},"cell_type":"code","source":"names( dtrain[, colnames(dtrain) != \"Class\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"473ea6029f62ff7982cdcd064a7f13b8173555a6","trusted":false,"_cell_guid":"af1d3d06-5e4f-4aed-b452-23d62c50d1fe"},"cell_type":"code","source":"lgb.unloader(wipe = TRUE) \nlibrary(lightgbm)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f1578061c0f9f448ad45086811728baa495c6d5","trusted":false,"_cell_guid":"2598e6a9-3bbc-4890-8bd5-2d33bf9465e9"},"cell_type":"code","source":"model <- lgb.train(params, dtrain, valids = list(validation = valid), nthread = 4,\n                   nrounds = 1000, verbose= 1,  eval_freq = 50)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fbf657938c0fe5f0a7f8f7aa78f46c0718f0928","trusted":false,"_cell_guid":"b7fa04d0-2256-4bbb-8789-5d8081cf71c7"},"cell_type":"code","source":"glimpse(valid)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7667d192358dbac40d4626ed8ba3b17b5dee727","trusted":false,"_cell_guid":"912fe1d9-05b5-4760-811a-829618cef993"},"cell_type":"code","source":"test<-test%>%mutate(year= year(ymd_hms(click_time)), month=month(ymd_hms(click_time)), day=day(ymd_hms(click_time)),hour_of_day=hour(ymd_hms(click_time)), week=week(ymd_hms(click_time)) )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61df44cc3a032c56fbd3709e383de7beedd94a7c","trusted":false,"_cell_guid":"aa186cc9-307e-4bf9-812f-c05a9d6733ba"},"cell_type":"code","source":"print(model)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bccb7d5e2dbdd66020fa6ae350be0bab0796aa49","trusted":false,"_cell_guid":"b5985f51-a2f4-416a-ada7-9f721d3d1409"},"cell_type":"code","source":"summary(model,\n        cBars=length(model$var.names),\n        n.trees=model$n.trees,\n        plotit=TRUE,\n        order=TRUE,\n        method=relative.influence,\n        normalize=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89c0d176150f95f7c70ec07db3d56fe8847f5a90","trusted":false,"_cell_guid":"f546cdb1-39ae-43a3-a8d7-c81cdec6100a"},"cell_type":"code","source":"top_predictors<-c('hour_of_day','channel','app')\ntest<-test[,top_predictors]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6aa1e87c6bf09fbb84482b8a3719497a0db8dab","trusted":false,"_cell_guid":"d0356bf2-7125-4610-a2cd-b7fdeac248ee"},"cell_type":"code","source":"\npredi<-predict(model,test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd90503fe2f75152fe89fb9ec864edd2b8c4de8d","trusted":false,"_cell_guid":"29283cdd-09b2-48c6-826a-379e90bef3a8"},"cell_type":"code","source":"print( predi)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}