{"cells":[{"metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true},"cell_type":"code","source":"\nlibrary(keras)\nlibrary(tidyverse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data <- read_csv(\"../input//landmark-recognition-2020//train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ggplot(data, aes(as.factor(landmark_id)))+\ngeom_bar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Preprocessing\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"head(data)\ndim(data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train data frame","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create train dataFrame\nfolder=\"train\"\n    folder_list <- c(list.dirs(paste(\"../input//landmark-recognition-2020/\",folder,\"/\",sep = \"\")))             \n    folder_list <- folder_list[str_length(string = folder_list)==48]     \n    # sum of total files\n    total_files <- 0\n    for(i in 1:length(folder_list)){\n    sum <-length(list.files(folder_list[i]))  \n    total_files <- total_files + sum \n    }\n    cat(\"total \",folder, \"files are :\",total_files)  \n    train_files <-tibble()\n    for(i in 1:length(folder_list)){\n        files <-list.files(folder_list[i])\n        file_path <- paste0(folder_list[i],\"/\",files,sep=\"\")\n        kouvas <- tibble(files = files, file_path = file_path)\n        train_files <- rbind(train_files,kouvas)\n    }\n    train_files$files <- str_remove(train_files$files,pattern = \".jpg\")   \n       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_csv(train_files, \"train_files.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim(train_files)\nhead(train_files)\ntail(train_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create test dataFrame\nfolder=\"test\"\n    folder_list <- c(list.dirs(paste(\"../input//landmark-recognition-2020/\",folder,\"/\",sep = \"\")))             \n    folder_list <- folder_list[str_length(string = folder_list)==47]  \n    # sum of total files\n    total_files <- 0\n    for(i in 1:length(folder_list)){\n    sum <-length(list.files(folder_list[i]))  \n    total_files <- total_files + sum \n    }       \n    cat(\"total \",folder, \"files are :\",total_files)\n      \ntest_files <- tibble()\n    for(i in 1:length(folder_list)){\n        files <-list.files(folder_list[i])\n        file_path <- paste0(folder_list[i],\"/\",files,sep=\"\")\n        kouvas <- tibble(files = files, file_path = file_path)\n        test_files <- rbind(test_files,kouvas)\n    }\n    test_files$files <- str_remove(test_files$files,pattern = \".jpg\")\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_csv(test_files, \"test_files.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim(test_files)\nhead(test_files)\ntail(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files <-train_files %>% rename(id = \"files\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainLabeled <- left_join(x = train_files,y = data,by = \"id\")\nhead(trainLabeled)\ndim(trainLabeled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_csv(trainLabeled, \"trainLabeled.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lower_limit_freq = 1000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainLabeledSorted <- trainLabeled %>% \ngroup_by(landmark_id) %>% \nmutate(count_class = n()) %>% \narrange(desc(count_class)) %>% \nfilter(count_class>=lower_limit_freq)%>%ungroup()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"head(trainLabeledSorted)\ndim(trainLabeledSorted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim(trainLabeledSorted)[1]\ndim(trainLabeledSorted)[1] %/% 1.3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_csv(trainLabeledSorted,\"trainLabeledSorted.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trainLabeledSorted <- trainLabeled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size <- 64\nsample_size = dim(trainLabeledSorted)[1]\n#sample_size = 100000\ntrain_size_portion = 0.7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train size\nx=(dim(trainLabeledSorted)[1] %/% (1+(1-train_size_portion)))/batch_size\ntrain_size=(round(x)-1)*batch_size\ncat(\"train size is..\",train_size, \"and \",train_size/batch_size, \"batches\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x=(sample_size*train_size_portion) %/% batch_size\n#train_size =(round(x)-1)*batch_size\n#cat(\"train size is..\",train_size, \"and \",train_size/batch_size, \"batches\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x=(sample_size-train_size) %/% batch_size\nvalidation_size=(round(x)-1)*batch_size\ncat(\" validation size is..\",validation_size, \"and\",validation_size/batch_size, \"batches\",\"\\n\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat(\" sample:\",sample_size, \"\\n\",\"train and validation:\", train_size+validation_size,\"\\n\",\n    \"difference is:\",\n    (sample_size-train_size-validation_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indices <- sample(1:nrow(trainLabeledSorted))\n\ntraining_indices <- indices[1:train_size]\nvalidation_indices <- indices[(train_size+1):(train_size+validation_size)]\n#test_indices <- indices[(training_size+validation_size+1):(training_size+validation_size+test_size)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"length(training_indices)\nlength(validation_indices)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataframe <- trainLabeledSorted[training_indices,]\ntrain_dataframe$landmark_id <- as.character(train_dataframe$landmark_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_csv(train_dataframe,\"train_dataframe.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output<-length(unique(x = train_dataframe$landmark_id))\noutput","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_dataframe <- trainLabeledSorted[validation_indices,]\nval_dataframe$landmark_id <- as.character(val_dataframe$landmark_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_csv(val_dataframe, \"val_dataframe.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nrow(distinct(val_dataframe, landmark_id))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"3.6.3"}},"nbformat":4,"nbformat_minor":4}