{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# load libraries\nlibrary(tidyverse, tidyverse_conflict()) # metapackage of all tidyverse packages\nlibrary(oro.dicom)\nlibrary(keras)\nlibrary(EBImage)\n#list.files(path = \"../input\") \n","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2021-07-17T16:16:43.673263Z","iopub.execute_input":"2021-07-17T16:16:43.6756Z","iopub.status.idle":"2021-07-17T16:16:43.698664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train files path\npath <- \"../input/siim-covid19-detection/train\"\n\n# train files list\ndir(path)  -> files_list\n\n# print sub dir file 1\ncat( \"first sub dir. file check : \", files_list [[1]],\"\\n\")\n\n# files list class\ncat(\"sub dir. class : \", class(files_list) )","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:16:43.718813Z","iopub.execute_input":"2021-07-17T16:16:43.720484Z","iopub.status.idle":"2021-07-17T16:16:43.778291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# job def : build images dir. list of train set images\nimg_dir_list <-list()\n\n# counter\ncnt = 0\n# build images dir. list  \nfor (file in seq(length(files_list)) ) {\n    # sub dir1 files : file1\n    dir(paste(path,files_list[[file]], sep =\"/\")) -> file1\n    \n    #sub dir.1 path : path1\n    paste(path,files_list[[file]], file1,sep =\"/\") -> path1\n    \n    # single file image(s)\n    paste(path1,dir(path1),sep =\"/\")  -> img\n    \n    # build images files list\n    img -> img_dir_list[[file]]\n    \n     # incr. counter\n    cnt = cnt + 1\n    }\n# job check\nn_files <-  length(files_list)\nifelse(cnt == n_files, \"all files red😉\",\" not all files red🤔\")\n\n# dir check\n# fisrt and last file dir \nimg_dir_list[1]\nimg_dir_list[6054]","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:16:43.782497Z","iopub.execute_input":"2021-07-17T16:16:43.784191Z","iopub.status.idle":"2021-07-17T16:16:49.674709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rename images directories files\nnames(img_dir_list) <- paste(\"File\",1:n_files,sep=\"_\")\n\n#img dir. list filename check\n# head\nhead(names(img_dir_list),3)\n\n# tail\ntail(names(img_dir_list),3)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:16:49.67869Z","iopub.execute_input":"2021-07-17T16:16:49.680297Z","iopub.status.idle":"2021-07-17T16:16:49.715484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read image from 1st dir. (File_1)\nreadDICOMFile(img_dir_list$File_1)$img %>% \nt() %>%\nimage(col = grey(0:8^2/8^2), xlab = paste(names(img_dir_list)[1],\n                            strsplit(img_dir_list$File_1,\"/\")[[1]][7],\n                             sep=\": \")\n                                                      )\n# read image from last dir. (File_6054)\nreadDICOMFile(img_dir_list$File_6054)$img %>% \nt() %>%\nimage(col = grey(0:8^2/8^2),\n         xlab = paste(names(img_dir_list)[n_files],\n         strsplit(img_dir_list$File_6054,\"/\")[[1]][7],\n            sep=\": \")\n                                                      )","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:16:49.719711Z","iopub.execute_input":"2021-07-17T16:16:49.721461Z","iopub.status.idle":"2021-07-17T16:17:40.473025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"due to memory limitation we are going to subset  train set to 50 samples of data","metadata":{}},{"cell_type":"code","source":"# split data into train and validation set\n# indexes\n#sample(n_files,floor((6/10) * n_files), replace = FALSE) -> train_sample_indx\n#img_dir_list[- train_sample_indx] -> val_sample_indx\n","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:17:40.477186Z","iopub.execute_input":"2021-07-17T16:17:40.479717Z","iopub.status.idle":"2021-07-17T16:17:40.498089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read  50 files from images list dir\n# create empty list and prealloc.\nimages <- vector(\"list\",50)\n\n# downsample img_dir_list\nset.seed(1234)\nsample(length(img_dir_list),50,  replace = FALSE) -> n_sample_indx\n\n# red file display, if any need to check red files \nred_file =\"\"\ncnt = 0\n# read in n_sample files with for loop\nfor (ind in  n_sample_indx) {\n     cnt = cnt + 1\n    \n    # get image name\n    red_file =\"\"\n    red_file <- names(img_dir_list)[ind]\n    \n    # create file name \n    paste0(\"File_\",ind) -> File\n    \n    # create image list\n    hdr <- list()\n    img <- list()\n    im <- list(hdr,img)\n    #im[[1]] <- hdr\n    #im[[2]] <- img\n    \n    # read images\n    try(\n        readDICOM(\n            strsplit(unlist(img_dir_list),\"File_%d\")[[ind]]\n        ) -> im\n            #Path)-> images [ind]\n            \n        )\n    # assign image to images list\n    images[[cnt]]<- im\n    \n    # set image names\n    names(images)[cnt] <- File\n    \n    }\n\n#print(red_file)\nprint(cnt)\n\n# check job\nifelse(cnt == length(n_sample_indx),\"all images red😉\",\"not all images red🤔\")","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:17:40.503111Z","iopub.execute_input":"2021-07-17T16:17:40.505098Z","iopub.status.idle":"2021-07-17T16:18:28.349329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"reading some files produces error ! \n\nwill check and fix it later 😉","metadata":{}},{"cell_type":"code","source":"# check subset \n#select image at random \nset.seed(2134)\nsample(50,2) -> indx\n\n# read image\nfor (ind in indx) {\n    img0 <- images[[ind]]$img  # a list\n    img0_name <- names(img0)\n    img0[[img0_name]] %>% \n    t() %>% \n    image(\n        col = grey(0:2^5/2^5),\n        xlab = paste0(\"File : \",strsplit(img0_name,\"/\")[[1]][7])\n)\n    \n}","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:18:28.353721Z","iopub.execute_input":"2021-07-17T16:18:28.355388Z","iopub.status.idle":"2021-07-17T16:19:04.092257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" diagnosis to be continued 😉","metadata":{}},{"cell_type":"code","source":"# inspect image \nimg0 <- readDICOMFile(img_dir_list$File_1)$img \n#t() %>%\n#image(col = grey(0:8^2/8^2), xlab = paste(names(img_dir_list)[1],\n                            #strsplit(img_dir_list$File_1,\"/\")[[1]][7],\n                             #sep=\": \"))","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.096683Z","iopub.execute_input":"2021-07-17T16:19:04.099199Z","iopub.status.idle":"2021-07-17T16:19:04.786023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inspect img0\n# class\nclass(img0)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.79054Z","iopub.execute_input":"2021-07-17T16:19:04.792225Z","iopub.status.idle":"2021-07-17T16:19:04.81153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure \nstr(img0)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.815418Z","iopub.execute_input":"2021-07-17T16:19:04.81722Z","iopub.status.idle":"2021-07-17T16:19:04.835144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dimension\ndim(img0)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.839322Z","iopub.execute_input":"2021-07-17T16:19:04.841035Z","iopub.status.idle":"2021-07-17T16:19:04.859315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next job , \n\nneed to merge csv files in train folder and get classes from it to gather to tf_imG classes.\n😉","metadata":{}},{"cell_type":"code","source":"# keras_array(img_dir_list, dtype = NULL) -> keras_img_list","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.863105Z","iopub.execute_input":"2021-07-17T16:19:04.864772Z","iopub.status.idle":"2021-07-17T16:19:04.87654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dirs data frame\ndata.frame(filename = paste0(\"File_\",seq(length(unlist(img_dir_list)))),\n              img_dir = unlist(img_dir_list)\n          ) -> img_dir_df\n\n# head(df)\nhead(img_dir_df,3)\n\n# tail df\ntail(img_dir_df,3)\n\n# dims\ndim(img_dir_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.880466Z","iopub.execute_input":"2021-07-17T16:19:04.88227Z","iopub.status.idle":"2021-07-17T16:19:04.936168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"joining train_img_level and train_study_level data frames from their csv files","metadata":{}},{"cell_type":"code","source":"# read train_img_level.csv\ntrain_img_level <- read_csv(\"../input/siim-covid19-detection/train_image_level.csv\") ","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:04.940121Z","iopub.execute_input":"2021-07-17T16:19:04.941887Z","iopub.status.idle":"2021-07-17T16:19:05.003348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# head\nhead(train_img_level,3)\n\n# tail\ntail(train_img_level,3)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:05.007739Z","iopub.execute_input":"2021-07-17T16:19:05.009662Z","iopub.status.idle":"2021-07-17T16:19:05.052935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read train study level csv\ntrain_study_level <- read_csv(\"../input/siim-covid19-detection/train_study_level.csv\") \n\n#head\nhead(train_study_level)\n\n# tail\ntail(train_study_level,3)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:05.057321Z","iopub.execute_input":"2021-07-17T16:19:05.059252Z","iopub.status.idle":"2021-07-17T16:19:05.135519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# table train_study_level : create id_new key \n\ntrain_study_level$id_new <- vector(\"character\",nrow(train_study_level)) \nfor (i in seq(nrow(train_study_level)) ) {\n    train_study_level$id_new[i] <- strsplit(train_study_level$id[i],\"_\")[[1]][1] \n}\n\n# check transf.\n# head\nhead(train_study_level,3)\n\n# tail\ntail(train_study_level,3)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:05.139823Z","iopub.execute_input":"2021-07-17T16:19:05.141654Z","iopub.status.idle":"2021-07-17T16:19:06.040308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inner join train_img_level, train_study_level csv files\ntrain_table <- inner_join(train_img_level, \n                          train_study_level, \n           by = c(\"StudyInstanceUID\" = \"id_new\" ),\n                          keep = FALSE\n)\n\n# unique\ntrain_table <- unique(train_table)\n\n# head\nhead(train_table,3)\n\n# tail\ntail(train_table,3)\n\n# dimension\ndim(train_table)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.04454Z","iopub.execute_input":"2021-07-17T16:19:06.046157Z","iopub.status.idle":"2021-07-17T16:19:06.142249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# column bind img_dir_list and train_table\n# prealloc.\n\nimg_dir_df$study <- vector(\"character\",nrow(img_dir_df))\nimg_dir_df$series <- vector(\"character\",nrow(img_dir_df))\nimg_dir_df$img <- vector(\"character\",nrow(img_dir_df))\n\nfor (i in seq(nrow(img_dir_df))){\n    \n   file_parts <- strsplit(img_dir_df$img_dir[i],\"/\")[[1]]  \n   \n    # assign\n    img_dir_df$study[i] <- file_parts[5]\n    img_dir_df$series[i]  <- file_parts[6]\n    #img_dir_df$img [i] <- strsplit(file_parts[7],\".dcm\")[[1]]\n    img_dir_df$img [i] <- file_parts[7]\n\n    \n}\n","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.146353Z","iopub.execute_input":"2021-07-17T16:19:06.148019Z","iopub.status.idle":"2021-07-17T16:19:06.850375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# head and tail img_dir_df \nhead(img_dir_df,3)\n\n#tail\ntail(img_dir_df,3)\n\n# dims\ndim(img_dir_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.854774Z","iopub.execute_input":"2021-07-17T16:19:06.856466Z","iopub.status.idle":"2021-07-17T16:19:06.903416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables \nimg_table <- inner_join(img_dir_df,\n           train_study_level,\n          by =  c(\"study\" = \"id_new\")\n          ) ","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.907426Z","iopub.execute_input":"2021-07-17T16:19:06.909085Z","iopub.status.idle":"2021-07-17T16:19:06.927998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim(img_table)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.931924Z","iopub.execute_input":"2021-07-17T16:19:06.933551Z","iopub.status.idle":"2021-07-17T16:19:06.951417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"head(img_table,2)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.955221Z","iopub.execute_input":"2021-07-17T16:19:06.956911Z","iopub.status.idle":"2021-07-17T16:19:06.984171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bind tables\ncbind(img_table,train_table) -> train_img_table\n\n# dimensions\ndim(train_img_table)\n\n# summary\nsummary(train_img_table)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:06.98808Z","iopub.execute_input":"2021-07-17T16:19:06.98971Z","iopub.status.idle":"2021-07-17T16:19:07.025243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_table$img_dir[1]","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:19:07.029142Z","iopub.execute_input":"2021-07-17T16:19:07.030754Z","iopub.status.idle":"2021-07-17T16:19:07.048488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image preprocessing\n# turn img data into array\nimg_list <- vector(\"list\",nrow(train_img_table))\n\n# boxes data\nboxes_list <- vector(\"list\",nrow(train_img_table))\n\n# label data\nlabels_list <- vector(\"list\",nrow(train_img_table))\n\n# counter\ncnt = 0\n\n# read image and reshape\n for (i in seq(nrow(train_img_table)) ){ \n     \n     # hdr : list()\n     hdr <- list()\n     \n     # img :  list()\n     img <-list()\n     \n    # single image : list of hdr and img \n     im <- list(hdr,img) \n     \n     im <- try(readDICOM(img_table$img_dir[i]))\n     im   <- try(\n                         resize(\n                             im$img[[1]],\n                             w = 28,\n                             h = 28)\n                          )\n     \n     # normalize im\n     if ( is.matrix(im)) {\n        img_list[[i]] <- im %/% 255 \n     } else {\n          img_list[[i]] <- \"NA\"\n     }\n     \n     \n  # read boxes data  \n   boxes_list[[i]] <- train_table$boxes[i] \n     \n  # read label data\n  labels_list[[i]] <- train_table$label[i]    \n     cnt = cnt + 1\n }\n\ncat( \"\\n cnt : \" ,cnt )","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:33:42.213469Z","iopub.execute_input":"2021-07-17T16:33:42.215148Z","iopub.status.idle":"2021-07-17T16:49:13.860274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_list[[1]] %>% t() %>% image ( col = grey (0:2^5/2^5) )","metadata":{"execution":{"iopub.status.busy":"2021-07-17T16:30:34.323072Z","iopub.execute_input":"2021-07-17T16:30:34.325036Z","iopub.status.idle":"2021-07-17T16:30:34.35457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check image resize : read sample image\nimg_list[[3]] %>% t() %>% image(col = grey(0:2^5/2^5))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#keras architecture","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"to be continued ! 😉","metadata":{}}]}