{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys, os\nimport cv2\nimport math\nfrom os import listdir, walk\nfrom os.path import isfile, join\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport matplotlib.patches as mpatches\nimport numpy as np\nimport pandas as pd\nimport csv\nimport re\nfrom skimage import data, util, measure\nfrom PIL import Image\nimport pickle\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T17:47:46.722572Z","iopub.execute_input":"2022-07-11T17:47:46.723103Z","iopub.status.idle":"2022-07-11T17:48:00.679207Z","shell.execute_reply.started":"2022-07-11T17:47:46.723001Z","shell.execute_reply":"2022-07-11T17:48:00.677950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#find indices of masked entries in train.csv\ndef findMaskInd(strList):\n    #create logical list: true if a mask exists, false if no mask is provided\n    masked = trainData.segmentation != ''\n    maskInd = [i for i, x in enumerate(masked) if x]\n    return maskInd\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:52:05.524036Z","iopub.execute_input":"2022-07-11T17:52:05.524522Z","iopub.status.idle":"2022-07-11T17:52:05.531017Z","shell.execute_reply.started":"2022-07-11T17:52:05.524475Z","shell.execute_reply":"2022-07-11T17:52:05.529699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\ndef get_case(row):  \n    return row.split('_')[0]\ndef get_day(row):  \n    return row.split('_')[1]\ndef get_slice(row):\n    return row.split('_',2)[2]\ndef get_case_day(row):  \n    return row.split('_')[0]+'_'+row.split('_')[1]\ndef get_full_path(row):\n    train_dir = '../input/uw-madison-gi-tract-image-segmentation/train/'\n    case=row.split('_')[0]\n    return os.path.join(train_dir,case, row,'scans')\n#res = test_string.startswith(tuple(pref_list))\ndef get_file_name_1(full_path, slice):   \n    filenames = os.listdir(full_path)\n    return  [f for f in filenames if f.startswith(slice)][0]\ndef get_file_name_2(full_path, slice):\n    os.chdir(full_path)\n    glob.glob(slice+\"*.png\")\n    os.chdir(\"/kaggle/working\")\ndef get_file_name(row):\n     return row.split('/')[-1]   \ndef get_file_list(image_dir):\n    os.chdir(image_dir)\n    filelist=glob.glob(\"*/*/*/*.png\", recursive=True)\n    filelist.sort()\n    os.chdir(\"/kaggle/working/\")\n    return filelist\ndef get_height(row):\n    return int(row.split('_')[2])\ndef get_width(row):\n    return int(row.split('_')[3])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:53:42.038668Z","iopub.execute_input":"2022-07-11T17:53:42.039217Z","iopub.status.idle":"2022-07-11T17:53:42.054565Z","shell.execute_reply.started":"2022-07-11T17:53:42.039182Z","shell.execute_reply":"2022-07-11T17:53:42.053224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define data root directory\nRoot = '../input/uw-madison-gi-tract-image-segmentation' \n\n#read train.csv data to list\nwith open(os.path.join(Root, \"train.csv\"), 'r') as file:\n    reader = csv.reader(file)\n    T = []\n    for row in reader:\n        T.append(row)\n\n#convert data (excluding column headers) to nparray\nnum = np.array(T[1:len(T)])\n\n#convert data to DataFrame\n#note that 'class' is a protected word in Python so the object class\n#column is renamed 'object'\ntrainData = pd.DataFrame(num, columns=['id','object','segmentation'])\n\n#Create list of all indices in trainData with a segmentation mask\nmaskInd = findMaskInd(trainData.segmentation)\n\nprint(f\"train.csv contains {len(trainData.segmentation)} entries\")\nprint(f\"{len(maskInd)} entries include masks, or {int(100*len(maskInd)/len(trainData.segmentation))} %.\")\n\n#print number of masks with each object label/class\nobjSet = set(trainData.object)\nfor obj in objSet:\n    print(f\"{obj}: {sum(trainData.object[maskInd]==obj)} masks\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:52:11.815791Z","iopub.execute_input":"2022-07-11T17:52:11.816249Z","iopub.status.idle":"2022-07-11T17:52:16.917251Z","shell.execute_reply.started":"2022-07-11T17:52:11.816213Z","shell.execute_reply":"2022-07-11T17:52:16.915745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trainData = pd.read_csv(\"../input/uw-madison-gi-tract-image-segmentation/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:50:35.893128Z","iopub.execute_input":"2022-07-11T17:50:35.893680Z","iopub.status.idle":"2022-07-11T17:50:36.528316Z","shell.execute_reply.started":"2022-07-11T17:50:35.893626Z","shell.execute_reply":"2022-07-11T17:50:36.526776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organiseTrainingData(trainData):\n    os.chdir(\"/kaggle/working\")\n    #uw_train = pd.DataFrame()\n    #uw_train['id'] = np.unique(trainData.id)\n    trainData.sort_values(by=['id']) \n    uw_train = pd.DataFrame({'id':trainData['id'][::3]})\n\n    uw_train['large_bowel'] = trainData['segmentation'][::3].values\n    uw_train['small_bowel'] = trainData['segmentation'][1::3].values\n    uw_train['stomach'] = trainData['segmentation'][2::3].values\n\n    folder = '../input/uw-madison-gi-tract-image-segmentation/train'\n    uw_train['case'] = uw_train['id'].apply(get_case)\n    uw_train['day'] = uw_train['id'].apply(get_day)\n    uw_train['case_day'] = uw_train['id'].apply(get_case_day)\n    uw_train['slice'] = uw_train['id'].apply(get_slice)\n    uw_train['path'] = uw_train['case_day'].apply(get_full_path)\n    uw_train = uw_train.sort_values(by='path')\n    uw_train['filename']=get_file_list(folder)\n    uw_train['filename']=uw_train['filename'].apply(get_file_name)\n    #uw_train['file_name'] = uw_train.apply(lambda x: get_file_name(x['full_path'], x['slice']), axis=1)\n    uw_train.drop('case_day',axis=1,inplace=True)\n    uw_train['width']=int(uw_train['filename'].apply(get_width)[0])\n    uw_train['height']=int(uw_train['filename'].apply(get_height)[0])\n   # case,day,sliceNo,path,filename,height,width = [],[],[],[],[],[],[]\n    \n   \n    uw_train.reset_index(inplace=True,drop=True)\n    uw_train.fillna('',inplace=True); \n    uw_train['count'] = np.sum(uw_train.iloc[:,1:4]!='',axis=1).values\n\n    return uw_train\n\nuw_train = organiseTrainingData(trainData)\n\n#Test : print 5 lines from the reorganised dataset\nuw_train.sample(5)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:00:46.376120Z","iopub.execute_input":"2022-07-11T18:00:46.376592Z","iopub.status.idle":"2022-07-11T18:00:47.788848Z","shell.execute_reply.started":"2022-07-11T18:00:46.376554Z","shell.execute_reply":"2022-07-11T18:00:47.787712Z"},"trusted":true},"execution_count":null,"outputs":[]}]}