{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os\nfrom tifffile import imread\nimport numpy as np\nimport cv2\nimport gc\nimport random\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess Data"},{"metadata":{},"cell_type":"markdown","source":"## Create training set\n\nCreate training set by reading, padding, downsizing & tiling images and according masks. Non-informative tiles are dismissed."},{"metadata":{"trusted":true},"cell_type":"code","source":"# thanks to: https://www.kaggle.com/iafoss/256x256-images\n\n# specify parameters\nTILE_SIZE = 512\nDOWNSIZE = 4\nX_TRAIN = []\nY_TRAIN = []\n\nprint('Starting to create training set ...')\nprint(f'Tile size is: {TILE_SIZE} px')\nprint(f'Downsizing images and masks by a factor of {DOWNSIZE}.')\n\n# read mask encodings\nMASKS = pd.read_csv('/kaggle/input/hubmap-kidney-segmentation/train.csv').set_index('id')\n\nstep = 1\nfor file in os.listdir('/kaggle/input/hubmap-kidney-segmentation/train/'):\n    if os.path.splitext(file)[1] == '.tiff':\n        # load image\n        print(f'--- STEP {step}/8 ---')\n        print(f'Reading image {file} ...')\n        img = imread('/kaggle/input/hubmap-kidney-segmentation/train/' + file)\n\n        # check if image is stored 3- or 5-dimensional and transform if necessary\n        if len(img.shape) == 5:\n            img = np.squeeze(img)\n            img = np.transpose(img, (1,2,0))\n\n        # define tiling parameters\n        HEIGHT_REMAIN = TILE_SIZE*DOWNSIZE - (img.shape[1] % (TILE_SIZE*DOWNSIZE))\n        WIDTH_REMAIN = TILE_SIZE*DOWNSIZE - (img.shape[0] % (TILE_SIZE*DOWNSIZE))\n        img_shape = img.shape\n\n        # pad image with zeros, such that image is divisible by tile size\n        print('Padding ...')\n        img_pad = np.pad(img, ((WIDTH_REMAIN//2, WIDTH_REMAIN-WIDTH_REMAIN//2),\n                               (HEIGHT_REMAIN//2, HEIGHT_REMAIN-HEIGHT_REMAIN//2),\n                               (0,0)), 'constant', constant_values=0)\n\n        # downsizing to fit into RAM\n        img_pad = cv2.resize(img_pad, (img_pad.shape[1]//DOWNSIZE, img_pad.shape[0]//DOWNSIZE),\n                             interpolation = cv2.INTER_AREA)\n\n        # devide image into tiles with 'reshape-transform-trick'\n        print('Tiling ...')\n        img_split = img_pad.reshape(img_pad.shape[0]//TILE_SIZE,\n                                    TILE_SIZE,\n                                    img_pad.shape[1]//TILE_SIZE,\n                                    TILE_SIZE,\n                                    3)\n        img_split = img_split.transpose(0, 2, 1, 3, 4).reshape(-1, TILE_SIZE, TILE_SIZE, 3)\n\n        # free memory\n        del img, img_pad\n        gc.collect()\n\n        # create empty array for mask with length 'total number of pixels'\n        print('Reading according mask ...')\n        mask = np.zeros(img_shape[0] * img_shape[1], dtype=np.uint8)\n\n        img_mask = MASKS.loc[os.path.splitext(file)[0]].encoding.split()\n\n        # fill empty array with mask encodings\n        for i in range(0, len(img_mask), 2):\n            start = int(img_mask[i])\n            stop = int(img_mask[i]) + int(img_mask[i+1])\n            mask[start:stop] = 1\n\n        # reshape into image form\n        mask = mask.reshape(img_shape[1], img_shape[0]).T\n\n        # pad mask with zeros, such that it is divisible by tile size\n        mask_pad = np.pad(mask, ((WIDTH_REMAIN//2, WIDTH_REMAIN-WIDTH_REMAIN//2),\n                                 (HEIGHT_REMAIN//2, HEIGHT_REMAIN-HEIGHT_REMAIN//2)),\n                          'constant', constant_values=0)\n\n        # downsizing to fit into RAM\n        mask_pad = cv2.resize(mask_pad, (mask_pad.shape[1]//DOWNSIZE, mask_pad.shape[0]//DOWNSIZE),\n                              interpolation = cv2.INTER_NEAREST)\n\n        # devide mask into tiles with 'reshape-transform-trick'\n        mask_split = mask_pad.reshape(mask_pad.shape[0]//TILE_SIZE,\n                                      TILE_SIZE,\n                                      mask_pad.shape[1]//TILE_SIZE,\n                                      TILE_SIZE)\n        mask_split = mask_split.transpose(0, 2, 1, 3).reshape(-1, TILE_SIZE, TILE_SIZE)\n\n        del  mask, img_mask, mask_pad\n        gc.collect()\n\n        # randomly drop images/masks to reduce training dataset size (30%)\n        rand = random.sample(range(0, len(img_split)), int(len(img_split)*0.3))\n        img_tiles = np.delete(img_split, rand, axis=0)\n        mask_tiles = np.delete(mask_split, rand, axis=0)\n        assert len(img_tiles) == len(mask_tiles), 'Number of image and mask tiles does not match!'\n\n        # append tiles and masks to final training set\n        print('No. of created images/masks:', len(img_tiles))\n        X_TRAIN = X_TRAIN + list(img_tiles)\n        Y_TRAIN = Y_TRAIN + list(mask_tiles)\n\n        del img_split, img_tiles, mask_split, mask_tiles\n        gc.collect()\n\n        step += 1\n\nprint(f'Training set created. Total number of sampels is {len(X_TRAIN)}.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot tile and mask (with glomerulus) for sanity check\nfor i, j in enumerate(X_TRAIN):\n    if 1 in j:\n        fig = plt.figure()\n        plt.imshow(X_TRAIN[i])\n        plt.imshow(Y_TRAIN[i], alpha=0.3)\n        plt.show()\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Write to txt\n\nFlatten arrays, so that each picture corresponds to one row. Reshape to (512, 512, 3) after import."},{"metadata":{"trusted":true},"cell_type":"code","source":"img_512 = [arr.flatten() for arr in X_TRAIN]\nmask_512 = [arr.flatten() for arr in Y_TRAIN]\n\nnp.savetxt('img_512.txt', np.asarray(img_512), fmt='%1.0f', delimiter=',')\nnp.savetxt('mask_512.txt', np.asarray(mask_512), fmt='%1.0f', delimiter=',')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}