{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import io\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport shapely  # for working with WKT format\nimport tifffile # for working with TIFF or TIF image format\nimport zipfile  # for working with massive .zip file","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# loading grid sizes data\n# not entirely sure what this data is - something to do with co-ordinate grids of images\ngrid_sizes = pd.read_csv('/kaggle/input/dstl-satellite-imagery-feature-detection/grid_sizes.csv.zip', index_col=0)\ngrid_sizes.index.name = 'image_id'\nprint('450 image IDs. These are the grid sizes:')\ngrid_sizes.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_wkt = pd.read_csv('/kaggle/input/dstl-satellite-imagery-feature-detection/train_wkt_v4.csv.zip', index_col=0)\nprint('Here is the training data \"labels\".')\nprint('25 training images, 10 classes per image, so 250 rows.')\ntrain_wkt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Data Summary\n<li>Have 450 ImageIDs, 25 of which correspond to training data, 425 test data. Each image is a 1km x 1km satellite image.</li>\n<li>Have 4 (types of) images per ImageID.</li>\n<li>Traditional RGB images are in the three_band.zip file. They are named {ImageID}.tif. Each is 3348 x 3403 pixels, with 3 channels (red, green, blue).</li>\n<li>The sixteen_band.zip file contains images capturing wider wavelength channels. There are three images per ImageID, named {ImageID}_{code}.tif where code = 'M', 'P', 'A'.</li>\n<li>Code 'P' corresponds to *panchromatic* waveband. Each image is 3348 x 3403 pixels. There is just one channel.</li>\n<li>Code 'M' corresponds to *multispectral* wavebands (red, red edge, coastal, blue, green, yellow, near-IR1 and near-IR2). Each image is 837 x 851 pixels. There are 8 channels.</li>\n<li>Code 'A' corresponds to *short-wave infra-red* (SWIR) wavebands. Each image is 134 x 137 pixels. There are 8 channels.</li>\n<li>Clear from the pixel sizes of the images that the RGB and panchromatic have the greatest spatial resolution (i.e. each pixel corresponds to smaller spatial region). Multispectral has 4x smaller spatial resolution, and SWIR has worst spatial resolution (each pixel covers a spatial area of size approximately 7.5m x 7.5m).</li>","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset should be 'three_band' or 'sixteen_band'; if 'sixteen_band' then need code 'P', 'M', or 'A'.\n# returns BytesIO object which can be passed as a file to any image reading function.\ndef get_img_file(dataset, imageid='6120_2_3', code='P'):\n    if dataset == 'sixteen_band':\n        if code in ['P', 'M', 'A']:\n            filename = f'{dataset}/{imageid}_{code}.tif'\n        else:\n            print('Need code to be P, M, or A.')\n            return\n    elif dataset == 'three_band':\n        filename = f'{dataset}/{imageid}.tif'\n    else:\n        print('Need dataset to be three_band or sixteen_band')\n    archive = zipfile.ZipFile(f'../input/dstl-satellite-imagery-feature-detection/{dataset}.zip', 'r')\n    img_as_bytes = archive.read(filename)\n    img_file = io.BytesIO(img_as_bytes) # can now pass this to any img reader as a file\n    return img_file\n\ndef stretch2(band, lower_percent=1, higher_percent=99):\n    a = 0 #np.min(band)\n    b = 255  #np.max(band)\n    c = np.percentile(band, lower_percent)\n    d = np.percentile(band, higher_percent)        \n    out = a + (band - c) * (b - a) / (d - c)    \n    out[out<a] = a\n    out[out>b] = b\n    return out\n\n# def plot_rgb(imageid='6120_2_3'):\n#     rgb_img_file = get_img_file('three_band', imageid)\n#     img_as_arr = tifffile.imread(rgb_img_file)  # dtype = uint16, shape (3, 3348, 3403)\n#     img_as_arr = np.rollaxis(img_as_arr, 0, 3)  # changing shape to (3348, 3403, 3) (necessary to plot with pyplot)\n#     # \n#     return img_as_arr\nimageid='6010_0_0'\nrgb_img_file = get_img_file('three_band', imageid)\nimg_as_arr = tifffile.imread(rgb_img_file)\nimg_as_arr = np.rollaxis(img_as_arr, 0, 3)\n# assuming pixel intensities range from 0 to 2^11 - 1 \nimg1 = (img_as_arr/(2**11) * 256).astype(np.uint8)\n\n\n\n# for i in range(3):\n#     img_as_arr[:, :, i] = img_as_arr[:, :, i].astype(float)/img_as_arr[:, :, i].max()\nimg2 = np.array([img_as_arr[:, :, i]/img_as_arr[:, :, i].max() for i in range(3)])\nimg2 = np.rollaxis(img2, 0, 3)\nplt.imshow(img2)\n\n# print(img_as_arr[:, :, 0].max()) \n# print(img_as_arr[:, :, 1].max()) \n# print(img_as_arr[:, :, 2].max()) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imageid='6010_0_0'\nrgb_img_file = get_img_file('three_band', imageid)\nimg_as_arr = tifffile.imread(rgb_img_file)\ntifffile.imshow(img_as_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_file = get_img_file('three_band', '6010_0_0')\nimg_file\n\nimport tifffile\nimport cv2\n\ntif_arr = np.rollaxis(tifffile.imread(img_file), 0, 3)\n\ntif_arr2 = tif_arr.astype(float)\nplt.imshow(stretch2(tif_arr2).astype(np.uint8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(((tif_arr/(2**11))*256).astype(np.uint8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"2**11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(rgb, vmin=0, vmax=2**16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.rollaxis(tif_arr, 0, 3).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nfile='../input/dstl-satellite-imagery-feature-detection/sixteen_band/6120_2_2_A.tif'\n#imBandA = Image.open(rb'../input/three_band/6120_2_2.tif')\n#imBandA.show()\n\n#from scipy import misc\n#raster = misc.imread('../input/sixteen_band/6120_2_2_A.tif')\n#type(raster)\n\nimport os\nos.listdir('../input/dstl-satellite-imagery-feature-detection/three_band')\n\nwith open('../input/dstl-satellite-imagery-feature-detection/three_band/6120_2_2.tif',encoding='utf-8', errors='ignore') as f:\n    print(f.readlines())\n\nimarray = plt.imread('image.tif')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'sfad '.strip(' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_rgb(imageid):\n    img_as_stream = get_img_file('three_band', 'image_id')\n    \n# for example, using tifffile:\nimport tifffile\nimg_as_arr = tifffile.imread(img_as_stream) #","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_as_arr.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"2**16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(img_as_arr[7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nplt.imshow(rgb[:, :, 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_as_arr[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_as_arr[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def stretch2(band, lower_percent=1, higher_percent=99):\n    a = 0 #np.min(band)\n    b = 255  #np.max(band)\n    c = np.percentile(band, lower_percent)\n    d = np.percentile(band, higher_percent)        \n    out = a + (band - c) * (b - a) / (d - c)    \n    out[out<a] = a\n    out[out>b] = b\n    return out\ndef adjust_contrast(x):    \n    for i in range(3):\n        x[:,:,i] = stretch2(x[:,:,i], 0, 100)\n    return x.astype(np.uint8) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(rgb2.astype(np.uint8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"help(plt.imshow)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rgb[:, :, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import io\n\nstream = io.BytesIO(imgdata)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# HEY LAKSH CHECK THIS","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nAuthor : amanbh\n\n- Set up some basic functions to load/manipulate image data\n- Visualize/Summarize cType counts, training data, and true classes\n- Plot Polygons with holes correctly by using descartes package\n\nBased on Kernel by\n    Author : Oleg Medvedev\n    Link   : https://www.kaggle.com/torrinos/dstl-satellite-imagery-feature-detection/exploration-and-plotting/run/553107\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\n\nfrom shapely.wkt import loads as wkt_loads  # for working with WKT format\nfrom matplotlib.patches import Polygon, Patch\n\n# decartes package makes plotting with holes much easier\nfrom descartes.patch import PolygonPatch\n\nimport matplotlib.pyplot as plt\nimport tifffile as tiff  # for working with .tif and .tiff files\n\nimport pylab\n# turn interactive mode on so that plots immediately\n# See: http://stackoverflow.com/questions/2130913/no-plot-window-in-matplotlib\n# pylab.ion()\nimport zipfile\n\ninDir = '../input/dstl-satellite-imagery-feature-detection'  \n\n# Give short names, sensible colors and z-orders to object types\nCLASSES = {\n        1 : 'Bldg',\n        2 : 'Struct',\n        3 : 'Road',\n        4 : 'Track',\n        5 : 'Trees',\n        6 : 'Crops',\n        7 : 'Fast H20',\n        8 : 'Slow H20',\n        9 : 'Truck',\n        10 : 'Car',\n        }\nCOLORS = {\n        1 : '0.7',\n        2 : '0.4',\n        3 : '#b35806',\n        4 : '#dfc27d',\n        5 : '#1b7837',\n        6 : '#a6dba0',\n        7 : '#74add1',\n        8 : '#4575b4',\n        9 : '#f46d43',\n        10: '#d73027',\n        }\nZORDER = {\n        1 : 5,\n        2 : 5,\n        3 : 4,\n        4 : 1,\n        5 : 3,\n        6 : 2,\n        7 : 7,\n        8 : 8,\n        9 : 9,\n        10: 10,\n        }\n# z-orders determine overlap order for the polygons\n\n# read the training data from train_wkt_v4.csv\ndf = pd.read_csv(inDir + '/train_wkt_v4.csv.zip')\nprint('training data:')\nprint(df.head())\nprint('\\n')\n\n# grid size will also be needed later..\ngs = pd.read_csv(inDir + '/grid_sizes.csv.zip', names=['ImageId', 'Xmax', 'Ymin'], skiprows=1)\nprint('grid sizes data:')\nprint(gs.head())\nprint('\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# imageIds in an array\nallImageIds = gs.ImageId.unique()\ntrainImageIds = df.ImageId.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_images(imageId):\n    '''\n    Load images correspoding to imageId\n\n    Parameters\n    ----------\n    imageId : str\n        imageId as used in grid_size.csv\n    img_key : {None, '3', 'A', 'M', 'P'}, optional\n        Specify this to load single image\n        None loads all images and returns in a dict\n        '3' loads image from three_band/\n        'A' loads '_A' image from sixteen_band/\n        'M' loads '_M' image from sixteen_band/\n        'P' loads '_P' image from sixteen_band/\n\n    Returns\n    -------\n    images : dict\n        A dict of image data from TIFF files as numpy array\n        Keys: ['3', 'A', 'M', 'P']\n    '''\n    images = dict()\n    # getting 3-band image\n    archive = zipfile.ZipFile(f'{inDir}/three_band.zip', 'r')\n    img_as_bytes = archive.read(f'three_band/{imageId}.tif')\n    img_file = io.BytesIO(img_as_bytes)  # byte stream\n    images['3'] = tiff.imread(img_file)\n    # getting 16-band images\n    archive = zipfile.ZipFile(f'{inDir}/sixteen_band.zip', 'r')\n    for k in ['P', 'M', 'A']:\n        img_as_bytes = archive.read(f'sixteen_band/{imageId}_{k}.tif')\n        img_file = io.BytesIO(img_as_bytes) \n        images[k] = tiff.imread(img_file)\n    return images\n\ndef get_size(imageId):\n    \"\"\"\n    Get the grid size of the image\n\n    Parameters\n    ----------\n    imageId : str\n        imageId as used in grid_size.csv\n    \"\"\"\n    xmax, ymin = gs[gs.ImageId == imageId].iloc[0,1:].astype(float)\n    W, H = get_images(imageId)['3'].shape[1:]\n    return (xmax, ymin, W, H)\n\ndef is_training_image(imageId):\n    '''\n    Returns\n    -------\n    is_training_image : bool\n        True if imageId belongs to training data\n    '''\n    return any(trainImageIds == imageId)\n\ndef plot_image(fig, ax, imageId, img_key, selected_channels=None):\n    '''\n    Plot get_images(imageId)[img_key] on axis/fig supplied\n    Optional: select which channels of the image are used (used for sixteen_band/ images)\n    Parameters\n    ----------\n    img_key : str, {'3', 'P', 'N', 'A'}\n        See get_images for description.\n    '''\n    images = get_images(imageId)\n    img = images[img_key]\n    title_suffix = ''\n    if selected_channels is not None:\n        img = img[selected_channels]\n        title_suffix = ' (' + ','.join([ repr(i) for i in selected_channels ]) + ')'\n    if len(img.shape) == 2:\n        new_img = np.zeros((3, img.shape[0], img.shape[1]))\n        new_img[0] = img\n        new_img[1] = img\n        new_img[2] = img\n        img = new_img\n    \n    tiff.imshow(img, figure=fig, subplot=ax)\n    ax.set_title(imageId + ' - ' + img_key + title_suffix)\n    ax.set_xlabel(img.shape[-2])\n    ax.set_ylabel(img.shape[-1])\n    ax.set_xticks([])\n    ax.set_yticks([])\n\ndef plot_polygons(fig, ax, polygonsList):\n    '''\n    Plot descrates.PolygonPatch from list of polygons objs for each CLASS\n    '''\n    legend_patches = []\n    for cType in polygonsList:\n        print('{} : {} \\tcount = {}'.format(cType, CLASSES[cType], len(polygonsList[cType])))\n        legend_patches.append(Patch(color=COLORS[cType],\n                                    label='{} ({})'.format(CLASSES[cType], len(polygonsList[cType]))))\n        for polygon in polygonsList[cType]:\n            mpl_poly = PolygonPatch(polygon,\n                                    color=COLORS[cType],\n                                    lw=0,\n                                    alpha=0.7,\n                                    zorder=ZORDER[cType])\n            ax.add_patch(mpl_poly)\n    # ax.relim()\n    ax.autoscale_view()\n    ax.set_title('Objects')\n    ax.set_xticks([])\n    ax.set_yticks([])\n    return legend_patches\n\ndef visualize_image(imageId, plot_all=True):\n    '''         \n    Plot all images and object-polygons\n    \n    Parameters\n    ----------\n    imageId : str\n        imageId as used in grid_size.csv\n    plot_all : bool, True by default\n        If True, plots all images (from three_band/ and sixteen_band/) as subplots.\n        Otherwise, only plots Polygons.\n    '''         \n    df_image = df[df.ImageId == imageId]\n    xmax, ymin, W, H = get_size(imageId)\n    \n    if plot_all:\n        fig, axArr = plt.subplots(figsize=(10, 10), nrows=3, ncols=3)\n        ax = axArr[0][0]\n    else:\n        fig, axArr = plt.subplots(figsize=(10, 10))\n        ax = axArr\n    if is_training_image(imageId):\n        print('ImageId : {}'.format(imageId))\n        polygonsList = {}\n        for cType in CLASSES.keys():\n            polygonsList[cType] = wkt_loads(df_image[df_image.ClassType == cType].MultipolygonWKT.values[0])\n        legend_patches = plot_polygons(fig, ax, polygonsList)\n        ax.set_xlim(0, xmax)\n        ax.set_ylim(ymin, 0)\n        ax.set_xlabel(xmax)\n        ax.set_ylabel(ymin)\n    if plot_all:\n        plot_image(fig, axArr[0][1], imageId, '3')\n        plot_image(fig, axArr[0][2], imageId, 'P')\n        plot_image(fig, axArr[1][0], imageId, 'A', [0, 3, 6])\n        plot_image(fig, axArr[1][1], imageId, 'A', [1, 4, 7])\n        plot_image(fig, axArr[1][2], imageId, 'A', [2, 5, 0])\n        plot_image(fig, axArr[2][0], imageId, 'M', [0, 3, 6])\n        plot_image(fig, axArr[2][1], imageId, 'M', [1, 4, 7])\n        plot_image(fig, axArr[2][2], imageId, 'M', [2, 5, 0])\n\n    if is_training_image(imageId):\n        ax.legend(handles=legend_patches,\n                   # loc='upper center',\n                   bbox_to_anchor=(0.9, 1),\n                   bbox_transform=plt.gcf().transFigure,\n                   ncol=5,\n                   fontsize='x-small',\n                   title='Objects-' + imageId,\n                   # mode=\"expand\",\n                   framealpha=0.3)\n    return (fig, axArr, ax)\n\n# # Loop over few training images and save to files\n# for imageId in trainImageIds:\n#     fig, axArr, ax = visualize_image(imageId, plot_all=False)\n#     plt.savefig('Objects--' + imageId + '.png')\n#     plt.clf()\n\n\n# Optionally, view images immediately:\n# pylab.show()\n# Uncomment to show plot when interactive mode is off \n# (this function blocks till fig is closed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainImageIds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_image('6120_2_2')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}