{ "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0, "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5506ce20-396b-1055-35d7-9a7ff547a5ca", "_active": false }, "outputs": [], "source": "# Summarized several notebooks of other Kagglers. Big Thanks to those guys\n# shawn:\n# https://www.kaggle.com/shawn775/dstl-satellite-imagery-feature-detection/polygon-transformation-to-match-image/comments\n# Oleg Medvedev:\n# https://www.kaggle.com/torrinos/dstl-satellite-imagery-feature-detection/exploration-and-plotting", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a68e9daa-5a5e-35ce-31b2-1c588a4dc8ab", "_active": false }, "outputs": [], "source": "import pandas as pd\nimport numpy as np\n\nfrom shapely.wkt import loads as wkt_loads\nfrom shapely import affinity\nfrom matplotlib.patches import Polygon\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nimport cv2\n\ndf = pd.read_csv('../input/train_wkt_v3.csv')\ndf.head(5)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "850c08eb-6d77-5434-b1ca-58a6a780c27b", "_active": false }, "outputs": [], "source": "# Class Type is Class of Objects:\n# 1. Buildings - large building, residential, non-residential, fuel storage facility, fortified building\n# 2. Misc. Manmade structures \n# 3. Road \n# 4. Track - poor/dirt/cart track, footpath/trail\n# 5. Trees - woodland, hedgerows, groups of trees, standalone trees\n# 6. Crops - contour ploughing/cropland, grain (wheat) crops, row (potatoes, turnips) crops\n# 7. Waterway \n# 8. Standing water\n# 9. Vehicle Large - large vehicle (e.g. lorry, truck,bus), logistics vehicle\n# 10. Vehicle Small - small vehicle (car, van), motorbike", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6bcd3539-46cb-88a2-f265-a82262e62737", "_active": false }, "outputs": [], "source": "gs = pd.read_csv('../input/grid_sizes.csv', names=['ImageId', 'Xmax', 'Ymin'], skiprows=1)\nprint(gs.head())", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9841dc33-0fc1-95dc-6ab8-f0fa72412302", "_active": false }, "outputs": [], "source": "df['ImageId'].unique()", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "da56bbca-1663-dd01-4d4a-c072eea50b8b", "_active": false }, "outputs": [], "source": "# First Image\npolygonsList ={}\nimage = df[df.ImageId == '6100_1_3']\nfor cType in image.ClassType.unique():\n polygonsList[cType] = wkt_loads(image[image.ClassType == cType].MultipolygonWKT.values[0])", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6fa1cc11-477a-eeaf-5f1e-c2a48d127c68", "_active": false }, "outputs": [], "source": "polygonsList", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c656a377-12fc-2558-7ac8-0571a9eb4284", "_active": false }, "outputs": [], "source": "# plot using matplotlib\nfig, ax = plt.subplots(figsize=(8, 8))\n\n# plotting, color by class type\nfor p in polygonsList:\n for polygon in polygonsList[p]:\n mpl_poly = Polygon(np.array(polygon.exterior), color=plt.cm.Set1(p*10), lw=0, alpha=0.3)\n ax.add_patch(mpl_poly)\n\nax.relim()\nax.autoscale_view()", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "10653179-3c9d-d182-3937-eae57cc8c349", "_active": false }, "outputs": [], "source": "# Anzahl Features\n\nfor p in polygonsList:\n print(\"Type: {:4d}, objects: {}\".format(p,len(polygonsList[p].geoms)))", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "31a94919-08f2-fd44-aaab-9a894deb39be", "_active": false }, "outputs": [], "source": "df['polygons'] = df.apply(lambda row: wkt_loads(row.MultipolygonWKT),axis=1)\ndf['nPolygons'] = df.apply(lambda row: len(row['polygons'].geoms), axis = 1)\n\npvt = df.pivot(index='ImageId', columns='ClassType', values='nPolygons')\npvt", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f1758991-22e0-1f14-dc2f-6313d90786e4", "_active": false }, "outputs": [], "source": "from os import listdir\nimagenames_16 = listdir('../input/sixteen_band') \nimagenames_13 = listdir('../input/three_band') ", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cdb50a6a-72f2-accd-c351-116fd40bd483", "_active": false }, "outputs": [], "source": "###############################\n# Convert polygons to pixels #\n###############################", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1c314807-0bf7-99e0-e622-746ccf985922", "_active": false }, "outputs": [], "source": "def _get_image_names(base_path, imageId):\n '''\n Get the names of the tiff files\n '''\n d = {'3': path.join(base_path,'three_band/{}.tif'.format(imageId)), # (3, 3348, 3403)\n 'A': path.join(base_path,'sixteen_band/{}_A.tif'.format(imageId)), # (8, 134, 137)\n 'M': path.join(base_path,'sixteen_band/{}_M.tif'.format(imageId)), # (8, 837, 851)\n 'P': path.join(base_path,'sixteen_band/{}_P.tif'.format(imageId)), # (3348, 3403)\n }\n return d", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "94674405-3151-292a-8b52-16ae8122774c", "_active": false }, "outputs": [], "source": "def _convert_coordinates_to_raster(coords, img_size, xymax):\n Xmax,Ymax = xymax\n H,W = img_size\n W1 = 1.0*W*W/(W+1)\n H1 = 1.0*H*H/(H+1)\n xf = W1/Xmax\n yf = H1/Ymax\n coords[:,1] *= yf\n coords[:,0] *= xf\n coords_int = np.round(coords).astype(np.int32)\n return coords_int", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3f1a2852-f4a8-e405-6795-88ad83ac4266", "_active": false }, "outputs": [], "source": "def _get_xmax_ymin(grid_sizes_panda, imageId):\n xmax, ymin = grid_sizes_panda[grid_sizes_panda.ImageId == imageId].iloc[0,1:].astype(float)\n return (xmax,ymin)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "365f64df-007f-9630-1d65-ba6006851f11", "_active": false }, "outputs": [], "source": "def _get_polygon_list(wkt_list_pandas, imageId, cType):\n df_image = wkt_list_pandas[wkt_list_pandas.ImageId == imageId]\n multipoly_def = df_image[df_image.ClassType == cType].MultipolygonWKT\n polygonList = None\n if len(multipoly_def) > 0:\n assert len(multipoly_def) == 1\n polygonList = wkt_loads(multipoly_def.values[0])\n return polygonList", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "be0ce404-b848-a553-ddf6-1ebd4c0d06d7", "_active": false }, "outputs": [], "source": "def _get_and_convert_contours(polygonList, raster_img_size, xymax):\n perim_list = []\n interior_list = []\n if polygonList is None:\n return None\n for k in range(len(polygonList)):\n poly = polygonList[k]\n perim = np.array(list(poly.exterior.coords))\n perim_c = _convert_coordinates_to_raster(perim, raster_img_size, xymax)\n perim_list.append(perim_c)\n for pi in poly.interiors:\n interior = np.array(list(pi.coords))\n interior_c = _convert_coordinates_to_raster(interior, raster_img_size, xymax)\n interior_list.append(interior_c)\n return perim_list,interior_list", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "99a193cb-4f12-18e4-780e-c62916bf18db", "_active": false }, "outputs": [], "source": "def _plot_mask_from_contours(raster_img_size, contours, class_value = 1):\n img_mask = np.zeros(raster_img_size,np.uint8)\n if contours is None:\n return img_mask\n perim_list,interior_list = contours\n cv2.fillPoly(img_mask,perim_list,class_value)\n cv2.fillPoly(img_mask,interior_list,0)\n return img_mask", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e313f4db-5cdb-d284-33c0-cf10901129ac", "_active": false }, "outputs": [], "source": "def generate_mask_for_image_and_class(raster_size, imageId, class_type, grid_sizes_panda,\n wkt_list_pandas):\n xymax = _get_xmax_ymin(grid_sizes_panda,imageId)\n polygon_list = _get_polygon_list(wkt_list_pandas,imageId,class_type)\n contours = _get_and_convert_contours(polygon_list,raster_size,xymax)\n mask = _plot_mask_from_contours(raster_size,contours,1)\n return mask", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e168a702-b9fa-9b5e-9ff1-5d76e17e6a60", "_active": false }, "outputs": [], "source": "set_of_mask = dict()\nmask_test = np.zeros((500,500))\n\nfor i in range(0,9):\n \n mask = generate_mask_for_image_and_class((500,500),\"6100_1_3\",i,gs,df)\n set_of_mask[i] = mask*255/9*i\n mask_test = mask_test + mask*255/9*i\n \ncv2.imwrite(\"mask.png\",mask_test)\nimg = mpimg.imread('mask.png')\nplt.imshow(img)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "caa21976-9acc-7c5d-49e6-ec79d3d0252b", "_active": false }, "outputs": [], "source": "import tifffile as tiff\n\nimg_filename = '../input/three_band/6100_1_3.tif'\n\nImage = tiff.imread(img_filename)\n\ntiff.imshow(Image)\nnp.shape(Image)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a4762847-fe93-34f1-0918-7481064332ac", "_active": false }, "outputs": [], "source": "#####################################################################\n# The 3 next In's show how to scale polygons to image size and back #\n#####################################################################\nimg_id = \"6100_1_3\"\ni_grid_size = gs[gs.ImageId == img_id]\nx_max = i_grid_size.Xmax.values[0]\ny_min = i_grid_size.Ymin.values[0]\n\n# Get just single class of trianing polyongs for this image\nclass_2 = df[(df.ImageId == img_id) & (df.ClassType == 2)]\n\n# WKT to shapely object\npolyg = wkt_loads(class_2.MultipolygonWKT.values[0])\n\nprint('Original Extent')\nprint(polyg.bounds)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cca81a4a-19cf-374c-d926-8f35fc2145ec", "_active": false }, "outputs": [], "source": "#Load the image and get its width and height\n\n#image = gdal.Open('three_band/6120_2_2.tif')\n#W = image.RasterXSize\n#H = image.RasterYSize\n#gdal is not loaded in kaggle yet, so I'll do these manually for now.\n\nW = 3403\nH = 3348\n\n# Transform the polygons\n\nW_ = W * (W / (W+1) )\nH_ = H * (H / (H+1) )\n\nx_scaler = W_ / x_max\ny_scaler = H_ / y_min\n\npolyg = affinity.scale(polyg, xfact = x_scaler, yfact = y_scaler, origin=(0,0,0))\n\nprint(\"New Extent to match raster\")\nprint(polyg.bounds)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fdb2dd89-f9de-28fe-db80-68948be3c817", "_active": false }, "outputs": [], "source": "# Now scale the shapely file back to its original coordinates for submission \n# The scaler is the inverse of the original scaler\nx_scaler = 1 / x_scaler\ny_scaler = 1 / y_scaler\n\npolyg = affinity.scale(polyg, xfact = x_scaler, yfact = y_scaler, origin=(0,0,0))\nprint(\"Back to original\")\nprint(polyg.bounds)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1dc390c3-f1cf-df3b-92eb-5a7b67ad50ad", "_active": true }, "outputs": [], "source": "############################\n# SOBEL EDGE DETECTION #\n# from https://www.kaggle.com/bkamphaus/draper-satellite-image-chronology/exploratory-image-analysis #\n############################\n\n\nimport skimage\nfrom skimage.feature import greycomatrix, greycoprops\nfrom skimage.filters import sobel, sobel_h, sobel_v\n\n# load the image and convert it to grayscale\nimage = Image\n\n\ndims = np.shape(image)\nprint(dims)\n\n# a sobel filter is a basic way to get an edge magnitude/gradient image\ntiff.imshow(image)\ntiff.imshow(sobel(image[2,:750,:750]))\n#tiff.imshow(sobel_h(image[2,:750,:750]), cmap='BuGn') \n#tiff.imshow(sobel_v(image[2,:750,:750]), cmap='BuGn') ", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d7462dbf-3d07-faea-38ca-ec3113b47443", "_active": false }, "outputs": [], "source": "from sklearn.decomposition import PCA\n\npca = PCA(3)\npca.fit(image.matrix)\nimage_pca = pca.transform(image.matrix)\nimage_pca_img = image.to_matched_img(image_pca)\n\ntiff.imshow(image_pca_img)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7bad157e-2e99-0814-df92-a272c2018186", "_active": false }, "outputs": [], "source": "#############################\n#\n#############################\n\nfrom skimage import color\n\nhsv = color.rgb2hsv(image)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "db0b39bd-da83-006f-4552-e54fe57eb04f", "_active": false }, "outputs": [], "source": null, "execution_state": "idle" } ] }