{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"b8b186e462cffd774a65c764ff6c5a4b93a6005a"},"cell_type":"code","source":"import os\nimport timeit\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom scipy.sparse import bsr_matrix\nimport dask.array as da\nimport dask\nimport dask.diagnostics as diag\nfrom multiprocessing.pool import ThreadPool\nimport h5py\nfrom bokeh.io import output_notebook\nfrom bokeh.resources import CDN\nimport os\nimport sys\nimport random\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.cluster import KMeans\nfrom scipy import signal\nimport cv2\nfrom PIL import Image\nimport pdb\nfrom tqdm import tqdm\nfrom glob import glob\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\noutput_notebook(CDN, hide_banner=True)\n\nship_dir = '../input'\ntrain_image_dir = os.path.join(ship_dir, 'train')\ntest_image_dir = os.path.join(ship_dir, 'test')\nimport gc; gc.enable() # memory is tight\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.bool)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    # Take the individual ship masks and create a single mask array for all ships\n    all_masks = np.zeros((768, 768), dtype = np.int16)\n    #if isinstance(in_mask_list, list):\n    count = 1\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks += count * rle_decode(mask)\n            count += 1\n    return np.expand_dims(all_masks, -1)\n\ndef get_file_size(is_train, id):\n    if is_train: \n        return os.stat(os.path.join(train_image_dir, id)).st_size/1024\n    return os.stat(os.path.join(test_image_dir, id)).st_size/1024\n\ndef get_file_path(is_train, id):\n    if is_train: \n        return os.path.join('train', id)\n    return os.path.join('test', id)\n    \ndef get_ship_size_and_sparse_image(df):\n    if not isinstance(df['EncodedPixels'], str):\n        df['size'] = 0\n        #df['bsr_matrix'] = bsr_matrix((768, 768), blocksize=[6, 6], dtype=int)\n        return df\n    decoded = rle_decode(df['EncodedPixels'], shape=(768, 768))\n    bsr = bsr_matrix(decoded, blocksize=[6,6], shape=(768, 768), dtype=int)\n    \n    # maybe, sometime in the future, i want it back - but for now - this version does not store the bsr_matrix\n    #df['bsr_matrix'] = bsr_matrix(decoded, blocksize=[6,6], shape=(768, 768), dtype=int)\n    df['size'] = bsr.sum()\n    return df\n\ndef aggregate_and_merge(df, what = 'size', do = np.sum, name = 'total'):\n    temp = df.groupby('ImageId').agg({what: do}).reset_index()\n    temp = pd.DataFrame(temp).rename(index=str, columns={what: name})\n    df = df.merge(temp, how='left', on='ImageId')\n    df[name] = df[name].fillna(0).astype(int)\n    return df\n\ndef dask_read_seg(in_batches, max_items = None):\n    d_mask_fun = dask.delayed(masks_as_image)\n    if max_items is None:\n        max_items = len(in_batches)\n    lazy_images = [d_mask_fun(c_masks['EncodedPixels'].values) \n                   for _, (_, c_masks) in zip(range(max_items), in_batches)\n                  ]     # Lazily evaluate on each group\n    s_img = lazy_images[0].compute()\n    arrays = [da.from_delayed(lazy_image,           # Construct a small Dask array\n                              dtype=s_img.dtype,   # for every lazy value\n                              shape=s_img.shape)\n              for lazy_image in lazy_images]\n\n    return da.stack(arrays, axis=0)                # Stack all small Dask arrays into one\n\n\ndef get_domimant_colors(fname, top_colors=2):\n    img = cv2.imread(os.path.join(ship_dir, fname))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n    img = cv2.resize(img, (32, 32))\n    img_l = img.reshape((img.shape[0] * img.shape[1], img.shape[2]))\n    clt = KMeans(n_clusters = top_colors)\n    clt.fit(img_l)\n    \n    # grab the number of different clusters and create a histogram\n    # based on the number of pixels assigned to each cluster\n    numLabels = np.arange(0, len(np.unique(clt.labels_)) + 1)\n    (hist, _) = np.histogram(clt.labels_, bins = numLabels)\n    # normalize the histogram, such that it sums to one\n    hist = hist.astype(\"float\")\n    hist /= hist.sum()\n    \n    return clt.cluster_centers_, hist\n\ndef get_colors_and_merge(df):\n    details = []\n    files = df['file_path'].values\n    \n    for imfile in files:\n        dominant_colors_hsv, dominant_rates_hsv = get_domimant_colors(imfile, top_colors=1)\n        dominant_colors_hsv = dominant_colors_hsv.reshape(1, dominant_colors_hsv.shape[0] \n                                                          * dominant_colors_hsv.shape[1])\n        details.append(dominant_colors_hsv.squeeze())\n    \n    kmeans = KMeans(n_clusters=10).fit(details)\n    df['dominant_colors'] = details\n    df['color_cluster'] = kmeans.predict(details)\n    \n    return df\n    \n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b6847e0218ce495caee116aeb1e30ce64b0d57b"},"cell_type":"markdown","source":"# What is this Kernal all about?\nInspired by the Kernel from [Kevin Mader](https://www.kaggle.com/kmader/package-segmentation-images) i decided to put together an updated Version, with a few usefull additional features. \nIn addition to the repacking of the test and train dataset, i added the dominant colors of each image. I think that could help classify images into \"boat\" and \"no-boat\". For that purpose, i just rip code from the great Kernel from [Costas Voglis](https://www.kaggle.com/voglinio/airbus-ship-detection-clustering-no-ship-images/notebook)\n\nIt's still a work in Progress, but it contains the complete dataset. \n\nIf you didn't have heard already, the annotations for the test-set got released. [Read all about the data-leak, and what happens now](https://www.kaggle.com/c/airbus-ship-detection/discussion/64388)\n\nWell, now get to the good stuff. I changed Kevins code a bit, so that the exported masks all have different numbers - i think its best, if you can reconstruct the original masks without some heavy postprocessing. So keep in mind that the generated masks are not binary anymore. For each mask i'll add to the array, i count one up. \n\nBesides that i thought it would be nice to have next to the count of found ships in the image, the sizes in pixel, and for every picture the median an std of the ship-sizes. This way it should be easy to balance the training-data for small and/or large vessels.\n\nBecause i'm lazy, if also put the image-filepath into the csv file, so that i dont have to worry anymore if an image is train or (former)-test.\n\nPuh, some last addition: I've filtered a few corrupted images out ,-)\nBut lets start.. with some code... \n\n\n*please wait while loading...*"},{"metadata":{"trusted":true,"_uuid":"3c99b165abc56a71238683331d03c711a5dbca4e","_kg_hide-input":true},"cell_type":"code","source":"# load train data and mark them as train \nmasks = pd.read_csv(os.path.join(ship_dir, 'train_ship_segmentations.csv'))\nmasks['train'] = True\n\n# load test data and tell every entry that it is not train data (at least not originaly ,-)\n# in case you wonder, we need that information - so that we know where to load the images\nappend = pd.read_csv(os.path.join(ship_dir, 'test_ship_segmentations.csv'))\nappend['train'] = False\n\n# do i need to comment this? (it gets late, and i need to finish this, i'm not THAT funny - normaly ,-)\nmasks = masks.append(append)\n\n# wuhhh, nice - i don't need to check anymore where i get those images\nmasks['file_path'] = masks.apply(lambda x: get_file_path(x['train'], x['ImageId']), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1557296719235890c20ac9d52922c23049b01564"},"cell_type":"markdown","source":"Ok, **feature compiling** - I'm very eager to know how long that did take: "},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"trusted":true,"_uuid":"8701c0d90de621a4b3e4f95d09a6de23f62cae8a"},"cell_type":"code","source":"%%time \n# here we generate ship_count\nmasks['has_ship'] = masks['EncodedPixels'].map(lambda x: 1 if isinstance(x, str) else 0)\nmasks = aggregate_and_merge(masks, what = 'has_ship', name = 'ship_count')\n\n# with the filesize, i can filter images that are corrupted\nmasks['file_size_kb'] = masks.apply(lambda x: get_file_size(x['train'], x['ImageId']), axis=1)\nmasks = masks[masks['file_size_kb'] > 50] # keep only +50kb files\n\n# shipsize - ahaaand... a blockwise sparse matrix \n# (which i used to calculate the size, a tiny little bit faster)\n# but besides that, it turned out to be useless - RLE encoding an decoding is way (not that much) - way faster\n# as i said, i'm lazy - so i keep it \nmasks = masks.apply(get_ship_size_and_sparse_image, axis=1)\n\n# here we have the total count of target-pixels, or total_ship_size for each image \nmasks = aggregate_and_merge(masks, what = 'size', name = 'total_ship_size')\n\n# with the total_ship_size, and the ship_count - its easy for you to get to the mean \n# but! sometimes, its more interesting, to look if there are outliers in size - or have a feature\n# that isn't that vulnerable for them - hence: here i present, the median_ship_size\nmasks = aggregate_and_merge(masks, what = 'size', do = np.median, name = 'median_ship_size')\n\n# did i mention that it could be interesting to know if there are outliers, to plan your training?\n# well, i think, the standard derivation can also help\nmasks = aggregate_and_merge(masks, what = 'size', do = np.std, name = 'std_ship_size')\nmasks.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"53551c342f3f4a664adabe4d633c9ac7eca5a3e3"},"cell_type":"markdown","source":"So, we are done for the day. If you are interested in all the data - even if i did only use them for a short time..\nand even the *author* does not exactly know what to do with them... \n\nYou find a generated file named **train.csv** containing at least one entry per image, but also one for every mask, if you click on the output-tab.\n\nWell, then take a peak at some *df.sample()*"},{"metadata":{"trusted":true,"_uuid":"9cbcd928fdce9d3f0e3726df18eb1597d11880f8","_kg_hide-input":true},"cell_type":"code","source":"# somehow the order got mixed up\n# if anyone finds where in my code that happend - i would be grateful for a hint\nmasks = masks.set_index('ImageId').reset_index()\nmasks.to_csv('train.csv', index=False)\n\n# since EncodedPixels are so long, we change the display-setting \npd.set_option('max_colwidth', 15)\npd.set_option('display.precision', 2)\n\nmasks.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"027709849b0e92d022a26e66f534c956c7031975"},"cell_type":"markdown","source":"Okay, i said i would keep all the features - but i dont know - RAM and diskspace - isn't *that* cheap \n\nSo if you are interested in the good stuff, but not the strange stuff i worked with.. \n\nIn contrast to the full train.csv, the other generated file only contains one entry per image.\n\nGo and fetch the **reduced_train.csv** from the output-tab, right after you had a look:"},{"metadata":{"trusted":true,"_uuid":"ff9691ba9eee4eaeb33b0d7a2f896d71e5c11f66","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"# preparing a bit early, because we are going to drop EncodedPixels, which we need for Kevins code \nall_batches = list(masks.groupby('ImageId'))\n\n# as i said, here comes the sugar \nmasks.drop_duplicates(['ImageId'], inplace=True)\nmasks.drop(['has_ship', 'file_size_kb', 'train', 'EncodedPixels', 'size'], axis=1, inplace=True)\nmasks.to_csv('reduced_train.csv', index=False)\n\nmasks.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f11a3b298d0aa02a9b67da75c73c0e836a2d744"},"cell_type":"markdown","source":"Did i say reduced? Lets add Color-Features... "},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"37bab7da954572d7d2e644303b11525f442f8543"},"cell_type":"code","source":"%%time \nmasks = get_colors_and_merge(masks)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a11753b80240b327be4b93cd54bf310ea9a99f2"},"cell_type":"markdown","source":"If you like these features, grab the file **color_train.csv**\n\nYou know the drill, lets have a look:"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"a463c13980685299b1980176aed0637ef0f45337"},"cell_type":"code","source":"masks.to_csv('color_train.csv', index=False)\nmasks.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae39bd0b06517e6563c4e3131ab3b59d70aacbfb"},"cell_type":"markdown","source":"It is time to let [Kevin](https://www.kaggle.com/kmader) have the last words..  because, the last bits of this kernel are completly his work. I think it was a nice and convinient idea to package all masks into one file. \n\nThank you Kevin, your Kernels are always a good and inspirational way to learn. \n\nAs Kevin in the original Kernel said: \n\n\n> **Now Package Everything**\n\n> instead of just using a small portion of the dataset we export all the results."},{"metadata":{"trusted":true,"_uuid":"38041b0c9d4b5336ea6e2bfa621fb4bb6a9f415e","_kg_hide-input":true},"cell_type":"code","source":"# larger chunks are more efficient for writing/compressing and make the paralellization more efficient\nlarger_chunker = lambda x: x.rechunk({0: x.shape[0]//400, 1: -1, 2: -1, 3: -1})\nall_img_ds = larger_chunker(dask_read_seg(all_batches))\n\nwith diag.ProgressBar(), diag.Profiler() as prof, diag.ResourceProfiler(0.5) as rprof:\n    with dask.config.set(pool=ThreadPool(4)):\n        all_img_ds.to_hdf5('segmentions.h5', '/image', compression = 'lzf')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}