{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Objective\nObjective of this kernel is \n- To explore the image data set \n- Understand the assets given\n- How to quickly(optimally) load the data\n- Basic exploratory analysis and statistics on the images"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(len(os.listdir(\"../input/test\")))\n\nDEV = False\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"truncated_df = pd.read_csv(\"../input/sample_truncated_submission.csv\")\nempty_df = pd.read_csv(\"../input/sample_empty_submission.csv\")\ntruncated_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom PIL import Image\n\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 9999\npd.options.display.float_format = '{:20, .2f}'.format","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from PIL import Image\nimg = Image.open(\"../input/test/d390310a4ce1c08a.jpg\")\nimg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reusable Methods\nThis section has 3 methods to render, read and explore the image files."},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLES_TO_EXAMINE = 15\nimport cv2\nimport time\nfrom PIL import Image\n\ndef render_images(files):\n    plt.figure(figsize=(50, 50))\n    row = 1\n    for an_image in files:\n        image = cv2.imread(an_image)[..., [2, 1, 0]]\n        plt.subplot(6, 5, row)\n        plt.imshow(image)\n        row += 1\n    plt.show()\n\ndef read_files(files):\n    images = []\n    shapes = []\n    for an_image in files:\n        #image = Image.imread(an_image)[..., [2, 1, 0]]\n        #images.append(image)\n        image = Image.open(an_image)\n        shapes.append((image.size[0], image.size[1], image.layers))\n        \n    return images, shapes\n\ndef get_images(task, files):\n    start_time = time.time()\n    images, shapes = read_files(files)\n    end_time = time.time()\n    print(\"Task: {0}, Duration: {1}, Image Count: {2}, Shape Count: {3}\".format(task, end_time - start_time, len(images), len(shapes)))\n\n    df = pd.DataFrame({\n        'ImageID': [file.split('/')[3].split('.')[0] for file in files],\n        'ImageHeight': [a_shape[0] for a_shape in shapes],\n        'ImageWidth': [a_shape[1] for a_shape in shapes],\n        'channels': [a_shape[2] for a_shape in shapes]\n    })\n    \n    df[\"location\"] = \"../input/test/\" + df[\"ImageID\"] + \".jpg\"\n    \n    return images, df\n#files_to_examine = test_image_files.sample(SAMPLES_TO_EXAMINE)\n#render_images(files_to_examine.files.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Classification by File Size\nThis section sorts the image files by size and classifies into 4 categories and creates a data frame of image properties.\n\n- Small Images\n- Medium Images\n- Large Images\n- Very Large Images"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#import jpeg4py as jpeg\nimport glob\nimport time\nimport random\nstart_time = time.time()\nsorted_files = sorted(glob.glob(\"../input/test/\" + \"*.jpg\"))\nprint(len(sorted_files))\nif(DEV == False): \n    small_files = sorted_files[:30000]\n    medium_files = sorted_files[30000:60000]\n    large_files = sorted_files[60000:90000]\n    very_large_files = sorted_files[90000:]\nelse:\n    small_files = sorted_files[:1000]\n    medium_files = sorted_files[1000:2000]\n    large_files = sorted_files[2000:3000]\n    very_large_files = sorted_files[3000:4000]\n\nlen(very_large_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"small_images, small_df = get_images(\"Read Small Files\", small_files)\nsmall_df[\"size\"] = \"small\"\ndel small_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"medium_images, medium_df = get_images(\"Read Medium Files\", medium_files)\nmedium_df[\"size\"] = \"medium\"\ndel medium_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"large_images, large_df = get_images(\"Read Large Files\", large_files)\nlarge_df[\"size\"] = \"large\"\ndel large_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"very_large_images, very_large_df = get_images(\"Read Very Large Files\", very_large_files)\nvery_large_df[\"size\"] = \"vlarge\"\ndel very_large_images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploration of Image Properties"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"len(small_files) + len(medium_files) + len(large_files) + len(very_large_files)\nimages_df = pd.concat([small_df, medium_df, large_df, very_large_df])\nimages_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"height_distribution = pd.DataFrame(images_df.ImageHeight.value_counts())\nheight_distribution.reset_index(inplace=True)\n\nwidth_distribution = pd.DataFrame(images_df.ImageWidth.value_counts())\nwidth_distribution.reset_index(inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"height_distribution.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly_express as px\npx.histogram(height_distribution, x=\"index\", y=\"ImageHeight\", \n            height=600, width=800)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"px.histogram(width_distribution, x=\"index\", y=\"ImageWidth\", \n            height=600, width=800)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images_df['ratio'] = np.round(images_df['ImageWidth'].divide(images_df['ImageHeight'], fill_value=1))\npx.scatter(images_df, x=\"ImageWidth\", y=\"ImageHeight\", \n           color=\"ratio\", height=1000, width=800, \n           marginal_x=\"histogram\", marginal_y=\"histogram\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(images_df))\nfiles_to_examine = random.sample(medium_files, SAMPLES_TO_EXAMINE)\nrender_images(files_to_examine)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}