{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# Credit to this kernel in Whale Identification - https://www.kaggle.com/stehai/duplicate-images. ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Importing the required Libraries. "},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport os\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport imagehash\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reading the CSV File"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking the Unique Image Names. There are 3662 unique training examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.id_code.nunique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Getting the path of the Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_image_dir = os.path.join('..', 'input/') # Joining the base directory Input\ntrain_dir = os.path.join(base_image_dir,'train_images/') # Training Directory Location\ndf = pd.read_csv(os.path.join(base_image_dir, 'train.csv')) # Reading the training file\ndf['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x))) # Getting the path of the image\ndf = df.drop(columns=['id_code']) # Dropping the Id code column\ndf = df.sample(frac=1).reset_index(drop=True) \ndf.head(10) # Getting the top 10 records","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating the Hash, Shape, Mode, Length and Ratio of each image"},{"metadata":{"trusted":true},"cell_type":"code","source":"def getImageMetaData(file_path):\n    with Image.open(file_path) as img:\n        img_hash = imagehash.phash(img)\n        return img.size, img.mode, img_hash\n\ndef get_train_input():\n    train_input = df.copy()\n        \n    m = train_input.path.apply(lambda x: getImageMetaData(x))\n    train_input[\"Hash\"] = [str(i[2]) for i in m]\n    train_input[\"Shape\"] = [i[0] for i in m]\n    train_input[\"Mode\"] = [str(i[1]) for i in m]\n    train_input[\"Length\"] = train_input[\"Shape\"].apply(lambda x: x[0]*x[1])\n    train_input[\"Ratio\"] = train_input[\"Shape\"].apply(lambda x: x[0]/x[1])\n    \n    \n    img_counts = train_input.path.value_counts().to_dict()\n    train_input[\"Id_Count\"] = train_input.path.apply(lambda x: img_counts[x])\n    return train_input\n\ntrain_input = get_train_input()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input1 = train_input[['Hash']] # Getting the Hash from the new data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input1['New']=1 # Creating a dummy column 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input2 = train_input1.groupby('Hash').count().reset_index() # Grouping the column by Hash to aggregate at Hash level","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input2 = train_input2[train_input2['New']>1] # Filtering those instances where the hash is occuring multiple times","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input2.shape # Checking the shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input2 = train_input2.sort_values('Hash') # Sorting the data by Hash ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input2.head(5) # Checking the top 5 records","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input[train_input['Hash']=='808d0f3513f33f2d'] # FIltering only one Hash from master data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom IPython import display\nimport time\n\n%matplotlib inline\n\nPATH = \"../input/train_images/1632c4311fc9.png\"\nimage = mpimg.imread(PATH) # images are color images\nplt.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom IPython import display\nimport time\n\n%matplotlib inline\n\nPATH = \"../input/train_images/a75bab2463d4.png\"\nimage = mpimg.imread(PATH) # images are color images\nplt.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}