{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nProject Title: Image Steganalysis to detect secret data based on similarity-probability prediction sourced from an RGB-triad scorer\n\nHeadrunner Title: Detection based on RGB-triad scorer \n\"\"\"\n\n\"\"\"\nKey employed libraries:\n\nos | Miscellaneous operating system interfaces: https://docs.python.org/3/library/os.html\npandas | https://pandas.pydata.org/\nmath | Mathematical Functions | https://docs.python.org/3/library/math.html\nNumPy | https://numpy.org/\nimageio | Interface to read and write a wide range of image data| https://pypi.org/project/imageio/\ncv2 |  Open Source Computer Vision (OpenCVopencv-python 4.2.0.34 | https://pypi.org/project/opencv-python/\n\n\"\"\"\n\n\"\"\"\nPros\n(1) Brute force approach with sampling option\n(2) Data reduction based on a scorer to estimate for detecting\n(2) Prediction with Machine Learning models for classification\n\n\"\"\"\n\n\"\"\"\nCons\n(1) Test data set (Cover images) would require to be processed with the algorithms JMiPOD, JUNIWARD, UERD to get true values.\n(2) AUC Filtering could be more accurate regarding the testing task if point 1 would be met since it can be input both true values and predicted values.\n(3) Data reduction is based on a RGB-triad scorer; the scorer can be excesively representative because of the real-pixel configuration; i.e. 1 condensed-RGB-triad (mean of image triads within an RGB image) would represent an RGB image of 100x100 whic was resized from a 512x512 one, previously.\n\n\"\"\"\n\n\n\"\"\"\nGeneral notes of the coder:\n\n(Note 1) The current code has been adapted to IaaS resources provided by Kaggle from 2 files created using on-premise resources to prototype.\n\n(Note 2) The files are as follows: (1) ImageSimilary.py; (2) KaggleSolution2.py.\n\n(Note 3) The current code does not pretend to process all the competition data.\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nSet default mode '/kaggle/working/'\n\"\"\"\nimport shutil\n\nprint('\\nHi there.')\ndefault = input('\\nDo you want to set the Kaggle resources to default mode? (y/n)')\n\nif (default == 'y' or normTask == 'Y'):\n    \n    try:\n        path1 = '/kaggle/working/output/'\n        shutil.rmtree(path1, ignore_errors=False, onerror=None)\n    except FileNotFoundError: #directory does not exist\n        pass\n    \n    try: \n        path2 = '/kaggle/working/input/'\n        shutil.rmtree(path2, ignore_errors=False, onerror=None) \n    except FileNotFoundError: #directory does not exist\n        pass\n    \n    print('\\nDefault mode has been set.')\n\nelse:\n    print('\\nThanks for your time... Bye')\n\n\"\"\"\nReference:\n\nHow do I remove/delete a folder that is not empty? (n.d.). Retrieved from\n    https://stackoverflow.com/questions/303200/how-do-i-remove-delete-a-folder-that-is-not-empty/303225#303225\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nLoad Input image files of the competition\n\nPlease add data manually coresponding to the following competition:\n\nALASKA2 Image Steganalysis\n\n\"\"\"\n\n# Input directories\n\ninput_Cover = '/kaggle/input/alaska2-image-steganalysis/Cover/'\ninput_JMiPOD = '/kaggle/input/alaska2-image-steganalysis/JMiPOD/'\ninput_JUNIWARD = '/kaggle/input/alaska2-image-steganalysis/JUNIWARD/'\ninput_UERD = '/kaggle/input/alaska2-image-steganalysis/UERD/'\ninput_Test = '/kaggle/input/alaska2-image-steganalysis/Test/'\ndirectoryListInput = [input_Cover]+[input_JMiPOD]+[input_JUNIWARD]+[input_UERD]+[input_Test]\n\n# Number of Input Images and set of sample number:\n\ndef SampleFilesToCompete(directoryListInput, trainSample, testSample):\n\n    import os\n    \n    print('\\n*******************************************************************************************************************')\n    print('Sampling Method: Aleatory extraction')\n    print('*******************************************************************************************************************')\n    \n    input_Test = '/kaggle/input/alaska2-image-steganalysis/Test/'\n    KaggleInputList = []\n    \n    for i in directoryListInput:\n        \n        listy = os.listdir(i)\n        \n        if i == input_Test:\n            listySample = listy[0:testSample]\n        else:\n            listySample = listy[0:trainSample]\n        \n        print('\\nSourced Directory of Kaggle: ',i)\n        print('\\nExpected Number of Files to be extracted: \\n',len(listySample), ' from ', len(listy), ' available instances')\n        \n        KaggleInputList = KaggleInputList + [listySample]\n        \n    \n    print('\\n*******************************************************************************************************************')\n    print('Sample Features:')\n    print('*******************************************************************************************************************')\n    print('\\nTest-sample instances number: ', testSample)\n    print('\\nTrain-sample instances number: ', trainSample)\n    print('\\nDirectory Input List to compete sourced from Kaggle',directoryListInput)\n    print('\\nSample Files per directory',KaggleInputList)\n    \n    #pairSample = [directoryListInput, KaggleInputList]\n    \n    return directoryListInput, KaggleInputList\n\ntrainSample = input('\\nPlease insert a sample-number of instances for training (0-1000) non-test-images:')\ntrainSample = int(trainSample)\ntestSample = input('\\nPlease insert a sample-number of instances for testing (0-5000) test-images:')\ntestSample = int(testSample)\n\n#trainSample = 10\n#testSample = 10\ndirectoryListInput, KaggleInputList = SampleFilesToCompete(directoryListInput,trainSample,testSample)\n              \n\n\"\"\"\nCreate directories to prototype with input image files of the competition data (prototype inputs)\n\"\"\"\n\n# Required directories\n\npath_generic1 = '/kaggle/working/input/'\npath_generic2 = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/'\n\npath_Cover = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Cover/'\npath_JMiPOD = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JMiPOD/'\npath_JUNIWARD = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JUNIWARD/'\npath_UERD = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/UERD/'\npath_Test = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Test/'\n\ndirectoryList = [path_generic1]+[path_generic2]+[path_Cover]+[path_JMiPOD]+[path_JUNIWARD]+[path_UERD]+[path_Test]\ndirectoryListG = [path_Cover]+[path_JMiPOD]+[path_JUNIWARD]+[path_UERD]+[path_Test]\n\nprint('\\n*******************************************************************************************************************')\nprint('Required directories to create')\nprint('*******************************************************************************************************************')\nprint('\\nDirectories to create:','\\n', directoryList)\n\n# Function directoryCreation(directoryList)\n\ndef directoryCreation(directoryList):\n    \n    import os\n\n    print('\\n*******************************************************************************************************************')\n    print('Created or existing directories')\n    print('*******************************************************************************************************************')\n    \n    for i in directoryList:\n    \n        try:\n            os.mkdir(i)\n        except FileExistsError: # directory already exists\n            pass\n        \n        print('\\n Created or existing directory: ',i)\n    \n    print('\\nDirectories are created and available to upload data...')\n    \n    return\n\n# Run\n\ndirectoryCreation(directoryList)\ndirectoryList.remove(path_generic1)\ndirectoryList.remove(path_generic2)\n\n\"\"\"\nLoad Input image files to prototype\n\"\"\"\n\n# Function\n\ndef SampleFilesExtraction(directoryListInput, directoryListOutput, fileList):\n\n    # Set working directory\n    \n    # Copy task\n    \n    # Based partially in Copy all JPG file in a directory to another directory in Python? (n.d.).\n    \n    print('\\n*******************************************************************************************************************')\n    print('Kaggle images extraction for sampling')\n    print('*******************************************************************************************************************')\n    print('\\n')\n    \n    import shutil, os\n    \n    for i in range (0,len(directoryListInput)):\n    \n        print('Origin Directory: ', directoryListInput[i])\n        print('Destination Directory: ', directoryListOutput[i])\n    \n        for f in fileList[i]:\n\n            try: \n                # Set origin directory\n                origin = directoryListInput[i]\n                os.chdir(origin)\n\n                # Copy task\n                # Based partially in Copy all JPG file in a directory to another directory in Python? (n.d.).\n                destination = directoryListOutput[i]\n                shutil.copy(f, destination)\n            \n            except FileExistsError: # file already exists\n                os.removedirs(f)\n                \n                # Set origin directory\n                origin = directoryListInput[i]\n                os.chdir(origin)\n\n                # Copy task\n                # Based partially in Copy all JPG file in a directory to another directory in Python? (n.d.).\n                destination = directoryListOutput[i]\n                shutil.copy(f, destination)\n                \n                pass\n            \n            print('  Extracted File: ', f)\n        print('\\n')\n    \n    return\n\n# Run\ndirectoryListOutput = directoryList\ndirectoryList, fileList = directoryListInput, KaggleInputList\nSampleFilesExtraction(directoryListInput, directoryListOutput, fileList)\n\n\n\"\"\"\nReference:\n\nCopy all JPG file in a directory to another directory in Python? (n.d.). Retrieved from\n    https://stackoverflow.com/questions/11903037/copy-all-jpg-file-in-a-directory-to-another-directory-in-python\n\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCreate directories to prototype with input image files of the competition data (prototype outputs)\n\"\"\"\n\n# Required directories\n\npath_generic1 = '/kaggle/working/output/'\npath_generic2 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/'\n\npath_Cover = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\npath_JMiPOD = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/'\npath_JUNIWARD = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/'\npath_UERD = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/'\npath_Test = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\n\ndirectoryList = [path_generic1]+[path_generic2]+[path_Cover]+[path_JMiPOD]+[path_JUNIWARD]+[path_UERD]+[path_Test]\ndirectoryListG = [path_Cover]+[path_JMiPOD]+[path_JUNIWARD]+[path_UERD]+[path_Test]\n\nprint('\\n*******************************************************************************************************************')\nprint('Required directories to create')\nprint('*******************************************************************************************************************')\nprint('\\nDirectories to create:','\\n', directoryList)\n\n# Function directoryCreation(directoryList)\n\ndef directoryCreation(directoryList):\n    \n    import os\n\n    print('\\n*******************************************************************************************************************')\n    print('Created or existing directories')\n    print('*******************************************************************************************************************')\n    \n    for i in directoryList:\n    \n        try:\n            os.mkdir(i)\n        except FileExistsError: # directory already exists\n            pass\n        \n        print('\\n Created or existing directory: ',i)\n    \n    print('\\nDirectories are created and available to upload data...')\n    \n    return\n\n# Run\n\ndirectoryCreation(directoryList)\ndirectoryList.remove(path_generic1)\ndirectoryList.remove(path_generic2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Thu Jun 03 14:47:16 2020\n\n@author: Mauricio Azálgara Bedoya\n\nKaggle user account: mazalgarab\n\n\"\"\"\n\n\n\"\"\"\nimageSimilarity.py\nversion adapted from orginal version\n\"\"\"\n\ndef image_path_input_list():\n    \n    coverImageTypeList = ['JMiPOD','JUNIWARD','UERD','Cover', 'Test']\n    \n    image_path_input_list = []\n    root = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/'\n    \n    for i in coverImageTypeList:\n        \n        component = root + i + '/'\n        image_path_input_list = image_path_input_list + [component]\n    \n    return image_path_input_list\n\n\ndef image_path(coverImageType):\n\n    coverImageTypeList = ['JMiPOD','JUNIWARD','UERD','Cover', 'Test']\n    \n    if(coverImageType == coverImageTypeList[0]):\n        image_path = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JMiPOD/'\n    if(coverImageType == coverImageTypeList[1]):\n        image_path = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JUNIWARD/'\n    if(coverImageType == coverImageTypeList[2]):\n        image_path = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/UERD/'\n    if(coverImageType == coverImageTypeList[3]):\n        image_path = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Cover/'\n    if(coverImageType == coverImageTypeList[4]):\n        image_path = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Test/'\n    \n    return image_path\n\n\ndef image_list_bulk(image_path_input_list):\n    \n    import os\n    \n    image_list_bulk = []\n    \n    for i in image_path_input_list:\n    \n        component = os.listdir(i)\n        image_list_bulk = image_list_bulk + [component]    \n    \n    return image_list_bulk\n\n\ndef reading_image_raw(filename, image_path):\n    \n    import imageio\n    ## Reading an image file | (Rashka & Mirjalili, 2019, pp.532-536)\n    ### Change1: Insert variable 'filename' for the filename in the form 'example-image.png'.\n    ### Change2: Built into Function reading_image_raw(filename)       \n    img = imageio.imread(image_path+filename)\n    \n    print('Image shape:', img.shape)\n    print('Number of channels:', img.shape[2])\n    print('Image data type:', img.dtype)\n    print(img[100:102, 100:102, :])\n    \n    return img\n\n\ndef normalizing_image(imageio):\n    \n    import cv2\n    ## Normalizing image | Convert numpy.ndarray into imageio.core.util.Image | (Rothman et al., 2018, pp. 470-479)\n    ## OpenCV | Smoothing Images | Image Blurring | (OpenCV, n.d.)\n    ## OpenCV | Geometric Transformations of Images | Scaling (OpenCV, n.d.)\n    ## OpenCV | Geometric Image Transformations | Image procecssing | resize()  | (OpenCV, n.d.)\n    ## From 512x512 pixels to 100x100 pixels | Adobe criteria to size (Adobe,n.d.) \n    imageio_re = cv2.resize(imageio, (100, 100), interpolation = cv2.INTER_AREA) \n    imageio_blur = cv2.GaussianBlur(imageio_re, (5, 5), 0)\n    \n    return imageio_blur\n\ndef imageio_array_to_DataFrame(imageio):\n    \n    import pandas as pd\n    import numpy as np\n    \n    si,sj,sk = np.shape(imageio)\n    numy = 0\n    \n    for i in range(0,si):\n        \n        if (i == 0):\n            colNames = [str(numy)] + [str(numy+1)] + [str(numy+2)]\n            numy = numy + 3\n            df_left = pd.DataFrame(imageio[i], columns = colNames)\n        \n        else:\n            colNames = [str(numy)] + [str(numy+1)] + [str(numy+2)]\n            numy = numy + 3\n            df_right = pd.DataFrame(imageio[i], columns = colNames) \n            df_left = pd.merge(df_left,df_right,left_index=True, right_index=True)\n    \n    return df_left\n\n\ndef image_resize_bulk_unitary(image_path_input_list,image_list_bulk,bulk):\n    \n    import pandas as pd\n    \n    # Set output directory for normalized images | image_path_output_list\n    \n    image_path1 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/'\n    image_path2 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/'\n    image_path3 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/'\n    image_path4 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\n    image_path5 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\n    image_path_output_list = [image_path1] +[image_path2] + [image_path3] + [image_path4] + [image_path5]\n    \n    ## Sequence: 'JMiPOD','JUNIWARD','UERD','Cover', 'Test'\n    ## bulk = [0,1,2,3,4]\n    \n    i_input = image_path_input_list[bulk]\n    i_output = image_path_output_list[bulk]\n    image_path_output = i_output\n    \n    j = image_list_bulk[bulk]\n    \n    for k in j:\n        \n        imageio = reading_image_raw(k,i_input)\n        image_array = normalizing_image(imageio)\n        df = imageio_array_to_DataFrame(image_array)\n        filename_output = k[:-4] + '.csv'\n        df.to_csv(image_path_output+filename_output,sep=',',index=False,header=False)\n                    \n    return\n\n\n\"\"\"\nRun (Batching task)\n\"\"\"\nnormTask = input('\\nDo you want to start the image normalizing task for the image sample ? : (y/n)')\nnormTask = str(normTask)\n\n\nif (normTask == 'y' or normTask == 'Y'):\n\n    ## image_path_input_list\n    image_path_input_list = image_path_input_list()\n    \n    ## image_list_bulk\n    image_list_bulk = image_list_bulk(image_path_input_list)\n    \n    ## image_resize_bulk_unitary\n    bulk = 0 #JMiPOD\n    image_resize_bulk_unitary(image_path_input_list,image_list_bulk,bulk)\n    bulk = 1 #JUNIWARD\n    image_resize_bulk_unitary(image_path_input_list,image_list_bulk,bulk)\n    bulk = 2 #UERD\n    image_resize_bulk_unitary(image_path_input_list,image_list_bulk,bulk)\n    bulk = 3 #Cover\n    image_resize_bulk_unitary(image_path_input_list,image_list_bulk,bulk)\n    bulk = 4 #test\n    image_resize_bulk_unitary(image_path_input_list,image_list_bulk,bulk)\n    \n    print('\\nImage normalizing complete for the sample.')\n\nelse:\n    print('\\nThanks for your time... Bye')\n    \n\"\"\"\nBibliography:\n\nRaschka, S., & Mirjalili, V. (2019). Python Machine Learning: Machine Learning\n    and Deep Learning with Python, scikit-learn, and TensorFlow 2 (3rd ed.).\n    Birmingham, United Kingdom: Packt Publishing Ltd.\n    \nRothman, D., Lamons, M., Kumar, R., Nagaraja, A., Ziai, A., & Dixit, A. (2018).\n    Python: Beginner's Guide to Artificial Intelligence: Build applications to\n    intelligently interact with the world around you using Python (1st ed.).\n    Birmingham, United Kingdom: Packt Publishing Ltd.\n    \nRefrences:\n\nAdobe. n.d. Resize and Crop images in Photoshop and Photoshop Elements. Retrieved from\n    https://www.adobe.com/support/techdocs/331327.html    \n\nConvert numpy.ndarray into imageio.core.util.Image. n.d. Retrieved from\n    https://stackoverflow.com/questions/49269327/convert-numpy-ndarray-into-imageio-core-util-image/60888547#60888547    \n\nOpenCV. n.d. Geometric Image Transformations. Retrieved from\n    https://docs.opencv.org/trunk/da/d54/group__imgproc__transform.html#ga47a974309e9102f5f08231edc7e7529d\n\nOpenCV. Geometric Transformations of Images. n.d. Retrieved from\n    https://docs.opencv.org/trunk/da/d6e/tutorial_py_geometric_transformations.html\n\nOpenCV. resize(). Retrieved from\n    https://docs.opencv.org/trunk/da/d54/group__imgproc__transform.html#ga47a974309e9102f5f08231edc7e7529d\n\nOpenCV. n.d. Smoothing Images. Retrieved from\n    https://docs.opencv.org/master/d4/d13/tutorial_py_filtering.html\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nRe set initial sample size\n\"\"\"\nprint('\\n************************************************************************************************************************************')\ntrainSample = input('\\nPlease re insert the initial sample-number of instances for training (0-1000) non-test-images:')\ntrainSample = int(trainSample)\ntestSample = input('\\nPlease re insert the initial sample-number of instances for testing (0-5000) test-images:')\ntestSample = int(testSample)\nprint('\\n************************************************************************************************************************************')\n\n\"\"\"\nInput Directory scheme and expected files\n\"\"\"\n\n# Competition Data\nCD = '/kaggle/working/input/alaska2-image-steganalysis/'\n\n# Competition Data to prototype (10 images per algorithm and 10 cover images)\nCD_ToPrototype = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/'\n\n## Cover\nCD_ToPrototype_Cover = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Cover/'\n\n## JUNIWARD\nCD_ToPrototype_JUNIWARD = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JUNIWARD/'\n\n## UERD\nCD_ToPrototype_UERD = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/UERD/'\n\n## JMiPOD\nCD_ToPrototype_JMiPOD = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JMiPOD/'\n\n## Test\nCD_ToPrototype_Test = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Test/'\n\n\n\ndef verification_Input(CD_ToPrototype_Cover,CD_ToPrototype_JUNIWARD,CD_ToPrototype_UERD,CD_ToPrototype_JMiPOD,CD_ToPrototype_Test,trainSample,testSample):\n\n    import os\n    \n    print('\\nVerification for Input 1: Image sample to prototype')\n    \n    InputList = [CD_ToPrototype_Cover]+[CD_ToPrototype_JUNIWARD]+[CD_ToPrototype_UERD]+[CD_ToPrototype_JMiPOD]\n\n    filesNumber = []\n\n    print('\\n Process: Non-Test Image verification status')\n\n    #### Non-Test Image verification status\n    \n    for i in InputList:\n\n        filesList = os.listdir(i)\n        filesNumber = filesNumber + [len(filesList)]\n        print('\\n   Directory: ',i)\n        print('\\n   Number of files:',len(filesList))\n\n    if sum(filesNumber)/4 == trainSample:\n        print('\\n   ||Input Non-Test-Image verification status: Ok')\n    else:\n        print('\\n   ||Files verfication status: Required to be reviewed')\n\n    #### Test Image verification status    \n\n    i = os.listdir(CD_ToPrototype_Test)\n    \n    print('\\n Process: Test Image verification status')\n    print('\\n   Directory: ',CD_ToPrototype_Test)\n    print('\\n   Number of files:',len(i))\n\n    if len(os.listdir(CD_ToPrototype_Test)) == testSample:\n        print('\\n   ||Input Test-Image verfication status: Ok')\n    else:\n        print('\\n   ||Files verfication status: Required to be reviewed')\n    \n    return\n    \n\"\"\"\nRun\n\"\"\"\nprint('\\n==================================================================================================================================================')\nverification_Input(CD_ToPrototype_Cover,CD_ToPrototype_JUNIWARD,CD_ToPrototype_UERD,CD_ToPrototype_JMiPOD,CD_ToPrototype_Test,trainSample,testSample)\nprint('\\n==================================================================================================================================================')    \n    \n\"\"\"\nOutput 1: Output Directory and expected files (/kaggle/working/)\n\"\"\"\n\n# /kaggle/working\n\nKW = '/kaggle/working/'\n\n# Generic Directory: OutPut \n\nGD = '/kaggle/working/output/'\n\n# Normalized Images (Generic)\nNI = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/'\n\n## Normalized Images | Cover\nNI_Cover = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\n\n## Normalized Images | JUNIWARD\nNI_JUNIWARD = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/'\n\n## Normalized Images | UERD\nNI_UERD = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/'\n\n## Normalized Images | JMiPOD\nNI_JMiPOD = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/'\n\n## Normalized Images | Test\nNI_Test = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\n\n### Files Verification\n\n\ndef verification_Output1(NI_Cover,NI_JUNIWARD,NI_UERD,NI_JMiPOD,NI_Test,trainSample,testSample):\n\n    import os\n    \n    print('\\nVerification for Output 2: Image instances after being normalized')\n    \n    OutputList = [NI_Cover]+[NI_JUNIWARD]+[NI_UERD]+[NI_JMiPOD]\n\n    filesNumber = []\n\n    print('\\n Process: Non-File Test verification status')\n\n    #### Non-File Test verification status\n    \n    for i in OutputList:\n\n        filesList = os.listdir(i)\n        filesNumber = filesNumber + [len(filesList)]\n        print('\\n   Directory: ',i)\n        print('\\n   Number of files:',len(filesList))\n\n    if sum(filesNumber)/4 == trainSample:\n        print('\\n   ||Input non-Test Files verfication status: Ok')\n    else:\n        print('\\n   ||Files verfication status: Required to be reviewed')\n\n    #### File Test verification status    \n\n    i = os.listdir(NI_Test)\n    \n    print('\\n Process: File Test verification status')\n    print('\\n   Directory: ',NI_Test)\n    print('\\n   Number of files:',len(i))\n\n    if len(os.listdir(NI_Test)) == testSample:\n        print('\\n   ||Input Test Files verfication status: Ok')\n    else:\n        print('\\n   ||Files verfication status: Required to be reviewed')\n    \n    return\n\n\"\"\"\nRun\n\"\"\"\nprint('\\n==================================================================================================================================================')\nverification_Output1(NI_Cover,NI_JUNIWARD,NI_UERD,NI_JMiPOD,NI_Test,trainSample,testSample)\nprint('\\n==================================================================================================================================================')\n\n\"\"\"\nOutput directory viewer\n\"\"\"\n\n\na = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JMiPOD/'\n   \nb = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/JUNIWARD/'\n   \nc = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/UERD/'\n    \nd = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Cover/'\n \ne = '/kaggle/working/input/inputtoprototype-alaska2imagesteganalysis/Test/'\n\n\n\n#################################################################\n\na1 = '/kaggle/working/OutPut'\n\na2 = '/kaggle/working/OutPut/JMiPOD/'\n\na3 = '/kaggle/working/OutPut/JUNIWARD/'\n\na4 = '/kaggle/working/OutPut/UERD/'\n\na5 = '/kaggle/working/OutPut/Cover/'\n\na6 = '/kaggle/working/OutPut/Test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCreate directories to prototype with input image files of the competition data (prototype outputs | scores)\n\"\"\"\n\n# Function directoryCreation(directoryList)\n\ndef directoryCreation(directoryList):\n    \n    import os\n\n    print('\\n*******************************************************************************************************************')\n    print('Created or existing directories')\n    print('*******************************************************************************************************************')\n    \n    for i in directoryList:\n    \n        try:\n            os.mkdir(i)\n        except FileExistsError: # directory already exists\n            pass\n        \n        print('\\n Created or existing directory: ',i)\n    \n    print('\\nDirectories are created and available to upload data...')\n    \n    return\n\n# Run\n\n## Required directories | Batch 1: Scores\n\npath_generic1 = '/kaggle/working/output/'\npath_generic2 = '/kaggle/working/output/scores/'\n\npath_ImageSimilarity = '/kaggle/working/output/scores/ImageSimilarity/'\npath_ImageSimilarityScore = '/kaggle/working/output/scores/ImageSimilarityScore/'\npath_PixelDeviationScore = '/kaggle/working/output/scores/PixelDeviationScore/'\npath_Fusion = '/kaggle/working/output/scores/Fusion/'\n\ndirectoryList1 = [path_generic1]+[path_generic2]+[path_ImageSimilarity]+[path_ImageSimilarityScore]+[path_PixelDeviationScore]+[path_Fusion]\ndirectoryListG1 = [path_ImageSimilarity]+[path_ImageSimilarityScore]+[path_PixelDeviationScore]\n\nprint('\\n*******************************************************************************************************************')\nprint('Required directories to create | Batch 1: Scores')\nprint('*******************************************************************************************************************')\nprint('\\nDirectories to create:','\\n', directoryList)\n\ndirectoryCreation(directoryList1)\ndirectoryList1.remove(path_generic1)\ndirectoryList1.remove(path_generic2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Thu Jun 08 13:38:00 2020\n\n@author: Mauricio Azálgara Bedoya\n\nKaggle user account: mazalgarab\n\n\"\"\"\n\n\"\"\"\nKaggleSolution2.py\nVersion adapted from orginal version\n\"\"\"\n\n\ndef imageio_path_input_list():\n    \n    coverImageTypeList = ['JMiPOD','JUNIWARD','UERD','Cover']\n    \n    imageio_path_input_list = []\n    root = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/'\n    \n    \n    for i in coverImageTypeList:\n        \n        component = root + i + '_ImageNormalized/'\n        imageio_path_input_list = imageio_path_input_list + [component]\n    \n    return imageio_path_input_list\n\n\ndef imageioFile_path(coverImageType):\n\n    coverImageTypeList = ['JMiPOD','JUNIWARD','UERD','Cover', 'Test']\n    \n    if(coverImageType == coverImageTypeList[0]):\n        imageioFile_path = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/'\n        \n    if(coverImageType == coverImageTypeList[1]):\n        imageioFile_path = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/'\n    if(coverImageType == coverImageTypeList[2]):\n        imageioFile_path = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/'\n    if(coverImageType == coverImageTypeList[3]):\n        imageioFile_path = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\n    if(coverImageType == coverImageTypeList[4]):\n        imageioFile_path = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\n    \n    return imageioFile_path\n\n\ndef imageio_list_bulk(imageioFile_path):\n    \n    import os\n    \n    imageio_list_bulk = []\n    \n    for i in [imageioFile_path]:\n    \n        component = os.listdir(i)\n        imageioFile_list_bulk = imageio_list_bulk + component\n    \n    return imageioFile_list_bulk\n\n\ndef PixelDeviationScore(start,termination,joinBaseField):\n\n    # TargetField\n    \n    BaseField = 'Cover'\n    TargetField = ['JMiPOD','JUNIWARD','UERD']\n    \n    import os\n    import pandas as pd\n\n    # Read CSV files\n\n    root_baseFieldInput = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/' #imageioFile_path(BaseField)\n    root_targeFieldtInput1 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/' #imageioFile_path(TargetField)\n    root_targeFieldtInput2 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/' #imageioFile_path(TargetField)\n    root_targeFieldtInput3 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/' #imageioFile_path(TargetField)\n    root_output = '/kaggle/working/output/scores/PixelDeviationScore/'\n    \n    filenameBaseFieldList = imageio_list_bulk(root_baseFieldInput)\n    \n    # Set batch\n    filenameBaseFieldList = filenameBaseFieldList[start:termination]\n    \n    \n    for i in filenameBaseFieldList:\n        \n        BaseField = pd.read_csv(root_baseFieldInput+i,index_col=None, header=None)\n        TargetField1 = pd.read_csv(root_targeFieldtInput1+i,index_col=None, header=None)\n        TargetField2 = pd.read_csv(root_targeFieldtInput2+i,index_col=None, header=None)\n        TargetField3 = pd.read_csv(root_targeFieldtInput3+i,index_col=None, header=None)\n        \n        # PixelDeviationScore computation\n        PixelDeviationScore1 = abs(BaseField - TargetField1)\n        PixelDeviationScore2 = abs(BaseField - TargetField2)\n        PixelDeviationScore3 = abs(BaseField - TargetField3)\n        \n        PixelDeviationScore = (PixelDeviationScore1 + PixelDeviationScore2 + PixelDeviationScore3)/(3*BaseField)\n        \n        # Build DataFrame\n        df = PixelDeviationScore\n        \n        # Join BaseField\n        if joinBaseField == True:\n            df = pd.merge(BaseField,df,left_index=True, right_index=True)\n            df.to_csv(root_output+i,sep=',',index=False,header=False)\n            print('File created (Pixel Deviation Score): '  ,i)\n        else:\n            df.to_csv(root_output+i,sep=',',index=False,header=False)\n            print('File created (Pixel Deviation Score): '  ,i)\n\n\n    return # print('Files generated:', os.listdir(root_output))\n\n\ndef round_up(n, decimals=0):\n    \n    import math\n    \n    ## Sourced (Amos, n.d.)\n    ## Sourced (Solving ValueError: cannot convert float NaN to integer, n.d.)\n    \n    multiplier = 10 ** decimals\n    \n    try:\n        result = math.ceil(n * multiplier) / multiplier\n    except ValueError:\n        result = int(0)\n    except OverflowError:\n        result = int(1)\n    \n    return result\n\n\ndef round_up_df(df, decimals=0):\n    \n    row,col = df.shape\n    \n    for i in range(0,col):\n        \n        for j in range(0,row):\n        \n            if df[i][j] != 0: df[i][j] = round_up(df[i][j],0)\n    \n    return df\n\n\ndef imageio_similarity_score(df):\n               \n    # Requires a binary Dataframe\n    \n    mean_series = df.mean()\n    score = mean_series.mean()\n    \n    return score\n\n\ndef image_fusion_score(df):\n    \n    # Theory 1: Average score per value within triad sets of the RGB image (Assumption: Predominant color)\n    \n    rows,cols = df.shape\n    \n    resultCols = []\n    resultTriad_1 = 0\n    resultTriad_2 = 0\n    resultTriad_3 = 0\n    numy = 0\n    \n    for i in range (0,cols):\n        \n        component = df[i].mean()\n        resultCols = resultCols + [component]\n    \n    limit = int(cols/3)\n    \n    for i in range(0,limit):    \n   \n        x = i + numy\n        y = i + 1 + numy\n        z = i + 2 + numy\n                \n        resultTriad_1 = resultTriad_1 + resultCols[x]\n        resultTriad_2 = resultTriad_2 + resultCols[y]\n        resultTriad_3 = resultTriad_3 + resultCols[z]\n        \n        numy = 2 * (i+1)\n    \n    triad = [int(resultTriad_1/cols)] + [int(resultTriad_2/cols)] + [int(resultTriad_3/cols)]\n            \n    return triad\n\n\ndef SimilarityScore(start,termination):\n\n    import os\n    import pandas as pd\n    import numpy as np\n\n    # Define data roots     \n\n    root_input = '/kaggle/working/output/scores/PixelDeviationScore/'    \n    root_input_basefield = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\n    root_output = '/kaggle/working/output/scores/ImageSimilarity/'\n    root_output_sc = '/kaggle/working/output/scores/ImageSimilarityScore/'\n    \n    \n        \n    # Set the input files (PixelDeviationScore)\n    \n    filenameBaseFieldList = imageio_list_bulk(root_input)\n    filenameBaseFieldList = filenameBaseFieldList[start:termination]\n    \n    ImageioSimilarityScoreList = []\n    ImageFusionScoreList_R = []\n    ImageFusionScoreList_G = []\n    ImageFusionScoreList_B = []\n    \n    selectedCols1 = []\n    selectedCols2 = []\n    newColumnsNames = {}\n    \n    \n    # Create List\n    \n    for i in range (300,600):\n        #num = str(i)\n        selectedCols1 = selectedCols1 + [i]\n        selectedCols2 = selectedCols2 + [i-300]\n        newColumnsNames[i]=i-300\n        \n    # Run Loop\n    \n    #print('\\n Cover images processed (i.e. BaseField) as follows: ')\n    #print('\\n')\n    \n    for i in filenameBaseFieldList:\n        \n        # ImageSimilarityScore Case\n        \n        dfImageio = pd.read_csv(root_input+i,index_col=None, header=None, usecols=selectedCols1) #Imported columns from 301 to 600 \n        dfImageio = dfImageio.rename(columns=newColumnsNames)\n        dfImageio = round_up_df(dfImageio,0)\n        df = dfImageio\n        df.to_csv(root_output+i,sep=',',index=False,header=False) #Binary DataFrame\n        ImageioSimilarityScore = imageio_similarity_score(df)\n        ImageioSimilarityScoreList = ImageioSimilarityScoreList + [ImageioSimilarityScore]\n        \n        # ImageFusionScore Case\n        \n        dfBaseField = pd.read_csv(root_input_basefield+i,index_col=None, header=None)\n        image_fusion_score_array = image_fusion_score(dfBaseField)\n        ImageFusionScoreList_R = ImageFusionScoreList_R + [image_fusion_score_array[0]]\n        ImageFusionScoreList_G = ImageFusionScoreList_G + [image_fusion_score_array[1]]\n        ImageFusionScoreList_B = ImageFusionScoreList_B + [image_fusion_score_array[2]]\n        #print(i)\n        \n    # Create array for lists: (1) ImageioSimilarityScoreList; (2) ImageFusionScoreList\n    \n    #arrayScore = np.array([ImageFusionScoreList]+[ImageioSimilarityScoreList])\n    #df2 = df2.T\n    \n    arrayScore1R = np.array(ImageFusionScoreList_R)\n    arrayScore1G = np.array(ImageFusionScoreList_G)\n    arrayScore1B = np.array(ImageFusionScoreList_B)\n    arrayScore2 = np.array(ImageioSimilarityScoreList)\n    \n    # Create DataFrame of the image set (batch) for image pair index: (image_fusion_score,imageio_similarity_score)\n    \n    df2_arrayScore1R = pd.DataFrame(arrayScore1R)\n    df2_arrayScore1G = pd.DataFrame(arrayScore1G)\n    df2_arrayScore1B = pd.DataFrame(arrayScore1B)\n    df2_arrayScore2 = pd.DataFrame(arrayScore2)\n    \n    df2_left = pd.merge(df2_arrayScore1R,df2_arrayScore1G,left_index=True, right_index=True)\n    df2_left = pd.merge(df2_left,df2_arrayScore1B,left_index=True, right_index=True)\n    df2 = pd.merge(df2_left,df2_arrayScore2,left_index=True, right_index=True)\n    \n    # Create CSV file | DataFrame of the image set (batch) for image pair index: (image_fusion_score,imageio_similarity_score)\n    \n    batch = 'batch_'+str(start)+'-'+str(termination)+'.csv'\n    df2.to_csv(root_output_sc+batch,sep=',',index=False,header=False)    \n\n    return # print('Files generated:', os.listdir(root_output))\n\n\ndef ImageFusionScoreTest(start,termination):\n    \n    import os\n    import pandas as pd\n    import numpy as np\n\n    # Define data roots     \n\n    root_input_test = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\n    root_output_f = '/kaggle/working/output/scores/Fusion/'\n    \n    # Set the input files (PixelDeviationScore)\n    \n    filenameTestList = imageio_list_bulk(root_input_test)\n    #filenameTestList.remove('Fusion') \n    \n    ImageFusionScoreList_R = []\n    ImageFusionScoreList_G = []\n    ImageFusionScoreList_B = []\n    \n    filenameTestList = filenameTestList[start:termination]\n    \n    # Run Loop\n    \n    for i in filenameTestList:\n        \n        # ImageFusionScore Case\n        \n        dfTest = pd.read_csv(root_input_test+i,index_col=None, header=None)\n        image_fusion_score_array = image_fusion_score(dfTest)\n        ImageFusionScoreList_R = ImageFusionScoreList_R + [image_fusion_score_array[0]]\n        ImageFusionScoreList_G = ImageFusionScoreList_G + [image_fusion_score_array[1]]\n        ImageFusionScoreList_B = ImageFusionScoreList_B + [image_fusion_score_array[2]]\n        \n    # Create array for lists: ImageFusionScoreList\n    \n    arrayScoreR = np.array(ImageFusionScoreList_R)\n    arrayScoreG = np.array(ImageFusionScoreList_G)\n    arrayScoreB = np.array(ImageFusionScoreList_B)\n    \n    # Create DataFrame of the image set (batch) for image pair index: (image_fusion_score,imageio_similarity_score)\n    \n    df2_arrayScoreR = pd.DataFrame(arrayScoreR)\n    df2_arrayScoreG = pd.DataFrame(arrayScoreG)\n    df2_arrayScoreB = pd.DataFrame(arrayScoreB)\n    \n    df2_left = pd.merge(df2_arrayScoreR,df2_arrayScoreG,left_index=True, right_index=True)\n    df2 = pd.merge(df2_left,df2_arrayScoreB,left_index=True, right_index=True)\n    \n    # Create CSV file | DataFrame of the image set (batch) for image pair index: (image_fusion_score,imageio_similarity_score)\n    \n    batch = 'batch_'+str(start)+'-'+str(termination)+'.csv'\n    df2.to_csv(root_output_f+batch,sep=',',index=False,header=False)    \n    \n    return # print('Files generated:', os.listdir(root_output_f))\n\n\n\"\"\"\nRe set initial sample size\n\"\"\"\nprint('\\n************************************************************************************************************************************')\ntrainSample = input('\\nPlease re insert the initial sample-number of instances for training (0-1000) non-test-images:')\ntrainSample = int(trainSample)\ntestSample = input('\\nPlease re insert the initial sample-number of instances for testing (0-5000) test-images:')\ntestSample = int(testSample)\nprint('\\n************************************************************************************************************************************')\n\n\ndef filesVerification():\n    \n    # Existing Files Verification\n\n    import os\n    \n    ## Input files (image normalized)\n    input_root1 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\n    input_root2 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/'\n    input_root3 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/'\n    input_root4 = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/'\n    inputList = [input_root1] + [input_root2] + [input_root3] + [input_root4]\n    \n    ## Input files (image normalized) | Test case\n    input_root_test = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'    \n    \n    ## Output roots\n    output_root1 = '/kaggle/working/output/scores/PixelDeviationScore/'    \n    output_root2 = '/kaggle/working/output/scores/ImageSimilarity/'    \n    outputList = [output_root1] + [output_root2]\n    \n    ## Output roots | Batch case\n    output_rootB1 = '/kaggle/working/output/scores/ImageSimilarityScore/'    \n    output_rootB2 = '/kaggle/working/output/scores/Fusion/'    \n    outputListB = [output_rootB1] + [output_rootB2]\n    \n    ## Input-files list verification\n    inputListEvaluation = []\n    \n    for i in inputList:\n        \n        numy = os.listdir(i)\n        \n        if len(numy) == trainSample:\n            inputListEvaluation = inputListEvaluation + [True]\n        else:\n            inputListEvaluation = inputListEvaluation + [False]\n    \n    \n    ## Input-files list verification | test case\n    numy = os.listdir(input_root_test)\n    #numy.remove('Fusion')\n    \n    if len(numy) == testSample:\n        inputListEvaluation = inputListEvaluation + [True]\n    else:\n        inputListEvaluation = inputListEvaluation + [False]\n    \n    \n    ## Output-files list verification\n    outputListEvaluation = []\n    numyBase =  os.listdir(outputList[1]) # Picked directory (criterion: files generation order): 2\n        \n    for i in outputList:\n        \n        numy = os.listdir(i)\n        \n        if len(numy) == len(numyBase):\n            outputListEvaluation = outputListEvaluation + [True]\n        else:\n            outputListEvaluation = outputListEvaluation + [False]        \n    \n    \n    ## Output-files list verification | Batch case\n    numyBaseB =  os.listdir(outputListB[1]) # Picked directory (criterion: files generation order): 2\n    \n    for i in outputListB:\n    \n        numy = os.listdir(i)\n    \n        if len(numy) == len(numyBaseB):\n            outputListEvaluation = outputListEvaluation + [True]\n        else:\n            if 0 >= len(numyBaseB)-testSample:\n                outputListEvaluation = outputListEvaluation + [True]\n            else:\n                outputListEvaluation = outputListEvaluation + [False]\n    \n    \n    ## Results\n    ResultsOutput = []    \n    inputList = inputList + [input_root_test]\n    place = 0\n    for i in  inputListEvaluation:\n        \n        if i != True:      \n            print('\\n --------------------------------------------')\n            print('\\n Input files does not meet the requirements.')\n            print('\\n Please check: ',inputList[place])\n            print('\\n --------------------------------------------')\n            ResultsOutput = ResultsOutput + [False]\n            break\n    \n        place = place + 1\n        \n    if (place == len(inputListEvaluation)):\n        print('\\n --------------------------------------------')\n        print('\\n Input files status: OK')\n        print('\\n --------------------------------------------')\n        ResultsOutput = ResultsOutput + [True]\n    \n    outputList = outputList + outputListB\n    place = 0\n    for i in  outputListEvaluation:\n        if i != True:      \n            print('\\n --------------------------------------------')\n            print('\\n Output files does not meet the requirements.')\n            print('\\n Please check: ',outputList[place])\n            print('\\n --------------------------------------------')\n            ResultsOutput = ResultsOutput + [False]\n            break\n    \n        place = place + 1\n    \n    if (place == len(outputListEvaluation)):\n        print('\\n --------------------------------------------')\n        print('\\n Output files status: OK')\n        print('\\n --------------------------------------------')\n        ResultsOutput = ResultsOutput + [True]\n\n    return ResultsOutput\n\ndef AUC_filtering():\n    \n    return\n\n\n\"\"\"\nKaggle Solution scheme\n\"\"\"\n\n# Milestone 1: P(E) = 1 - weighted similarity score\n# Milestone 2: Build Model | Cover + JMiPOD + JUNIWARD + UERD = P(E)\n# Milestone 3: Evaluate Test\n# Milestone 4: Accomplishment level of AUC constraint (i.e. AUC filtering)\n# Kaggle Submission\n\n\n\"\"\"\nBatching testing mode\n\"\"\" \n# Set Batch | Training dataset\nprint('\\n Please set a batch scope of 1000 units.')\nprint('\\n Remark 1: Do not insert the same number, please.')\nprint('\\n Remark 2: Images Normalized to be fused from test data set allow batch numbers from 0 to 5001 batches')\nstart = input('\\n Enter the batch-start number (training dataset): ')\ntermination = input('\\n Enter the batch-termination number (training dataset): ')\nstart = int(start)\ntermination = int(termination)\njoinBaseField = True\n\n\n# Set Batch | Testingining dataset\nprint('\\n Please set a batch scope of 1000 units.')\nprint('\\n Remark 1: Do not insert the same number, please.')\nprint('\\n Remark 2: Images Normalized to be fused from test data set allow batch numbers from 0 to 5001 batches')\nstartT = input('\\n Enter the batch-start number (testing dataset): ')\nterminationT = input('\\n Enter the batch-termination number (testing dataset): ')\nstartT = int(startT)\nterminationT = int(terminationT)\njoinBaseField = True\n\n# Verification\na,b = filesVerification()\n#a, b = True, True\n\n# Files creation | Training\n\nif (a == True and b == True):\n    \n    for i in range(start,termination):\n    \n        PixelDeviationScore(i,i+1,joinBaseField)\n        SimilarityScore(i,i+1)\n\n    print('\\n Processed Batches (Training Dataset): '+'From '+str(start)+' to '+ str(termination-1))\n    print('\\n Work done.')\n    print('\\n --------------------------------------------')\n    print('\\n Start next time from the following batch number: '+ str(termination))\n    print('\\n --------------------------------------------')\n    print('\\n --------------------------------------------')\n        \n# Files creation | Testing\n        \nif (a == True and b == True):        \n        \n    for i in range(startT,terminationT):\n        \n        if (i<= 5001):ImageFusionScoreTest(i,i+1)\n        print('Processed Batch:', str(i), ' of ', str(terminationT-1), 'ones.')\n      \n    print('\\n Processed Batches (Testing Dataset): '+'From '+str(startT)+' to '+ str(terminationT-1))\n    print('\\n Work done.')\n    print('\\n --------------------------------------------')\n    print('\\n Start next time from the following batch number: '+ str(terminationT))\n    print('\\n --------------------------------------------')\n    print('\\n --------------------------------------------')\n\n\"\"\"\nBibliography:\n\n    \nRefrences:\n\nAmos,D. n.d. How to Round Numbers in Python. Retrieved from\n    https://realpython.com/python-rounding/\n\nSolving ValueError: cannot convert float NaN to integer. Retrieved from\n    https://stackoverflow.com/questions/59390764/solving-valueerror-cannot-convert-float-nan-to-integer\n\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nOutcome Verifier of KaggleSolution2.py\n\"\"\"\nimport os\n\nprint('\\n******************')\nprint('Input files')\nprint('******************')\n\nprint('\\nImage Normalized | Cover')\nrootToTest = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Cover_ImageNormalized/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nImage Normalized | JMiPOD')\nrootToTest = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JMiPOD_ImageNormalized/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nImage Normalized | JUNIWARD')\nrootToTest = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/JUNIWARD_ImageNormalized/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nImage Normalized | UERD')\nrootToTest = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/UERD_ImageNormalized/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nImage Normalized | Test')\nrootToTest = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\n******************')\nprint('Output files')\nprint('******************')\n\nprint('\\nPixel Deviation Score | Matrix 1 (deviation-values matrix): 300x100 | Matrix 2:300x100 (deviation-score-values matrix)')\nrootToTest = '/kaggle/working/output/scores/PixelDeviationScore/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nImage Similarity | Binary Matrix (0:non-altered value | 1:altered value)')\nrootToTest = '/kaggle/working/output/scores/ImageSimilarity/' \nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nImage Similarity Score (RGB triads for non-Test)')\nrootToTest = '/kaggle/working/output/scores/ImageSimilarityScore/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nFusion (RGB triads for Test)')\nrootToTest = '/kaggle/working/output/scores/Fusion/' \nnumy = os.listdir(rootToTest)\nprint(numy)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCreate directories to prototype with input image files of the competition data (prototype outputs | consolidation)\n\"\"\"\n\n# Function directoryCreation(directoryList)\n\ndef directoryCreation(directoryList):\n    \n    import os\n\n    print('\\n*******************************************************************************************************************')\n    print('Created or existing directories')\n    print('*******************************************************************************************************************')\n    \n    for i in directoryList:\n    \n        try:\n            os.mkdir(i)\n        except FileExistsError: # directory already exists\n            pass\n        \n        print('\\n Created or existing directory: ',i)\n    \n    print('\\nDirectories are created and available to upload data...')\n    \n    return\n\n## Required directories | Batch 2: Consolidation\n\npath_generic1 = '/kaggle/working/output/'\npath_generic3 = '/kaggle/working/output/consolidation/'\n\npath_ImageSimilarityScoreConsolidation = '/kaggle/working/output/consolidation/ImageSimilarityScoreConsolidation/'\npath_FusionConsolidation = '/kaggle/working/output/consolidation/FusionConsolidation/'\npath_Submission = '/kaggle/working/output/consolidation/Submission/'\n\ndirectoryList2 = [path_generic1]+[path_generic3]+[path_ImageSimilarityScoreConsolidation]+[path_FusionConsolidation]+[path_Submission]\ndirectoryListG2 = [path_ImageSimilarityScoreConsolidation]+[path_FusionConsolidation]+[path_Submission]\n\nprint('\\n*******************************************************************************************************************')\nprint('Required directories to create | Batch 2: Consolidation')\nprint('*******************************************************************************************************************')\nprint('\\nDirectories to create:','\\n', directoryList2)\n\ndirectoryCreation(directoryList2)\ndirectoryList2.remove(path_generic1)\ndirectoryList2.remove(path_generic3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Thu May 30 11:49:00 2020\n\n@author: Mauricio Azálgara Bedoya\n\nKaggle user account: mazalgarab\n\n\"\"\"\n\n\"\"\"\nAUCEvaluation.py\nVersion adapted from orginal version\nPart 1: Batches' consolidation\n\"\"\"\n\n\n\"\"\"\nTensorflow Approach | AUC (Area under the curve) via a Riemann sum\n\"\"\"\n\ndef AUCvalue(num_thresholds,thresholds,y_true,y_pred):\n\n    # tf.keras.metrics.AUC\n    ## Remark: weights = [2, 1] not included\n    import tensorflow as tf\n    \n    m = tf.keras.metrics.AUC(\n        num_thresholds=3, curve='ROC', summation_method='interpolation', name=None,\n        dtype=None, thresholds=[0.0, 0.4, 1.0], multi_label=False, label_weights=None\n    )\n    _ = m.update_state(y_true, y_pred)\n    \n    return m.result().numpy()\n\ndef AUCFiltering():\n    \n    import pandas as pd\n    \n    # Load Predictions file as DataFrame\n    \n    rootInput = 'C:/Users/mazal/Downloads/ALASKA2 Image Steganalysis/DataToPrototype/TestPrototype2/Output/Consolidation/Test_Predictions/'\n    InputFileName = 'Predictions.csv'\n    df = pd.read_csv(rootInput+InputFileName)\n    \n    # Set List of Prediction Models\n    PredictionModelList = list(df.columns)\n    PredictionModelList.remove('Feature 1')\n    PredictionModelList.remove('Feature 2')\n    PredictionModelList.remove('Feature 3')\n    items = len(PredictionModelList)\n    itemList = []\n    \n    for i in range(0,items): itemList = itemList + [i]\n    \n    d = {'Item':itemList,'Model':PredictionModelList}\n    dfModel = pd.DataFrame(d)\n    \n    # Select Models\n    \n    print('\\Prediction Model List:')\n    print(dfModel)\n    print('\\nInsert 2 numbers corresponding to 2 predicion models to be AUC filtered:')\n    model1 = input('\\nModel 1: ')\n    model2 = input('\\nModel 2: ')\n    \n    # AUC values\n    ## AUC values should turn out over 0.85.\n    \n        \n    return print(model1, model2)\n\ndef batchMerger(start, termination, test):\n    \n    import pandas as pd\n    import os\n        \n    # test = 0 -> work with non-Test dataset (Image Normalized and Fused | ImageSimilarityScore computed from Cover, JMiPOD, UERD, JUNIWARD)\n    # test = 1 -> work with Test dataset (Image Normalized and Fused)\n    \n    # Set working directory\n    path = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/'\n    os.chdir(path)\n    \n    # Set input directory\n    if(test == 0):\n        rootInput = '/kaggle/working/output/scores/ImageSimilarityScore/'\n    else:\n        rootInput = '/kaggle/working/output/scores/Fusion/' \n                \n    # Set output directory\n    if(test == 0): \n        rootOutput = '/kaggle/working/output/consolidation/ImageSimilarityScoreConsolidation/'\n        datasetName = 'Image Normalized and Fused from Cover, JMiPOD, UERD, and JUNIWARD'\n    else:\n        rootOutput = '/kaggle/working/output/consolidation/FusionConsolidation/'\n        datasetName = 'Image Normalized and Fused from Test'\n    \n    # Consolidation\n    dfList = []\n    numy = 1\n    if test == 0:\n        T = 'Non-Test'\n    else:\n        T = 'Test'\n            \n    for i in range(start,termination):\n        \n                \n        InputFileName = 'batch_'+str(i)+'-'+str(i+1)+'.csv'\n        df = pd.read_csv(rootInput+InputFileName,index_col=None, header=None)\n        dfList = dfList + [df]\n        print('Processed Batches', T,':', numy, ' from ', termination, ' ones.')\n        numy = numy + 1\n        \n    df = pd.concat(dfList, ignore_index=False, keys=None,levels=None, names=None, verify_integrity=False, copy=True)\n    \n    # Create consolidated file\n    fileName = 'batch_'+str(start)+'-'+str(termination)+'.csv'\n    print('\\nDataset:',datasetName,'\\nNumber of merged instances: ', str(termination-start), '\\nInstances available at: ', rootOutput)\n    df.to_csv(rootOutput+fileName,sep=',',index=False,header=False)\n    \n    return \n\n\n\"\"\"\nBatching\n\"\"\"\nprint('\\n*******************************************************************')\n\n# Set Batch Training Data Set\nprint('\\n Please set a batch scope to include as a training data set:')\nprint('\\n Upper limit: 1000')\nprint('\\n Lower limit: 0')\n\nstartTrain = input('\\n Enter the batch-start number: ')\nterminationTrain = input('\\n Enter the batch-termination number: ')\nstartTrain = int(startTrain)\nterminationTrain = int(terminationTrain)\n\nprint('\\n*******************************************************************')\n\n# Set Batch Training Data Set\nprint('\\n Please set a batch scope to include as a testing data set:')\nprint('\\n Upper limit: 5000')\nprint('\\n Lower limit: 0')\n\nstartTest = input('\\n Enter the batch-start number: ')\nterminationTest = input('\\n Enter the batch-termination number: ')\nstartTest = int(startTest)\nterminationTest = int(terminationTest)\n\nprint('\\n*******************************************************************')\n\n# Running inputs\n\nstart = startTrain\ntermination = terminationTrain\ntest = 0\nbatchMerger(start,termination,test)\nprint('\\n*******************************************************************')\nstart = startTest\ntermination = terminationTest\ntest = 1\nbatchMerger(start,termination,test)\nprint('\\n*******************************************************************')\n\"\"\"\nNotes:\nOther approaches\n(1) Best practices for model evaluation and Hyperparameter Tuning (Rashka et al., 2019  pp. 216-218)\n(2) Scikit-learn approach (Scikit-learn Developers, 2019, pp. 2013-2015)\n\nWork with the following Tensorflow versions: TensorFlow 2.x\n\nUninstall tensorflow:\npip uninstall tensorflow\npip uninstall tensorflow-gpu\n\nInstall Tensorflow: \npip install tensorflow\npip install tensorflow-gpu   \n\n\n\"\"\"\n\n\n\"\"\"\nBibliography:\n    \nRaschka, S., & Mirjalili, V. (2019). Python Machine Learning: Machine Learning\n    and Deep Learning with Python, scikit-learn, and TensorFlow 2 (3rd ed.).\n    Birmingham, United Kingdom: Packt Publishing Ltd.\n\nScikit-learn Developers. (2019). Scikit-learn user guide | Release 0.21.3.\n    Retrieved from https://scikit-learn.org/stable/_downloads/scikit-learn-docs.pdf\n\nReferences:\n\n    tf.keras.metrics.AUC. Retrieved from \n    https://www.tensorflow.org/api_docs/python/tf/keras/metrics/AUC\n\nStackoverflow:\n    \n    Error on tensorflow cannot import name 'export_saved_model'. Retrieved from\n    https://stackoverflow.com/questions/61833301/error-on-tensorflow-cannot-import-name-export-saved-model\n\n    Uninstall tensorflow 2.1.0. Retrieved from\n    https://stackoverflow.com/questions/59824224/uninstall-tensorflow-2-1-0\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nOutcome Verifier of AUCEvaluation.py\n\"\"\"\nimport os\n\nprint('\\n******************')\nprint('Outcome 1 of AUCEvaluation.py')\nprint('Datasets to build a prediction model')\nprint('******************')\n\nprint('\\nTraining dataset')\nrootToTest = '/kaggle/working/output/consolidation/ImageSimilarityScoreConsolidation/'\nnumy = os.listdir(rootToTest)\nprint(numy)\n\nprint('\\nTesting dataset')\nrootToTest = '/kaggle/working/output/consolidation/FusionConsolidation/'\nnumy = os.listdir(rootToTest)\nprint(numy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nPrediction Model\n\"\"\"\n\n\"\"\"\nPredictionModels.py\nversion adapted from orginal version\n\"\"\"\n\n# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Tue Jul  7 14:09:39 2020\n\n@author: Mauricio Azálgara Bedoya\n\nKaggle user account: mazalgarab\n\"\"\"\n\n\"\"\"\nRandom Forest Classification Using scikit-learn\n\"\"\"\n\n\"\"\"\nRe set initial sample size\n\"\"\"\nprint('\\n************************************************************************************************************************************')\ntrainSample = input('\\nPlease re insert the initial sample-number of instances for training (0-1000) non-test-images:')\ntrainSample = int(trainSample)\ntestSample = input('\\nPlease re insert the initial sample-number of instances for testing (0-5000) test-images:')\ntestSample = int(testSample)\nprint('\\n************************************************************************************************************************************')\n\n# Set working directory\nimport os\npath = '/kaggle/working/output/consolidation/Submission/'\nos.chdir(path)\n\n# Load training dataset\nimport pandas as pd\nimport numpy as np\nfileNameTrainSample = '/kaggle/working/output/consolidation/ImageSimilarityScoreConsolidation/' + 'batch_0-' + str(trainSample) + '.csv'\ntrainSample = pd.read_csv(fileNameTrainSample,index_col=None, header=None)\ntrainSampleFeatures = trainSample.loc[:,[0,1,2]]\ntrainSampleLabels = trainSample.loc[:,[3]]\n\n# Load testing dataset\nfileNameTestSample = '/kaggle/working/output/consolidation/FusionConsolidation/' + 'batch_0-' + str(testSample) + '.csv'\ntestSample = pd.read_csv(fileNameTestSample,index_col=None, header=None)\n\n# Concat training and testing dataset\n#dfT = trainSample + [testSample]\n#dfT = pd.concat(df, ignore_index=False, keys=None,levels=None, names=None, verify_integrity=False, copy=True)\n\n# Set features and label\n\n## Training dataset\nfeatures = np.array(trainSampleFeatures)\nlabel = np.array(trainSampleLabels)\n\n## Testing dataset\nfeaturesTest = np.array(testSample)\n\n# Train-test splitting\n# Source: (Nagy, 2018, pp. 191 - 192)\n\nfrom sklearn import model_selection\n\nfeatures_train, features_test, label_train, label_test = model_selection.train_test_split(features,label,test_size=0.1)\n\n\n## Preprocessing task for label_train\n### Require to check with Orange prototype\nfrom sklearn import preprocessing\nle = preprocessing.LabelEncoder()\nlabel_train = le.fit_transform(label_train)\n\n\n# Classifier\n# Source: (Nagy, 2018, pp. 191 - 192)\n\nfrom sklearn.ensemble import RandomForestClassifier\n\n## Prediction using just the training dataset\nrandom_forest_classifier = RandomForestClassifier(n_estimators=10,max_depth=5)\nrandom_forest_classifier.fit(features_train, label_train)\nlabels_predicted = random_forest_classifier.predict(features_test)\n\n\n## Prediction using just the testing dataset\nlabels_predicted = random_forest_classifier.predict(featuresTest)\n\n# Kaggle Submission\n\n## List\nimport os\n\nroot = '/kaggle/working/output/prototypeOutput-alaska2imagesteganalysis/Test_ImageNormalized/'\nimageList = os.listdir(root)\n\nimageListSubmission = []\n\nfor i in imageList:\n    file = i[0:4]\n    file = str(file)\n    file = file + '.jpg'\n    imageListSubmission = imageListSubmission + [file]\n    \n## Submission Sample\n\n## File generation\nlistSubmission = list(labels_predicted)\ndata = {'Id':imageListSubmission, 'Label':listSubmission}\ndf = pd.DataFrame(data)\n\nrootOutput1 = '/kaggle/working/output/consolidation/Submission/'\nrootOutput2 = '/kaggle/working/'\nfileName = 'submission.csv'\ndf.to_csv(rootOutput1+fileName,sep=',',index=False)\ndf.to_csv(rootOutput2+fileName,sep=',',index=False)\n\nprint('Kaggle Submission')\nprint('************************************************************************************************************************************')\nprint(df)\n\n\"\"\"\nBibliography:\n\nNagy, Z. (2018). Artificial Intelligence and Machine Learning Fundamentals: Develop\n    real-world applications powered by the latest AI advances (1st ed.). Birmingham,\n    United Kingdom: Packt Publishing Ltd.\n\n\"\"\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}