{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook I build a flexible function which reads and processes a large number of images by managing the RAM memory and by saving the result later uses.\n\nI use this function to train a LightGBM for that I applied an image pooling in 4x4 and flattened the result and then I summed the RGB colors, which gives me a dataframe of 65,536 columns + 1 columns of out put and 24,000 rows which corresponds to 24,000 images stored on 6 GB of RAM.\n\nThis can be modified according to your needs. if you want to train classic models and not CNNs.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport tqdm\nfrom PIL import Image\nimport glob\nimport skimage.measure\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# nbr_images : the quantity of images that we want to import\n\n# df : - If a DataFrame that contains images exists and we want to concatenate \n#        with the images that this function is going to import, we add it. \n#        the return of the function is the existing dataframe concatenated with the new.\n#      - If NaN the function creates a new DataFrame which contains the imported images.\n\n# file_name : The name of the file into which we want to import these images. Cover, JMiPOD, JUNIWARD or UERD.\n\n# from_ : From which image you can start importing. If 1000 entered, the function will start importing from the thousandth image.\n\n# status : It is a gadget, it takes 'neg' or 'pos', if 'neg' it adds an output column at the end of the DataFrame equal to 0,\n#          if pos it adds an output column equal to 1. it is to distinguish whether the image hides a message or not.\n\n\n\ndef img_reader(nbr_images = 10, df = None, file_name = 'Cover', from_ = 0, status = 'neg') :\n    from_ = from_\n    nbr_images  = nbr_images\n    image_list = []\n    i=0\n    j=0\n    df = df\n    file_name = file_name\n    for filename in tqdm.tqdm(glob.glob('../input/alaska2-image-steganalysis/'+file_name+'/*.jpg')): \n        if j >= from_ :\n            im=mpimg.imread(filename)\n            im=skimage.measure.block_reduce(im, (2,2,1), np.max) # Drop this step to not apply the image pooling.\n            image_list.append(np.sum(im.reshape((d3, d1*d2)), axis = 0).tolist()) # d3 = 3, d1 and d2 = 256, without pooling this may be 3 * 512 * 512, np.sum() is for sum rgb. \n            i+=1\n            if i%1000 == 0 : # is for concat DataFrame by batch of 100 images.\n                if df is None:\n                    df = pd.DataFrame(image_list).astype('int16')\n                    del image_list\n                    gc.collect()\n                    image_list = []\n                else :\n                    df = pd.concat([df , pd.DataFrame(image_list).astype('int16')])\n                    del image_list\n                    gc.collect()\n                    image_list = []\n                    if i == nbr_images :    \n                        del image_list\n                        gc.collect()\n                        break\n        j=j+1\n        \n    if status == 'neg' :\n        df['output'] = 0\n        df['output'] = df['output'].astype('int16')\n        gc.collect()\n    else :\n        df['output'] = 1\n        df['output'] = df['output'].astype('int16')\n        gc.collect()\n        \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here i recover an image and I apply the pooling on it in order to recover the final dimensions d1, d2 and d3.\n\nimg=mpimg.imread('../input/alaska2-image-steganalysis/Cover/00001.jpg')\ntest_pool = skimage.measure.block_reduce(img, (2,2,1), np.max)\nd1, d2, d3 = test_pool.shape\ndel test_pool\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I call this function 4 times:\n\n# the first to receive 12,000 images from image 1 of the folder Cover, label them 0 and store them in df_neg.\n\ndf_neg = img_reader(nbr_images = 12000, df = None, file_name = 'Cover', from_ = 0, status = 'neg')\nprint('import Cover Done !')\n\n# Thereafter i recover 4,000 images from JMiPOD starting with image number 1, then 4,000 from JUNIWARD starting from image number 4,000 and 4,000 from UERD starting from image number 8000.\n\n# At the end i have a dataframe 'df_pos' with 4000 images of JMiPOD followed by 4000 images of JUNIWARD followed by 4000 images of UERD which makes 12000 images in total labeled 1.\n\ndf_pos = img_reader(nbr_images = 4000, df = None, file_name = 'JMiPOD', from_ = 0, status = 'pos')\nprint('JMiPOD Done!')\ndf_pos = img_reader(nbr_images = 4000, df = df_pos, file_name = 'JUNIWARD', from_ = 4000, status = 'pos')\nprint('JUNIWARD Done!')\ndf_pos = img_reader(nbr_images = 4000, df = df_pos, file_name = 'UERD', from_ = 8000, status = 'pos')\nprint('UERD Done!')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('df_neg info :')\ndisplay(df_neg.info())\n\nprint('df_pos info :')\ndisplay(df_pos.info())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1.5 GB each, No Break =)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print('df_neg head :')\ndisplay(df_neg.head())\n\nprint('df_pos head :')\ndisplay(df_pos.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Everything Alright","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Lets save them in pkl format for next.\n\ndf_neg.to_pickle('df_neg.pkl')\ndf_pos.to_pickle('df_pos.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# concatenate all, free up space.\n\ndf_train = pd.concat([df_pos, df_neg], ignore_index = True).astype('int16')\ndel df_pos, df_neg\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we have 24,000 images we can do what we want with =)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"If you have suggestions to improve the importation of images I am interested.\n\nIf it helped you make an UpVote it will be chic on your part =)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}