{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, I will preprocess the images in the training set. I will:\n- Split the data into train/test set 4:1\n- Shrink image size to 256*256\n- Reduce color the channel","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nfrom pathlib import Path\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.image as img\nfrom tqdm.notebook import tqdm\n\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-07T22:02:30.324199Z","iopub.execute_input":"2022-02-07T22:02:30.324840Z","iopub.status.idle":"2022-02-07T22:02:30.865361Z","shell.execute_reply.started":"2022-02-07T22:02:30.324730Z","shell.execute_reply":"2022-02-07T22:02:30.864398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the DataFrame","metadata":{}},{"cell_type":"code","source":"labels = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/train.csv')\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T22:02:32.346923Z","iopub.execute_input":"2022-02-07T22:02:32.347257Z","iopub.status.idle":"2022-02-07T22:02:32.483568Z","shell.execute_reply.started":"2022-02-07T22:02:32.347217Z","shell.execute_reply":"2022-02-07T22:02:32.482938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of Training Dataset:', labels.shape[0])\nprint(labels.shape[1], ' Columns')","metadata":{"execution":{"iopub.status.busy":"2022-02-07T22:02:38.648286Z","iopub.execute_input":"2022-02-07T22:02:38.648989Z","iopub.status.idle":"2022-02-07T22:02:38.655513Z","shell.execute_reply.started":"2022-02-07T22:02:38.648935Z","shell.execute_reply":"2022-02-07T22:02:38.654602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define the pathes to the image files","metadata":{}},{"cell_type":"code","source":"ROOT_PATH = Path('/kaggle/input/happy-whale-and-dolphin/train_images/')\nSAVE_PATH = Path('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2022-02-07T22:02:41.786694Z","iopub.execute_input":"2022-02-07T22:02:41.787238Z","iopub.status.idle":"2022-02-07T22:02:41.791129Z","shell.execute_reply.started":"2022-02-07T22:02:41.787203Z","shell.execute_reply":"2022-02-07T22:02:41.789980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Take an Example of the Basic Preprocess Logic\n\nLet's see a preprocessing example first:","metadata":{}},{"cell_type":"code","source":"ex_path = ROOT_PATH/'00021adfb725ed.jpg'\nex_arry = cv2.imread(str(ex_path), 0)/255\nex_arry = cv2.resize(ex_arry, (256,256)).astype(np.float16)\n\nprint('Type of ex_arry: ', type(ex_arry))\nprint('Shape of ex_arry:', ex_arry.shape)\nplt.imshow(ex_arry, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T22:02:50.404000Z","iopub.execute_input":"2022-02-07T22:02:50.404454Z","iopub.status.idle":"2022-02-07T22:02:50.868332Z","shell.execute_reply.started":"2022-02-07T22:02:50.404420Z","shell.execute_reply":"2022-02-07T22:02:50.867402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that following the process above, we shrink the image to 256 * 256. From 3 channels of RGB to 1 channel which is gray. Now I am going to follow the same logit and write a loop to process all the images. But before that, I am going to split the train/test set.","metadata":{}},{"cell_type":"markdown","source":"# Train/Test Split, Standardize and Normalize\n- train/test split\n- resize to 256 * 256\n- gray scale to reduce the complexity\n- standardize all images by the maximum pixel value in the provided dataset, 255\n- standard normalization","metadata":{}},{"cell_type":"code","source":"# shuffle the labels dataframe\nlabels = labels.sample(frac=1, random_state=SEED).reset_index(drop=True)\n\n# split point\nsplit = int(labels.shape[0]*4/5)\nprint('training size:', split)\nprint('test size    :', labels.shape[0] - split)\n\n# flag train/test\n# col is_train == 1 --> belongs to training set\n# col is_train == 0 --> belongs to test set\nlabels['is_train'] = np.concatenate((np.ones(split), np.zeros(labels.shape[0] - split)), axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T22:02:59.092390Z","iopub.execute_input":"2022-02-07T22:02:59.092848Z","iopub.status.idle":"2022-02-07T22:02:59.121160Z","shell.execute_reply.started":"2022-02-07T22:02:59.092814Z","shell.execute_reply":"2022-02-07T22:02:59.120167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's loop over and preprocess all the images:","metadata":{}},{"cell_type":"code","source":"sums = 0\nsums_squared = 0\n\nfor c, image_id in enumerate(tqdm(labels.image)):\n    image_path = ROOT_PATH/image_id                        # create the path to the .jpg file\n    image_arry = cv2.imread(str(image_path),0) / 255       # read image as array, and standardize\n    image_arry = cv2.resize(image_arry, (256,256)).astype(np.float16) # resize to 256*256\n        \n    label = labels.individual_id.iloc[c]                   # retrieve the corresponding label\n    \n    # 4/5 train split, 1/5 test split\n    train_or_test = 'train' if c < split else 'test'\n    \n    current_save_path = SAVE_PATH/train_or_test/str(label) # define save path and create if necessary\n    current_save_path.mkdir(parents=True, exist_ok=True)\n    np.save(current_save_path/image_id, image_arry)        # save the aray in the corresponding directory\n    \n    normalizer = image_arry.shape[0]*image_arry.shape[1]\n    if train_or_test == 'train':\n        sums += np.sum(image_arry) / normalizer\n        sums_squared += (np.power(image_arry, 2).sum())/normalizer","metadata":{"execution":{"iopub.status.busy":"2022-02-07T22:03:43.159621Z","iopub.execute_input":"2022-02-07T22:03:43.160110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = sums / split\nstd = np.sqrt(sums_squared / split) - (mean**2)\nprint(f'Mean of Dataset: {mean:.4f}')\nprint(f'STD of Dataset : {std:.4f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"🚧 To be continued...","metadata":{}}]}