{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom keras.models import Sequential\nfrom keras.layers import Convolution2D, MaxPooling2D\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom tensorflow.keras import utils\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"path = '../input/understanding_cloud_organization'\nos.listdir(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'{path}/train.csv')\nsub = pd.read_csv(f'{path}/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_train = len(os.listdir(f'{path}/train_images'))\nn_test = len(os.listdir(f'{path}/test_images'))\nprint(f'There are {n_train} images in train dataset')\nprint(f'There are {n_test} images in test dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"value_counts = train_df.loc[train_df['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[1]).value_counts()\n\nfish_height = value_counts['Fish']\ngravel_height = value_counts['Gravel']\nflower_height = value_counts['Flower']\nsugar_height = value_counts['Sugar']\n\nheight = [fish_height, gravel_height, flower_height, sugar_height]\nbars = ('Fish', 'Gravel', 'Flower', 'Sugar')\ny_pos = np.arange(len(bars))\nplt.bar(y_pos, height, color=(0.2, 0.4, 0.6, 0.6))\n\nplt.xticks(y_pos, bars, color='orange', rotation=45, fontweight='bold', fontsize='17', horizontalalignment='right')\n# Custom Axis title\nplt.xlabel('Labels', fontweight='bold', color = 'orange', fontsize='17', horizontalalignment='center')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"frequency_count = train_df.loc[train_df['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[0]).value_counts().value_counts()\n\nonce = frequency_count[1]\ntwice = frequency_count[2]\nthrice = frequency_count[3]\nfour_times = frequency_count[4]\n\nheight = [once, twice, thrice, four_times]\nbars = ('1', '2', '3', '4')\ny_pos = np.arange(len(bars))\nplt.bar(y_pos, height, color=(0.2, 0.4, 0.6, 0.6))\n\nplt.xticks(y_pos, bars, color='orange', rotation=45, fontweight='bold', fontsize='17', horizontalalignment='right')\n# Custom Axis title\nplt.xlabel('Images with n number of labels', fontweight='bold', color = 'orange', fontsize='17', horizontalalignment='center')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preperation"},{"metadata":{},"cell_type":"markdown","source":"# Get Labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[~train_df['EncodedPixels'].isnull()]\ntrain_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])\ntrain_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])\nclasses = train_df['Class'].unique()\ntrain_df = train_df.groupby('Image')['Class'].agg(set).reset_index()\nfor class_name in classes:\n    train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs, val_imgs = train_test_split(train_df['Image'].values, \n                                        test_size=0.2, \n                                        stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))),\n                                        random_state=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Generators"},{"metadata":{"trusted":true},"cell_type":"code","source":"from copy import deepcopy\n\nclass DataGenenerator(utils.Sequence):\n    def __init__(self, images_list=None, folder_imgs=f'{path}/train_images', \n                 batch_size=32, shuffle=True, augmentation=None,\n                 resized_height=224, resized_width=224, num_channels=3):\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augmentation = augmentation\n        if images_list is None:\n            self.images_list = os.listdir(folder_imgs)\n        else:\n            self.images_list = deepcopy(images_list)\n        self.folder_imgs = folder_imgs\n        self.len = len(self.images_list) // self.batch_size\n        self.resized_height = resized_height\n        self.resized_width = resized_width\n        self.num_channels = num_channels\n        self.num_classes = 4\n        self.is_test = not 'train' in folder_imgs\n        if not shuffle and not self.is_test:\n            self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]\n\n    def __len__(self):\n        return self.len\n    \n    def on_epoch_start(self):\n        if self.shuffle:\n            random.shuffle(self.images_list)\n\n    def __getitem__(self, idx):\n        current_batch = self.images_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n        y = np.empty((self.batch_size, self.num_classes))\n\n        for i, image_name in enumerate(current_batch):\n            path = os.path.join(self.folder_imgs, image_name)\n            img = cv2.resize(cv2.imread(path), (self.resized_height, self.resized_width)).astype(np.float32)\n            if not self.augmentation is None:\n                augmented = self.augmentation(image=img)\n                img = augmented['image']\n            X[i, :, :, :] = img/255.0\n            if not self.is_test:\n                y[i, :] = img_2_ohe_vector[image_name]\n        return X, y\n\n    def get_labels(self):\n        if self.shuffle:\n            images_current = self.images_list[:self.len*self.batch_size]\n            labels = [img_2_ohe_vector[img] for img in images_current]\n        else:\n            labels = self.labels\n        return np.array(labels)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator_train = DataGenenerator(train_imgs)\ndata_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)\ndata_generator_val = DataGenenerator(val_imgs, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CNN model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(224,224,3)))\nmodel.add(Convolution2D(32, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25)) \nmodel.add(Flatten())\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(4, activation='softmax'))\nmodel.compile(loss='categorical_crossentropy',metrics=['accuracy'], optimizer='adam')\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"from multiprocessing import cpu_count\nimport cv2\n\nnum_cores = cpu_count()\n\nhistory = model.fit_generator(generator=data_generator_train,\n                              validation_data=data_generator_val,\n                              epochs=10,\n                              workers=num_cores,\n                              verbose=1\n                             )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy = history.history['accuracy']\nloss = history.history['loss']\nepochs = range(len(accuracy))\nplt.plot(epochs, accuracy, 'b-', label='Training accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, 'r-', label='Training loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN Testing"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix\n\ny_pred = model.predict_generator(data_generator_val, workers=num_cores)\ny_true = data_generator_val.get_labels()\n\nprint(multilabel_confusion_matrix(y_true, y_pred))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}