{"cells":[{"metadata":{},"cell_type":"markdown","source":"## imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nimport cv2\n\nimport numpy as np\nimport pandas as pd\n\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.optimizers import *\nfrom keras.utils import *\nfrom keras.callbacks import *\n\nfrom keras import backend as K\nfrom keras.applications.densenet import DenseNet121, preprocess_input\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import fbeta_score\n\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = os.listdir(\"../input/imet-2019-fgvc6/train/\")\ntest_images = os.listdir(\"../input/imet-2019-fgvc6/test/\")\n\nprint(\"number of train images: \", len(train_images))\nprint(\"number of test  images: \", len(test_images))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/imet-2019-fgvc6/train.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = pd.read_csv(\"../input/imet-2019-fgvc6/labels.csv\")\nlabels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cultures = [x for x in labels.attribute_name.values if x.startswith(\"culture\")]\ntags = [x for x in labels.attribute_name.values if x.startswith(\"tag\")]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(cultures), len(tags)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_culture_tag(x):\n    cultures_ = list()\n    tags_ = list()\n    for i in x.split(\" \"):\n        if int(i) <= len(cultures):\n            cultures_.append(i)\n        else:\n            tags_.append(str(int(i) - len(cultures)))\n    if not cultures_:\n        cultures_.append(str(len(cultures)))\n    if not tags_:\n        tags_.append(str(len(tags)))\n    return \" \".join(cultures_), \" \".join(tags_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"culture_ids = list()\ntag_ids = list()\n\nfor v in tqdm(train.attribute_ids.values):\n    c, t = split_culture_tag(v)\n    culture_ids.append(c)\n    tag_ids.append(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"culture_ids\"] = culture_ids\ntrain[\"tag_ids\"] = tag_ids\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes_c = len(cultures) + 1\nnum_classes_t = len(tags) + 1\n\nprint(num_classes_c, num_classes_t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_map = {v:i for i, v in zip(labels.attribute_id.values, labels.attribute_name.values)}\nlabels_map_rev = {i:v for i, v in zip(labels.attribute_id.values, labels.attribute_name.values)}\n\nnum_classes = len(labels_map)\nprint(\"{} categories\".format(num_classes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/imet-2019-fgvc6/sample_submission.csv\")\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## prepare X and y"},{"metadata":{"trusted":true},"cell_type":"code","source":"def obtain_y_c(ids):\n    y = np.zeros(num_classes_c)\n    for idx in ids.split(\" \"):\n        y[int(idx)] = 1\n    return y\n\ndef obtain_y_t(ids):\n    y = np.zeros(num_classes_t)\n    for idx in ids.split(\" \"):\n        y[int(idx)] = 1\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = [\"../input/imet-2019-fgvc6/train/{}.png\".format(x) for x in train.id.values]\n\ntargets_c = np.array([obtain_y_c(y) for y in train.culture_ids.values])\ntargets_t = np.array([obtain_y_t(y) for y in train.tag_ids.values])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### image generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageGenerator(Sequence):\n    \n    def __init__(self, paths, targets_c, targets_t, batch_size, shape, augment=False):\n        self.paths = paths\n        self.targets_c = targets_c\n        self.targets_t = targets_t\n        self.batch_size = batch_size\n        self.shape = shape\n        self.augment = augment\n        \n    def __len__(self):\n        return int(np.ceil(len(self.paths) / float(self.batch_size)))\n    \n    def __getitem__(self, idx):\n        batch_paths = self.paths[idx * self.batch_size : (idx + 1) * self.batch_size]\n        x = np.zeros((len(batch_paths), self.shape[0], self.shape[1], self.shape[2]), dtype=np.float32)\n        y = np.zeros((self.batch_size, num_classes, 1))\n        for i, path in enumerate(batch_paths):\n            x[i] = self.__load_image(path)\n        y_c = self.targets_c[idx * self.batch_size : (idx + 1) * self.batch_size]\n        y_t = self.targets_t[idx * self.batch_size : (idx + 1) * self.batch_size]\n        return x, [y_c, y_t]\n    \n    def __iter__(self):\n        for item in (self[i] for i in range(len(self))):\n            yield item\n            \n    def __load_image(self, path):\n        image = cv2.imread(path)\n        image = cv2.resize(image, (self.shape[0], self.shape[1]))\n        image = preprocess_input(image)\n        if self.augment:\n            seq = iaa.Sequential([\n                iaa.OneOf([\n                    iaa.Fliplr(0.5),\n                    iaa.Flipud(0.5),\n                    iaa.CropAndPad(percent=(-0.25, 0.25)),\n                    iaa.Crop(percent=(0, 0.1)),\n                    iaa.Sometimes(0.5,\n                        iaa.GaussianBlur(sigma=(0, 0.5))\n                    ),\n                    iaa.Affine(\n                        scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n                        translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n                        rotate=(-180, 180),\n                        shear=(-8, 8)\n                    )\n                ])\n            ], random_order=True)\n            image = seq.augment_image(image)\n        return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train test split"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\n\ntrain_paths, val_paths, train_targets_c, val_targets_c, train_targets_t, val_targets_t = train_test_split(paths, \n                                                                      targets_c,\n                                                                      targets_t,\n                                                                      test_size=0.1, \n                                                                      random_state=1029)\n\ntrain_gen = ImageGenerator(train_paths, train_targets_c, train_targets_t, batch_size=batch_size, shape=(224,224,3), augment=False)\nval_gen = ImageGenerator(val_paths, val_targets_c, val_targets_t, batch_size=batch_size, shape=(224,224,3), augment=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input((224, 224, 3))\nbackbone = DenseNet121(input_tensor=inp,\n                       weights=\"../input/densenet-keras/DenseNet-BC-121-32-no-top.h5\",\n                       include_top=False)\nx = backbone.output\nx = GlobalAveragePooling2D()(x)\n\ny_c = Dense(1024, activation=\"relu\")(x)\ny_c = Dropout(0.5)(y_c)\ny_c = Dense(num_classes_c, activation=\"sigmoid\", name=\"cultures_out\")(y_c)\n\ny_t = Dense(2048, activation=\"relu\")(x)\ny_t = Dropout(0.5)(y_t)\ny_t = Dense(num_classes_t, activation=\"sigmoid\", name=\"tags_out\")(y_t)\n\n\nmodel = Model(inp, [y_c, y_t])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" losses = {\n     \"cultures_out\": 'binary_crossentropy',\n     \"tags_out\": 'binary_crossentropy'\n }\n    \nloss_weights = {\n    \"cultures_out\": 1.0,\n    \"tags_out\": 4.0\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### f_score for Keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"def f_score(y_true, y_pred, threshold=0.1, beta=2):\n    tp = tp_score(y_true, y_pred, threshold)\n    fp = fp_score(y_true, y_pred, threshold)\n    fn = fn_score(y_true, y_pred, threshold)\n    precision = tp / (tp + fp)\n    recall = tp / (tp + fn)\n    return (1+beta**2) * ((precision * recall) / ((beta**2)*precision + recall))\n\n\ndef tp_score(y_true, y_pred, threshold=0.1):\n    tp_3d = K.concatenate(\n        [\n            K.cast(K.expand_dims(K.flatten(y_true)), 'bool'),\n            K.cast(K.expand_dims(K.flatten(K.greater(y_pred, K.constant(threshold)))), 'bool'),\n            K.cast(K.ones_like(K.expand_dims(K.flatten(y_pred))), 'bool')\n        ], axis=1\n    )\n    tp = K.sum(K.cast(K.all(tp_3d, axis=1), 'int32'))\n    return tp\n\n\ndef fp_score(y_true, y_pred, threshold=0.1):\n    fp_3d = K.concatenate(\n        [\n            K.cast(K.expand_dims(K.flatten(K.abs(y_true - K.ones_like(y_true)))), 'bool'),\n            K.cast(K.expand_dims(K.flatten(K.greater(y_pred, K.constant(threshold)))), 'bool'),\n            K.cast(K.ones_like(K.expand_dims(K.flatten(y_pred))), 'bool')\n        ], axis=-1\n    )\n    fp = K.sum(K.cast(K.all(fp_3d, axis=1), 'int32'))\n    return fp\n\n\ndef fn_score(y_true, y_pred, threshold=0.1):\n    fn_3d = K.concatenate(\n        [\n            K.cast(K.expand_dims(K.flatten(y_true)), 'bool'),\n            K.cast(K.expand_dims(K.flatten(K.abs(K.cast(K.greater(y_pred, K.constant(threshold)), 'float') - K.ones_like(y_pred)))), 'bool'),\n            K.cast(K.ones_like(K.expand_dims(K.flatten(y_pred))), 'bool')\n        ], axis=1\n    )\n    fn = K.sum(K.cast(K.all(fn_3d, axis=1), 'int32'))\n    return fn\n\n\ndef precision_score(y_true, y_pred, threshold=0.1):\n    tp = tp_score(y_true, y_pred, threshold)\n    fp = fp_score(y_true, y_pred, threshold)\n    return tp / (tp + fp)\n\n\ndef recall_score(y_true, y_pred, threshold=0.1):\n    tp = tp_score(y_true, y_pred, threshold)\n    fn = fn_score(y_true, y_pred, threshold)\n    return tp / (tp + fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = ModelCheckpoint('model.h5', \n                             monitor='val_tags_out_f_score', \n                             verbose=1, \n                             save_best_only=True, \n                             mode='max', \n                             save_weights_only=True)\n\nreduce_lr = ReduceLROnPlateau(monitor='val_tags_out_f_score', factor=0.2,\n                              patience=1, verbose=1, mode='max',\n                              min_delta=0.0001, cooldown=2, min_lr=1e-7)\n\nearly_stop = EarlyStopping(monitor=\"val_tags_out_f_score\", mode=\"max\", patience=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(\n    loss=losses,\n    loss_weights=loss_weights,\n    optimizer=Adam(1e-03),\n    metrics=['acc', f_score])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(generator=train_gen, \n                              steps_per_epoch=len(train_gen), \n                              validation_data=val_gen, \n                              validation_steps=len(val_gen),\n                              epochs=20,\n                              callbacks=[checkpoint, reduce_lr, early_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams['figure.figsize'] = (6,6)\n\nc_fscore = history.history['cultures_out_f_score']\nval_c_fscore = history.history['val_cultures_out_f_score']\nt_fscore = history.history['tags_out_f_score']\nval_t_fscore = history.history['val_tags_out_f_score']\n\nepochs = range(1, len(c_fscore) + 1)\n\nplt.title('Training and validation culture f2 score')\nplt.plot(epochs, c_fscore, 'red', label='Training f_score')\nplt.plot(epochs, val_c_fscore, 'blue', label='Validation f_score')\nplt.legend()\n\nplt.title('Training and validation tag f2 score')\nplt.plot(epochs, t_fscore, 'red', label='Training f_score')\nplt.plot(epochs, val_t_fscore, 'blue', label='Validation f_score')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"./model.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## prediction"},{"metadata":{},"cell_type":"markdown","source":"### test image generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestImageGenerator(Sequence):\n    \n    def __init__(self, paths, batch_size, shape):\n        self.paths = paths\n        self.batch_size = batch_size\n        self.shape = shape\n        \n    def __len__(self):\n        return int(np.ceil(len(self.paths) / float(self.batch_size)))\n    \n    def __getitem__(self, idx):\n        batch_paths = self.paths[idx * self.batch_size : (idx + 1) * self.batch_size]\n        x = np.zeros((len(batch_paths), self.shape[0], self.shape[1], self.shape[2]), dtype=np.float32)\n        for i, path in enumerate(batch_paths):\n            x[i] = self.__load_image(path)\n        return x\n    \n    def __iter__(self):\n        for item in (self[i] for i in range(len(self))):\n            yield item\n            \n    def __load_image(self, path):\n        image = cv2.imread(path)\n        image = cv2.resize(image, (self.shape[0], self.shape[1]))\n        image = preprocess_input(image)\n        return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### do prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_paths = [\"../input/imet-2019-fgvc6/test/{}.png\".format(x) for x in submission.id.values]\ntest_gen = TestImageGenerator(test_paths, batch_size=batch_size, shape=(224,224,3))\n\npredicts = model.predict_generator(test_gen, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicts[0].shape, predicts[1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_predicts = model.predict_generator(val_gen, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_threshold_c = 0.\nbest_score_c = 0.\n\nfor threshold in tqdm(np.arange(0, 0.5, 0.01)):\n    f2_score = fbeta_score(val_targets_c, np.array(val_predicts[0]) > threshold, beta=2, average='samples')\n    if f2_score > best_score_c:\n        best_score_c = f2_score\n        best_threshold_c = threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_threshold_t = 0.\nbest_score_t = 0.\n\nfor threshold in tqdm(np.arange(0, 0.5, 0.01)):\n    f2_score = fbeta_score(val_targets_t, np.array(val_predicts[1]) > threshold, beta=2, average='samples')\n    if f2_score > best_score_t:\n        best_score_t = f2_score\n        best_threshold_t = threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"culture classifier: best threshold: {} best score: {}\".format(best_threshold_c, best_score_c))\nprint(\"tag     classifier: best threshold: {} best score: {}\".format(best_threshold_t, best_score_t))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def classifier(probs, th_c, th_t):\n    c = list()\n    \n    # culture classifier\n    a = np.array(probs[0] > th_c, dtype=np.int8)\n    b = np.where(a == 1)[0]\n    for idx in b.tolist():\n        if idx != len(cultures):\n            c.append(str(idx))\n            \n    # tag classifier\n    a = np.array(probs[1] > th_t, dtype=np.int8)\n    b = np.where(a == 1)[0]\n    for idx in b.tolist():\n        if idx != len(cultures) + len(tags):\n            c.append(str(idx + len(cultures)))\n\n    return \" \".join(c)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = list()\n\nfor probs in tqdm(zip(predicts[0], predicts[1])):\n    predictions.append(classifier(probs, best_threshold_c, best_threshold_t))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 6\n\nimg = cv2.imread(test_paths[n])\nplt.imshow(img)\n\na = np.array(predicts[0][n]>best_score_c, dtype=np.int8)\nb = np.where(a==1)[0]\nfor idx in b.tolist():\n    if idx != len(cultures):\n        print(labels_map_rev[idx])\n    \na = np.array(predicts[1][n]>best_score_t, dtype=np.int8)\nb = np.where(a==1)[0]\nfor idx in b.tolist():\n    if idx != len(cultures) + len(tags):\n        print(labels_map_rev[idx + len(cultures)])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[\"attribute_ids\"] = np.array(predictions)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = submission.copy()\nsubmission_df.n_cate = submission.attribute_ids.apply(lambda x: len(x.split(\" \")))\n_ = submission_df.n_cate.value_counts().sort_index().plot.bar()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}