{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\nModified from https://www.kaggle.com/mathormad/pretrained-resnet50-focal-loss/notebook?scriptVersionId=12746542\n(we have tried to catch up the Pytorch :) To see all my personal modification, please use `diff` from the original kernel to the latest version.\n\nThe best score is at version 17. But that version used `ResNet50` instead of 'should-be-better' `ResNet50V2`\n\nIn the current version, I try to improve the performance by change back from `DenseNet121` to `ResNet50`. For the previous DenseNet version with Acknowledgement, please take a look at version 9.\n\n### Modification list\n\n### ResNet50 Change LOG\n* V13 [LB553, CV546] PixSize224, LR1e-4, Batch64, 2048Dense-Head, 30Epochs\n* V14 [LB, CV549] Try VALID_SPLIT0.1, 29Epochs\n* V15 Try batch96 << runtime exceed, from LOG file, there is a memory issue that degrades performance too\n* V16 [CV528] Batch32, Epoch24 << finish too early & final LR too small\n* V17 [CV551, LB560] Batch88, Epoch27 \n* V18 [CV Bad] gamma=3\n* V19 [CV Bad] gamma=4\n* V20 [CV 524] change back to gamma=1 and use ResNetV2 weights -- thanks @mathormad again!\n* V21[CV 381!] change Hyperparameter\n* V22[CV526] batch64, LR1e-4\n* V23[CV511] batch48, LR8e-5\n* V24 Fix bugs on f2, my_f2 functions, try a bit deeper head to digest more information at the end\n* V25-27 try V17 again [FAILED]\n* V28 [] make a 'variable-size' input augmentation modification (resize later) - ResNetV1\n* V28-34 Bugs\n* V34 Bug Fixed [V2]\n* V35-36[CV558 / 531] increase LR, adjust ReduceOnPlateau for both V1&V2 -- After play around with Lopuhin's great kernel on pytorch, I try to make a learning process converge faster and better << Better results\n* V37-38 increase LR a bit more to force the network to converge earlier\n\n** ToDo ** Deal with this issue : http://blog.datumbox.com/the-batch-normalization-layer-of-keras-is-broken/"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score\nfrom keras.utils import Sequence\nWORKERS = 2\nCHANNEL = 3\nNUM_CLASSES = 1103\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nSIZE = 224\n\nRESNET_VER = 2 # Choose between V1 and V2\nepochs = 27\nbatch_size = 88\nSPLIT_RATIO = 0.1\nPOST_PROCESS = False\nLR = 5e-4\nLR_FACTOR = 0.2\nPATIENCE=2\nLR_WARM = 1e-3\ngamma = 0.0 # focal loss\ncheckpoint_file = '../working/resnet50_focal.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load dataset info\npath_to_train = '../input/imet-2019-fgvc6/train/'\ndata = pd.read_csv('../input/imet-2019-fgvc6/train.csv')\n\ntrain_dataset_info = []\nfor name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')):\n    train_dataset_info.append({\n        'path':os.path.join(path_to_train, name),\n        'labels':np.array([int(label) for label in labels])})\ntrain_dataset_info = np.array(train_dataset_info)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/lopuhin/imet-2019-submission/output\nimport argparse\nfrom collections import defaultdict, Counter\nimport random\n\nimport pandas as pd\n\nDATA_ROOT = '../input/imet-2019-fgvc6/'\n\n\ndef make_folds(n_folds: int) -> pd.DataFrame:\n    df = pd.read_csv(DATA_ROOT+ 'train.csv')\n    cls_counts = Counter(cls for classes in df['attribute_ids'].str.split()\n                         for cls in classes)\n    fold_cls_counts = defaultdict(int)\n    folds = [-1] * len(df)\n    for item in tqdm(df.sample(frac=1, random_state=42).itertuples(),\n                          total=len(df)):\n        cls = min(item.attribute_ids.split(), key=lambda cls: cls_counts[cls])\n        fold_counts = [(f, fold_cls_counts[f, cls]) for f in range(n_folds)]\n        min_count = min([count for _, count in fold_counts])\n        random.seed(item.Index)\n        fold = random.choice([f for f, count in fold_counts\n                              if count == min_count])\n        folds[item.Index] = fold\n        for cls in item.attribute_ids.split():\n            fold_cls_counts[fold, cls] += 1\n    df['fold'] = folds\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"\nepsilon = K.epsilon()\ndef focal_loss(y_true, y_pred):\n    pt = y_pred * y_true + (1-y_pred) * (1-y_true)\n    pt = K.clip(pt, epsilon, 1-epsilon)\n    CE = -K.log(pt)\n    FL = K.pow(1-pt, gamma) * CE\n    loss = K.sum(FL, axis=1)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"beta_f2=2\n\n# if gamma == 0.0:\n#     F2_THRESHOLD = 0.1\n# elif gamma == 1.0:\n#     F2_THRESHOLD = 0.2\n# else:\n#     F2_THRESHOLD = 0.3\n\n# print(F2_THRESHOLD)\n    \ndef f2(y_true, y_pred):\n    #y_pred = K.round(y_pred)\n#     y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), F2_THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=1)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=1)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=1)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=1)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f2 = (1+beta_f2**2)*p*r / (p*beta_f2**2 + r + K.epsilon())\n    f2 = tf.where(tf.is_nan(f2), tf.zeros_like(f2), f2)\n    return K.mean(f2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.resnet50 import preprocess_input\n\nclass data_generator(Sequence):\n    \n    def create_train(dataset_info, batch_size, shape, augument=True):\n        assert shape[2] == 3\n        while True:\n            dataset_info = shuffle(dataset_info)\n            for start in range(0, len(dataset_info), batch_size):\n                end = min(start + batch_size, len(dataset_info))\n                batch_images = []\n                X_train_batch = dataset_info[start:end]\n                batch_labels = np.zeros((len(X_train_batch), NUM_CLASSES))\n                for i in range(len(X_train_batch)):\n                    image = data_generator.load_image(\n                        X_train_batch[i]['path'], shape)   \n                    if augument:\n                        image = data_generator.augment(image)\n                        \n#                     image = cv2.resize(image, (SIZE, SIZE)) # resize later\n                    batch_images.append(preprocess_input(image))\n                    batch_labels[i][X_train_batch[i]['labels']] = 1\n                    \n                yield np.array(batch_images, np.float32), batch_labels\n\n    def create_valid(dataset_info, batch_size, shape, augument=False):\n        assert shape[2] == 3\n        while True:\n            # dataset_info = shuffle(dataset_info)\n            for start in range(0, len(dataset_info), batch_size):\n                end = min(start + batch_size, len(dataset_info))\n                batch_images = []\n                X_train_batch = dataset_info[start:end]\n                batch_labels = np.zeros((len(X_train_batch), NUM_CLASSES))\n                for i in range(len(X_train_batch)):\n                    image = data_generator.load_image(\n                        X_train_batch[i]['path'], shape)   \n                    if augument:\n                        image = data_generator.augment(image)\n                        \n#                     image = cv2.resize(image, (SIZE, SIZE)) # resize later\n                    batch_images.append(preprocess_input(image))\n                    batch_labels[i][X_train_batch[i]['labels']] = 1\n                yield np.array(batch_images, np.float32), batch_labels\n\n\n    def load_image(path, shape):\n        image = cv2.imread(path+'.png')\n        image = cv2.resize(image, (SIZE, SIZE)) # resize later\n        return image\n\n    def augment(image):\n        augment_img = iaa.Sequential([\n            iaa.SomeOf((0,4),[\n#             iaa.OneOf([\n#                 iaa.Affine(rotate=0),\n#                 iaa.Affine(rotate=90),\n#                 iaa.Affine(rotate=180),\n#                 iaa.Affine(rotate=270),\n                iaa.Crop(percent=(0, 0.25)),\n                iaa.ContrastNormalization((0.8, 1.2)),\n                iaa.Multiply((0.9, 1.1), per_channel=0.2),\n                iaa.Fliplr(0.5),\n#                 iaa.GaussianBlur(sigma=(0, 0.6)),\n#                 iaa.Affine(\n#                         scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n#                         translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n#                         rotate=(-180, 180),\n#                     )\n            ])], random_order=True)\n\n        image_aug = augment_img.augment_image(image)\n        return image_aug","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,GlobalAveragePooling2D,\n                          BatchNormalization, Input, Conv2D, Concatenate)\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from keras_applications import imagenet_utils as utils\n\ndef ResNet(stack_fn,\n           preact,\n           use_bias,\n           model_name='resnet',\n           include_top=True,\n           weights='imagenet',\n           input_tensor=None,\n           input_shape=None,\n           pooling=None,\n           classes=1000,\n           **kwargs):\n    \"\"\"Instantiates the ResNet, ResNetV2, and ResNeXt architecture.\n    Optionally loads weights pre-trained on ImageNet.\n    Note that the data format convention used by the model is\n    the one specified in your Keras config at `~/.keras/keras.json`.\n    # Arguments\n        stack_fn: a function that returns output tensor for the\n            stacked residual blocks.\n        preact: whether to use pre-activation or not\n            (True for ResNetV2, False for ResNet and ResNeXt).\n        use_bias: whether to use biases for convolutional layers or not\n            (True for ResNet and ResNetV2, False for ResNeXt).\n        model_name: string, model name.\n        include_top: whether to include the fully-connected\n            layer at the top of the network.\n        weights: one of `None` (random initialization),\n              'imagenet' (pre-training on ImageNet),\n              or the path to the weights file to be loaded.\n        input_tensor: optional Keras tensor\n            (i.e. output of `layers.Input()`)\n            to use as image input for the model.\n        input_shape: optional shape tuple, only to be specified\n            if `include_top` is False (otherwise the input shape\n            has to be `(224, 224, 3)` (with `channels_last` data format)\n            or `(3, 224, 224)` (with `channels_first` data format).\n            It should have exactly 3 inputs channels.\n        pooling: optional pooling mode for feature extraction\n            when `include_top` is `False`.\n            - `None` means that the output of the model will be\n                the 4D tensor output of the\n                last convolutional layer.\n            - `avg` means that global average pooling\n                will be applied to the output of the\n                last convolutional layer, and thus\n                the output of the model will be a 2D tensor.\n            - `max` means that global max pooling will\n                be applied.\n        classes: optional number of classes to classify images\n            into, only to be specified if `include_top` is True, and\n            if no `weights` argument is specified.\n    # Returns\n        A Keras model instance.\n    # Raises\n        ValueError: in case of invalid argument for `weights`,\n            or invalid input shape.\n    \"\"\"\n    global backend, layers, models, keras_utils\n    # backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs)\n    backend, layers, models, keras_utils = keras.backend, keras.layers, keras.models, keras.utils\n\n    if not (weights in {'imagenet', None} or os.path.exists(weights)):\n        raise ValueError('The `weights` argument should be either '\n                         '`None` (random initialization), `imagenet` '\n                         '(pre-training on ImageNet), '\n                         'or the path to the weights file to be loaded.')\n\n    if weights == 'imagenet' and include_top and classes != 1000:\n        raise ValueError('If using `weights` as `\"imagenet\"` with `include_top`'\n                         ' as true, `classes` should be 1000')\n\n    # Determine proper input shape\n    input_shape = utils._obtain_input_shape(input_shape,\n                                          default_size=224,\n                                          min_size=32,\n                                          data_format=backend.image_data_format(),\n                                          require_flatten=include_top,\n                                          weights=weights)\n\n    if input_tensor is None:\n        img_input = layers.Input(shape=input_shape)\n    else:\n        if not backend.is_keras_tensor(input_tensor):\n            img_input = layers.Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n\n    bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n    x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input)\n    x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x)\n\n    if preact is False:\n        x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                                      name='conv1_bn')(x)\n        x = layers.Activation('relu', name='conv1_relu')(x)\n\n    x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x)\n    x = layers.MaxPooling2D(3, strides=2, name='pool1_pool')(x)\n\n    x = stack_fn(x)\n\n    if preact is True:\n        x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                                      name='post_bn')(x)\n        x = layers.Activation('relu', name='post_relu')(x)\n\n    if include_top:\n        x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n        x = layers.Dense(classes, activation='softmax', name='probs')(x)\n    else:\n        if pooling == 'avg':\n            x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n        elif pooling == 'max':\n            x = layers.GlobalMaxPooling2D(name='max_pool')(x)\n\n    # Ensure that the model takes into account\n    # any potential predecessors of `input_tensor`.\n    if input_tensor is not None:\n        inputs = keras_utils.get_source_inputs(input_tensor)\n    else:\n        inputs = img_input\n\n    # Create model.\n    model = models.Model(inputs, x, name=model_name)\n\n    # Load weights.\n    if (weights == 'imagenet') and (model_name in WEIGHTS_HASHES):\n        if include_top:\n            file_name = model_name + '_weights_tf_dim_ordering_tf_kernels.h5'\n            file_hash = WEIGHTS_HASHES[model_name][0]\n        else:\n            file_name = model_name + '_weights_tf_dim_ordering_tf_kernels_notop.h5'\n            file_hash = WEIGHTS_HASHES[model_name][1]\n        weights_path = keras_utils.get_file(file_name,\n                                            BASE_WEIGHTS_PATH + file_name,\n                                            cache_subdir='models',\n                                            file_hash=file_hash)\n        model.load_weights(weights_path)\n    elif weights is not None:\n        model.load_weights(weights)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\ndef block2(x, filters, kernel_size=3, stride=1,\n           conv_shortcut=False, name=None):\n    \"\"\"A residual block.\n    # Arguments\n        x: input tensor.\n        filters: integer, filters of the bottleneck layer.\n        kernel_size: default 3, kernel size of the bottleneck layer.\n        stride: default 1, stride of the first layer.\n        conv_shortcut: default False, use convolution shortcut if True,\n            otherwise identity shortcut.\n        name: string, block label.\n    # Returns\n        Output tensor for the residual block.\n    \"\"\"\n    bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n    preact = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                                       name=name + '_preact_bn')(x)\n    preact = layers.Activation('relu', name=name + '_preact_relu')(preact)\n\n    if conv_shortcut is True:\n        shortcut = layers.Conv2D(4 * filters, 1, strides=stride,\n                                 name=name + '_0_conv')(preact)\n    else:\n        shortcut = layers.MaxPooling2D(1, strides=stride)(x) if stride > 1 else x\n\n    x = layers.Conv2D(filters, 1, strides=1, use_bias=False,\n                      name=name + '_1_conv')(preact)\n    x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                                  name=name + '_1_bn')(x)\n    x = layers.Activation('relu', name=name + '_1_relu')(x)\n\n    x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x)\n    x = layers.Conv2D(filters, kernel_size, strides=stride,\n                      use_bias=False, name=name + '_2_conv')(x)\n    x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                                  name=name + '_2_bn')(x)\n    x = layers.Activation('relu', name=name + '_2_relu')(x)\n\n    x = layers.Conv2D(4 * filters, 1, name=name + '_3_conv')(x)\n    x = layers.Add(name=name + '_out')([shortcut, x])\n    return x\n\n\ndef stack2(x, filters, blocks, stride1=2, name=None):\n    \"\"\"A set of stacked residual blocks.\n    # Arguments\n        x: input tensor.\n        filters: integer, filters of the bottleneck layer in a block.\n        blocks: integer, blocks in the stacked blocks.\n        stride1: default 2, stride of the first layer in the first block.\n        name: string, stack label.\n    # Returns\n        Output tensor for the stacked blocks.\n    \"\"\"\n    x = block2(x, filters, conv_shortcut=True, name=name + '_block1')\n    for i in range(2, blocks):\n        x = block2(x, filters, name=name + '_block' + str(i))\n    x = block2(x, filters, stride=stride1, name=name + '_block' + str(blocks))\n    return x\n\ndef ResNet50V2(include_top=True,\n               weights='imagenet',\n               input_tensor=None,\n               input_shape=None,\n               pooling=None,\n               classes=1000,\n               **kwargs):\n    def stack_fn(x):\n        x = stack2(x, 64, 3, name='conv2')\n        x = stack2(x, 128, 4, name='conv3')\n        x = stack2(x, 256, 6, name='conv4')\n        x = stack2(x, 512, 3, stride1=1, name='conv5')\n        return x\n    return ResNet(stack_fn, True, True, 'resnet50v2',\n                  include_top, weights,\n                  input_tensor, input_shape,\n                  pooling, classes,\n                  **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/keras-pretrain-model-weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/resnet50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.resnet50 import ResNet50\n\n\n# pretrained model is of 3 channels\ndef create_model_resnet50(n_out=NUM_CLASSES):\n    print(RESNET_VER)\n    if RESNET_VER == 1:\n        base_model =ResNet50(weights=None, include_top=False)\n        base_model.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n    else:\n        base_model =ResNet50V2(weights=None, include_top=False)\n        base_model.load_weights('../input/keras-pretrain-model-weights/resnet50v2_weights_tf_dim_ordering_tf_kernels_notop.h5')\n    \n    x0 = base_model.output\n    x1 = GlobalAveragePooling2D()(x0)\n    x2 = GlobalMaxPooling2D()(x0)\n    x = Concatenate()([x1,x2])\n    \n    x = BatchNormalization()(x)\n#     x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(512, activation='relu')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n\n    predictions = Dense(n_out, activation='sigmoid')(x)\n\n    # this is the model we will train\n    model = Model(inputs=base_model.input, outputs=predictions)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%script false\n# warm up model\nmodel = create_model_resnet50(\n    n_out=NUM_CLASSES)\n\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-10,0):\n    model.layers[i].trainable = True\n\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=Adam(LR_WARM),metrics=['acc',f2])\n\nmodel.summary()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create callbacks list\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\n                             \nfrom sklearn.model_selection import train_test_split\n\ncheckpoint = ModelCheckpoint(checkpoint_file, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = False)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=LR_FACTOR, patience=PATIENCE, \n                                   verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=1e-5)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n                       separator=',',\n                       append=True)\n\n\n# split data into train, valid\nindexes = np.arange(train_dataset_info.shape[0])\ntrain_indexes, valid_indexes = train_test_split(indexes, test_size=SPLIT_RATIO, random_state=8)\n\n# create train and valid datagens\ntrain_generator = data_generator.create_train(\n    train_dataset_info[train_indexes], batch_size, (SIZE,SIZE,3), augument=True)\ntrain_generator_warmup = data_generator.create_train(\n    train_dataset_info[train_indexes], batch_size, (SIZE,SIZE,3), augument=False)\nvalidation_generator = data_generator.create_valid(\n    train_dataset_info[valid_indexes], batch_size, (SIZE,SIZE,3), augument=False)\n\n# callbacks_list = [checkpoint, csv_logger, lrsched]\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.ceil(float(len(train_indexes)) / float(batch_size)))\nprint(np.ceil(float(len(valid_indexes)) / float(batch_size)))\nprint(len(train_indexes), batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"\n# %%script false\nmodel.fit_generator(\n    train_generator_warmup,\n    steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)),\n    epochs=1,\n    max_queue_size=16, workers=WORKERS, use_multiprocessing=True,\n    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(K.eval(model.optimizer.lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor layer in model.layers:\n    layer.trainable = True\n\nmodel.compile(# loss='binary_crossentropy',\n            loss=focal_loss,\n            optimizer=Adam(lr=LR),\n            metrics=['acc',f2])\n\nhist = model.fit_generator(\n    train_generator,\n    steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)),\n    validation_data=validation_generator,\n    validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)),\n    epochs=epochs,\n    verbose=1,\n    max_queue_size=16, workers=WORKERS, use_multiprocessing=True,\n    callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hist.epoch, hist.history[\"loss\"], label=\"Train loss\")\nax[0].plot(hist.epoch, hist.history[\"val_loss\"], label=\"Validation loss\")\nax[1].set_title('f2')\nax[1].plot(hist.epoch, hist.history[\"f2\"], label=\"Train F2\")\nax[1].plot(hist.epoch, hist.history[\"val_f2\"], label=\"Validation F2\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(K.eval(model.optimizer.lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../working/'))\nmodel.load_weights(checkpoint_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''Search for the best threshold regarding the validation set'''\n\nBATCH = 512\nfullValGen = data_generator.create_valid(\n    train_dataset_info[valid_indexes], BATCH, (SIZE,SIZE,3))\n\nn_val = round(train_dataset_info.shape[0]*0.2)//BATCH\nprint(n_val)\n\nlastFullValPred = np.empty((0, NUM_CLASSES))\nlastFullValLabels = np.empty((0, NUM_CLASSES))\nfor i in tqdm(range(n_val+1)): \n    im, lbl = next(fullValGen)\n    scores = model.predict(im)\n    lastFullValPred = np.append(lastFullValPred, scores, axis=0)\n    lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)\nprint(lastFullValPred.shape, lastFullValLabels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def my_f2(y_true, y_pred):\n    assert y_true.shape[0] == y_pred.shape[0]\n\n    tp = np.sum((y_true == 1) & (y_pred == 1),axis=1)\n    tn = np.sum((y_true == 0) & (y_pred == 0),axis=1)\n    fp = np.sum((y_true == 0) & (y_pred == 1),axis=1)\n    fn = np.sum((y_true == 1) & (y_pred == 0),axis=1)\n    \n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f2 = (1+beta_f2**2)*p*r / (p*beta_f2**2 + r + 1e-15)\n\n    return np.mean(f2)\n\ndef find_best_fixed_threshold(preds, targs, do_plot=True):\n    score = []\n    thrs = np.arange(0, 0.5, 0.01)\n    for thr in tqdm(thrs):\n        score.append(my_f2(targs, (preds > thr).astype(int) ))\n    score = np.array(score)\n    pm = score.argmax()\n    best_thr, best_score = thrs[pm], score[pm].item()\n    print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')\n    if do_plot:\n        plt.plot(thrs, score)\n        plt.vlines(x=best_thr, ymin=score.min(), ymax=score.max())\n        plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);\n        plt.show()\n    return best_thr, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')\npredicted = []\n\nfor i, name in tqdm(enumerate(submit['id'])):\n    path = os.path.join('../input/imet-2019-fgvc6/test/', name)\n    image = data_generator.load_image(path, (SIZE,SIZE,3))\n    score_predict = model.predict(preprocess_input(image[np.newaxis]))\n    # print(score_predict)\n    label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]\n    # print(label_predict)\n    str_predict_label = ' '.join(str(l) for l in label_predict)\n    predicted.append(str_predict_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['attribute_ids'] = predicted\nif POST_PROCESS:\n    submit.to_csv('submission_not_process.csv', index=False)\nelse:\n    submit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit2 = submit.copy()\nsubmit2.n_cate = submit.attribute_ids.apply(lambda x: len(x.split(\" \")))\n_ = submit2.n_cate.value_counts().sort_index().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id_rare = set([199,81,221,230,366,11,104,396,262,805,112,146,281,\n               293,328,987,71,271,201,240,108,142,389,904,312,187])\n\nid_bool = (lastFullValPred[0] > 2) # all False\nprint(sum(id_bool))\nfor ii in id_rare:\n    id_bool[ii] = True\nprint(sum(id_bool))\n\nlastFullValPred2 = lastFullValPred.copy()\n\nlastFullValPred2[0][id_bool] = 0\nthresh = np.percentile(lastFullValPred2[0],99.2)\nprint(sum(lastFullValPred2[0] < thresh))\nlastFullValPred2[0][lastFullValPred2[0] < thresh] = 0\nprint(sum(lastFullValPred2[0] == 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(lastFullValPred2[0].shape, (lastFullValPred2[i] < 0.1).shape)\nlastFullValPred2 = lastFullValPred.copy()\nfor i in tqdm(range(len(lastFullValPred2))):\n    lastFullValPred2[i][id_bool] = 0\n    thresh = np.percentile(lastFullValPred2[i],99.2)\n    lastFullValPred2[i][lastFullValPred2[i] < thresh] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_thr, best_score = find_best_fixed_threshold(lastFullValPred2, lastFullValLabels, do_plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsubmit = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')\npredicted_post = []\n\nfor i, name in tqdm(enumerate(submit['id'])):\n    path = os.path.join('../input/imet-2019-fgvc6/test/', name)\n    image = data_generator.load_image(path, (SIZE,SIZE,3))\n    score_predict = model.predict(preprocess_input(image[np.newaxis]))\n    \n    thresh = np.percentile(score_predict,99.2)\n    score_predict[score_predict < thresh] = 0\n    \n    \n    # print(score_predict)\n    label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]\n    # print(label_predict)\n    str_predict_label = ' '.join(str(l) for l in label_predict)\n    predicted_post.append(str_predict_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['attribute_ids'] = predicted_post\nif POST_PROCESS:\n    submit.to_csv('submission.csv', index=False)\nelse:\n    submit.to_csv('submission_post_process.csv', index=False)\n\nsubmit2 = submit.copy()\nsubmit2.n_cate = submit.attribute_ids.apply(lambda x: len(x.split(\" \")))\n_ = submit2.n_cate.value_counts().sort_index().plot.bar()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}