{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[{"output_type":"stream","text":"['pet-keras-models', 'imet-2019-fgvc6']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score, fbeta_score\nfrom keras.utils import Sequence\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import *\nfrom keras.applications import *\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.utils.vis_utils import plot_model\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nSIZE = 224\nNUM_CLASSES = 1103\nbeta_f2=2","execution_count":2,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load dataset info\npath_to_train = '../input/imet-2019-fgvc6/train/'\ndata = pd.read_csv('../input/imet-2019-fgvc6/train.csv')\n\ntrain_dataset_info = []\nfor name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')):\n train_dataset_info.append({\n 'path':os.path.join(path_to_train, name),\n 'labels':np.array([int(label) for label in labels])})\ntrain_dataset_info = np.array(train_dataset_info)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gamma = 2.0\nepsilon = K.epsilon()\ndef focal_loss(y_true, y_pred):\n pt = y_pred * y_true + (1-y_pred) * (1-y_true)\n pt = K.clip(pt, epsilon, 1-epsilon)\n CE = -K.log(pt)\n FL = K.pow(1-pt, gamma) * CE\n loss = K.sum(FL, axis=1)\n return loss","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n\nclass data_generator(Sequence):\n \n def create_train(dataset_info, batch_size, shape, augument=True):\n assert shape[2] == 3\n while True:\n dataset_info = shuffle(dataset_info)\n for start in range(0, len(dataset_info), batch_size):\n end = min(start + batch_size, len(dataset_info))\n batch_images = []\n X_train_batch = dataset_info[start:end]\n batch_labels = np.zeros((len(X_train_batch), NUM_CLASSES))\n for i in range(len(X_train_batch)):\n image = data_generator.load_image(\n X_train_batch[i]['path'], shape) \n if augument:\n image = data_generator.augment(image)\n batch_images.append(image/255.)\n batch_labels[i][X_train_batch[i]['labels']] = 1\n \n yield np.array(batch_images, np.float32), batch_labels\n\n def create_valid(dataset_info, batch_size, shape, augument=False):\n assert shape[2] == 3\n while True:\n # dataset_info = shuffle(dataset_info)\n for start in range(0, len(dataset_info), batch_size):\n end = min(start + batch_size, len(dataset_info))\n batch_images = []\n X_train_batch = dataset_info[start:end]\n batch_labels = np.zeros((len(X_train_batch), NUM_CLASSES))\n for i in range(len(X_train_batch)):\n image = data_generator.load_image(\n X_train_batch[i]['path'], shape) \n if augument:\n image = data_generator.augment(image)\n batch_images.append(image/255.)\n batch_labels[i][X_train_batch[i]['labels']] = 1\n yield np.array(batch_images, np.float32), batch_labels\n\n\n def load_image(path, shape):\n image = cv2.imread(path+'.png')\n image = cv2.resize(image, (SIZE, SIZE))\n return image\n\n def augment(image):\n augment_img = iaa.Sequential([\n# sometimes(\n# iaa.OneOf([\n# # iaa.AddToHueAndSaturation((-20, 20)),\n# # iaa.Add((-10, 10), per_channel=0.5),\n# # iaa.Multiply((0.9, 1.1), per_channel=0.5),\n# # # iaa.GaussianBlur((0, 0.5)), # blur images with a sigma between 0 and 3.0\n# # iaa.ContrastNormalization((0.8, 1.2), per_channel=0.5), # improve or worsen the contrast\n# # iaa.Sharpen(alpha=(0, 0.2), lightness=(0.8, 1.2)), # sharpen images\n# # iaa.Emboss(alpha=(0, 0.5), strength=(0, 0.5)), # emboss images\n# iaa.Crop(percent=(0, 0.1))\n# ])\n# ),\n iaa.OneOf([\n iaa.Affine(rotate=0),\n iaa.Affine(rotate=90),\n iaa.Affine(rotate=180),\n iaa.Affine(rotate=270),\n iaa.Fliplr(0.5),\n # iaa.Flipud(0.5),\n ])], random_order=True)\n\n image_aug = augment_img.augment_image(image)\n return image_aug\n","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import *\nfrom keras.applications.xception import Xception\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reference link: https://gist.github.com/drscotthawley/d1818aabce8d1bf082a6fb37137473ae\nfrom keras.callbacks import Callback\n\ndef get_1cycle_schedule(lr_max=1e-3, n_data_points=8000, epochs=200, batch_size=40, verbose=0): \n \"\"\"\n Creates a look-up table of learning rates for 1cycle schedule with cosine annealing\n See @sgugger's & @jeremyhoward's code in fastai library: https://github.com/fastai/fastai/blob/master/fastai/train.py\n Wrote this to use with my Keras and (non-fastai-)PyTorch codes.\n Note that in Keras, the LearningRateScheduler callback (https://keras.io/callbacks/#learningratescheduler) only operates once per epoch, not per batch\n So see below for Keras callback\n\n Keyword arguments:\n lr_max chosen by user after lr_finder\n n_data_points data points per epoch (e.g. size of training set)\n epochs number of epochs\n batch_size batch size\n Output: \n lrs look-up table of LR's, with length equal to total # of iterations\n Then you can use this in your PyTorch code by counting iteration number and setting\n optimizer.param_groups[0]['lr'] = lrs[iter_count]\n \"\"\"\n if verbose > 0:\n print(\"Setting up 1Cycle LR schedule...\")\n pct_start, div_factor = 0.3, 25. # @sgugger's parameters in fastai code\n lr_start = lr_max/div_factor\n lr_end = lr_start/1e4\n n_iter = (n_data_points * epochs // batch_size) + 1 # number of iterations\n a1 = int(n_iter * pct_start)\n a2 = n_iter - a1\n\n # make look-up table\n lrs_first = np.linspace(lr_start, lr_max, a1) # linear growth\n lrs_second = (lr_max-lr_end)*(1+np.cos(np.linspace(0,np.pi,a2)))/2 + lr_end # cosine annealing\n lrs = np.concatenate((lrs_first, lrs_second))\n return lrs\n\n\nclass OneCycleScheduler(Callback):\n \"\"\"My modification of Keras' Learning rate scheduler to do 1Cycle learning\n which increments per BATCH, not per epoch\n Keyword arguments\n **kwargs: keyword arguments to pass to get_1cycle_schedule()\n Also, verbose: int. 0: quiet, 1: update messages.\n\n Sample usage (from my train.py):\n lrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=X_train.shape[0],\n epochs=epochs, batch_size=batch_size, verbose=1)\n \"\"\"\n def __init__(self, **kwargs):\n super(OneCycleScheduler, self).__init__()\n self.verbose = kwargs.get('verbose', 0)\n self.lrs = get_1cycle_schedule(**kwargs)\n self.iteration = 0\n\n def on_batch_begin(self, batch, logs=None):\n lr = self.lrs[self.iteration]\n K.set_value(self.model.optimizer.lr, lr) # here's where the assignment takes place\n if self.verbose > 0:\n print('\\nIteration %06d: OneCycleScheduler setting learning '\n 'rate to %s.' % (self.iteration, lr))\n self.iteration += 1\n\n def on_epoch_end(self, epoch, logs=None): # this is unchanged from Keras LearningRateScheduler\n logs = logs or {}\n logs['lr'] = K.get_value(self.model.optimizer.lr)\n self.iteration = 0\n\n","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def squeeze_excite_block(input, ratio=16):\n init = input\n channel_axis = 1 if K.image_data_format() == \"channels_first\" else -1\n filters = init._keras_shape[channel_axis]\n se_shape = (1, 1, filters)\n\n se = GlobalAveragePooling2D()(init)\n se = Reshape(se_shape)(se)\n se = Dense(filters // ratio, activation='relu', kernel_initializer='he_normal', use_bias=False)(se)\n se = Dense(filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(se)\n\n if K.image_data_format() == 'channels_first':\n se = Permute((3, 1, 2))(se)\n\n x = multiply([init, se])\n return x\n\n\ndef spatial_squeeze_excite_block(input):\n se = Conv2D(1, (1, 1), activation='sigmoid', use_bias=False,\n kernel_initializer='he_normal')(input)\n x = multiply([input, se])\n return x\n\n\ndef channel_spatial_squeeze_excite(input, ratio=16):\n cse = squeeze_excite_block(input, ratio)\n sse = spatial_squeeze_excite_block(input)\n x = add([cse, sse])\n return x\n\n\nfrom __future__ import print_function\nfrom __future__ import absolute_import\nfrom __future__ import division\n\nfrom keras.regularizers import l2\nfrom keras.utils import conv_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.engine.topology import get_source_inputs\nfrom keras_applications.imagenet_utils import _obtain_input_shape\nfrom keras.applications.resnet50 import preprocess_input\nfrom keras.applications.imagenet_utils import decode_predictions\nfrom keras import backend as K\n\n__all__ = ['SEResNet', 'SEResNet50', 'SEResNet101', 'SEResNet154', 'preprocess_input', 'decode_predictions']\n\nWEIGHTS_PATH = \"\"\nWEIGHTS_PATH_NO_TOP = \"\"\n\n\ndef SEResNet(input_shape=None,\n initial_conv_filters=64,\n depth=[3, 4, 6, 3],\n filters=[64, 128, 256, 512],\n width=1,\n bottleneck=False,\n weight_decay=1e-4,\n include_top=True,\n weights=None,\n input_tensor=None,\n pooling=None,\n classes=1000):\n\n if weights not in {'imagenet', None}:\n raise ValueError('The `weights` argument should be either '\n '`None` (random initialization) or `imagenet` '\n '(pre-training on ImageNet).')\n\n if weights == 'imagenet' and include_top and classes != 1000:\n raise ValueError('If using `weights` as imagenet with `include_top`'\n ' as true, `classes` should be 1000')\n\n assert len(depth) == len(filters), \"The length of filter increment list must match the length \" \\\n \"of the depth list.\"\n\n # Determine proper input shape\n input_shape = _obtain_input_shape(input_shape,\n default_size=224,\n min_size=32,\n data_format=K.image_data_format(),\n require_flatten=False)\n\n if input_tensor is None:\n img_input = Input(shape=input_shape)\n else:\n if not K.is_keras_tensor(input_tensor):\n img_input = Input(tensor=input_tensor, shape=input_shape)\n else:\n img_input = input_tensor\n\n x = _create_se_resnet(classes, img_input, include_top, initial_conv_filters,\n filters, depth, width, bottleneck, weight_decay, pooling)\n\n # Ensure that the model takes into account\n # any potential predecessors of `input_tensor`.\n if input_tensor is not None:\n inputs = get_source_inputs(input_tensor)\n else:\n inputs = img_input\n # Create model.\n model = Model(inputs, x, name='resnext')\n\n # load weights\n\n return model\n\n\ndef SEResNet50(input_shape=None,\n width=1,\n bottleneck=True,\n weight_decay=1e-4,\n include_top=True,\n weights=None,\n input_tensor=None,\n pooling=None,\n classes=1000):\n return SEResNet(input_shape,\n width=width,\n bottleneck=bottleneck,\n weight_decay=weight_decay,\n include_top=include_top,\n weights=weights,\n input_tensor=input_tensor,\n pooling=pooling,\n classes=classes)\n\n\ndef SEResNet101(input_shape=None,\n width=1,\n bottleneck=True,\n weight_decay=1e-4,\n include_top=True,\n weights=None,\n input_tensor=None,\n pooling=None,\n classes=1000):\n return SEResNet(input_shape,\n depth=[3, 6, 23, 3],\n width=width,\n bottleneck=bottleneck,\n weight_decay=weight_decay,\n include_top=include_top,\n weights=weights,\n input_tensor=input_tensor,\n pooling=pooling,\n classes=classes)\n\n\ndef SEResNet154(input_shape=(SIZE,SIZE,3),\n width=1,\n bottleneck=True,\n weight_decay=1e-4,\n include_top=True,\n weights=None,\n input_tensor=None,\n pooling=None,\n classes=1103):\n return SEResNet(input_shape,\n depth=[3, 8, 36, 3],\n width=width,\n bottleneck=bottleneck,\n weight_decay=weight_decay,\n include_top=include_top,\n weights=weights,\n input_tensor=input_tensor,\n pooling=pooling,\n classes=classes)\n\n\ndef _resnet_block(input, filters, k=1, strides=(1, 1)):\n init = input\n channel_axis = 1 if K.image_data_format() == \"channels_first\" else -1\n\n x = BatchNormalization(axis=channel_axis)(input)\n x = Activation('relu')(x)\n\n if strides != (1, 1) or init._keras_shape[channel_axis] != filters * k:\n init = Conv2D(filters * k, (1, 1), padding='same', kernel_initializer='he_normal',\n use_bias=False, strides=strides)(x)\n\n x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal',\n use_bias=False, strides=strides)(x)\n x = BatchNormalization(axis=channel_axis)(x)\n x = Activation('relu')(x)\n\n x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal',\n use_bias=False)(x)\n\n # squeeze and excite block\n x = squeeze_excite_block(x)\n\n m = add([x, init])\n return m\n\n\ndef _resnet_bottleneck_block(input, filters, k=1, strides=(1, 1)):\n ''' Adds a pre-activation resnet block with bottleneck layers\n Args:\n input: input tensor\n filters: number of output filters\n k: width factor\n strides: strides of the convolution layer\n Returns: a keras tensor\n '''\n init = input\n channel_axis = 1 if K.image_data_format() == \"channels_first\" else -1\n bottleneck_expand = 4\n\n x = BatchNormalization(axis=channel_axis)(input)\n x = Activation('relu')(x)\n\n if strides != (1, 1) or init._keras_shape[channel_axis] != bottleneck_expand * filters * k:\n init = Conv2D(bottleneck_expand * filters * k, (1, 1), padding='same', kernel_initializer='he_normal',\n use_bias=False, strides=strides)(x)\n\n x = Conv2D(filters * k, (1, 1), padding='same', kernel_initializer='he_normal',\n use_bias=False)(x)\n x = BatchNormalization(axis=channel_axis)(x)\n x = Activation('relu')(x)\n\n x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal',\n use_bias=False, strides=strides)(x)\n x = BatchNormalization(axis=channel_axis)(x)\n x = Activation('relu')(x)\n\n x = Conv2D(bottleneck_expand * filters * k, (1, 1), padding='same', kernel_initializer='he_normal',\n use_bias=False)(x)\n\n # squeeze and excite block\n x = squeeze_excite_block(x)\n\n m = add([x, init])\n return m\n\n\ndef _create_se_resnet(classes, img_input, include_top, initial_conv_filters, filters,\n depth, width, bottleneck, weight_decay, pooling):\n channel_axis = 1 if K.image_data_format() == 'channels_first' else -1\n N = list(depth)\n # block 1 (initial conv block)\n x = Conv2D(initial_conv_filters, (7, 7), padding='same', use_bias=False, strides=(2, 2),\n kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(img_input)\n x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)\n # block 2 (projection block)\n for i in range(N[0]):\n if bottleneck:\n x = _resnet_bottleneck_block(x, filters[0], width)\n else:\n x = _resnet_block(x, filters[0], width)\n\n # block 3 - N\n for k in range(1, len(N)):\n if bottleneck:\n x = _resnet_bottleneck_block(x, filters[k], width, strides=(2, 2))\n else:\n x = _resnet_block(x, filters[k], width, strides=(2, 2))\n for i in range(N[k] - 1):\n if bottleneck:\n x = _resnet_bottleneck_block(x, filters[k], width)\n else:\n x = _resnet_block(x, filters[k], width)\n x = BatchNormalization(axis=channel_axis)(x)\n x = Activation('relu')(x)\n if include_top:\n x = GlobalAveragePooling2D()(x)\n x = Dense(classes, use_bias=False, kernel_regularizer=l2(weight_decay),\n activation='softmax')(x)\n else:\n if pooling == 'avg':\n x = GlobalAveragePooling2D()(x)\n elif pooling == 'max':\n x = GlobalMaxPooling2D()(x)\n return x\n","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create callbacks list\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n EarlyStopping, ReduceLROnPlateau,CSVLogger)\n \nfrom sklearn.model_selection import train_test_split\n\nepochs = 40; batch_size = 16\ncheckpoint = ModelCheckpoint('../working/Resnet50_focal.h5', monitor='val_loss', verbose=1, \n save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, \n verbose=1, mode='auto', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n mode=\"min\", \n patience=9)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n separator=',',\n append=True)\n\n\n# split data into train, valid\nindexes = np.arange(train_dataset_info.shape[0])\ntrain_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)\n\n# create train and valid datagens\ntrain_generator = data_generator.create_train(\n train_dataset_info[train_indexes], batch_size, (SIZE,SIZE,3), augument=True)\ntrain_generator_warmup = data_generator.create_train(\n train_dataset_info[train_indexes], batch_size, (SIZE,SIZE,3), augument=False)\nvalidation_generator = data_generator.create_valid(\n train_dataset_info[valid_indexes], batch_size, (SIZE,SIZE,3), augument=False)\n\nlrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=len(train_indexes),\n epochs=1, batch_size=batch_size, verbose=0)\n# callbacks_list = [checkpoint, csv_logger, lrsched]\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat]","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# warm up model\nmodel = SEResNet154()\n\nfor layer in model.layers:\n layer.trainable = True\n\n\nmodel.compile(\n loss='binary_crossentropy',\n optimizer=Adam(1e-3))\n\n# model.summary()\n\nmodel.fit_generator(\n train_generator_warmup,\n steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)),\n epochs=2,\n max_queue_size=16, workers=WORKERS, use_multiprocessing=True,\n verbose=1)","execution_count":10,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\nEpoch 1/3\n1280/5804 [=====>........................] - ETA: 32:12 - loss: 0.0217","name":"stdout"},{"output_type":"stream","text":"Process ForkPoolWorker-1:\nProcess ForkPoolWorker-2:\nTraceback (most recent call last):\nTraceback (most recent call last):\n File \"/opt/conda/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n self.run()\n File \"/opt/conda/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n self.run()\n File \"/opt/conda/lib/python3.6/multiprocessing/process.py\", line 93, in run\n self._target(*self._args, **self._kwargs)\n File \"/opt/conda/lib/python3.6/multiprocessing/process.py\", line 93, in run\n self._target(*self._args, **self._kwargs)\n File \"/opt/conda/lib/python3.6/multiprocessing/pool.py\", line 108, in worker\n task = get()\n File \"/opt/conda/lib/python3.6/multiprocessing/queues.py\", line 335, in get\n res = self._reader.recv_bytes()\n File \"/opt/conda/lib/python3.6/multiprocessing/pool.py\", line 108, in worker\n task = get()\n File \"/opt/conda/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n buf = self._recv_bytes(maxlength)\n File \"/opt/conda/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n with self._rlock:\n File \"/opt/conda/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n return self._semlock.__enter__()\nKeyboardInterrupt\n File \"/opt/conda/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n buf = self._recv(4)\n File \"/opt/conda/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n chunk = read(handle, remaining)\nKeyboardInterrupt\n","name":"stderr"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWORKERS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m verbose=1)\n\u001b[0m","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m 90\u001b[0m 'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 1416\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1417\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1420\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 215\u001b[0m outs = model.train_on_batch(x, y,\n\u001b[1;32m 216\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m class_weight=class_weight)\n\u001b[0m\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1215\u001b[0m \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1216\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1217\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1218\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2713\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2716\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2717\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2673\u001b[0m \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2674\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2676\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1437\u001b[0m ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m 1438\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 1440\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1441\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train all layers\n\n\nmodel.compile(loss='binary_crossentropy',\n # loss=focal_loss,\n optimizer=Adam(lr=1e-4))\n\nmodel.fit_generator(\n train_generator,\n steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)),\n validation_data=validation_generator,\n validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)),\n epochs=5,\n verbose=1,\n max_queue_size=16, workers=WORKERS, use_multiprocessing=True,\n callbacks=callbacks_list)\n","execution_count":null,"outputs":[{"output_type":"stream","text":"Epoch 1/3\n 377/5804 [>.............................] - ETA: 48:17 - loss: 0.0168","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../working/'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')\nmodel.load_weights('../working/Resnet50_focal.h5')\npredicted = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''Search for the best threshold regarding the validation set'''\n\nBATCH = 512\nfullValGen = data_generator.create_valid(\n train_dataset_info[valid_indexes], BATCH, (SIZE,SIZE,3))\n\nn_val = round(train_dataset_info.shape[0]*0.15)//BATCH\nprint(n_val)\n\nlastFullValPred = np.empty((0, NUM_CLASSES))\nlastFullValLabels = np.empty((0, NUM_CLASSES))\nfor i in tqdm(range(n_val+1)): \n im, lbl = next(fullValGen)\n scores = model.predict(im)\n lastFullValPred = np.append(lastFullValPred, scores, axis=0)\n lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)\nprint(lastFullValPred.shape, lastFullValLabels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def my_f2(y_true, y_pred):\n assert y_true.shape[0] == y_pred.shape[0]\n\n tp = np.sum((y_true == 1) & (y_pred == 1))\n tn = np.sum((y_true == 0) & (y_pred == 0))\n fp = np.sum((y_true == 0) & (y_pred == 1))\n fn = np.sum((y_true == 1) & (y_pred == 0))\n \n p = tp / (tp + fp + K.epsilon())\n r = tp / (tp + fn + K.epsilon())\n\n f2 = (1+beta_f2**2)*p*r / (p*beta_f2**2 + r + 1e-15)\n\n return f2\n\ndef find_best_fixed_threshold(preds, targs, do_plot=True):\n score = []\n thrs = np.arange(0, 0.5, 0.01)\n for thr in tqdm(thrs):\n score.append(my_f2(targs, (preds > thr).astype(int) ))\n score = np.array(score)\n pm = score.argmax()\n best_thr, best_score = thrs[pm], score[pm].item()\n print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')\n if do_plot:\n plt.plot(thrs, score)\n plt.vlines(x=best_thr, ymin=score.min(), ymax=score.max())\n plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);\n plt.show()\n return best_thr, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor i, name in tqdm(enumerate(submit['id'])):\n path = os.path.join('../input/imet-2019-fgvc6/test/', name)\n image = data_generator.load_image(path, (SIZE,SIZE,3))\n score_predict = model.predict(image[np.newaxis]/255.)\n # print(score_predict)\n label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]\n # print(label_predict)\n str_predict_label = ' '.join(str(l) for l in label_predict)\n predicted.append(str_predict_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['attribute_ids'] = predicted\nsubmit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}