{"cells":[{"metadata":{"_uuid":"bf22c215-2ee9-4b78-99f8-aab8adf9084b","_cell_guid":"4eec7317-0eab-4b31-8d72-f80d8a8dd7d9","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"51e5bcd5-2f9c-4e7c-a315-dd377b0b53d1","_cell_guid":"b5eaa12f-fe0d-4186-bd2b-585b04743fe9","trusted":true},"cell_type":"code","source":"proj_dir = '../input/cassava-leaf-disease-classification/train_images/'\ntrain = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.loc[:,'label'] = train.loc[:,'label'].astype('str')\nBATCH_SIZE = 64\nSPLIT = 0.2\nIMG_SIZE = 448\nCHNL = 3\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(rescale=1./255,\n#         rotation_range=40,\n#         width_shift_range=0.2,\n#         height_shift_range=0.2,\n#         shear_range=0.2,\n#         zoom_range=0.2,\n#         horizontal_flip=True,\n#         fill_mode='nearest',\n#         brightness_range=[0.3,1.0],\n# #         zca_whitening=True,\n        validation_split=SPLIT)\n# train_datagen = ImageDataGenerator(rescale=1/.255,\n#                               validation_split=0.2)\n\ntrain_generator = train_datagen.flow_from_dataframe(\n        train,\n        directory = proj_dir,\n        x_col = 'image_id',\n        y_col = 'label',\n        target_size=(IMG_SIZE,IMG_SIZE),\n        batch_size=BATCH_SIZE,\n        subset = 'training',\n        class_mode='categorical')\n\nval_generator = train_datagen.flow_from_dataframe(\n        train,\n        directory = proj_dir,\n        x_col = 'image_id',\n        y_col = 'label',\n        target_size=(IMG_SIZE,IMG_SIZE),\n        batch_size=BATCH_SIZE,\n        subset = 'validation',\n        class_mode='categorical')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"142d351f-bd33-4d03-9b4d-889523cfd5b9","_cell_guid":"d4d2dbd4-32bb-452c-b645-7b0b537ba2d0","trusted":true},"cell_type":"markdown","source":"## Model"},{"metadata":{"_uuid":"ba61ad3b-f9e4-444d-9f9f-545897aed300","_cell_guid":"fbf1c47e-b455-4589-8fa3-eb2ddf1a0a81","trusted":true},"cell_type":"code","source":"# from keras.applications import Xception,VGG16\nimport tensorflow as tf\n# conv_base = Xception(weights='imagenet',include_top=False,input_shape=(150,150,3))\nconv_base = tf.keras.models.load_model('../input/pretrained-models/efficientnet')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a5d8ebc-0da7-4916-9f83-cfb9dd446af9","_cell_guid":"08086d43-e42f-42d5-adc1-99d67987ca79","trusted":true},"cell_type":"code","source":"# conv_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69d77e9c-3259-4d19-8b5c-a6bccc47d3ea","_cell_guid":"00d4c36f-6e9b-4f8d-b7ee-ffd3771b4960","trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\n\n# model = models.Sequential()\n# # model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(128,128,3)))\n# # model.add(layers.MaxPooling2D((2,2)))\n# # model.add(layers.Conv2D(64,(3,3),activation='relu'))\n# # model.add(layers.MaxPooling2D((2,2)))\n# # model.add(layers.Conv2D(128,(3,3),activation='relu'))\n# # model.add(layers.MaxPooling2D((2,2)))\n# # model.add(layers.Conv2D(128,(3,3),activation='relu'))\n# # model.add(layers.MaxPooling2D((2,2)))\n# model.add(conv_base)\n# model.add(layers.MaxPooling2D((2,2)))\n# model.add(layers.Flatten())\n# # model.add(layers.Dense(1024,activation='relu',kernel_regularizer=tf.keras.regularizers.l2(l2=0.01)))\n# # model.add(layers.Dense(256,activation='relu',kernel_regularizer=tf.keras.regularizers.l2(l2=0.01)))\n# model.add(layers.Dense(1024,activation='relu'))\n# model.add(layers.Dense(256,activation='relu'))\n# model.add(layers.Dense(5,activation='softmax'))\n\ninput_tensor = layers.Input(shape=(IMG_SIZE,IMG_SIZE,CHNL))\nx = conv_base(input_tensor)\nx = layers.MaxPooling2D((2,2))(x)\nx = layers.Flatten()(x)\nx = layers.Dense(1024,activation='relu')(x)\nx = layers.Dense(256,activation='relu')(x)\noutput = layers.Dense(5,activation='softmax')(x)\n\nmodel = models.Model(input_tensor,output)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27adc597-d3cf-41f1-8920-8c5097ae0a57","_cell_guid":"9ff04472-ad04-4b7b-aa22-40395dfae20b","trusted":true},"cell_type":"code","source":"print(len(conv_base.trainable_weights))\nconv_base.trainable=True\n\nset_trainable=False\n# print(len(model.trainable_weights))\n\nfor layer in conv_base.layers :\n#     if layer.name == 'block5_conv1' :\n    if layer.name == 'block7b_project_conv' :\n        set_trainable=True\n    if set_trainable:\n        layer.trainable=True\n    else:\n        layer.trainable=False\n\nprint(len(conv_base.trainable_weights))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5bedfda4-e11d-4edf-aea5-909c53637a87","_cell_guid":"73fdbcb1-7b00-4823-b853-ebc8fb314585","trusted":true},"cell_type":"code","source":"# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7b73683-0358-4a07-9d23-8805274fabdd","_cell_guid":"631b1d32-482a-4310-b301-f8cac404d77e","trusted":true},"cell_type":"code","source":"# coding=utf1-8\n# Copyright 2019 The Google Research Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Robust Bi-Tempered Logistic Loss Based on Bregman Divergences.\n\nSource: https://bit.ly/3jSol8T\n\"\"\"\n\nimport functools\nimport tensorflow.compat.v1 as tf1\n\n\ndef for_loop(num_iters, body, initial_args):\n  \"\"\"Runs a simple for-loop with given body and initial_args.\n\n  Args:\n    num_iters: Maximum number of iterations.\n    body: Body of the for-loop.\n    initial_args: Args to the body for the first iteration.\n\n  Returns:\n    Output of the final iteration.\n  \"\"\"\n  for i in range(num_iters):\n    if i == 0:\n      outputs = body(*initial_args)\n    else:\n      outputs = body(*outputs)\n  return outputs\n\n\ndef log_t(u, t):\n  \"\"\"Compute log_t for `u`.\"\"\"\n\n  def _internal_log_t(u, t):\n    return (u**(1.0 - t) - 1.0) / (1.0 - t)\n\n  return tf1.cond(\n      tf1.equal(t, 1.0), lambda: tf1.log(u),\n      functools.partial(_internal_log_t, u, t))\n\n\ndef exp_t(u, t):\n  \"\"\"Compute exp_t for `u`.\"\"\"\n\n  def _internal_exp_t(u, t):\n    return tf1.nn.relu(1.0 + (1.0 - t) * u)**(1.0 / (1.0 - t))\n\n  return tf1.cond(\n      tf1.equal(t, 1.0), lambda: tf1.exp(u),\n      functools.partial(_internal_exp_t, u, t))\n\n\ndef compute_normalization_fixed_point(activations, t, num_iters=5):\n  \"\"\"Returns the normalization value for each example (t > 1.0).\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (> 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n  Return: A tensor of same rank as activation with the last dimension being 1.\n  \"\"\"\n\n  mu = tf1.reduce_max(activations, -1, keep_dims=True)\n  normalized_activations_step_0 = activations - mu\n  shape_normalized_activations = tf1.shape(normalized_activations_step_0)\n\n  def iter_body(i, normalized_activations):\n    logt_partition = tf1.reduce_sum(\n        exp_t(normalized_activations, t), -1, keep_dims=True)\n    normalized_activations_t = tf1.reshape(\n        normalized_activations_step_0 * tf1.pow(logt_partition, 1.0 - t),\n        shape_normalized_activations)\n    return [i + 1, normalized_activations_t]\n\n  _, normalized_activations_t = for_loop(num_iters, iter_body,\n                                         [0, normalized_activations_step_0])\n  logt_partition = tf1.reduce_sum(\n      exp_t(normalized_activations_t, t), -1, keep_dims=True)\n  return -log_t(1.0 / logt_partition, t) + mu\n\n\ndef compute_normalization_binary_search(activations, t, num_iters=10):\n  \"\"\"Returns the normalization value for each example (t < 1.0).\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (< 1.0 for finite support).\n    num_iters: Number of iterations to run the method.\n  Return: A tensor of same rank as activation with the last dimension being 1.\n  \"\"\"\n  mu = tf1.reduce_max(activations, -1, keep_dims=True)\n  normalized_activations = activations - mu\n  shape_activations = tf1.shape(activations)\n  effective_dim = tf1.cast(\n      tf1.reduce_sum(\n          tf1.cast(\n              tf1.greater(normalized_activations, -1.0 / (1.0 - t)), tf1.int32),\n          -1,\n          keep_dims=True), tf1.float32)\n  shape_partition = tf1.concat([shape_activations[:-1], [1]], 0)\n  lower = tf1.zeros(shape_partition)\n  upper = -log_t(1.0 / effective_dim, t) * tf1.ones(shape_partition)\n\n  def iter_body(i, lower, upper):\n    logt_partition = (upper + lower)/2.0\n    sum_probs = tf1.reduce_sum(exp_t(\n        normalized_activations - logt_partition, t), -1, keep_dims=True)\n    update = tf1.cast(tf1.less(sum_probs, 1.0), tf1.float32)\n    lower = tf1.reshape(lower * update + (1.0 - update) * logt_partition,\n                       shape_partition)\n    upper = tf1.reshape(upper * (1.0 - update) + update * logt_partition,\n                       shape_partition)\n    return [i + 1, lower, upper]\n\n  _, lower, upper = for_loop(num_iters, iter_body, [0, lower, upper])\n  logt_partition = (upper + lower)/2.0\n  return logt_partition + mu\n\n\ndef compute_normalization(activations, t, num_iters=5):\n  \"\"\"Returns the normalization value for each example.\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (< 1.0 for finite support, > 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n  Return: A tensor of same rank as activation with the last dimension being 1.\n  \"\"\"\n  return tf1.cond(\n      tf1.less(t, 1.0),\n      functools.partial(compute_normalization_binary_search, activations, t,\n                        num_iters),\n      functools.partial(compute_normalization_fixed_point, activations, t,\n                        num_iters))\n\n\ndef _internal_bi_tempered_logistic_loss(activations, labels, t1, t2):\n  \"\"\"Computes the Bi-Tempered logistic loss.\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    labels: batch_size\n    t1: Temperature 1 (< 1.0 for boundedness).\n    t2: Temperature 2 (> 1.0 for tail heaviness).\n\n  Returns:\n    A loss tensor for robust loss.\n  \"\"\"\n  if t2 == 1.0:\n    normalization_constants = tf1.log(\n        tf1.reduce_sum(tf1.exp(activations), -1, keep_dims=True))\n    if t1 == 1.0:\n      return normalization_constants + tf1.reduce_sum(\n          tf1.multiply(labels, tf1.log(labels + 1e-10) - activations), -1)\n    else:\n      shifted_activations = tf1.exp(activations - normalization_constants)\n      one_minus_t1 = (1.0 - t1)\n      one_minus_t2 = 1.0\n  else:\n    one_minus_t1 = (1.0 - t1)\n    one_minus_t2 = (1.0 - t2)\n    normalization_constants = compute_normalization(\n        activations, t2, num_iters=5)\n    shifted_activations = tf1.nn.relu(1.0 + one_minus_t2 *\n                                     (activations - normalization_constants))\n\n  if t1 == 1.0:\n    return tf1.reduce_sum(\n        tf1.multiply(\n            tf1.log(labels + 1e-10) -\n            tf1.log(tf1.pow(shifted_activations, 1.0 / one_minus_t2)), labels),\n        -1)\n  else:\n    beta = 1.0 + one_minus_t1\n    logt_probs = (tf1.pow(shifted_activations, one_minus_t1 / one_minus_t2) -\n                  1.0) / one_minus_t1\n    return tf1.reduce_sum(\n        tf1.multiply(log_t(labels, t1) - logt_probs, labels) - 1.0 / beta *\n        (tf1.pow(labels, beta) -\n         tf1.pow(shifted_activations, beta / one_minus_t2)), -1)\n\n\ndef tempered_sigmoid(activations, t, num_iters=5):\n  \"\"\"Tempered sigmoid function.\n\n  Args:\n    activations: Activations for the positive class for binary classification.\n    t: Temperature tensor > 0.0.\n    num_iters: Number of iterations to run the method.\n\n  Returns:\n    A probabilities tensor.\n  \"\"\"\n  t = tf1.convert_to_tensor(t)\n  input_shape = tf1.shape(activations)\n  activations_2d = tf1.reshape(activations, [-1, 1])\n  internal_activations = tf1.concat(\n      [tf1.zeros_like(activations_2d), activations_2d], 1)\n  normalization_constants = tf1.cond(\n      # pylint: disable=g-long-lambda\n      tf1.equal(t, 1.0),\n      lambda: tf1.log(\n          tf1.reduce_sum(tf1.exp(internal_activations), -1, keep_dims=True)),\n      functools.partial(compute_normalization, internal_activations, t,\n                        num_iters))\n  internal_probabilities = exp_t(internal_activations - normalization_constants,\n                                 t)\n  one_class_probabilities = tf1.split(internal_probabilities, 2, axis=1)[1]\n  return tf1.reshape(one_class_probabilities, input_shape)\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n  \"\"\"Tempered softmax function.\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature tensor > 0.0.\n    num_iters: Number of iterations to run the method.\n\n  Returns:\n    A probabilities tensor.\n  \"\"\"\n  t = tf1.convert_to_tensor(t)\n  normalization_constants = tf1.cond(\n      tf1.equal(t, 1.0),\n      lambda: tf1.log(tf1.reduce_sum(tf1.exp(activations), -1, keep_dims=True)),\n      functools.partial(compute_normalization, activations, t, num_iters))\n  return exp_t(activations - normalization_constants, t)\n\n\ndef bi_tempered_binary_logistic_loss(activations,\n                                     labels,\n                                     t1,\n                                     t2,\n                                     label_smoothing=0.0,\n                                     num_iters=5):\n  \"\"\"Bi-Tempered binary logistic loss.\n\n  Args:\n    activations: A tensor containing activations for class 1.\n    labels: A tensor with shape and dtype as activations.\n    t1: Temperature 1 (< 1.0 for boundedness).\n    t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n    label_smoothing: Label smoothing\n    num_iters: Number of iterations to run the method.\n\n  Returns:\n    A loss tensor.\n  \"\"\"\n  with tf1.name_scope('binary_bitempered_logistic'):\n    t1 = tf1.convert_to_tensor(t1)\n    t2 = tf1.convert_to_tensor(t2)\n    out_shape = tf1.shape(labels)\n    labels_2d = tf1.reshape(labels, [-1, 1])\n    activations_2d = tf1.reshape(activations, [-1, 1])\n    internal_labels = tf1.concat([1.0 - labels_2d, labels_2d], 1)\n    internal_logits = tf1.concat([tf1.zeros_like(activations_2d), activations_2d],\n                                1)\n    losses = bi_tempered_logistic_loss(internal_logits, internal_labels, t1, t2,\n                                       label_smoothing, num_iters)\n    return tf1.reshape(losses, out_shape)\n\n\ndef bi_tempered_logistic_loss(labels,\n                              activations,\n                              t1=0.2,\n                              t2=1.0,\n                              label_smoothing=0.0,\n                              num_iters=10):\n  \"\"\"Bi-Tempered Logistic Loss with custom gradient.\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    labels: A tensor with shape and dtype as activations.\n    t1: Temperature 1 (< 1.0 for boundedness).\n    t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n    label_smoothing: Label smoothing parameter between [0, 1).\n    num_iters: Number of iterations to run the method.\n\n  Returns:\n    A loss tensor.\n  \"\"\"\n  with tf1.name_scope('bitempered_logistic'):\n    t1 = tf1.convert_to_tensor(t1)\n    t2 = tf1.convert_to_tensor(t2)\n    if label_smoothing > 0.0:\n      num_classes = tf1.cast(tf1.shape(labels)[-1], tf1.float32)\n      labels = (\n          1 - num_classes /\n          (num_classes - 1) * label_smoothing) * labels + label_smoothing / (\n              num_classes - 1)\n\n    @tf1.custom_gradient\n    def _custom_gradient_bi_tempered_logistic_loss(activations):\n      \"\"\"Bi-Tempered Logistic Loss with custom gradient.\n\n      Args:\n        activations: A multi-dimensional tensor with last dim `num_classes`.\n\n      Returns:\n        A loss tensor, grad.\n      \"\"\"\n      with tf1.name_scope('gradient_bitempered_logistic'):\n        probabilities = tempered_softmax(activations, t2, num_iters)\n        loss_values = tf1.multiply(\n            labels,\n            log_t(labels + 1e-10, t1) -\n            log_t(probabilities, t1)) - 1.0 / (2.0 - t1) * (\n                tf1.pow(labels, 2.0 - t1) - tf1.pow(probabilities, 2.0 - t1))\n\n        def grad(d_loss):\n          \"\"\"Explicit gradient calculation.\n\n          Args:\n            d_loss: Infinitesimal change in the loss value.\n          Returns:\n            Loss gradient.\n          \"\"\"\n          delta_probs = probabilities - labels\n          forget_factor = tf1.pow(probabilities, t2 - t1)\n          delta_probs_times_forget_factor = tf1.multiply(delta_probs,\n                                                        forget_factor)\n          delta_forget_sum = tf1.reduce_sum(\n              delta_probs_times_forget_factor, -1, keep_dims=True)\n          escorts = tf1.pow(probabilities, t2)\n          escorts = escorts / tf1.reduce_sum(escorts, -1, keep_dims=True)\n          derivative = delta_probs_times_forget_factor - tf1.multiply(\n              escorts, delta_forget_sum)\n          return tf1.multiply(d_loss, derivative)\n\n        return loss_values, grad\n\n    loss_values = tf1.cond(tf1.logical_and(tf1.equal(t1, 1.0), tf1.equal(t2, 1.0)),\n                          functools.partial(\n                              tf1.nn.softmax_cross_entropy_with_logits,\n                              labels=labels,\n                              logits=activations),\n                          functools.partial(\n                              _custom_gradient_bi_tempered_logistic_loss,\n                              activations))\n    reduce_sum_last = lambda x: tf1.reduce_sum(x, -1)\n    loss_values = tf1.cond(tf1.logical_and(tf1.equal(t1, 1.0), tf1.equal(t2, 1.0)),\n                          functools.partial(tf1.identity, loss_values),\n                          functools.partial(reduce_sum_last, loss_values))\n#     _,temp = functools.partial(_custom_gradient_bi_tempered_logistic_loss,activations)\n    return loss_values\n\n\ndef sparse_bi_tempered_logistic_loss(activations, labels, t1, t2, num_iters=5):\n  \"\"\"Sparse Bi-Tempered Logistic Loss with custom gradient.\n\n  Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    labels: A tensor with dtype of int32.\n    t1: Temperature 1 (< 1.0 for boundedness).\n    t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n    num_iters: Number of iterations to run the method.\n\n  Returns:\n    A loss tensor.\n  \"\"\"\n  with tf1.name_scope('sparse_bitempered_logistic'):\n    t1 = tf1.convert_to_tensor(t1)\n    t2 = tf1.convert_to_tensor(t2)\n    num_classes = tf1.shape(activations)[-1]\n\n    @tf1.custom_gradient\n    def _custom_gradient_sparse_bi_tempered_logistic_loss(activations):\n      \"\"\"Sparse Bi-Tempered Logistic Loss with custom gradient.\n\n      Args:\n        activations: A multi-dimensional tensor with last dim `num_classes`.\n\n      Returns:\n        A loss tensor, grad.\n      \"\"\"\n      with tf1.name_scope('gradient_sparse_bitempered_logistic'):\n        probabilities = tempered_softmax(activations, t2, num_iters)\n        # TODO(eamid): Replace one hot with gather.\n        loss_values = -log_t(\n            tf1.reshape(\n                tf1.gather_nd(probabilities,\n                             tf1.where(tf1.one_hot(labels, num_classes))),\n                tf1.shape(activations)[:-1]), t1) - 1.0 / (2.0 - t1) * (\n                    1.0 - tf1.reduce_sum(tf1.pow(probabilities, 2.0 - t1), -1))\n\n        def grad(d_loss):\n          \"\"\"Explicit gradient calculation.\n\n          Args:\n            d_loss: Infinitesimal change in the loss value.\n          Returns:\n            Loss gradient.\n          \"\"\"\n          delta_probs = probabilities - tf1.one_hot(labels, num_classes)\n          forget_factor = tf1.pow(probabilities, t2 - t1)\n          delta_probs_times_forget_factor = tf1.multiply(delta_probs,\n                                                        forget_factor)\n          delta_forget_sum = tf1.reduce_sum(\n              delta_probs_times_forget_factor, -1, keep_dims=True)\n          escorts = tf1.pow(probabilities, t2)\n          escorts = escorts / tf1.reduce_sum(escorts, -1, keep_dims=True)\n          derivative = delta_probs_times_forget_factor - tf1.multiply(\n              escorts, delta_forget_sum)\n          return tf1.multiply(d_loss, derivative)\n\n        return loss_values, grad\n\n    loss_values = tf1.cond(\n        tf1.logical_and(tf1.equal(t1, 1.0), tf1.equal(t2, 1.0)),\n        functools.partial(tf1.nn.sparse_softmax_cross_entropy_with_logits,\n                          labels=labels, logits=activations),\n        functools.partial(_custom_gradient_sparse_bi_tempered_logistic_loss,\n                          activations))\n    return loss_values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10673e93-157b-4cb6-83b7-b6fd2a527058","_cell_guid":"bee06cb8-abef-41fe-a9e6-6fff9b4c209d","trusted":true},"cell_type":"code","source":"# def f2(y, z):\n#     return y + z\n\n# tf.cond(1==2,functools.partial(f2,2,3),functools.partial(f2,3,3))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a92adaa-b4f6-419a-80ff-17aece25e4bd","_cell_guid":"5fb82795-7d3d-4f10-8040-a6eda7442201","trusted":true},"cell_type":"code","source":"# a=3\n# b = lambda x : x\n# b(a)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75858bd1-62dc-4d58-bb2f-f48313361e28","_cell_guid":"bf54f1d8-4381-46dd-b79a-d0d1bd233f77","trusted":true},"cell_type":"code","source":"# import keras.backend as K\n# def gambler_loss(y_actual,y_pred):\n#     O = tf.constant([19.68,9.77,8.97,1.63,8.30])\n#     relu = tf.keras.layers.ReLU()\n# #     mul = tf.keras.layers.Multiply()\n# #     add = tf.keras.layers.Add()\n#     a = tf.add(tf.multiply(O,y_actual),-1)\n#     b = tf.add(tf.multiply(O,y_pred),-1)\n#     relu_b = relu(b)\n#     loss = -1*tf.multiply(tf.add(tf.multiply(O,y_actual),-1),relu(tf.add(tf.multiply(O,y_pred),-1)))\n#     return tf.reduce_sum(loss,axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efce15b3-6fed-40a7-93ee-9a35c67510c5","_cell_guid":"25d82bfe-ccb2-4144-baf8-ca6dd1c9f0cc","trusted":true},"cell_type":"code","source":"# a = tf.constant([0.,0.,0.,1.,0.])                 \n# b = tf.constant([0.01,0.04,0.02,0.9,0.03])\n# # out = tf.multiply(a, b)\n# print(gambler_loss(a,b))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"018dd653-bc3e-458b-af9e-8e3c899369ba","_cell_guid":"a3cc46fb-3d57-42d2-af08-63378a76c419","trusted":true},"cell_type":"code","source":"from keras import optimizers\n\n# model.compile(loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),optimizer='Adamax',metrics=['acc'])\nmodel.compile(loss=bi_tempered_logistic_loss,optimizer='Adamax',metrics=['acc'])\n\ncallback = tf.keras.callbacks.EarlyStopping(\n    monitor='val_acc', min_delta=0.01, patience=5, verbose=0,\n    mode='max', baseline=None, restore_best_weights=True\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37518c48-1683-43f3-9159-07caeb59a069","_cell_guid":"e462460c-4f94-4862-9203-65a32205f361","trusted":true},"cell_type":"code","source":"# tf.compat.v1.reset_default_graph()\nhistory = model.fit_generator(train_generator,epochs=5,steps_per_epoch=int(len(train)*(1-SPLIT)/BATCH_SIZE),callbacks=[callback],validation_data=val_generator,validation_steps=int(len(train)*SPLIT/BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = models.Model(input_tensor,x)\nmodel2.save('/kaggle/working/bitempered_activations')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # model2.summary()\n# act_datagen = ImageDataGenerator(rescale=1./255)\n# act = act_datagen.flow_from_dataframe(\n#         train,\n#         directory = proj_dir,\n#         x_col = 'image_id',\n# #         y_col = 'label',\n#         target_size=(IMG_SIZE,IMG_SIZE),\n#         batch_size=BATCH_SIZE,\n# #         subset = 'training',\n#         class_mode=None,\n#         shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# act.reset()\n\n# pred = model2.predict_generator(act,verbose=1,steps = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(len(pred)):\n#     if i == 1 :\n#         print(pred[i])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6e3508a-6b05-4d63-b088-3ab8157e83d9","_cell_guid":"cc569f8c-21a9-4f40-a192-ae1739ac684d","trusted":true},"cell_type":"code","source":"# # TODO : TTA\n# from tqdm import tqdm\n# tta_steps = 5\n# predictions = []\n\n# for i in tqdm(range(tta_steps)):\n#     preds = model.predict_generator(train_datagen.flow_from_dataframe(train,proj_dir,x_col = 'image_id',target_size = (448, 448),batch_size = len(train), shuffle=False,class_mode=None), steps = 1)\n#     predictions.append(preds)\n\n# pred = np.mean(predictions, axis=0)\n\n# acc = np.mean(np.equal(train['label'], np.argmax(pred, axis=-1)))\n# print(acc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78591528-c410-45f7-9953-41a5c84c12ec","_cell_guid":"472482b6-f284-4fd2-96ba-6a0b2af3c4bb","trusted":true},"cell_type":"code","source":"# history = model.fit_generator(train_generator,epochs=2,steps_per_epoch=265,validation_data=val_generator,validation_steps=65)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8285809a-5a2d-4aee-ba6d-cbf8680332da","_cell_guid":"86ba8db3-a8f6-496e-9abb-a275e6192b09","trusted":true},"cell_type":"code","source":"# history = model.fit_generator(train_generator,epochs=1,steps_per_epoch=265,validation_data=val_generator,validation_steps=65)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15d708c6-4e5f-43fc-a01f-470300c1e48c","_cell_guid":"4428f0f3-a829-4f91-95f9-19f25d0fa99c","trusted":true},"cell_type":"code","source":"# def smooth_points(points,factor=0.85):\n#     smoothed_points=[]\n#     for point in points:\n#         if smoothed_points:\n#             previous=smoothed_points[-1]\n#             smoothed_points.append(previous*factor+point*(1-factor))\n#         else:\n#             smoothed_points.append(point)\n#     return smoothed_points\n\n# import matplotlib.pyplot as plt\n\n# acc=history.history['acc']\n# val_acc=history.history['val_acc']\n\n# loss=history.history['loss']\n# val_loss=history.history['val_loss']\n\n# epochs=range(1,len(acc)+1)\n\n# plt.plot(epochs,acc,'bo',label='Training acc')\n# plt.plot(epochs,val_acc,'b',label='Validation acc')\n# plt.title('Training and validation accuracy')\n# plt.legend()\n\n# plt.figure()\n\n# plt.plot(epochs,loss,'bo',label='Training loss')\n# plt.plot(epochs,val_loss,'b',label='Validation loss')\n# plt.title('Training and validation loss')\n# plt.legend()\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26703ee7-86ce-4645-a159-4ffa7d12a1d5","_cell_guid":"1f2825fb-abb8-43e9-ada3-def152892b35","trusted":true},"cell_type":"markdown","source":"> ## Based on the accuracy and validation loss curve 15 epochs are to be used"},{"metadata":{"_uuid":"75a6fa04-1433-4dca-b4c4-a7ce8bf456d7","_cell_guid":"92d278c6-2ee3-4e98-8445-7766a1b3f26b","trusted":true},"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# from keras import optimizers\n# from keras import models\n# from keras import layers\n# # from keras.applications import Xception,VGG16\n\n# import tensorflow as tf\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n# import functools","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"499fe475-ede4-4e8d-82f0-89a35c9ac4f8","_cell_guid":"5cba9350-d37d-40b1-8858-e21329128793","trusted":true},"cell_type":"code","source":"# train_datagen_v2 = ImageDataGenerator(rescale=1./255,\n# #         rotation_range=40,\n# #         width_shift_range=0.2,\n# #         height_shift_range=0.2,\n# #         shear_range=0.2,\n# #         zoom_range=0.2,\n#         horizontal_flip=True\n# #         fill_mode='nearest',\n# #         brightness_range=[0.3,1.0]\n#         )\n\n# test_datagen_v2 = ImageDataGenerator(rescale=1./255)\n\n# train_generator = train_datagen_v2.flow_from_dataframe(\n#         train,\n#         directory = proj_dir,\n#         x_col = 'image_id',\n#         y_col = 'label',\n#         target_size=(448, 448),\n#         batch_size=BATCH_SIZE,\n# #         subset = 'training',\n#         class_mode='categorical')\n\n# test_dir = '../input/cassava-leaf-disease-classification/test_images/'\n\n# test=pd.DataFrame()\n# test['image_id']=os.listdir('../input/cassava-leaf-disease-classification/test_images/')\n\n# test_generator = test_datagen_v2.flow_from_dataframe(\n#         test,\n#         directory = test_dir,\n#         x_col = 'image_id',\n#         target_size = (448, 448),\n#         batch_size = 1,\n#         class_mode = None,\n#         shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a99f810-078c-4992-a058-13d42ffa9dcf","_cell_guid":"99f2ebfe-8103-4699-b570-43ee9cee6d50","trusted":true},"cell_type":"code","source":"# train_generator.class_indices","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e983ed7-75df-4e06-bff0-15f1f4f94874","_cell_guid":"253767e2-bfd6-4d5c-b800-4126056647f4","trusted":true},"cell_type":"code","source":"# with tf.device('/GPU:0'):\n#     model = models.Sequential()\n#     model.add(conv_base)\n#     model.add(layers.MaxPooling2D((2,2)))\n#     model.add(layers.Flatten())\n#     model.add(layers.Dense(1024,activation='relu'))\n#     model.add(layers.Dense(256,activation='relu'))\n#     model.add(layers.Dense(5,activation='softmax'))\n    \n#     conv_base.trainable=True\n\n#     set_trainable=False\n#     # print(len(model.trainable_weights))\n\n#     for layer in conv_base.layers :\n#         if layer.name == 'block5_conv1' :\n#             set_trainable=True\n#         if set_trainable:\n#             layer.trainable=True\n#         else:\n#             layer.trainable=False\n            \n#     from keras import optimizers\n\n# #     model.compile(loss='categorical_crossentropy',optimizer=optimizers.RMSprop(lr=2e-5),metrics=['acc'])\n#     model.compile(loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),optimizer='Adamax',metrics=['acc'])\n    \n#     history = model.fit_generator(train_generator,epochs=15,steps_per_epoch=len(train)/BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4ef8993-d5f7-4d89-8813-f925334bc0bf","_cell_guid":"1ba4cb27-6f01-4984-84e4-7c87b1f6d711","trusted":true},"cell_type":"code","source":"# test_generator.reset()\n\n# pred = model.predict_generator(test_generator,verbose=1,steps = len(test))\n\n# predicted_class_indices = np.argmax(pred,axis=1)\n\n# labels = (train_generator.class_indices)\n# labels = dict((v,k) for k,v in labels.items())\n# predictions = [labels[k] for k in predicted_class_indices]\n\n# filenames=test_generator.filenames\n# results=pd.DataFrame({\"image_id\":filenames,\n#                       \"label\":predictions})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bdb310f-3fec-4346-8ed1-e6a6f0085ce0","_cell_guid":"cf386bdd-1374-4464-ac1a-68797583d45e","trusted":true},"cell_type":"code","source":"# # results.to_csv('/kaggle/working/submission.csv',index=False)\n# model.save('/kaggle/working/model')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}