{
  "id": 114721,
  "title": "AdamAccumulate",
  "url": "/competitions/understanding_cloud_organization/discussion/114721",
  "author_name": "",
  "post_date": "2019-10-28T22:14:44.806968700Z",
  "votes": 3,
  "comment_count": 6,
  "views": 0,
  "content": "<p>My question is do you know some working version of AdamAccumulate for Keras 2.3.0?\nBecause this version only works with keras==2.2.5. Thanks in advance for any help! </p>\n\n<p>```\nimport keras.backend as K\nfrom keras.legacy import interfaces\nfrom keras.optimizers import Optimizer</p>\n\n<p>class AdamAccumulate(Optimizer):</p>\n\n<pre><code>def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n             epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):\n    if accum_iters &lt; 1:\n        raise ValueError('accum_iters must be &gt;= 1')\n    super(AdamAccumulate, self).__init__(**kwargs)\n    with K.name_scope(self.__class__.__name__):\n        self.iterations = K.variable(0, dtype='int64', name='iterations')\n        self.lr = K.variable(lr, name='lr')\n        self.beta_1 = K.variable(beta_1, name='beta_1')\n        self.beta_2 = K.variable(beta_2, name='beta_2')\n        self.decay = K.variable(decay, name='decay')\n    if epsilon is None:\n        epsilon = K.epsilon()\n    self.epsilon = epsilon\n    self.initial_decay = decay\n    self.amsgrad = amsgrad\n    self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n    self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n@interfaces.legacy_get_updates_support\ndef get_updates(self, loss, params):\n    grads = self.get_gradients(loss, params)\n    self.updates = [K.update_add(self.iterations, 1)]\n\n    lr = self.lr\n\n    completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())\n\n    if self.initial_decay &gt; 0:\n        lr = lr * (1. / (1. + self.decay * completed_updates))\n\n    t = completed_updates + 1\n\n    lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n    # self.iterations incremented after processing a batch\n    # batch:              1 2 3 4 5 6 7 8 9\n    # self.iterations:    0 1 2 3 4 5 6 7 8\n    # update_switch = 1:        x       x    (if accum_iters=4)  \n    update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n    update_switch = K.cast(update_switch, K.floatx())\n\n    ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n    vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n    gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n    if self.amsgrad:\n        vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n    else:\n        vhats = [K.zeros(1) for _ in params]\n\n    self.weights = [self.iterations] + ms + vs + vhats\n\n    for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n        sum_grad = tg + g\n        avg_grad = sum_grad / self.accum_iters_float\n\n        m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n        v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n        if self.amsgrad:\n            vhat_t = K.maximum(vhat, v_t)\n            p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n            self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n        else:\n            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n        self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n        self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n        self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n        new_p = p_t\n\n        # Apply constraints.\n        if getattr(p, 'constraint', None) is not None:\n            new_p = p.constraint(new_p)\n\n        self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n    return self.updates\n\ndef get_config(self):\n    config = {'lr': float(K.get_value(self.lr)),\n              'beta_1': float(K.get_value(self.beta_1)),\n              'beta_2': float(K.get_value(self.beta_2)),\n              'decay': float(K.get_value(self.decay)),\n              'epsilon': self.epsilon,\n              'amsgrad': self.amsgrad}\n    base_config = super(AdamAccumulate, self).get_config()\n    return dict(list(base_config.items()) + list(config.items()))\n</code></pre>\n\n<p>```</p>",
  "messages": [
    {
      "id": "660218",
      "postDate": "10/28/2019 22:14:44",
      "content": "<p>My question is do you know some working version of AdamAccumulate for Keras 2.3.0?\nBecause this version only works with keras==2.2.5. Thanks in advance for any help! </p>\n\n<p>```\nimport keras.backend as K\nfrom keras.legacy import interfaces\nfrom keras.optimizers import Optimizer</p>\n\n<p>class AdamAccumulate(Optimizer):</p>\n\n<pre><code>def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n             epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):\n    if accum_iters &lt; 1:\n        raise ValueError('accum_iters must be &gt;= 1')\n    super(AdamAccumulate, self).__init__(**kwargs)\n    with K.name_scope(self.__class__.__name__):\n        self.iterations = K.variable(0, dtype='int64', name='iterations')\n        self.lr = K.variable(lr, name='lr')\n        self.beta_1 = K.variable(beta_1, name='beta_1')\n        self.beta_2 = K.variable(beta_2, name='beta_2')\n        self.decay = K.variable(decay, name='decay')\n    if epsilon is None:\n        epsilon = K.epsilon()\n    self.epsilon = epsilon\n    self.initial_decay = decay\n    self.amsgrad = amsgrad\n    self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n    self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n@interfaces.legacy_get_updates_support\ndef get_updates(self, loss, params):\n    grads = self.get_gradients(loss, params)\n    self.updates = [K.update_add(self.iterations, 1)]\n\n    lr = self.lr\n\n    completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())\n\n    if self.initial_decay &gt; 0:\n        lr = lr * (1. / (1. + self.decay * completed_updates))\n\n    t = completed_updates + 1\n\n    lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n    # self.iterations incremented after processing a batch\n    # batch:              1 2 3 4 5 6 7 8 9\n    # self.iterations:    0 1 2 3 4 5 6 7 8\n    # update_switch = 1:        x       x    (if accum_iters=4)  \n    update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n    update_switch = K.cast(update_switch, K.floatx())\n\n    ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n    vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n    gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n    if self.amsgrad:\n        vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n    else:\n        vhats = [K.zeros(1) for _ in params]\n\n    self.weights = [self.iterations] + ms + vs + vhats\n\n    for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n        sum_grad = tg + g\n        avg_grad = sum_grad / self.accum_iters_float\n\n        m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n        v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n        if self.amsgrad:\n            vhat_t = K.maximum(vhat, v_t)\n            p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n            self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n        else:\n            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n        self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n        self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n        self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n        new_p = p_t\n\n        # Apply constraints.\n        if getattr(p, 'constraint', None) is not None:\n            new_p = p.constraint(new_p)\n\n        self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n    return self.updates\n\ndef get_config(self):\n    config = {'lr': float(K.get_value(self.lr)),\n              'beta_1': float(K.get_value(self.beta_1)),\n              'beta_2': float(K.get_value(self.beta_2)),\n              'decay': float(K.get_value(self.decay)),\n              'epsilon': self.epsilon,\n              'amsgrad': self.amsgrad}\n    base_config = super(AdamAccumulate, self).get_config()\n    return dict(list(base_config.items()) + list(config.items()))\n</code></pre>\n\n<p>```</p>",
      "rawMarkdown": "My question is do you know some working version of AdamAccumulate for Keras 2.3.0?\nBecause this version only works with keras==2.2.5. Thanks in advance for any help! \n\n```\nimport keras.backend as K\nfrom keras.legacy import interfaces\nfrom keras.optimizers import Optimizer\n\nclass AdamAccumulate(Optimizer):\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):\n        if accum_iters &lt; 1:\n            raise ValueError('accum_iters must be &gt;= 1')\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n        self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())\n\n        if self.initial_decay &gt; 0:\n            lr = lr * (1. / (1. + self.decay * completed_updates))\n\n        t = completed_updates + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n        # self.iterations incremented after processing a batch\n        # batch:              1 2 3 4 5 6 7 8 9\n        # self.iterations:    0 1 2 3 4 5 6 7 8\n        # update_switch = 1:        x       x    (if accum_iters=4)  \n        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n        update_switch = K.cast(update_switch, K.floatx())\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n            sum_grad = tg + g\n            avg_grad = sum_grad / self.accum_iters_float\n\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n            else:\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n```",
      "votes": null
    },
    {
      "id": "660260",
      "postDate": "10/29/2019 00:03:15",
      "content": "<p>I simply uninstall both tensorflow 2.0 and keras 2.3 and install the old versions...</p>",
      "rawMarkdown": "I simply uninstall both tensorflow 2.0 and keras 2.3 and install the old versions...",
      "votes": null
    },
    {
      "id": "660304",
      "postDate": "10/29/2019 01:14:19",
      "content": "<p><a href=\"/gogo827jz\">@gogo827jz</a> what versions of Keras and Tensorflow do you use on the kernels?</p>",
      "rawMarkdown": "gogo827jz what versions of Keras and Tensorflow do you use on the kernels?",
      "votes": null
    },
    {
      "id": "660308",
      "postDate": "10/29/2019 01:25:10",
      "content": "<p>Tensorflow-gpu 1.14.0\nKeras 2.2.5</p>",
      "rawMarkdown": "Tensorflow-gpu 1.14.0\nKeras 2.2.5",
      "votes": null
    },
    {
      "id": "660626",
      "postDate": "10/29/2019 11:45:15",
      "content": "<p>Update.</p>\n\n<p>this issue has been reported and solved, <a href=\"https://github.com/CyberZHG/keras-gradient-accumulation/issues/2\">https://github.com/CyberZHG/keras-gradient-accumulation/issues/2</a></p>",
      "rawMarkdown": "Update.\n\nthis issue has been reported and solved, https://github.com/CyberZHG/keras-gradient-accumulation/issues/2",
      "votes": null
    },
    {
      "id": "660638",
      "postDate": "10/29/2019 12:05:58",
      "content": "<p>Thanks.</p>",
      "rawMarkdown": "Thanks.",
      "votes": null
    },
    {
      "id": "661024",
      "postDate": "10/29/2019 21:54:02",
      "content": "<p>Thanks a lot!</p>",
      "rawMarkdown": "Thanks a lot!",
      "votes": null
    }
  ],
  "comments": [
    {
      "id": 660260,
      "author_name": "gogo827jz",
      "author_url": "",
      "post_date": "10/29/2019 00:03:15",
      "content": "<p>I simply uninstall both tensorflow 2.0 and keras 2.3 and install the old versions...</p>",
      "votes": null,
      "replies": [
        {
          "id": 660304,
          "author_name": "dimitreoliveira",
          "author_url": "",
          "post_date": "10/29/2019 01:14:19",
          "content": "<p><a href=\"/gogo827jz\">@gogo827jz</a> what versions of Keras and Tensorflow do you use on the kernels?</p>",
          "votes": null,
          "replies": []
        },
        {
          "id": 660308,
          "author_name": "gogo827jz",
          "author_url": "",
          "post_date": "10/29/2019 01:25:10",
          "content": "<p>Tensorflow-gpu 1.14.0\nKeras 2.2.5</p>",
          "votes": null,
          "replies": []
        }
      ]
    },
    {
      "id": 660626,
      "author_name": "dimitreoliveira",
      "author_url": "",
      "post_date": "10/29/2019 11:45:15",
      "content": "<p>Update.</p>\n\n<p>this issue has been reported and solved, <a href=\"https://github.com/CyberZHG/keras-gradient-accumulation/issues/2\">https://github.com/CyberZHG/keras-gradient-accumulation/issues/2</a></p>",
      "votes": null,
      "replies": [
        {
          "id": 660638,
          "author_name": "gogo827jz",
          "author_url": "",
          "post_date": "10/29/2019 12:05:58",
          "content": "<p>Thanks.</p>",
          "votes": null,
          "replies": []
        },
        {
          "id": 661024,
          "author_name": "jessepinkman21",
          "author_url": "",
          "post_date": "10/29/2019 21:54:02",
          "content": "<p>Thanks a lot!</p>",
          "votes": null,
          "replies": []
        }
      ]
    }
  ],
  "raw_markdown_by_id": {
    "660218": "My question is do you know some working version of AdamAccumulate for Keras 2.3.0?\nBecause this version only works with keras==2.2.5. Thanks in advance for any help! \n\n```\nimport keras.backend as K\nfrom keras.legacy import interfaces\nfrom keras.optimizers import Optimizer\n\nclass AdamAccumulate(Optimizer):\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):\n        if accum_iters &lt; 1:\n            raise ValueError('accum_iters must be &gt;= 1')\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n        self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())\n\n        if self.initial_decay &gt; 0:\n            lr = lr * (1. / (1. + self.decay * completed_updates))\n\n        t = completed_updates + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n        # self.iterations incremented after processing a batch\n        # batch:              1 2 3 4 5 6 7 8 9\n        # self.iterations:    0 1 2 3 4 5 6 7 8\n        # update_switch = 1:        x       x    (if accum_iters=4)  \n        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n        update_switch = K.cast(update_switch, K.floatx())\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n            sum_grad = tg + g\n            avg_grad = sum_grad / self.accum_iters_float\n\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n            else:\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n```",
    "660260": "I simply uninstall both tensorflow 2.0 and keras 2.3 and install the old versions...",
    "660304": "gogo827jz what versions of Keras and Tensorflow do you use on the kernels?",
    "660308": "Tensorflow-gpu 1.14.0\nKeras 2.2.5",
    "660626": "Update.\n\nthis issue has been reported and solved, https://github.com/CyberZHG/keras-gradient-accumulation/issues/2",
    "660638": "Thanks.",
    "661024": "Thanks a lot!"
  },
  "source": "meta"
}