{"cells":[{"metadata":{},"cell_type":"markdown","source":"**in this kernel i will try to implement [this kernel](https://www.kaggle.com/jiageng/segmentation-cls) for understanding cloud competiton and for inference [this kernel](https://www.kaggle.com/jiageng/segmentation-cls) can be used**\n\n# please upvote that original authors kernel by visiting those 2 links above!"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir('../input/understanding_cloud_organization'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np, pandas as pd, os, gc\nimport matplotlib.pyplot as plt, time\nfrom PIL import Image \nimport warnings\nimport random\nwarnings.filterwarnings(\"ignore\")\n\npath = '../input/understanding_cloud_organization/'\ntrain = pd.read_csv(path + 'train.csv')\n\n# RESTRUCTURE TRAIN DATAFRAME\ntrain['ImageId'] = train['Image_Label'].map(lambda x: x.split('.')[0]+'.jpg')\ntrain2 = pd.DataFrame({'ImageId':train['ImageId'][::4]})\ntrain2['e1'] = train['EncodedPixels'][::4].values\ntrain2['e2'] = train['EncodedPixels'][1::4].values\ntrain2['e3'] = train['EncodedPixels'][2::4].values\ntrain2['e4'] = train['EncodedPixels'][3::4].values\ntrain2.reset_index(inplace=True,drop=True)\ntrain2.fillna('',inplace=True); \ntrain2['count'] = np.sum(train2.iloc[:,1:]!='',axis=1).values\n\nindexes = list(range(len(train2)))\nrandom.shuffle(indexes)\ntrain_ratio = 0.99\npartio = int(len(train2) * train_ratio)\ntrain_indexes = indexes[:partio]\nval_indexes = indexes[partio:]\ntrain_df = train2.iloc[train_indexes, :]\nval_df = train2.iloc[val_indexes, :]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data augument with albumentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    Compose, HorizontalFlip, ShiftScaleRotate, PadIfNeeded, RandomCrop,\n    RGBShift, RandomBrightness, RandomContrast, VerticalFlip\n)\ntrain_augmentator = Compose([\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        ShiftScaleRotate(shift_limit=0.03, scale_limit=0,\n                         rotate_limit=(-3, 3), border_mode=0, p=0.75),\n        RandomBrightness(limit=(-0.25, 0.25), p=0.75),\n        RandomContrast(limit=(-0.15, 0.4), p=0.75),\n        RGBShift(r_shift_limit=10, g_shift_limit=10, b_shift_limit=10, p=0.75)\n    ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"h = 400\nw = 400","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bd366818-0631-4453-9e26-7d25b7eeecba","_uuid":"06b65374-ac2f-4396-bb92-c4c923712061","trusted":true},"cell_type":"code","source":"import mxnet as mx\nfrom mxnet.gluon import data, HybridBlock, nn\nimport pandas as pd\nimport cv2\nimport os\nimport numpy as np\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.gluon.model_zoo import vision\nfrom mxnet.lr_scheduler import CosineScheduler\nfrom mxnet.gluon import loss, Trainer\nfrom mxnet import autograd\nimport random\nfrom PIL import Image, ImageOps, ImageFilter\nfrom mxnet import nd as F, lr_scheduler as lrs\nfrom mxnet.gluon.contrib.estimator import Estimator\nimport gluoncv.model_zoo  as gm\n\ndef scale_func(image_shape):\n    return random.uniform(0.5, 1.2)\n\n\nclass cloudDataset(data.Dataset):\n    def __init__(self, df, img_dir, debug=False):\n        \n        self.train_df = df\n        self.root_dir = img_dir\n        self.transform = transforms.Compose(\n            [\n                transforms.ToTensor(),\n                transforms.Normalize(\n                    mean=(0.485, 0.456, 0.406),\n                   std=(0.229, 0.224, 0.225)\n                )\n            ]\n        )\n        \n        self.debug = debug\n        \n    def __getitem__(self, i):\n        if self.debug:\n            curr_df = self.train_df.head(20)\n        masks = np.zeros((h, w), np.uint8)\n        img_names = []\n        item = self.train_df.iloc[i, :]\n        img_name = item['ImageId']\n        defect_label = np.zeros((1, 4), dtype=np.float32)\n        for j in range(4):\n            curr_item = item[\"e{}\".format(j+1)]\n            if len(curr_item) > 0:\n\n                rle_pixels = curr_item\n                label = rle_pixels.split(\" \")\n                positions = list(map(int, label[0::2]))\n                length = list(map(int, label[1::2]))\n                mask = np.zeros(h * w, dtype=np.uint8)\n                for pos, le in zip(positions, length):\n                    mask[pos - 1:(pos + le - 1)] = j+1\n                count = np.sum(np.where(mask==(j+1), 1, 0))\n                if count < 8:\n                    mask = np.where(mask==(j+1), -1, 0)\n                defect_label[:, j] = 1\n                masks[ :, :] = masks[ :, :] + mask.reshape(h, w, order='F')\n                \n        oimg = cv2.imread(os.path.join(self.root_dir, img_name))[:, :, ::-1]\n        # oimg, masks = self.rescale_sample(oimg, masks)\n        aug_out = train_augmentator(image=oimg, mask=masks)\n        oimg = aug_out['image']\n        masks = aug_out['mask']\n      \n        img = F.array(oimg)\n        img = self.transform(img)\n        \n        if self.debug:\n            return img, F.array(masks[::4, ::4]), oimg, masks, curr_df\n        else:\n            return img, F.array(masks), F.array(defect_label)\n        \n    def __len__(self):\n        return len(self.train_df)\n\n\n    def rescale_sample(self, image, mask):\n\n        scale = scale_func(image.shape)\n        image = cv2.resize(image, (0, 0), fx=scale, fy=scale)\n        new_size = (image.shape[1], image.shape[0])\n\n        mask = cv2.resize(mask, new_size, interpolation=cv2.INTER_NEAREST)\n\n        return image, mask\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for test\nimport matplotlib.pyplot as plt\ncsv_file = '../input/understanding_cloud_organization/train.csv'\nimg_dir = '../input/understanding_cloud_organization/train_images/'\ncloud_dataset = cloudDataset(train2, img_dir, debug=True)\nprint(len(cloud_dataset))\n_, mm, im, mask, curr_df = cloud_dataset[16]\nplt.figure(figsize=(20, 20))\nplt.subplot(2, 1, 1)\nplt.imshow(im)\nplt.subplot(2, 1, 2)\nplt.imshow(mask[::4, ::4])\nmm.flatten().shape\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### FPN-based segmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gluoncv.model_zoo.resnetv1b import resnet50_v1s, resnet101_v1s, resnet152_v1s\nimport mxnet as mx\n\nclass ResNetBackbone(mx.gluon.HybridBlock):\n    def __init__(self, backbone='resnet50', pretrained_base=True,dilated=True, **kwargs):\n        super(ResNetBackbone, self).__init__()\n\n        with self.name_scope():\n            if backbone == 'resnet50':\n                pretrained = resnet50_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs)\n            elif backbone == 'resnet101':\n                pretrained = resnet101_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs)\n            elif backbone == 'resnet152':\n                pretrained = resnet152_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs)\n            else:\n                raise RuntimeError(f'unknown backbone: {backbone}')\n\n            self.conv1 = pretrained.conv1\n            self.bn1 = pretrained.bn1\n            self.relu = pretrained.relu\n            self.maxpool = pretrained.maxpool\n            self.layer1 = pretrained.layer1\n            self.layer2 = pretrained.layer2\n            self.layer3 = pretrained.layer3\n            self.layer4 = pretrained.layer4\n\n    def hybrid_forward(self, F, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n        c1 = self.layer1(x)\n        c2 = self.layer2(c1)\n        c3 = self.layer3(c2)\n        c4 = self.layer4(c3)\n\n        return c1, c2, c3, c4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import mxnet as mx\nfrom mxnet.gluon import nn\nfrom mxnet.gluon.nn import HybridBlock\n\nclass ResNetSteel(mx.gluon.HybridBlock):\n    def __init__(self, backbone= 'resnet50', num_classes=4, backbone_lr_mult=0.1, **kwargs):\n        super(ResNetSteel, self).__init__()\n\n        self.backbone_name = backbone\n        self.backbone_lr_mult = backbone_lr_mult\n        self._kwargs = kwargs\n\n        with self.name_scope():\n            self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)\n\n            self.head = Classification_head(output_channels=256, num_classes=num_classes)\n\n    def load_pretrained_weights(self):\n        pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)\n        backbone_params = self.backbone.collect_params()\n        pretrained_weights = pretrained.collect_params()\n        for k, v in pretrained_weights.items():\n            param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]\n            backbone_params[param_name].set_data(v.data())\n\n        self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)\n\n    def hybrid_forward(self,F, x):\n        c1, c2, c3, c4 = self.backbone(x)\n        logits = self.head(c4)\n\n        return logits\n\nclass Classification_head(HybridBlock):\n    def __init__(self, output_channels=256, num_classes=4):\n        super(Classification_head, self).__init__()\n\n        with self.name_scope():\n            self.cls_head = nn.HybridSequential()\n            self.cls_head.add(ConvBlock(output_channels, kernel_size=1))\n            self.cls_head.add(nn.GlobalAvgPool2D())\n            self.cls_head.add(nn.Conv2D(num_classes, kernel_size=1))\n\n    def hybrid_forward(self, F, x):\n        logits = self.cls_head(x)\n\n        return F.squeeze(logits)\n\n\nclass ConvBlock(HybridBlock):\n    def __init__(self, output_channels, kernel_size, padding=0, activation='relu', norm_layer=nn.BatchNorm):\n        super().__init__()\n        self.body = nn.HybridSequential()\n        self.body.add(\n            nn.Conv2D(output_channels, kernel_size=kernel_size, padding=padding, activation=activation),\n            norm_layer(in_channels=output_channels)\n        )\n\n    def hybrid_forward(self, F, x):\n        return self.body(x)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet = ResNetSteel(num_classes=4)\nunet.collect_params().initialize()\nunet.load_pretrained_weights()\na = mx.nd.normal(shape=(4, 3, h, w))\nlogits = unet(a)\nprint(logits.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### loss function\n- focal loss\n- normalize focal loss\n- dice loss\n- bce loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet import nd\nfrom mxnet.gluon.loss import Loss, _apply_weighting, _reshape_like\n\nclass NormalizedFocalLossSoftmax(Loss):\n    def __init__(self, sparse_label=True, batch_axis=0, ignore_label=-1,\n                 size_average=True, detach_delimeter=True, gamma=2, eps=1e-10, **kwargs):\n        super(NormalizedFocalLossSoftmax, self).__init__(None, batch_axis, **kwargs)\n        self._sparse_label = sparse_label\n        self._ignore_label = ignore_label\n        self._size_average = size_average\n        self._detach_delimeter = detach_delimeter\n        self._eps = eps\n        self._gamma = gamma\n        self._k_sum = 0\n\n    def hybrid_forward(self, F, pred, label):\n        label = F.expand_dims(label, axis=1)\n        softmaxout = F.softmax(pred, axis=1)\n\n        t = label != self._ignore_label\n        pt = F.pick(softmaxout, label, axis=1, keepdims=True)\n        pt = F.where(t, pt, F.ones_like(pt))\n        beta = (1 - pt) ** self._gamma\n\n        t_sum = F.cast(F.sum(t, axis=(-2, -1), keepdims=True), 'float32')\n        beta_sum = F.sum(beta, axis=(-2, -1), keepdims=True)\n        mult = t_sum / (beta_sum + self._eps)\n        if self._detach_delimeter:\n            mult = mult.detach()\n        beta = F.broadcast_mul(beta, mult)\n        self._k_sum = 0.9 * self._k_sum + 0.1 * mult.asnumpy().mean()\n\n        loss = -beta * F.log(F.minimum(pt + self._eps, 1))\n\n        if self._size_average:\n            bsum = F.sum(t_sum, axis=self._batch_axis, exclude=True)\n            loss = F.sum(loss, axis=self._batch_axis, exclude=True) / (bsum + self._eps)\n        else:\n            loss = F.sum(loss, axis=self._batch_axis, exclude=True)\n\n        return loss\n\n    def log_states(self, sw, name, global_step):\n        sw.add_scalar(tag=name + '_k', value=self._k_sum, global_step=global_step)\n\n\nclass NormalizedFocalLossSigmoid(gluon.loss.Loss):\n    def __init__(self, axis=-1, alpha=0.25, gamma=2,\n                 from_logits=False, batch_axis=0,\n                 weight=None, size_average=True, detach_delimeter=True,\n                 eps=1e-12, scale=1.0,\n                 ignore_label=-1, **kwargs):\n        super(NormalizedFocalLossSigmoid, self).__init__(weight, batch_axis, **kwargs)\n        self._axis = axis\n        self._alpha = alpha\n        self._gamma = gamma\n        self._ignore_label = ignore_label\n\n        self._scale = scale\n        self._from_logits = from_logits\n        self._eps = eps\n        self._size_average = size_average\n        self._detach_delimeter = detach_delimeter\n        self._k_sum = 0\n\n    def hybrid_forward(self, F, pred, label, sample_weight=None):\n        one_hot = label > 0\n        t = F.ones_like(one_hot)\n\n        if not self._from_logits:\n            pred = F.sigmoid(pred)\n\n        alpha = F.where(one_hot, self._alpha * t, (1 - self._alpha) * t)\n        pt = F.where(one_hot, pred, 1 - pred)\n        pt = F.where(label != self._ignore_label, pt, F.ones_like(pt))\n\n        beta = (1 - pt) ** self._gamma\n\n        t_sum = F.sum(t, axis=(-2, -1), keepdims=True)\n        beta_sum = F.sum(beta, axis=(-2, -1), keepdims=True)\n        mult = t_sum / (beta_sum + self._eps)\n        if self._detach_delimeter:\n            mult = mult.detach()\n        beta = F.broadcast_mul(beta, mult)\n\n        ignore_area = F.sum(label == -1, axis=0, exclude=True).asnumpy()\n        sample_mult = F.mean(mult, axis=0, exclude=True).asnumpy()\n        if np.any(ignore_area == 0):\n            self._k_sum = 0.9 * self._k_sum + 0.1 * sample_mult[ignore_area == 0].mean()\n\n        loss = -alpha * beta * F.log(F.minimum(pt + self._eps, 1))\n        sample_weight = label != self._ignore_label\n\n        loss = _apply_weighting(F, loss, self._weight, sample_weight)\n        if self._size_average:\n            bsum = F.sum(sample_weight, axis=self._batch_axis, exclude=True)\n            loss = F.sum(loss, axis=self._batch_axis, exclude=True) / (bsum + self._eps)\n        else:\n            loss = F.sum(loss, axis=self._batch_axis, exclude=True)\n\n        return self._scale * loss\n\n    def log_states(self, sw, name, global_step):\n        sw.add_scalar(tag=name + '_k', value=self._k_sum, global_step=global_step)\n\n\nclass FocalLoss(gluon.loss.Loss):\n    def __init__(self, axis=-1, alpha=0.25, gamma=2,\n                 from_logits=False, batch_axis=0,\n                 weight=None, num_class=None,\n                 eps=1e-9, size_average=True, scale=1.0, **kwargs):\n        super(FocalLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._axis = axis\n        self._alpha = alpha\n        self._gamma = gamma\n\n        self._scale = scale\n        self._num_class = num_class\n        self._from_logits = from_logits\n        self._eps = eps\n        self._size_average = size_average\n\n    def hybrid_forward(self, F, pred, label, sample_weight=None):\n        if not self._from_logits:\n            pred = F.sigmoid(pred)\n\n        one_hot = label > 0\n        pt = F.where(one_hot, pred, 1 - pred)\n\n        t = label != -1\n        alpha = F.where(one_hot, self._alpha * t, (1 - self._alpha) * t)\n        beta = (1 - pt) ** self._gamma\n\n        loss = -alpha * beta * F.log(F.minimum(pt + self._eps, 1))\n        sample_weight = label != -1\n\n        loss = _apply_weighting(F, loss, self._weight, sample_weight)\n        if self._size_average:\n            tsum = F.sum(label == 1, axis=self._batch_axis, exclude=True)\n            loss = F.sum(loss, axis=self._batch_axis, exclude=True) / (tsum + self._eps)\n        else:\n            loss = F.sum(loss, axis=self._batch_axis, exclude=True)\n\n        return self._scale * loss\n\n\nclass SoftmaxCrossEntropyLoss(Loss):\n    def __init__(self, sparse_label=True, batch_axis=0, ignore_label=-1,\n                 size_average=True, grad_scale=1.0, **kwargs):\n        super(SoftmaxCrossEntropyLoss, self).__init__(None, batch_axis, **kwargs)\n        self._sparse_label = sparse_label\n        self._ignore_label = ignore_label\n        self._size_average = size_average\n        self._grad_scale = grad_scale\n\n    def hybrid_forward(self, F, pred, label):\n        softmaxout = F.SoftmaxOutput(\n            pred, label.astype(pred.dtype), ignore_label=self._ignore_label,\n            multi_output=self._sparse_label,\n            use_ignore=True, normalization='valid' if self._size_average else 'null',\n            grad_scale=self._grad_scale,\n        )\n        loss = -F.pick(F.log(softmaxout), label, axis=1, keepdims=True)\n        loss = F.where(label.expand_dims(axis=1) == self._ignore_label,\n                       F.zeros_like(loss), loss)\n        return F.mean(loss, axis=self._batch_axis, exclude=True)\n\n\nclass SigmoidBinaryCrossEntropyLoss(Loss):\n    def __init__(self, from_sigmoid=False, weight=None, batch_axis=0, ignore_label=-1, **kwargs):\n        super(SigmoidBinaryCrossEntropyLoss, self).__init__(\n            weight, batch_axis, **kwargs)\n        self._from_sigmoid = from_sigmoid\n        self._ignore_label = ignore_label\n\n    def hybrid_forward(self, F, pred, label):\n        label = _reshape_like(F, label, pred)\n        sample_weight = label != self._ignore_label\n        label = F.where(sample_weight, label, F.zeros_like(label))\n\n        if not self._from_sigmoid:\n            loss = F.relu(pred) - pred * label + \\\n                F.Activation(-F.abs(pred), act_type='softrelu')\n        else:\n            eps = 1e-12\n            loss = -(F.log(pred + eps) * label\n                     + F.log(1. - pred + eps) * (1. - label))\n\n        loss = _apply_weighting(F, loss, self._weight, sample_weight)\n        return F.mean(loss, axis=self._batch_axis, exclude=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ac46c49b-aa68-4973-b730-45ebd60f8ef3","_uuid":"9ec0a844-05a5-4594-93eb-b376d2548250","trusted":true},"cell_type":"code","source":"def compute_iou(label, pred):\n    union = np.logical_or(label, pred)\n    intersection = np.logical_and(label, pred)\n    iou = intersection / (union + 1e-5)\n    return np.mean(iou)\n\ndef iou_metric(labels, preds):\n    \n#     labels = F.array(labels)\n#     preds = F.array(preds)\n    labels = labels.asnumpy()\n    preds = F.argmax(F.softmax(preds, axis=1), axis=1).asnumpy()\n    ious = []\n    for i in range(5):\n        curr_pred = np.where(preds==i, 1, 0)\n        curr_labels = np.where(labels==i, 1, 0)\n        curr_iou = compute_iou(curr_labels, curr_pred)\n        ious.append(curr_iou)\n    mean_iou = np.mean(ious)\n    ious.append(mean_iou)\n#     print(\"IOU_INFO:: bg:{}, 1:{}, 2:{}, 3:{}, 4:{}, mean_iou:{}\".format(*ious))\n    cls = ['bg', '1', '2', '3', '4', 'mean_iou']\n    return {k:v for k, v in zip(cls, ious)}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def training(epoch, data, net, cls_loss, trainer, ctx):\n    train_loss = 0.0\n    hybridize = False\n    tbar = tqdm(data)\n    for i, batch_data in enumerate(tbar):\n        image, mask, label = batch_data\n        image = image.as_in_context(ctx)\n        mask = mask.as_in_context(ctx)\n        label = label.as_in_context(ctx)\n        with autograd.record():\n            logits = net(image)\n            cls_losses = cls_loss(logits, label)\n            losses = cls_losses\n        losses.backward()\n        global_step = epoch * len(data) + i\n        trainer.step(len(batch_data))\n\n        batch_loss = sum(loss.asnumpy().mean() for loss in losses) / len(losses)\n        train_loss += batch_loss\n       \n        if i % 6:\n            tbar.set_description(f'Epoch {epoch}, training loss {train_loss/(i+1):.6f}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluation(data, net, ctx):\n    \n    val_acc = 0.0\n    hybridize = False\n    tbar = tqdm(data)\n    for i, batch_data in enumerate(tbar):\n        image, mask, label = batch_data\n        image = image.as_in_context(ctx)\n        mask = mask.as_in_context(ctx)\n        label = label.as_in_context(ctx)\n        logits = net(image)\n        probs = F.sigmoid(logits)\n        probs = F.where(probs > 0.5, F.ones_like(probs), F.zeros_like(probs))\n        val_acc += F.mean(label==probs).asscalar()\n        if i % 6:\n            tbar.set_description(f'val_accs {val_acc/(i+1):.6f}')\n    return val_acc * 1.0 /(i+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom tqdm import tqdm\ndef train_from_manual(train_df, val_df, img_dir, batch_size, epoches, lr=0.001, ctx=mx.cpu()):\n    # TODO: finish trainer .etc, add ctx\n    cloud_dataset = cloudDataset(train_df, img_dir)\n    cloud_data = data.DataLoader(cloud_dataset, batch_size=batch_size, num_workers=4, shuffle=True)\n    \n    val_steel_dataset = cloudDataset(val_df, img_dir)\n    val_steel_data = data.DataLoader(val_steel_dataset, batch_size=batch_size, num_workers=4, shuffle=False)\n    \n    normal_focal_loss = NormalizedFocalLossSoftmax(ignore_label=-1, gamma=1)\n    bce_loss = SigmoidBinaryCrossEntropyLoss()\n    \n    unet = ResNetSteel(num_classes=4)\n    unet.initialize(mx.init.Xavier(rnd_type='gaussian', magnitude=2), ctx=ctx)\n    unet.load_pretrained_weights()\n    for k, v in unet.collect_params('.*beta|.*gamma|.*bias').items():\n        v.wd_mult = 0.0\n    lr_sche = lrs.FactorScheduler(step=5, base_lr=lr, factor=0.7,  warmup_steps=2, warmup_begin_lr=0.00002)\n    trainer = Trainer(unet.collect_params(), 'adam', \n                        {'learning_rate': lr,\n                         'wd':1e-5,\n#                          'lr_scheduler': lr_sche\n                        })\n    for epoch in range(epoches):\n        max_acc = -1\n        if epoch in [10, 15, 20, 25, 35]:\n            lr = lr * 0.7\n            trainer.set_learning_rate(lr=lr)\n        training(epoch, cloud_data, unet, bce_loss, trainer, ctx)\n        if epoch % 2 == 0:\n            val_acc = evaluation(val_steel_data, unet, ctx)\n            if val_acc > max_acc:\n                print(\"acc from {} improve to {}\".format(max_acc, val_acc))\n                max_acc = val_acc\n                \n            unet.save_parameters('unet_{}_{}.params'.format(epoch, max_acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 4\ncsv_file = '../input/understanding_cloud_organization/train.csv'\nimg_dir = '../input/understanding_cloud_organization/train_images/'\n\nepoches = 1\ntrain_from_manual(train_df, val_df, img_dir, batch_size, epoches, ctx=mx.gpu())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import pandas as pd\nsample_submission = pd.read_csv(\"../input/understanding_cloud_organization/sample_submission.csv\")\ntrain = pd.read_csv(\"../input/understanding_cloud_organization/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3},"nbformat":4,"nbformat_minor":1}