{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a59a645049c52c1abcdb182493ec2f3344425272"},"cell_type":"code","source":"! pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom pretrainedmodels import xception\n\nimport PIL\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"686bb705fce2e832aba8c71c9d0c3ce178020aad"},"cell_type":"markdown","source":"Set some contants..."},{"metadata":{"trusted":true,"_uuid":"0f870861486f649503520a12c096b7fc09b80188"},"cell_type":"code","source":"IMG_SIZE = 128\nNUM_CHANNEL = 3\n\nPATH_TO_IMAGES = '../input/train/'\nPATH_TO_TEST_IMAGES = '../input/test/'\nPATH_TO_META = '../input/train.csv'\nSAMPLE_SUBMIT = '../input/sample_submission.csv'\n\nSEED = 666\nDEVICE = 'cuda'\n\nLABEL_MAP = {\n    0: \"Nucleoplasm\",\n    1: \"Nuclear membrane\",\n    2: \"Nucleoli\",\n    3: \"Nucleoli fibrillar center\",\n    4: \"Nuclear speckles\",\n    5: \"Nuclear bodies\",\n    6: \"Endoplasmic reticulum\",\n    7: \"Golgi apparatus\",\n    8: \"Peroxisomes\",\n    9: \"Endosomes\",\n    10: \"Lysosomes\",\n    11: \"Intermediate filaments\",\n    12: \"Actin filaments\",\n    13: \"Focal adhesion sites\",\n    14: \"Microtubules\",\n    15: \"Microtubule ends\",\n    16: \"Cytokinetic bridge\",\n    17: \"Mitotic spindle\",\n    18: \"Microtubule organizing center\",\n    19: \"Centrosome\",\n    20: \"Lipid droplets\",\n    21: \"Plasma membrane\",\n    22: \"Cell junctions\",\n    23: \"Mitochondria\",\n    24: \"Aggresome\",\n    25: \"Cytosol\",\n    26: \"Cytoplasmic bodies\",\n    27: \"Rods & rings\"}\n\n\n# Originally from https://www.kaggle.com/crimzoid/4-channel-resnet-from-scratch-pytorch-lb-0-16\n# Modified by Ji Yang\n\nimage_transform = transforms.Compose([\n    transforms.Resize(IMG_SIZE),\n    transforms.RandomChoice([\n        transforms.RandomVerticalFlip(p=0.5),\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.RandomAffine((0, 0)),\n        transforms.RandomAffine((90, 90)),\n        transforms.RandomAffine((180, 180)),\n        transforms.RandomAffine((270, 270)),\n    ]),\n    transforms.ToTensor(),\n])\n\nsub_image_transform = transforms.Compose([\n    transforms.Resize(IMG_SIZE),\n    transforms.ToTensor(),\n\n])\n\n\nclass MultiBandMultiLabelDataset(Dataset):\n    # BANDS_NAMES = ['_red.png', '_green.png', '_blue.png']\n    BANDS_NAMES = ['_red.png', '_green.png', '_blue.png', '_yellow.png']\n\n    def __len__(self):\n        return len(self.images_df)\n\n    def __init__(self, images_df, base_path, image_transform, train_mode=True):\n        if not isinstance(base_path, pathlib.Path):\n            base_path = pathlib.Path(base_path)\n\n        self.images_df = images_df.copy()\n        self.image_transform = image_transform\n        self.images_df.Id = self.images_df.Id.apply(lambda x: base_path / x)\n        self.mlb = MultiLabelBinarizer(classes=list(LABEL_MAP.keys()))\n        self.train_mode = train_mode\n\n    def __getitem__(self, index):\n        y = None\n        X = self._load_multiband_image(index)\n        if self.train_mode:\n            y = self._load_multilabel_target(index)\n\n        X = self.image_transform(X)\n\n        return X, y\n\n    def _load_multiband_image(self, index):\n        row = self.images_df.iloc[index]\n        image_bands = []\n        for band_name in self.BANDS_NAMES:\n            p = str(row.Id.absolute()) + band_name\n            img = PIL.Image.open(p)\n            image_bands.append(img)\n\n        # lets pretend its a RBGA image to support 4 channels\n        band4image = PIL.Image.merge('RGBA', bands=image_bands)\n\n        return band4image\n\n    def _load_multilabel_target(self, index):\n        return list(map(int, self.images_df.iloc[index].Target.split(' ')))\n\n    def collate_func(self, batch):\n        labels = None\n        images = [x[0] for x in batch]\n\n        if self.train_mode:\n            labels = [x[1] for x in batch]\n            labels_one_hot = self.mlb.fit_transform(labels)\n            labels = torch.FloatTensor(labels_one_hot)\n\n        return torch.stack(images)[:, :4, :, :], labels\n\n\ndf = pd.read_csv(PATH_TO_META)\ndf_train, df_test = train_test_split(df, test_size=.2, random_state=SEED)\ndf_submission = pd.read_csv(SAMPLE_SUBMIT)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c4f6a6d7f24fd2437bc9bcb25cc0717c8804f39"},"cell_type":"code","source":"debugging = True\nif debugging:\n    EPOCHS = 3\n    BATCH_SIZE = 24\n    BASE_LR = 0.001\n\nif debugging:\n    df_train = df_train[:300]\n    df_test = df_test[:50]\n    df_submission = df_submission\n\ngtrain = MultiBandMultiLabelDataset(df_train, base_path=PATH_TO_IMAGES, image_transform=image_transform)\ntrain_load = DataLoader(gtrain, collate_fn=gtrain.collate_func, batch_size=BATCH_SIZE, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bff7696c23e13bf40c3ffc04887d9695221e5b0a"},"cell_type":"code","source":"# You may want to use this to find the best value of number of workers...\nfor i in range(1, 9):\n    train_load = DataLoader(gtrain, collate_fn=gtrain.collate_func, batch_size=BATCH_SIZE, num_workers=i)\n    start = time.time()\n    for batch_idx, batch in enumerate(train_load):\n        if batch_idx == 50:\n            break\n    print('# of workers: {} | {:4f}'.format(i, (time.time() - start) / 20))\n\nfor batch_idx, batch in enumerate(train_load):\n    break\nimages, labels = batch\n\nfig, ax = plt.subplots(3, 4, figsize=(15, 9))\nfor r in range(3):\n    for c in range(4):\n        ax[r][c].imshow(images[r * 4 + c].permute(1, 2, 0))\nprint('min: {0}, max: {1}'.format(images.min(), images.max()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7fd8442f12ee30e1b3239547751472b0ab75a21"},"cell_type":"markdown","source":"Build an Xception model"},{"metadata":{"trusted":true,"_uuid":"868f42485e7de5615d67e24f0900bdde9f7c6ec9"},"cell_type":"code","source":"class ConvBnRelu2d(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3, padding=1):\n        super().__init__()\n        self.conv = nn.Sequential(nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding),\n                                  nn.BatchNorm2d(out_channels),\n                                  nn.ReLU(inplace=True)\n                                  )\n\n    def forward(self, x):\n        return self.conv(x)\n\n\nclass Xception(nn.Module):\n    def __init__(self, num_classes=28, pretrained='imagenet', dropout_rate=0.):\n        super(Xception, self).__init__()\n        self.pretrained_model = xception(pretrained=pretrained)\n        self.process_input = nn.Sequential(\n            ConvBnRelu2d(4, 16),\n            ConvBnRelu2d(16, 3),\n        )\n        self.fc_input = 2048\n        self.activation = nn.ReLU()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.dropout = nn.Dropout(p=dropout_rate)\n        self.last_linear = nn.Linear(self.fc_input, num_classes)\n\n    def forward(self, x):\n        x = self.process_input(x)\n        x = self.pretrained_model.features(x)\n        x = self.activation(x)\n        x = self.pooling(x)\n        x = x.view(x.size(0), -1)\n        x = self.dropout(x)\n        x = self.last_linear(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6f8a1f748a33842e8c284437fe67dc1581581ae"},"cell_type":"code","source":"def f1_loss(target, output, epsilon=1e-7):\n    y_pred = nn.Sigmoid()(output)\n    y_true = target.double()\n\n    TP = (y_pred * y_true).sum(1)\n    prec = TP / (y_pred.sum(1) + epsilon)\n    rec = TP / (y_true.sum(1) + epsilon)\n    res = 2 * prec * rec / (prec + rec + epsilon)\n\n    f1 = res\n    f1 = f1.clamp(min=0)\n    return 1 - f1.mean()\n\n\nfrom sklearn.metrics import f1_score\n\nsk_f1_score = f1_score\n\n\nclass AverageMeter:\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def init(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n    def __repr__(self):\n        return str(self.avg)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfe23d3553257f54948edfea2d85e31dcf4b2334"},"cell_type":"code","source":"EPOCHS, BATCH_SIZE, BASE_LR, DEVICE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ceef8cb43bbdf65a188c136ab7e3d4ac386f882"},"cell_type":"code","source":"model = Xception(num_classes=28, pretrained=None).to(DEVICE)\n\n\ngtrain = MultiBandMultiLabelDataset(df_train, base_path=PATH_TO_IMAGES, image_transform=image_transform)\ngtest = MultiBandMultiLabelDataset(df_test, base_path=PATH_TO_IMAGES, image_transform=sub_image_transform)\n\ntrain_load = DataLoader(gtrain, collate_fn=gtrain.collate_func, batch_size=BATCH_SIZE, num_workers=6)\ntest_load = DataLoader(gtest, collate_fn=gtest.collate_func, batch_size=BATCH_SIZE, num_workers=6)\n\n\nlr = BASE_LR\noptimizer = torch.optim.Adam(model.parameters(), lr=lr)\n\npatience = 0\nbest_validation = 0\nrecord = []\nval_record = []\nlrs = []\n\nfor epoch in range(1, EPOCHS):\n    if epoch % 2 == 0:\n        lr *= 0.5\n        for param_group in optimizer.param_groups:\n            param_group['lr'] = lr\n            print('Epoch {} with starting LR {}\\n'.format(epoch, lr))\n    lrs.append(lr)\n    epoch_loss = AverageMeter()\n    epoch_scr = AverageMeter()\n    epoch_skl = AverageMeter()\n    start = time.time()\n    for batch_idx, batch in enumerate(train_load):\n        data_, target = batch\n        data = data_.to(DEVICE)\n        target = target.to(DEVICE)\n        output = model(data)\n        loss = f1_loss(target.double(), output.double())\n        # f1 = f1_score(target, output)\n        f1 = sk_f1_score(target.cpu().data.numpy(), (output > 0.5).cpu().data.numpy(), average='macro')\n        # sk_f1 = sk_f1_score(target.cpu().data.numpy(), (output > 0.5).cpu().data.numpy(), average='macro')\n        loss.backward()\n\n        optimizer.step()\n        optimizer.zero_grad()\n        epoch_loss.update(loss.item())\n        epoch_scr.update(f1)\n    print('Epoch {:3} Trn --> Loss: {:.5f}, F1: {:.5f}'.format(epoch, epoch_loss.avg, epoch_scr.avg), end=' | ')\n    record.append((epoch_loss, epoch_scr))\n\n    with torch.no_grad():\n        val_epoch_loss = AverageMeter()\n        val_epoch_scr = AverageMeter()\n        for batch_idx, batch in enumerate(test_load):\n            data_, target = batch\n            data = data_.to(DEVICE)\n            target = target.to(DEVICE)\n            output = model(data)\n            loss = f1_loss(target.double(), output.double())\n            f1 = sk_f1_score(target.cpu().data.numpy(), (output > 0.5).cpu().data.numpy(), average='macro')\n            val_epoch_loss.update(loss.item())\n            val_epoch_scr.update(f1)\n        print('Epoch {:3} Val --> Loss: {:.5f}, F-1: {:.5f}'.format(epoch, val_epoch_loss.avg, val_epoch_scr.avg),\n              end=' | \\n')\n        val_record.append((val_epoch_loss, val_epoch_scr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a254920c22483ddeacd527f98aeadfe3ffca860"},"cell_type":"code","source":"with torch.no_grad():\n    for thres in range(1, 10):\n        thres /= 10\n        val_epoch_loss = AverageMeter()\n        val_epoch_scr = AverageMeter()\n        for batch_idx, batch in enumerate(test_load):\n            data_, target = batch\n            data = data_.to(DEVICE)\n            target = target.to(DEVICE)\n            output = model(data)\n            loss = f1_loss(target.double(), output.double())\n            f1 = sk_f1_score(target.cpu().data.numpy(), (output > 0.5).cpu().data.numpy(), average='macro')\n            val_epoch_loss.update(loss.item())\n            val_epoch_scr.update(f1)\n        print('Threshold: {} | Val --> Loss: {:.5f}, F-1: {:.5f}'.format(thres, val_epoch_loss.avg,\n                                                                         val_epoch_scr.avg), end=' | \\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3b414e08ce387ec45d785874ec22de685712753"},"cell_type":"code","source":"gsub = MultiBandMultiLabelDataset(df_submission, base_path=PATH_TO_TEST_IMAGES, train_mode=False,\n                                  image_transform=sub_image_transform)\nsubmission_load = DataLoader(gsub, collate_fn=gsub.collate_func, batch_size=BATCH_SIZE, num_workers=2)\n\nresult = []\nwith torch.no_grad():\n    for batch_idx, batch in tqdm(enumerate(submission_load)):\n        data_, _ = batch\n        data = data_.to(DEVICE)\n        output = model(data)\n        result.append(output.sigmoid().cpu().data.numpy())\n\nresult = np.concatenate(result)\nresult.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b87b2aecd09de6381186e1ee06b923b20bf32df"},"cell_type":"code","source":"def make_submission_file(sample_submission_df, predictions):\n    submissions = []\n    for row in predictions:\n        subrow = ' '.join(list([str(i) for i in np.nonzero(row)[0]]))\n        submissions.append(subrow)\n\n    sample_submission_df['Predicted'] = submissions\n    sample_submission_df.to_csv('submission.csv', index=None)\n\n    return sample_submission_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae98d04dda8ff1b79d3eb4387a2f8baa5a07f5a0"},"cell_type":"code","source":"# prepare the submission file and\nTHRESHOLD = 0.4\np = result > THRESHOLD\n\nsubmission_file = make_submission_file(sample_submission_df=df_submission,\n                                       predictions=p)\n\nsubmission_file = submission_file.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}