{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Recursion: How to taking over the kernel for long time training"},{"metadata":{},"cell_type":"markdown","source":"I don't have any GPU machine. So, I use kaggle's kernels a lot in this competition. But, in this competition, it takes a long time to train enough epochs.\nThis kernel makes it easy to train in time-limited environments such as kaggle's kernel or google colab(GPU 9 hours) by taking over the kernel repeatedly.\n\nHow to:\n1. training first stage\n  - conf.stage = 1\n2. Add Data and continuous training\n  - add previous stage output data\n  - change checkpoint_path\n  - conf.stage = 2〜\n\nBy training all celltype 70 epoch and each celltype 50 stage in this kernel, you may get 0.65〜0.7 in the public LB.\n\n- model:densenet121\n- all cell_type training → each cell_type training\n- augmentation:Horizontal, Vartical flip (no tta)\n- no model.eval() when predicting (mini batch normalization)\n- each experiment predict\n- use plate leak\n- No Hungarian algorithm"},{"metadata":{"nbpresent":{"id":"407b7c2c-7072-44ef-93a1-24abb1995de9"}},"cell_type":"markdown","source":"- fold number 0\n- stage1\n- all cell_type training → each cell_type training\n- densenet121\n- 10epoch(about 7 hour)\n- learning rate 1e-4～1e-5 per 20 epoch"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","nbpresent":{"id":"61bec088-c15e-4e2f-b237-c44fa8149b9f"},"trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"812bbdd3-cee3-43a0-9d3a-de14d34d6ba5"},"trusted":true},"cell_type":"code","source":"class conf:\n    #particular to this competition\n    \"\"\"\n        False:all cell_type trainig\n        True :each cell_type training\n    \"\"\"\n    training_each_experiment = False\n    \"\"\"\n        all cell_type training weight path\n    \"\"\"\n    all_experiment_pretrain_path = '../input/all-experiment-densenet121-brightness-fold0/weight_best_all_3.pt'\n    \"\"\"\n        please chage stages due to kernel 9 hours limit.\n    \"\"\"\n    stage = 1\n    n_splits = 5\n    fold_number = 0\n    if stage == 1:\n        resume_training = False\n    else:\n        resume_training = True\n    \"\"\"\n        continuing stage check point path\n    \"\"\"\n    checkpoint_path = '../input/all-experiment-densenet121-brightness/'\n    \n    DEFAULT_CHANNELS = [1, 2, 3, 4, 5, 6]\n    \n    #common configs\n    SEED = 717\n    path_data = '../input/recursion-cellular-image-classification/'\n    device = 'cuda'\n\n    num_classes = 1108\n    num_channels = 6\n    input_size = 512    \n    model_type = 'densenet121'\n    use_pretrained = True # image net pretrain\n    unflozen_epoch = 2\n\n    num_epochs = 10 #about 8 hours\n    batch_size = 16\n    test_batch_size = 16\n    gamma= 1\n    lr = 1e-4 * (gamma ** (stage - 1))\n    eta_min = 1e-5 * (gamma ** (stage - 1))\n    t_max = 10\n    cycle = t_max * 2 # for snapshot ensemble\n    \n    debug = False\n    predict = False","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","nbpresent":{"id":"2b439cbd-d8cf-45cd-85c6-49f96fc456a5"},"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport gc\nimport os\nimport sys\nimport pickle\nimport random\nimport time\nimport logging\nfrom IPython.display import FileLink\n\nfrom collections import Counter, defaultdict\nfrom functools import partial\nfrom pathlib import Path\nfrom psutil import cpu_count\nimport datetime as dt\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau, MultiStepLR, ExponentialLR\nfrom fastprogress import master_bar, progress_bar\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\nimport torchvision.models as models\n#from imgaug import augmenters as iaa\n\nimport matplotlib.pyplot as plt\nfrom fastprogress import master_bar, progress_bar\n\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"a9a0855b-0e14-4309-a5e1-1c87c715f35d"}},"cell_type":"markdown","source":"## Utils"},{"metadata":{"nbpresent":{"id":"9481a70a-3db8-45e0-bcdc-6f19009d364d"},"trusted":true},"cell_type":"code","source":"def get_logger(name=\"Main\", tag=\"exp\", log_dir=\"log/\"):\n    log_path = Path(log_dir)\n    path = log_path / tag\n    path.mkdir(exist_ok=True, parents=True)\n\n    logger = logging.getLogger(name)\n    logger.setLevel(logging.INFO)\n\n    fh = logging.FileHandler(\n        path / (dt.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\") + \".log\"))\n    sh = logging.StreamHandler(sys.stdout)\n    formatter = logging.Formatter(\n        \"%(asctime)s %(name)s %(levelname)s %(message)s\")\n\n    fh.setFormatter(formatter)\n    sh.setFormatter(formatter)\n    logger.addHandler(fh)\n    logger.addHandler(sh)\n    return logger","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"f4e3add8-df3d-4a6d-a5ee-2a15a7fa233a"},"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = conf.SEED\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"3a6e1f55-7ae9-416f-a614-954f6e64754e"},"trusted":true},"cell_type":"code","source":"def save_checkpoint(conf, model, optimizer, scheduler, epoch, best_acc, best_epoch, cell_type):\n    #checkpoint_path = 'checkpoint.pth.tar'\n    checkpoint_path = 'checkpoint_{}.pth.tar'.format(cell_type)\n    \n    weights_dict = {\n        'model_state_dict': model.state_dict(),\n        'optimizer_state_dict' : optimizer.state_dict(),\n        'scheduler_state_dict' : scheduler.state_dict(),\n        'epoch' : epoch,\n        'best_acc': best_acc,\n        'best_epoch': best_epoch\n    }\n    torch.save(weights_dict, checkpoint_path)\n\ndef load_checkpoint(conf, model, optimizer, scheduler, cell_type):\n\n    checkpoint = torch.load(conf.checkpoint_path + 'checkpoint_{}.pth.tar'.format(cell_type))\n    print(\"=> loaded checkpoint '{}' (trained for {} epochs)\".format(conf.checkpoint_path + 'checkpoint_{}.pth.tar'.format(cell_type), checkpoint['epoch']+1))\n    end_epoch = checkpoint['epoch']\n    best_acc = checkpoint['best_acc']\n    best_epoch = checkpoint['best_epoch']\n    \n    model.load_state_dict(checkpoint['model_state_dict'])\n    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n    scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n    return end_epoch, best_acc, best_epoch, model, optimizer, scheduler","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"028d2115-72b8-4018-9066-98760ece8897"}},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"nbpresent":{"id":"772f6dc0-a0c4-4b2e-b9d8-6de6569f7069"},"trusted":true},"cell_type":"code","source":"def image_path(experiment,\n               plate,\n               well,\n               site,\n               channel,\n               base_path=conf.path_data,\n               mode='train'):\n        \n    return os.path.join(base_path, mode, experiment, \"Plate{}\".format(plate),\n                        \"{}_s{}_w{}.png\".format(well, site, channel))\n\ndef image_paths(experiment,\n                plate,\n                well,\n                site,\n                channels=conf.DEFAULT_CHANNELS,\n                base_path=conf.path_data,\n                mode='train'):\n    \n    channel_paths = [\n        image_path(\n            experiment, plate, well, site, c, base_path=base_path, mode=mode)\n        for c in channels\n    ]\n    \n    return channel_paths\n\ndef load_image(file_name):\n    img = Image.open(file_name)\n    return img\n\ndef transform_image(transforms, img):\n    img = transforms(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"687d0ce6-5105-4bf6-9f84-05dece0cbb34"},"trusted":true},"cell_type":"code","source":"class ImagesDS(Dataset):\n    def __init__(self, conf, df, transforms, mode='train'):\n        \n        #df = pd.read_csv(csv_file)\n        self.records = df.to_records(index=False)\n        self.conf = conf\n        self.channels = self.conf.DEFAULT_CHANNELS\n        self.path_data = self.conf.path_data\n        self.mode = mode\n        self.transforms = transforms\n        self.len = df.shape[0]\n        \n    def __getitem__(self, index):\n        paths = image_paths(self.records[index].experiment,self.records[index].plate,\n                            self.records[index].well,self.records[index].site,\n                            channels=self.channels,base_path=self.path_data, mode=self.mode)\n\n        img = torch.cat([transform_image(self.transforms, load_image(img_path)) for img_path in paths])\n        if self.mode == 'train':\n            return img, self.records[index].sirna\n        else:\n            return img, self.records[index].id_code\n\n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"32fb7d1d-a27e-4b35-8338-8f6a0bcee596"}},"cell_type":"markdown","source":"## Model"},{"metadata":{"nbpresent":{"id":"4ce55610-96e0-45b2-b2f8-e52f5c1c3685"},"trusted":true},"cell_type":"code","source":"def initialize_model(conf):\n    # Initialize these variables which will be set in this if statement. Each of these\n    #   variables is model specific.\n    model_ft = None\n\n    if conf.model_type == \"resnet18\":\n        \"\"\" Resnet18\n        \"\"\"\n        model_ft = models.resnet18(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, conf.num_classes)\n        input_size = conf.input_size\n\n    elif conf.model_type == \"resnet34\":\n        \"\"\" Resnet34\n        \"\"\"\n        model_ft = models.resnet34(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, conf.num_classes)\n        input_size = conf.input_size\n        \n    elif conf.model_type == \"densenet121\":\n        \"\"\" densenet121\n        \"\"\"\n        model_ft = models.densenet121(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.classifier.in_features\n        model_ft.classifier = nn.Linear(num_ftrs, conf.num_classes)\n        input_size = conf.input_size\n        \n    elif conf.model_type == \"densenet201\":\n        \"\"\" densenet201\n        \"\"\"\n        model_ft = models.densenet201(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.classifier.in_features\n        model_ft.classifier = nn.Linear(num_ftrs, conf.num_classes)\n        input_size = conf.input_size\n        \n    elif conf.model_type == \"resnext50_32x4d\":\n        \"\"\" resnext50_32x4d\n        \"\"\"\n        model_ft = models.resnext50_32x4d(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, conf.num_classes)\n        input_size = conf.input_size\n        \n    elif conf.model_type == \"wide_resnet50_2\":\n        \"\"\" wide_resnet50_2\n        \"\"\"\n        model_ft = models.wide_resnet50_2(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, conf.num_classes)\n        input_size = conf.input_size\n\n    elif conf.model_type == \"alexnet\":\n        \"\"\" Alexnet\n        \"\"\"\n        model_ft = models.alexnet(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,conf.num_classes)\n        input_size = conf.input_size\n\n    elif conf.model_type == \"vgg\":\n        \"\"\" VGG11_bn\n        \"\"\"\n        model_ft = models.vgg11_bn(pretrained=conf.use_pretrained)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,conf.num_classes)\n        input_size = conf.input_size\n\n    elif conf.model_type == \"squeezenet\":\n        \"\"\" Squeezenet\n        \"\"\"\n        model_ft = models.squeezenet1_0(pretrained=conf.use_pretrained)\n        model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))\n        model_ft.num_classes = conf.num_classes\n        input_size = conf.input_size\n\n    elif conf.model_type == \"inception\":\n        \"\"\" Inception v3\n        Be careful, expects (299,299) sized images and has auxiliary output\n        \"\"\"\n        model_ft = models.inception_v3(pretrained=conf.use_pretrained)\n        # Handle the auxilary net\n        num_ftrs = model_ft.AuxLogits.fc.in_features\n        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, conf.num_classes)\n        # Handle the primary net\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs,conf.num_classes)\n        input_size = 299\n\n    else:\n        print(\"Invalid model name, exiting...\")\n        exit()\n\n    return model_ft, input_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # To tune or design nn here.\nmodel_ft, input_size = initialize_model(conf)\n\nif (conf.model_type == \"densenet121\") | (conf.model_type == \"densenet201\"):\n    trained_kernel = model_ft.features.conv0.weight\n    new_conv = nn.Conv2d(6, 64, kernel_size=7, stride=2, padding=3, bias=False)\n    with torch.no_grad():\n        new_conv.weight[:,:] = torch.stack([torch.mean(trained_kernel, 1)]*6, dim=1)\n    model_ft.features.conv0 = new_conv\nelse:    \n    trained_kernel = model_ft.conv1.weight\n    new_conv = nn.Conv2d(6, 64, kernel_size=7, stride=2, padding=3, bias=False)\n    with torch.no_grad():\n        new_conv.weight[:,:] = torch.stack([torch.mean(trained_kernel, 1)]*6, dim=1)\n    model_ft.conv1 = new_conv\n\n# Print the model we just instantiated\nprint(model_ft)","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"0dd31b67-229c-442c-97bf-f4eba0a44ab6"}},"cell_type":"markdown","source":"## Train"},{"metadata":{"nbpresent":{"id":"877bc114-5020-4088-96c4-5612c7cbbaf3"},"trusted":true},"cell_type":"code","source":"def train_single_epoch(conf, model, train_loader, criterion, optimizer, mb):\n    avg_loss = 0.\n\n    # train process\n    for x_batch, y_batch in progress_bar(train_loader, parent=mb):\n\n        if conf.model_type != 'inception':\n            preds = model(x_batch.cuda())\n            loss = criterion(preds, y_batch.cuda())\n        else:\n            outputs, aux_outputs = model(x_batch.cuda())\n            loss1 = criterion(outputs, y_batch.cuda())\n            loss2 = criterion(aux_outputs, y_batch.cuda())\n            loss = loss1 + 0.4*loss2\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        avg_loss += loss.item() / len(train_loader)\n    \n    return avg_loss\n\ndef evaluate_single_epoch(conf, model, valid_loader, criterion, optimizer):\n    avg_val_loss = 0.              \n    correct = 0\n    total = 0\n    for x_batch, y_batch in valid_loader:\n        preds = model(x_batch.cuda()).detach()\n        loss_ = criterion(preds, y_batch.cuda())\n\n        preds = torch.sigmoid(preds)\n        preds = preds.max(dim=-1)[1].cpu().numpy()\n        avg_val_loss += loss_.item() / len(valid_loader)                \n        #_, predicted = torch.max(outputs.data, 1)\n\n        #total += len(valid_loader)\n        correct += (preds == y_batch.cpu().numpy()).sum().item()\n    acc = correct / len(valid_loader.dataset)\n        \n    return avg_val_loss, acc\n\ndef set_requires_grad(conf, model, epoch):\n    if (conf.stage == 1) & (conf.training_each_experiment == False):\n        #fine tuning\n        if epoch + 1 == 1:\n            for name, child in model.named_children():\n                #print(name, child)\n                if (name == 'fc') | (name == 'classifier'):\n                    for param in child.parameters():\n                        param.requires_grad = True\n                else:\n                    for param in child.parameters():\n                        param.requires_grad = False\n\n        #unflozen all layer for epoch 2\n        if epoch + 1 == 2:\n            for name, child in model.named_children():\n                for param in child.parameters():\n                    param.requires_grad = True\n    return model","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"22910abd-99cd-4196-b1aa-95e6070f1cd8"},"trusted":true},"cell_type":"code","source":"def train_model(conf, df, train_transforms, valid_transforms, cell_type='all'):\n    #logger\n    #logger = get_logger(\"Main\", tag=\"train\", log_dir=\"log/\")\n    print('fold: {}'.format(conf.fold_number))\n    print('Training {}'.format(cell_type))\n    print('Stage: {}'.format(conf.stage))\n    logger.info('fold {}'.format(conf.fold_number))\n    logger.info('Training {}'.format(cell_type))\n    logger.info('Stage: {}'.format(conf.stage))\n    \n    #initialize\n    trn_loss = []\n    val_loss = []\n    val_acc = []\n    lr_log = []\n    bests = []\n    loss_list = []\n    acc_list = []\n    \n    #train_test_split\n    idx = np.arange(len(df))\n    #trn_idx, val_idx = train_test_split(idx, test_size=0.2, random_state=SEED)\n    folds = KFold(n_splits=conf.n_splits, shuffle=True, random_state=SEED)\n    for fold, (trn_idx_, test_idx_) in enumerate(folds.split(idx)):\n        if fold == conf.fold_number:\n            trn_idx = trn_idx_\n            val_idx = test_idx_\n    \n    df_trn = df.iloc[trn_idx, :]\n    df_val = df.iloc[val_idx, :]\n    #extract cell_type\n    if conf.training_each_experiment:\n        df_trn = df_trn[df_trn.cell_type == cell_type]\n        df_val = df_val[df_val.cell_type == cell_type]\n\n    #Dataset\n    \"\"\"\n    sample(frac=1, random_state=conf.stage)\n    This code means shuffling train data in each stage. Train images are viewed in the same order wihout shuffling, because seed is static.\n    \"\"\"\n    train_dataset = ImagesDS(conf, df_trn.sample(frac=1, random_state=conf.stage), train_transforms)\n    valid_dataset = ImagesDS(conf, df_val, valid_transforms)\n\n    #DataLoader\n    train_loader = DataLoader(train_dataset,batch_size=conf.batch_size, shuffle=True, num_workers=4)\n    valid_loader = DataLoader(valid_dataset, batch_size=conf.batch_size, shuffle=False, num_workers=4)\n    \n    #model\n    model = model_ft.cuda()\n    \n    #loss function\n    criterion = nn.CrossEntropyLoss().cuda()\n    #optimizer\n    optimizer = Adam(params=model.parameters(), lr=conf.lr, amsgrad=False)\n    #learning rate\n    scheduler = CosineAnnealingLR(optimizer, T_max=conf.t_max, eta_min=conf.eta_min)\n    #initialize\n    best_epoch = -1\n    best_acc = 0.\n    start_epoch = 0\n    cycle_count = (conf.stage - 1) * conf.num_epochs // conf.cycle\n    torch.cuda.empty_cache()\n    #resume training\n    if conf.resume_training:\n        end_epoch, best_acc, best_epoch, model, optimizer, scheduler = load_checkpoint(conf, model, optimizer, scheduler, cell_type)\n        start_epoch = end_epoch + 1\n        \n    #all_experiment_pretrain\n    if (conf.training_each_experiment == True) & (conf.stage == 1):\n        model.load_state_dict(torch.load(conf.all_experiment_pretrain_path))\n        \n    mb = master_bar(range(start_epoch, start_epoch + conf.num_epochs))\n    torch.cuda.empty_cache()\n    \n    #---\n    accumurated_time = 0\n    max_elapsed_time = 0\n    for epoch in mb:\n        start_time = time.time()\n        if epoch + 1 <= conf.unflozen_epoch:\n            model = set_requires_grad(conf, model, epoch)\n                        \n        #cycleごとにlearning rateを減衰\n        if (epoch != 0) & (epoch % conf.cycle == 0):\n            conf.lr = conf.lr * conf.gamma\n            conf.eta_min = conf.eta_min * conf.gamma\n            optimizer = Adam(params=model.parameters(), lr=conf.lr, amsgrad=False)\n            scheduler = CosineAnnealingLR(optimizer, T_max=conf.t_max, eta_min=conf.eta_min)\n\n        #train process\n        model.train()\n        avg_loss = train_single_epoch(conf, model, train_loader, criterion, optimizer, mb)\n\n        # validation process\n        model.eval()\n        avg_val_loss, acc = evaluate_single_epoch(conf, model, valid_loader, criterion, optimizer)\n    \n    \n        # record the metrics\n        for param_group in optimizer.param_groups:\n            lr_temp = param_group['lr']\n        lr_log.append(lr_temp)\n        #print(\"Learning rate: {}\".format(lr_temp))\n        \n        # scheduler step\n        scheduler.step()\n        #scheduler.step(avg_val_loss) # for reduceLR \n        \n        # cycle for snapshot ensemple\n        if (epoch != 0) & (epoch % conf.cycle == 0):\n            cycle_count += 1\n            #reset acc\n            best_acc = 0.\n        if epoch % conf.cycle == 0:\n            print('Cycle {}'.format(cycle_count))           \n        \n        # log\n        if (epoch + 1) % 1 == 0:\n            elapsed = time.time() - start_time\n            print('Epoch {} -> Train Loss: {:.4f} Valid Loss: {:.4f}, ACC: {:.4f}'.format(epoch + 1, avg_loss, avg_val_loss, acc))\n            logger.info('Epoch {} -> LR: {:.6f} Train Loss: {:.4f} Valid Loss: {:.4f}, ACC: {:.4f}, time: {:.0f}s'.format(epoch + 1, lr_temp, avg_loss, avg_val_loss, acc, elapsed))\n            trn_loss.append(avg_loss)\n            val_loss.append(avg_val_loss)\n            val_acc.append(acc)\n        \n        # save best weight\n        if acc > best_acc:\n            best_epoch = epoch + 1\n            best_acc = acc\n            torch.save(model.state_dict(), 'weight_best_{}_{}.pt'.format(cell_type, cycle_count))\n        #save checkpoint\n        model.train()\n        save_checkpoint(conf, model, optimizer, scheduler, epoch, best_acc, best_epoch, cell_type)\n        loss_list.append([avg_loss, avg_val_loss])\n        acc_list.append(acc)\n        \n        #--\n        \"\"\"\n        When the kernel is slow, your kernle will time out.\n        This code will prevent time out when the kernel is slow.\n        \"\"\"\n        #accumurated_time += elapsed\n        #if elapsed > max_elapsed_time:\n        #    max_elapsed_time = elapsed\n        #if accumurated_time >= 9 * 60 * 60 - max_elapsed_time:\n        #    break\n\n    bests.append([best_epoch, best_acc])\n    logger.info(f\"Best: {bests}\")\n    return bests, loss_list, acc_list, lr_log","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"45fa6583-90de-4878-a855-cb5e679729f7"},"trusted":true},"cell_type":"code","source":"class ImgAugTransform:\n    def __init__(self):\n        self.aug = iaa.Sequential([\n            iaa.CoarseDropout(0.1,size_percent=0.02)\n        ])\n        \n    def __call__(self, img):\n        img = np.array(img)\n        return self.aug.augment_image(img)\n\ntransforms_dict = {\n        'train': transforms.Compose([\n            #transforms.RandomCrop(input_size),\n            transforms.RandomHorizontalFlip(0.5),\n            transforms.RandomVerticalFlip(0.5),\n            #transforms.ColorJitter(brightness=0.5), #caution: work well in all cell_type trainig. But,It is overfitting in each cell_type training.\n            #transforms.RandomRotation((-90,90)),\n            #ImgAugTransform(),\n            transforms.ToTensor()\n        ]),\n        'valid': transforms.Compose([\n            #transforms.RandomCrop(input_size),\n            transforms.RandomHorizontalFlip(0.5),\n            transforms.RandomVerticalFlip(0.5),\n            #ImgAugTransform(),\n            transforms.ToTensor()\n        ]),\n        'test': transforms.Compose([\n            #transforms.RandomCrop(input_size),\n            transforms.RandomHorizontalFlip(0.5),\n            transforms.RandomVerticalFlip(0.5),\n            #ImgAugTransform(),\n            transforms.ToTensor()\n        ])\n    }","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"36cf8fe0-2e14-4ee8-98b4-de5e65afcb80"},"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(conf.path_data+'/train.csv')\nif conf.debug:\n    train_df = train_df[0:300]\ntrain_df['cell_type'] = train_df.experiment.str.split(\"-\").apply(lambda a: a[0])\ntrain_df.cell_type.unique()","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"82f926df-f43a-4cea-b69f-3851cc6660e7"},"trusted":true},"cell_type":"code","source":"train_df['site'] = 1 \ntrain_df_2 = train_df.copy()\ntrain_df_2['site'] = 2\ntrain_df = pd.concat([train_df, train_df_2]).sort_index().reset_index(drop=True)\ndel train_df_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"d1096964-a4f7-4c6e-b92a-c38ba8a05091"},"trusted":true},"cell_type":"code","source":"#logger\nlogger = get_logger(\"Main\", tag=\"train\", log_dir=\"log/\")","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"08a82107-21cd-4193-b1c3-14a403a97f5b"},"trusted":true},"cell_type":"code","source":"#run\nif conf.training_each_experiment:\n    result_HEPG2, loss_list_HEPG2, acc_list_HEPG2, lr_log_HEPG2 = train_model(conf, train_df, transforms_dict['train'], transforms_dict['valid'], 'HEPG2')\n    result_HUVEC, loss_list_HUVEC, acc_list_HUVEC, lr_log_HUVEC = train_model(conf, train_df, transforms_dict['train'], transforms_dict['valid'], 'HUVEC')\n    result_RPE, loss_list_RPE, acc_list_RPE, lr_log_RPE = train_model(conf, train_df, transforms_dict['train'], transforms_dict['valid'], 'RPE')\n    result_U2OS, loss_list_U2OS, acc_list_U2OS, lr_log_U2OS = train_model(conf, train_df, transforms_dict['train'], transforms_dict['valid'], 'U2OS')\nelse:\n    result_all, loss_list_all, acc_list_all, lr_log_all = train_model(conf, train_df, transforms_dict['train'], transforms_dict['valid'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\"\"\"\nimport requests\n\ndef send_line_notification(message):\n    line_token = 'token'  # set your token\n    endpoint = 'https://notify-api.line.me/api/notify'\n    message = \"\\n{}\".format(message)\n    payload = {'message': message}\n    headers = {'Authorization': 'Bearer {}'.format(line_token)}\n    requests.post(endpoint, data=payload, headers=headers)\n\nresult = 'training finished'\n\nsend_line_notification(result)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"5a180127-fe49-459a-9e00-89b16e432070"},"trusted":false},"cell_type":"code","source":"if conf.training_each_experiment:\n    print(train_df.cell_type.value_counts().HEPG2/len(train_df) * max(acc_list_HEPG2) + \\\n          train_df.cell_type.value_counts().HUVEC/len(train_df) * max(acc_list_HUVEC) + \\\n          train_df.cell_type.value_counts().RPE/len(train_df) * max(acc_list_RPE) + \\\n          train_df.cell_type.value_counts().U2OS/len(train_df) * max(acc_list_U2OS))","execution_count":null,"outputs":[]},{"metadata":{"nbpresent":{"id":"ddf90d42-e9d0-4e5a-9374-6c60d54644f3"},"trusted":true},"cell_type":"code","source":"#plot\nif conf.training_each_experiment:\n    loss_list_HEPG2 = pd.DataFrame(loss_list_HEPG2)\n    loss_list_HEPG2['cell_type'] = 'HEPG2'\n    loss_list_HUVEC = pd.DataFrame(loss_list_HUVEC)\n    loss_list_HUVEC['cell_type'] = 'HUVEC'\n    loss_list_RPE = pd.DataFrame(loss_list_RPE)\n    loss_list_RPE['cell_type'] = 'RPE'\n    loss_list_U2OS = pd.DataFrame(loss_list_U2OS)\n    loss_list_U2OS['cell_type'] = 'U2OS'\n    acc_list_HEPG2 = pd.DataFrame(acc_list_HEPG2)\n    acc_list_HEPG2['cell_type'] = 'HEPG2'\n    acc_list_HUVEC = pd.DataFrame(acc_list_HUVEC)\n    acc_list_HUVEC['cell_type'] = 'HUVEC'\n    acc_list_RPE = pd.DataFrame(acc_list_RPE)\n    acc_list_RPE['cell_type'] = 'RPE'\n    acc_list_U2OS = pd.DataFrame(acc_list_U2OS)\n    acc_list_U2OS['cell_type'] = 'U2OS'\n    lr_log_HEPG2 = pd.DataFrame(lr_log_HEPG2)\n    lr_log_HEPG2['cell_type'] = 'HEPG2'\n    lr_log_HUVEC = pd.DataFrame(lr_log_HUVEC)\n    lr_log_HUVEC['cell_type'] = 'HUVEC'\n    lr_log_RPE = pd.DataFrame(lr_log_RPE)\n    lr_log_RPE['cell_type'] = 'RPE'\n    lr_log_U2OS = pd.DataFrame(lr_log_U2OS)\n    lr_log_U2OS['cell_type'] = 'U2OS'\n    \n    loss_list = pd.concat([loss_list_HEPG2, loss_list_HUVEC, loss_list_RPE, loss_list_U2OS])\n    acc_list = pd.concat([acc_list_HEPG2, acc_list_HUVEC, acc_list_RPE, acc_list_U2OS])\n    lr_log = pd.concat([lr_log_HEPG2, lr_log_HUVEC, lr_log_RPE, lr_log_U2OS])\n    \n    for i, cell_type in enumerate(['HEPG2', 'HUVEC', 'RPE', 'U2OS']):\n        loss = loss_list[loss_list.cell_type == cell_type][0]\n        val_loss = loss_list[loss_list.cell_type == cell_type][1]\n        acc = acc_list[acc_list.cell_type == cell_type][0]\n        lr = lr_log[lr_log.cell_type == cell_type][0]\n\n        epochs = range((conf.stage-1) * conf.num_epochs + 1, (conf.stage-1) * conf.num_epochs + len(loss) + 1)\n\n        #lossとaccをプロット\n        fig, ax1 = plt.subplots()\n        ax1.plot(epochs, loss, color = 'royalblue', label = \"Training loss\")\n        ax1.plot(epochs, val_loss, color='r', label = \"Validation loss\")\n        ax1.set_ylim([0, 7])\n        ax2 = ax1.twinx()\n        ax2.plot(epochs, acc, 'bo',color='r', label = \"ACC\")\n        ax2.set_ylim([0, 1])\n        ax3 = ax2.twinx()\n        ax3.plot(epochs, lr, color='c', label = \"LR\")\n        ax3.set_ylim([0, 0.0001])\n        plt.title('Loss and ACC and Learning rate {}'.format(cell_type))\n        h1, l1 = ax1.get_legend_handles_labels()\n        h2, l2 = ax2.get_legend_handles_labels()\n        h3, l3 = ax3.get_legend_handles_labels()\n        ax1.legend(h1+h2+h3, l1+l2+l3, loc='upper right')\n        plt.tight_layout()\n        plt.savefig('figure_{}.png'.format(cell_type))\n        plt.show()\nelse:\n    loss_list_all = pd.DataFrame(loss_list_all)\n    acc_list_all = pd.DataFrame(acc_list_all)\n    lr_log_all = pd.DataFrame(lr_log_all)\n\n    loss = loss_list_all[0]\n    val_loss = loss_list_all[1]\n    acc = acc_list_all\n    lr = lr_log_all\n\n    epochs = range((conf.stage-1) * conf.num_epochs + 1, (conf.stage-1) * conf.num_epochs + len(loss) + 1)\n\n    #plot loss and acc\n    fig, ax1 = plt.subplots()\n    ax1.plot(epochs, loss, color = 'royalblue', label = \"Training loss\")\n    ax1.plot(epochs, val_loss, color='r', label = \"Validation loss\")\n    ax1.set_ylim([0, 10])\n    ax2 = ax1.twinx()\n    ax2.plot(epochs, acc, 'bo',color='r', label = \"ACC\")\n    ax2.set_ylim([0, 1])\n    ax3 = ax2.twinx()\n    ax3.plot(epochs, lr, color='c', label = \"LR\")\n    ax3.set_ylim([0, 0.0001])\n    plt.title('Loss and ACC and Learning rate')\n    h1, l1 = ax1.get_legend_handles_labels()\n    h2, l2 = ax2.get_legend_handles_labels()\n    h3, l3 = ax3.get_legend_handles_labels()\n    ax1.legend(h1+h2+h3, l1+l2+l3, loc='upper right')\n    plt.tight_layout()\n    plt.savefig('figure_all.png')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"if conf.predict:\n    train_df = pd.read_csv(conf.path_data+'/train.csv')\n    test_df = pd.read_csv(conf.path_data + '/test.csv')\n    if conf.debug:\n        test_df = test_df[0:100]\n    test_df['cell_type'] = test_df.experiment.str.split(\"-\").apply(lambda a: a[0])\n    test_df.cell_type.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_model(conf, df, model, test_transforms, weight_cycle, cell_type='all'):\n    \n    #extract cell_type\n    df = df[df.cell_type == cell_type]\n    print('Predict {}'.format(cell_type))\n    \n    preds_1_all = []\n    preds_2_all = []\n    \n    for experiment in df.experiment.unique():\n        print('experiment {}'.format(experiment))\n        \n        df_ex = df[df.experiment == experiment]\n        \n        test_dataset_site1 = ImagesDS(conf, df_ex, test_transforms, mode='test', site=1)\n        test_loader_site1 = DataLoader(test_dataset_site1,batch_size=conf.test_batch_size, shuffle=False, num_workers=4)\n        test_dataset_site2 = ImagesDS(conf, df_ex, test_transforms, mode='test', site=2)\n        test_loader_site2 = DataLoader(test_dataset_site2,batch_size=conf.test_batch_size, shuffle=False, num_workers=4)\n\n        model.load_state_dict(torch.load(conf.checkpoint_path + 'weight_best_{}_{}.pt'.format(cell_type, weight_cycle)))\n        model.cuda()\n        #model.eval()\n        model.train()  #caution: usually use model.eval()\n        \n        \n        #preds_1_all = []\n        #preds_2_all = []\n        #preds_all = np.empty(0)\n\n        pb = progress_bar(test_loader_site1)\n        for images, id_code in pb:\n            with torch.no_grad():\n                preds_1 = torch.sigmoid(model(images.cuda()).detach())\n                preds_1_all.append(preds_1)\n        \n        pb = progress_bar(test_loader_site2)\n        for images, id_code in pb:\n            with torch.no_grad():\n                preds_2 = torch.sigmoid(model(images.cuda()).detach())\n                preds_2_all.append(preds_2)\n        \n    preds_1_all = torch.cat(preds_1_all)     \n    preds_2_all = torch.cat(preds_2_all)\n    \n    preds = (preds_1_all + preds_2_all) / 2\n\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if conf.predict:\n    preds_HEPG2_1 = predict_model(conf, test_df, model_ft, transforms_dict['test'], 5, 'HEPG2')\n    preds_HUVEC_1 = predict_model(conf, test_df, model_ft, transforms_dict['test'], 5,'HUVEC')\n    preds_RPE_1 = predict_model(conf, test_df, model_ft, transforms_dict['test'], 5,'RPE')\n    preds_U2OS_1 = predict_model(conf, test_df, model_ft, transforms_dict['test'], 5,'U2OS')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# apply plate leak\n\"\"\"\n    https://www.kaggle.com/zaharch/keras-model-boosted-with-plates-leak\n\"\"\"\nplate_groups = np.zeros((1108,4), int)\nfor sirna in range(1108):\n    grp = train_df.loc[train_df.sirna==sirna,:].plate.value_counts().index.values\n    assert len(grp) == 3\n    plate_groups[sirna,0:3] = grp\n    plate_groups[sirna,3] = 10 - grp.sum()    \n\ndef post_processing(preds_HEPG2, preds_HUVEC, preds_RPE, preds_U2OS, name):\n    preds_HEPG2_sirna = preds_HEPG2.max(dim=-1)[1].cpu().numpy()\n    preds_HUVEC_sirna = preds_HUVEC.max(dim=-1)[1].cpu().numpy()\n    preds_RPE_sirna = preds_RPE.max(dim=-1)[1].cpu().numpy()\n    preds_U2OS_sirna = preds_U2OS.max(dim=-1)[1].cpu().numpy()\n    predicted = np.concatenate([preds_HEPG2.cpu().numpy(), preds_HUVEC.cpu().numpy(), preds_RPE.cpu().numpy(), preds_U2OS.cpu().numpy()]).squeeze()\n    preds = np.concatenate([preds_HEPG2_sirna, preds_HUVEC_sirna, preds_RPE_sirna, preds_U2OS_sirna])\n    sub = pd.read_csv(conf.path_data + '/test.csv')\n    sub['sirna'] = preds.astype(int)\n    \n    all_test_exp = test_df.experiment.unique()\n\n    group_plate_probs = np.zeros((len(all_test_exp),4))\n    for idx in range(len(all_test_exp)):\n        preds_sirna = sub.loc[test_df.experiment == all_test_exp[idx],'sirna'].values\n        pp_mult = np.zeros((len(preds_sirna),1108))\n        pp_mult[range(len(preds_sirna)),preds_sirna] = 1\n\n        sub_test = test_df.loc[test_df.experiment == all_test_exp[idx],:]\n        assert len(pp_mult) == len(sub_test)\n\n        for j in range(4):\n            mask = np.repeat(plate_groups[np.newaxis, :, j], len(pp_mult), axis=0) == \\\n                   np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)\n\n            group_plate_probs[idx,j] = np.array(pp_mult)[mask].sum()/len(pp_mult)\n            \n    exp_to_group = group_plate_probs.argmax(1)\n    \n    for idx in range(len(all_test_exp)):\n        #print('Experiment', idx)\n        indices = (test_df.experiment == all_test_exp[idx])\n\n        preds = predicted[indices,:].copy()\n\n        preds = select_plate_group(preds, idx, exp_to_group)\n        #preds = preds.argmax(1)\n        sub.loc[indices,'sirna'] = preds.argmax(1)\n        \n        sub.to_csv('submission_{}.csv'.format(name), index=False, columns=['id_code','sirna'])\n    \n    return sub['sirna']\n\ndef select_plate_group(pp_mult, idx, exp_to_group):\n    all_test_exp = test_df.experiment.unique()\n    sub_test = test_df.loc[test_df.experiment == all_test_exp[idx],:]\n    assert len(pp_mult) == len(sub_test)\n    mask = np.repeat(plate_groups[np.newaxis, :, exp_to_group[idx]], len(pp_mult), axis=0) != \\\n           np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)\n    pp_mult[mask] = 0\n    return pp_mult","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#densenet121\nif conf.predict:\n    sub_densenet121_1 = post_processing(preds_HEPG2_1, preds_HUVEC_1, preds_RPE_1, preds_U2OS_1, 'densenet121_1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if conf.predict:\n    sub = pd.read_csv(conf.path_data + '/test.csv')\n    sub['sirna'] = sub_densenet121_1.astype(int)\n    sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if conf.predict:\n    sub[['id_code','sirna']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nimport requests\n\ndef send_line_notification(message):\n    line_token = 'token'  # set your token\n    endpoint = 'https://notify-api.line.me/api/notify'\n    message = \"\\n{}\".format(message)\n    payload = {'message': message}\n    headers = {'Authorization': 'Bearer {}'.format(line_token)}\n    requests.post(endpoint, data=payload, headers=headers)\n\nif conf.predict:\n    result = 'predict finished'\n\n    send_line_notification(result)\n\"\"\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}