{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os, sys\nsys.path.insert(0,\"/kaggle/input/our-env/lib\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d4c1534961343b1ac4a2c09d0c02dce27e32baa","_kg_hide-output":true},"cell_type":"code","source":"!pip install fastai==1.0.34","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport numpy as np\nimport scipy.optimize as opt\n# common\nfrom lib.data.utils import read_dataset_info, save_pred, Oversampling\nfrom lib.constants import *\nfrom lib.data.data_visualization import visualize_samples\n# torch\nfrom torchvision.transforms import *\nfrom torch.utils.data import DataLoader, WeightedRandomSampler\nimport fastai\nimport fastai.vision\nfrom fastai.vision.learner import create_cnn, cnn_config, create_body, create_head\nfrom fastai.torch_core import *\nfrom fastai import DatasetType\nfrom fastai.metrics import accuracy_thresh\n\nfrom lib.torch.dataset import ProteinDataset, get_sample_weights\nfrom lib.torch.losses import FocalLoss\nfrom lib.torch.metrics import f1_macro, fbeta, fit_thresholds\nfrom lib.torch.augmentation import tta, train_transforms\nfrom lib.torch.fit_custom_cycle import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"807802f6c228763ae3f5991fc694573f43c1a798"},"cell_type":"code","source":"labeled_dataset_info = read_dataset_info(\"../input/human-protein-atlas-image-classification/train\", \"../input/human-protein-atlas-image-classification/train.csv\", target_col='Target')\ntest_dataset_info = read_dataset_info(\"../input/human-protein-atlas-image-classification/test\", \"../input/human-protein-atlas-image-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"7a7ee5346db006e229b4f19ce21d64be86a18255"},"cell_type":"code","source":"#visualize_samples(labeled_dataset_info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dab1d8d6fe2244fc775d918bda830d1638a27eac"},"cell_type":"code","source":"label_count = np.zeros(len(CLASS_LABEL_DICT))\nfor info in labeled_dataset_info:\n    label_count[info['labels']] += 1\nlabeled_fractions = torch.cuda.FloatTensor((label_count / len(labeled_dataset_info)).astype(np.float32))\n    \nimage_size = (256, 256)\nbatch_size = 256\ntrain_stats = ([0.08069, 0.05258, 0.05487, 0.08282], [0.13704, 0.10145, 0.15313, 0.13814])\ntest_stats = ([0.05913, 0.0454, 0.04066, 0.05928], [0.11734, 0.09503, 0.129, 0.11528])\n\n\ny = [x['labels'] for x in labeled_dataset_info]\nstratifiing_y = [labels[np.argmin(label_count[labels])] for labels in y]\ntrain_indices, validation_indices, _, _  = train_test_split(range(len(y)), y, test_size=0.08, stratify=stratifiing_y, \n                                                            random_state=13)\n#train_dataset_info = Oversampling().apply(labeled_dataset_info[train_indices])\ntrain_dataset_info = labeled_dataset_info #[train_indices]\n\nprint('Train dataset len = {} / {} (oversampling)'.format(len(train_indices), len(train_dataset_info)))\nprint('Validation dataset len = {}'.format(len(validation_indices)))\nprint('Test dataset len = {}'.format(len(test_dataset_info)))\n\ntrain_dataset = ProteinDataset(train_dataset_info, image_size, train_stats, train_transforms)\nvalidation_dataset = ProteinDataset(labeled_dataset_info[validation_indices], image_size, train_stats)\ntest_dataset = ProteinDataset(test_dataset_info, image_size, train_stats)\n\nweigths = get_sample_weights(train_dataset_info, label_count)\ntrain_loader = DataLoader(train_dataset, batch_size, sampler=WeightedRandomSampler(weigths, len(train_dataset_info)))\nvalidation_loader = DataLoader(validation_dataset, batch_size)\ntest_loader = DataLoader(test_dataset, batch_size)\n\ndata_bunch = fastai.DataBunch(train_loader, validation_loader, test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94f29b0d2b0cc0b01dbb38598f2493dbc901c5a3"},"cell_type":"code","source":"classes_number = 28\n\ndef create_model():\n    pretrain_model = fastai.vision.models.resnet34\n    meta = cnn_config(pretrain_model)\n    body = create_body(pretrain_model(True), meta['cut'])\n\n    w = body[0].weight\n    body[0] = nn.Conv2d(4,64,kernel_size=(7,7),stride=(2,2),padding=(3, 3), bias=False)\n    body[0].weight = torch.nn.Parameter(torch.cat((w, w[:,:1,:,:]),dim=1))\n\n    resnet_body_output = 512\n    head = create_head(resnet_body_output*2, nc=classes_number)\n    head[3].p = 0.5 # make dropout with p=0.5\n    model = nn.Sequential(body, head)\n    return model\n\nmodel = create_model()\nlayer_groups = split_model_idx(model, [0,41,92,100])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"484d4ad0c09a38f9a54b2d5fb6d9b589b2b2fde6"},"cell_type":"code","source":"learn = fastai.Learner(data_bunch, model, loss_func=FocalLoss(), metrics=[accuracy_thresh, f1_macro], layer_groups=layer_groups)\nlearn.freeze()\nlearn.clip_grad(1.)\nlr = 5e-3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bafce30bed5fe376c978d7dbd4cd7ac63131b76b"},"cell_type":"markdown","source":"lr = 5e-3\nlearn.fit(1, lr)\nlearn.save('ResNet34_optimized_lr_freezed')"},{"metadata":{"trusted":true,"_uuid":"4fd24c22418e4e477979eab3602835b7a9742010"},"cell_type":"markdown","source":"weight_history = []\nlearn.unfreeze()\nlrs=np.array([lr/2,lr/2,lr])\nfor _ in range(4):\n    fit_custom_cycle(learn, 2, lrs/4, div_factor=20, pct_start=0.05)\n    \n    pred_y, true_y = learn.get_preds()\n    class_f1 = fbeta(pred_y, true_y, beta=1, thresh=0.5).numpy()\n    weights = get_sample_weights(train_dataset_info, label_count, class_f1)\n    learn.data.train_dl.sampler.weights = torch.tensor(weights)\n    weight_history.append(class_f1)\n    #print(np.mean(class_f1))\n\nlearn.save('ResNet34_1st_phase')"},{"metadata":{"trusted":true,"_uuid":"423165d6586ef849ab3afa01b424c237d269aef9"},"cell_type":"code","source":"weight_history = []\nlearn.load('../../input/resnet34-optimized-lr-freezedpth/ResNet34_1st_phase')\nlearn.unfreeze()\nlrs=np.array([lr/2,lr/2, lr])\nfor i in range(2):\n    pred_y, true_y = learn.get_preds(DatasetType.Train)\n    sample_loss = (pred_y.sigmoid() - true_y).abs().sum(dim=1).numpy()\n    weights = get_sample_weights(train_dataset_info, label_count, sample_loss=sample_loss)\n    learn.data.train_dl.sampler.weights = torch.tensor(weights)\n    \n    fit_custom_cycle(learn, 4, lrs/16, div_factor=15, pct_start=0.05)\nlearn.save('ResNet34_2nd_phase')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"668ae630d144f623bd398698320ed34ce84440bf"},"cell_type":"code","source":"pred_y, true_y = learn.get_preds()\nclass_f1 = fbeta(pred_y, true_y, beta=1, thresh=0.5).numpy()\nprint(class_f1)\nprint(np.mean(class_f1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53cc97c63baa2eea229fbbfcab85bbeab6515f79"},"cell_type":"markdown","source":"pred_y, true_y = learn.get_preds()\nclass_f1 = fbeta(pred_y, true_y, beta=1, thresh=0.5).numpy()\nprint(np.mean(class_f1))\nprint(class_f1)"},{"metadata":{"trusted":true,"_uuid":"465bb47bd0236a58dfcbaa7100afa34404206c86"},"cell_type":"markdown","source":"for i in range(2):\n    fit_custom_cycle(learn, 2, lrs/16, div_factor=15, pct_start=0.05)"},{"metadata":{"trusted":true,"_uuid":"00ed948fee26067accf5345afee23c0382a2b146"},"cell_type":"markdown","source":"fit_custom_cycle(learn, 4, lrs/32, div_factor=15, pct_start=0.05)\nlearn.save('./trained_learner')"},{"metadata":{"trusted":true,"_uuid":"63da124a4299220f06c8532134b9ab4c081fe233","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"markdown","source":"pred_val, y_val = tta(learn)"},{"metadata":{"trusted":true,"_uuid":"1d60658d980a0fdb5b70d96e23ce02ed96cc88a5"},"cell_type":"markdown","source":"th_val = fit_thresholds(pred_val, labeled_fractions)\n\nprint('Thresholds: ',th_val)\nprint('F1 macro: ',f1_macro(pred_val, y_val, th_val))\nprint('F1 macro (th = 0.5): ',f1_macro(pred_val, y_val))\nprint('Fractions: ',(pred_val.sigmoid() > th_val).float().mean(dim=0))\nprint('Fractions (all labeled): ',labeled_fractions)\nprint('Fractions (true): ',y_val.mean(dim=0))"},{"metadata":{"trusted":true,"_uuid":"99394f76e1aa31fba554a0f237bda5d8037bca31"},"cell_type":"markdown","source":"pred_test, _ = tta(learn, DatasetType.Test)\nth_test = fit_thresholds(pred_test, labeled_fractions)\nth_test"},{"metadata":{"trusted":true,"_uuid":"cfd59908608e80375cfe4d0acf21f56aa8af0875"},"cell_type":"markdown","source":"save_pred('./submission.csv', test_dataset_info, pred_test.sigmoid().numpy(), th_test.numpy())\nsave_pred('./submission05.csv', test_dataset_info, pred_test.sigmoid().numpy())"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}