{"cells":[{"metadata":{"_uuid":"1d55a793bada76c11521ae968af539738795678a"},"cell_type":"markdown","source":"# The Idea of this Kernel\nIn this kernel i largely merged the fastai v1 starter by Horton and the fastai rgby by iafoss.\nI (in my opinion) simplified some stuffabout the dataset and loading.\nI added:\n1. Iterative Stratified splitting the dataset into train and validation \n2. Class weighted Focalloss (the alpha parameter)\n3. Oversampling of the dataset\n4. The now standart onecycle instead of multicycle learning\n\nNext I would like to optimize the augmentation. So far TTA also is less performant than the regular predict which I find confusing.\nSo far I got into the top 20% with this which is nice."},{"metadata":{"trusted":true,"_uuid":"8489b8564cfd9c29f5dbf70756719334d873a301","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport os\nfrom sklearn.metrics import f1_score\n\nfrom fastai import *\nfrom fastai.vision import *\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport cv2\n\nfrom tqdm import tqdm\nfrom skmultilearn.model_selection import iterative_train_test_split\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n%load_ext autoreload\n%autoreload","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e08a08c1e6fa1376de9a14aec18cec46f38053c","_kg_hide-input":true},"cell_type":"code","source":"def open_4_channel(self,fname):\n    fname = str(fname)\n    # strip extension before adding color\n    if fname.endswith('.png'):\n        fname = fname[:-4]\n    colors = ['red','green','blue','yellow']\n    flags = cv2.IMREAD_GRAYSCALE\n    img = [cv2.imread(fname+'_'+color+'.png', flags).astype(np.float32)/255\n           for color in colors]\n\n    x = np.stack(img, axis=-1)\n    return Image(pil2tensor(x, np.float32).float())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bd61bbd3aa99fc9508ebcd33ce44c903e1a2e59","_kg_hide-input":true},"cell_type":"code","source":"def show_batch_cstm(self, rows:int=5, ds_type:DatasetType=DatasetType.Train, **kwargs)->None:\n        \"Show a batch of data in `ds_type` on a few `rows`.\"\n        x,y = self.one_batch(ds_type, True, True)\n        x=x[:,:3,:,:]\n        \n        if self.train_ds.x._square_show: rows = rows ** 2\n        xs = [self.train_ds.x.reconstruct(grab_idx(x, i, self._batch_first)) for i in range(rows)]\n        #TODO: get rid of has_arg if possible\n        if has_arg(self.train_ds.y.reconstruct, 'x'):\n            ys = [self.train_ds.y.reconstruct(grab_idx(y, i), x=x) for i,x in enumerate(xs)]\n        else : ys = [self.train_ds.y.reconstruct(grab_idx(y, i)) for i in range(rows)]\n        print(ys)\n        self.train_ds.x.show_xys(xs, ys, **kwargs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d616059af6aacfcd314d26c0f822661784922b5"},"cell_type":"code","source":"path='.'\ninput_fldr=Path('../input/')\ntrn_fldr=f'{input_fldr}/train/'\ntest_fldr=f'{input_fldr}/test/'\ntrn_lbl=f'{input_fldr}/train.csv'\nsample_csv = f'{input_fldr}/sample_submission.csv'\ndf_trn=pd.read_csv(f'{input_fldr}/train.csv')\ndf_trn.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce6b5f4c38a595535d3813ee828eaf7598207293"},"cell_type":"code","source":"sz=512\nbs=16\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bcca6b0c8e031163be652e1ce794f3a23eb3057"},"cell_type":"code","source":"targets=[np.array(target.split(' ')).astype(float) for target in df_trn.Target.values]\nmapping=MultiLabelBinarizer()\nmapping.fit(targets)\ntargets_mapped=mapping.transform(targets)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d759cb279ed9979b1dde8c2872c7d0c22c11d48"},"cell_type":"markdown","source":"### The iterative splitting into val and trn set is alot better"},{"metadata":{"trusted":true,"_uuid":"f443b8f685a7acb9add14a48c80bfd3dc31d2d59"},"cell_type":"code","source":"X_train, y_train, X_test, y_test = iterative_train_test_split(df_trn.Id[:,None], targets_mapped, test_size = 0.5)\nplt.plot(np.sum(y_train,axis=0)-np.sum(y_test,axis=0),label='iterative')\nX_train, X_test,y_train , y_test = train_test_split(df_trn.Id[:,None], targets_mapped, test_size = 0.5)\nplt.plot(np.sum(y_train,axis=0)-np.sum(y_test,axis=0),label='split')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ebbc9071bac7bdea7b0ea78149d86d93f5ee992f"},"cell_type":"markdown","source":"There was a weird bug that if the last batch is size=1 the batchnorm throws an error that took me a way too long to debug so I make sure that the split leads to a size of the last batch larger than 1"},{"metadata":{"trusted":true,"_uuid":"da78f68b3f71a6c645addc2192ab6d4e34f000f4"},"cell_type":"code","source":"p=1.\nn=int(p*df_trn.shape[0])\nwhile True:\n    if p<1.0:\n        idx=np.random.choice(len(df_trn.index),n)\n    else:\n        idx=range(len(df_trn.index))\n    X_train, y_train, X_val, y_val = iterative_train_test_split(df_trn.index[idx,None], targets_mapped[idx,:], test_size = 0.2)\n    if (np.sum(targets_mapped[idx,:],axis=0)>0).all() and ((X_train.shape[0]%bs!=1) and (X_val.shape[0]%bs!=1)):\n        break\nX_train.shape,(X_train.shape[0]%bs),X_val.shape,(X_val.shape[0]%bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7ce31fea7e97f4f4bbe871631fc0d3cd494b3c7","_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\n\nname_label_dict = {\n    0:   ('Nucleoplasm', 12885),\n    1:   ('Nuclear membrane', 1254),\n    2:   ('Nucleoli', 3621),\n    3:   ('Nucleoli fibrillar center', 1561),\n    4:   ('Nuclear speckles', 1858),\n    5:   ('Nuclear bodies', 2513),\n    6:   ('Endoplasmic reticulum', 1008),   \n    7:   ('Golgi apparatus', 2822),\n    8:   ('Peroxisomes', 53), \n    9:   ('Endosomes', 45),\n    10:  ('Lysosomes', 28),\n    11:  ('Intermediate filaments', 1093), \n    12:  ('Actin filaments', 688),\n    13:  ('Focal adhesion sites', 537),  \n    14:  ('Microtubules', 1066), \n    15:  ('Microtubule ends', 21),\n    16:  ('Cytokinetic bridge', 530),\n    17:  ('Mitotic spindle', 210),\n    18:  ('Microtubule organizing center', 902),\n    19:  ('Centrosome', 1482),\n    20:  ('Lipid droplets', 172),\n    21:  ('Plasma membrane', 3777),\n    22:  ('Cell junctions', 802),\n    23:  ('Mitochondria', 2965),\n    24:  ('Aggresome', 322),\n    25:  ('Cytosol', 8228),\n    26:  ('Cytoplasmic bodies', 328),   \n    27:  ('Rods &amp; rings', 11)\n    }\n\nn_labels = 50782\n\ndef cls_wts(label_dict, mu=0.5):\n    prob_dict, prob_dict_bal = {}, {}\n    max_ent_wt = 1/28\n    for i in range(28):\n        prob_dict[i] = label_dict[i][1]/n_labels\n        if prob_dict[i] > max_ent_wt:\n            prob_dict_bal[i] = prob_dict[i]-mu*(prob_dict[i] - max_ent_wt)\n        else:\n            prob_dict_bal[i] = prob_dict[i]+mu*(max_ent_wt - prob_dict[i])            \n    return prob_dict, prob_dict_bal","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a9e7362d4f3ef862b7b44aecae7192b183a956d"},"cell_type":"markdown","source":"### The choices for the oversampling and weight scaling parameters are pretty arbitrary.\nFor me the log scaled class weights worked best, the randomoversampling I did not really test yet since I frequently get a CUDA out of memory when the overall dataset is to large (which I dont really understand)\nI scaled the oversampling linearly but cut off the classes that are not too imbalanced to not have too large of a dataset for sampling.\nI initially tried to change the sampler of the training set to torch's weightedrandomsampler but somehow it never sampled as I intended it."},{"metadata":{"trusted":true,"_uuid":"f1fa5e905295ec12e8e57f3db1ddac65c1c6c3fb","_kg_hide-input":true},"cell_type":"code","source":"balance_p=.5\nprob_d,prob_bal=cls_wts(name_label_dict,balance_p)\nalpha=torch.cuda.FloatTensor(1/np.array([prob_bal[i] for i in prob_bal]))\nalpha_log=torch.cuda.FloatTensor([-np.log(prob_bal[i])/8 for i in prob_bal])\nplt.plot(to_np(alpha)/min(to_np(alpha)),label='Linear scaled weights')\nplt.plot(to_np(alpha_log)/min(to_np(alpha_log)),label='Log scaled weights')\nplt.legend()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46bee1b241700577676dd6f6098414255a3e7277","_kg_hide-input":true},"cell_type":"code","source":"alpha_norm=to_np(alpha)/np.min(to_np(alpha))\nplt.plot(alpha_norm,label='Linear scaled oversampling')\nalpha_norm[alpha_norm<4]=1\nalpha_norm[(alpha_norm>4) & (alpha_norm<6) ]=1.5\nalpha_norm[(alpha_norm>6) & (alpha_norm<7) ]=3\nalpha_norm[alpha_norm>7]=alpha_norm[alpha_norm>7]\nplt.plot(alpha_norm,label='Linear damped oversampling')\n#plt.plot(targets_mapped[:n,:].sum(axis=0)/1000)\nplt.legend()\nover_sample={i:int(item*alpha_norm[i]) for i,item in enumerate(np.unique(np.argmin(-y_train*(to_np(alpha_log)/np.min(to_np(alpha_log))),axis=1),return_counts=True)[1])}\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a809a1ef7d7e3f4a111555f85dcaa6272de7a3a"},"cell_type":"markdown","source":"This is a weird hacky version to the oversampler going. I did not manage to get it to work using the proper y_train nhot encoded targets but I had to throw out all classes except the most imbalanced one. I'll try to do this in a non stupid way later"},{"metadata":{"trusted":true,"_uuid":"a308e1711a64367f9a9e2a62f1e928264096c753"},"cell_type":"code","source":"from imblearn.over_sampling import RandomOverSampler\nros = RandomOverSampler(random_state=42,sampling_strategy=over_sample)\nX_res, y_res = ros.fit_resample(X_train, np.argmin(-y_train*(to_np(alpha_log)/np.min(to_np(alpha_log))),axis=1))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34bef12afbc5a75cffdc6607fae12d7606b17dc9"},"cell_type":"markdown","source":"### I adapted the open function to load 4 layers and the show batch function to only show 3 layers, the fourth wouldve been interpreted as alpha value (transparency) which looks bad"},{"metadata":{"trusted":true,"_uuid":"72ac6b0285889f476ce180eecf92b17d18203e91"},"cell_type":"code","source":"ImageItemList.open=open_4_channel\nImageDataBunch.show_batch=show_batch_cstm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0876034cca75d7717ad8c25eee89068224d3676"},"cell_type":"code","source":"tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=30., max_zoom=1,\n                      max_lighting=0.05, max_warp=0.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b745f4c4d783457c54fce45712add27719b73440"},"cell_type":"code","source":"test_ids = list(sorted({fname.split('_')[0] for fname in os.listdir(input_fldr/'test')}))\ntest_fnames = [input_fldr/'test'/test_id for test_id in test_ids]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f1bccb48406ce0546d0e4dfd19ff7cb55cf749e"},"cell_type":"code","source":"src = (ImageItemList.from_csv(input_fldr, 'train.csv', folder='train', suffix='.png',num_workers=0)\n        .split_by_idxs(X_res[:,0],X_val[:,0])\n        .label_from_df(sep=' ',  classes=[str(i) for i in range(28)]))\nsrc.add_test(test_fnames, label='0');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6edcd75487cc4a48bbc972cd9ccc5a086d0e2ad"},"cell_type":"markdown","source":"Load dataset with transformations calculate image stats and then normalize them by it. One should use larger sample sizes since the variance in the dataset is quite large, but my tests show this is not too important so Ill keept it at one batch"},{"metadata":{"trusted":true,"_uuid":"8cbd4859b756e01cbc0d2953c8a95bba96772921"},"cell_type":"code","source":"data = (src.transform(tfms, size=sz)\n        .databunch(num_workers=0))\nstats=data.batch_stats()        \ndata.normalize(stats)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7345ef695004d19824b21d366366ab20406bd2c5"},"cell_type":"markdown","source":"## These pictures look awesome"},{"metadata":{"trusted":true,"_uuid":"f4881e99c71422a3c270827d6c7360d93a3e921a"},"cell_type":"code","source":"data.show_batch(rows=3, figsize=(12,9))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"268ca6db2dbd74783d865f4de678f296202f4c1e"},"cell_type":"markdown","source":"### Definition of the 4layer resnet, fastai version by W Horton"},{"metadata":{"trusted":true,"_uuid":"c5a1c10c275001e391dc828f730aaeeb9baefc35","_kg_hide-input":true},"cell_type":"code","source":"RESNET_ENCODERS = {\n    34: torchvision.models.resnet34,\n    50: torchvision.models.resnet50,\n    101: torchvision.models.resnet101,\n    152: torchvision.models.resnet152,\n}\nclass Resnet4Channel(nn.Module):\n    def __init__(self, encoder_depth=34, pretrained=True, num_classes=28):\n        super().__init__()\n\n        encoder = RESNET_ENCODERS[encoder_depth](pretrained=pretrained)\n        \n        # we initialize this conv to take in 4 channels instead of 3\n        # we keeping corresponding weights and initializing new weights with zeros\n        # this trick taken from https://www.kaggle.com/iafoss/pretrained-resnet34-with-rgby-0-460-public-lb\n        w = encoder.conv1.weight\n        self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        \n        self.conv1.weight = nn.Parameter(torch.cat((w,w[:,:1,:,:]),dim=1))\n        \n        self.bn1 = encoder.bn1\n        self.relu = nn.ReLU(inplace=True) \n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.layer1 = encoder.layer1\n        self.layer2 = encoder.layer2\n        self.layer3 = encoder.layer3\n        self.layer4 = encoder.layer4\n        \n        self.avgpool = encoder.avgpool\n        self.fc = nn.Linear(512 * (1 if encoder_depth==34 else 4), num_classes)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n    \n\ndef resnet34(pretrained):\n    return Resnet4Channel(encoder_depth=34)\n\ndef resnet50(pretrained):\n    return Resnet4Channel(encoder_depth=50)\n\n\n\n# copied from https://github.com/fastai/fastai/blob/master/fastai/vision/learner.py\ndef _resnet_split(m): return (m[0][6],m[1])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39d084c3500ed4f3970871c5dbadfe316a83966d"},"cell_type":"code","source":"f1_score = partial(fbeta, thresh=0.35, beta=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51b91b078735c85c3960db37729a882d61a58cc6"},"cell_type":"code","source":"def _resnet_split(m): return (m[0][6],m[1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de857ee6c70d90e0444b3ed5f46a82fe2288eb99"},"cell_type":"markdown","source":"Focal loss implementation taken from the salt identification challenge"},{"metadata":{"trusted":true,"_uuid":"845d18d5b3c44d7f7aa6b5c30aeee78e683c9042"},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss\n        ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3f45f4022d1abded65e4849829c538d82c54a9c"},"cell_type":"markdown","source":"### Here I add the class weighting in the loss function"},{"metadata":{"trusted":true,"_uuid":"f1fa5e905295ec12e8e57f3db1ddac65c1c6c3fb"},"cell_type":"code","source":"learn = create_cnn(\n    data,\n    resnet34,\n    cut=-2,\n    split_on=_resnet_split,\n    loss_func=FocalLoss(logits=True,alpha=alpha_log),\n    path=path,    \n    metrics=[f1_score], \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5ea8b65242fc3c564747faa3c8f357051770c43"},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5ea8b65242fc3c564747faa3c8f357051770c43"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3172e7e07e870f0be3fe27dbe7de5ced37451272"},"cell_type":"code","source":"learn.fit_one_cycle(1,4e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"634af0f4a5a33da7a132ad5835a1fd5d9453ebcf"},"cell_type":"code","source":"learn.recorder.plot_losses()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aafdfd79e1b4dd28a36f407ff6011a9bacca51e5"},"cell_type":"code","source":"learn.unfreeze()\nlearn.lr_find(num_it=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9cafd5fd622892bd606298d0f9a70cdb0022d74"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a1512bab803a822acbeffc6ffd19bb6b7ba3aef","scrolled":true},"cell_type":"code","source":"lr=1e-3\nlearn.fit_one_cycle(4, slice(lr/10,lr))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89953247ae9cb26a883890313dce6b20bbf26029"},"cell_type":"code","source":"learn.save('4_epochs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12286fbb1841eec2bf5883dc639bde125b273ec5"},"cell_type":"code","source":"learn.recorder.plot(skip_start=0,skip_end=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef0407725853005cf33de34055be8c51c68cfe13"},"cell_type":"code","source":"learn.recorder.plot_losses()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7723e8fe86c248447fdfe01c0dad1d4a5790b1b4"},"cell_type":"code","source":"learn.recorder.plot_lr()\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16394e160c0a6762ba5c82f68243dcbec281c570"},"cell_type":"markdown","source":"### Now first I predict on the validation set and then fit a threshold for the whole dataset"},{"metadata":{"trusted":true,"_uuid":"f2ce513e19c8bec4994eb26757e8e408009a796a"},"cell_type":"code","source":"pred,y=learn.get_preds()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03ae413567a145778d713213d2d0d013ae287ebd"},"cell_type":"code","source":"pred_test,_=learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f62b65c5622b5c8ed6e39207682f1cef24e5249d"},"cell_type":"code","source":"pred_test_tta,_=learn.TTA(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"51ebb11c1e5fd04697bab69c116e05876e91d343"},"cell_type":"markdown","source":"### These Thresholds are shamelessly stolen from Iafoss kernel**"},{"metadata":{"trusted":true,"_uuid":"ec2d386cc85197e592adfca2d03b2339a6ae9dc7"},"cell_type":"code","source":"th_t = np.array([0.565,0.39,0.55,0.345,0.33,0.39,0.33,0.45,0.38,0.39,\n               0.34,0.42,0.31,0.38,0.49,0.50,0.38,0.43,0.46,0.40,\n               0.39,0.505,0.37,0.47,0.41,0.545,0.32,0.1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"022619077ccf2f7988bb907890b5e8531ae4eda3"},"cell_type":"code","source":"from sklearn.metrics import f1_score as f1_sc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10d17f2875d15f2c11a9e0b92e800cc2a474c13e"},"cell_type":"code","source":"def eval_pred(pred,y):\n    ths=np.arange(0.001,1,0.01)\n    preds_s=F.sigmoid(pred)\n    th_val=ths[np.argmax([f1_sc(y,preds_s>th,average='macro') for th in ths])]\n    print('F1 macro: ',f1_sc(to_np(y),to_np(preds_s)>th_t,average='macro'))\n    print(f'F1 macro (th = {th_val}): ',f1_sc(to_np(y),to_np(preds_s)>th_val,average='macro'))\n    plt.plot(f1_sc(to_np(y),to_np(preds_s)>th_t,average=None),label='opt')\n    plt.plot(f1_sc(to_np(y),to_np(preds_s)>th_val,average=None),label='valid')\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c08ae371971d7bb4b8e7f759de6de2b1dcd8f7b"},"cell_type":"code","source":"ths=np.arange(0.001,1,0.01)\npreds_s=F.sigmoid(pred)\nth_val=ths[np.argmax([f1_sc(y,preds_s>th,average='macro') for th in ths])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10d17f2875d15f2c11a9e0b92e800cc2a474c13e"},"cell_type":"code","source":"eval_pred(pred,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"204e57652eb14102cd52e0d91c1c1c455a84672a"},"cell_type":"code","source":"def save_pred(pred, th=0.5, fname='protein_classification.csv'):\n    pred_list = []\n    for line in pred:\n        s = ' '.join(list([str(i) for i in np.nonzero(line>th)[0]]))\n        pred_list.append(s)\n        \n    sample_df = pd.read_csv(sample_csv)\n    sample_list = list(sample_df.Id)\n    #fnames_=[fname.split('/')[-1] for fname in learn.data.test_ds.fnames]\n    pred_dic = dict((key, value) for (key, value) \n                in zip(test_ids,pred_list))\n    pred_list_cor = [pred_dic[id] for id in sample_list]\n    df = pd.DataFrame({'Id':sample_list,'Predicted':pred_list_cor})\n    df.to_csv(fname, header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"204e57652eb14102cd52e0d91c1c1c455a84672a"},"cell_type":"code","source":"save_pred(to_np(F.sigmoid(pred_test)), th=th_val, fname=f'protein_classification_{np.around(th_val,decimals=2)}.csv')\n\nsave_pred(to_np(F.sigmoid(pred_test)), th=th_t, fname='protein_classification_customth.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"204e57652eb14102cd52e0d91c1c1c455a84672a"},"cell_type":"code","source":"save_pred(to_np(F.sigmoid(pred_test_tta)), th=th_val, fname=f'protein_classification_{np.around(th_val,decimals=2)}_tta.csv')\n\nsave_pred(to_np(F.sigmoid(pred_test_tta)), th=th_t, fname='protein_classification_customth_tta.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}