{"cells":[{"metadata":{},"cell_type":"markdown","source":"# FULL FOLD NEW BASELINE"},{"metadata":{"trusted":true},"cell_type":"code","source":"CNF={\n#     'isTrain':False,\n    'plotCAM':False,\n    'isTrain':True,\n#     'targetFold':[0],\n    'targetFold':[0,1,2,3,4],\n    'validateSequence':True,\n    'batch_size':8,\n    #'epoch':30,\n    'epoch':10,#10,30\n    'lr':0.001,\n    'n_tta':3, # unimplemented\n    'EMA':True,\n    # 'use_fp':True,\n    'use_fp':False,\n    'nhead':4,#4  8はいまいちな予感\n    'nmel':128, # 128 or 256\n    'skf_random_state':0, # fold0が比較的バランスが良くなる\n    'num_workers':2,\n    'loadPthPath':f\"../input/pthresnet1229/resnet\",\n    'backbone':'resnest50', # 'resnest50' or'resnest101' or 'resnet34'\n#     'mixup':False,\n    'mixup':True,\n    'multiLabel':True,\n#     'multiLabel':False,\n    'trainCUT':6, # 先生は8\n    #'trainCUT':5,\n    'vgg_sub_sys':True,\n    #'pseudo':True,\n    'pseudo':False,\n    \n}\n\n\nCNF['meltrainTpPath']=f\"../input/mymelspec/melspec_tp_NMEL{CNF['nmel']}_FMIN20_FMAX16000_SR32000.npy\"\nCNF['meltrainFpPath']=f\"../input/mymelspec/melspec_fp_NMEL{CNF['nmel']}_FMIN20_FMAX16000_SR32000.npy\"\nCNF['meltestPath']   =f\"../input/mymelspec/melspec_test_NMEL{CNF['nmel']}_FMIN20_FMAX16000_SR32000.npy\"\n\nLABEL_COLS = [f\"s{_i}\" for _i in range(24)]\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport gc\nimport time\nfrom scipy.interpolate import interp1d\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom scipy.stats import rankdata\n\nfrom sklearn.linear_model import LogisticRegression\n\n\nimport soundfile as sf\n# Librosa Libraries\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\ntry:\n    import cuml as cm\n    import cupy as cp\nexcept ModuleNotFoundError:\n    pass\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torchvision import datasets, models, transforms\nfrom transformers import *\nfrom math import floor, ceil\n\nfrom rfcxutil import lwlrap,rfcx_appendFold,rfcx_showSplit,rfcx_foldSplitSanityCheck,rfcx_cut_melspec,rfcx_SimpleDataset\nfrom osicutil import ModelEMA\nimport pickle\nimport os,gc,random,glob \n\nimport albumentations as A\nfrom albumentations.core.transforms_interface import BasicTransform\n\n\n#from resnest.torch import resnest50\n#from resnest.torch import resnest101\n\ndef seed_everything(seed: int):\n    random.seed(seed);os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed);torch.manual_seed(seed);torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \n\nimport pickle\ndef _save(x,fname):\n    pickle.dump(x, open(fname, 'wb'))\ndef _load(fname):\n    return pickle.load(open(fname, 'rb'))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntraint.shape, trainf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_unique_id(df): # melspec読み込みのためのid\n    \n    if 'mymelid' in df.columns:\n        print(f\"WARNING mymelid already in columns\")\n        df = df.drop(columns='mymelid')\n    \n    udf = df.drop_duplicates(subset='recording_id', keep='first', ignore_index=True).copy()\n    udf['mymelid'] = udf.index.tolist()\n    \n    return df.merge(udf[['recording_id','mymelid']],how='left',on='recording_id')\n    \ntraint = set_unique_id(traint)\ntrainf = set_unique_id(trainf)\ntraint['tp_or_fp'] = 'tp'\ntrainf['tp_or_fp'] = 'fp'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Append fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = rfcx_appendFold(traint,random_state=CNF['skf_random_state'])\nrfcx_showSplit(traint)\nprint(f\"Sanity Check [ same recording id -> same fold ] : {rfcx_foldSplitSanityCheck(traint)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNF['trainCUT']秒間での鳥の種類を記録"},{"metadata":{"trusted":true},"cell_type":"code","source":"traint['t_center'] = (traint.t_min+traint.t_max)/2\ntraint['t_start'] = traint['t_center'] - CNF['trainCUT']/2\ntraint['t_end']   = traint['t_center'] + CNF['trainCUT']/2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Labeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def label_keepindex(_df): # keepindexはdf applyしたときにindexはそのままの意味\n    \n    ret_df = pd.DataFrame({ f\"s{_spid}\":  (_df.species_id == _spid).astype(float)  for _spid in range(24)},index=_df.index)\n    \n    species_ids = [[]]*len(_df)\n    \n    for idx0,row0 in _df.iterrows():\n        \n        for idx1,row1 in _df.iterrows():\n            \n            # 鳴いてる時間row0とrow1に重複があればrow1にrow0.species_idを記録する\n            \n            if row0.t_end < row1.t_start: # | row0 | | row1 | \n                pass\n            elif row1.t_end < row0.t_start: # | row1 | | row0 | \n                pass\n            else: # | ro|w r|ow| \n    \n                if idx0==idx1:\n                    ret_df.loc[idx1,f\"s{row0.species_id}\"] = 1\n                else:\n                    if CNF['multiLabel']:\n                        ret_df.loc[idx1,f\"s{row0.species_id}\"] = 1\n            \n    return ret_df        \n    \nrenew_species_df = traint.groupby('recording_id').apply(label_keepindex)\n\ntraint = pd.concat([traint,renew_species_df],axis=1)\ntraint","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## traintをレコード毎にマルチラベル化したtrainu"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train tp -> レコード事のマルチラベル化 -> train u \ntrainu = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' );trainu = set_unique_id(trainu)\ntrainu = rfcx_appendFold(trainu,random_state=CNF['skf_random_state'])\ndef multi_label_keepindex(_df): # keepindexはdf applyしたときにindexはそのままの意味\n\n    ret_df = pd.DataFrame({ f\"s{_spid}\":  (_df.species_id == _spid).astype(float)  for _spid in range(24)},index=_df.index)\n    species_ids = [[]]*len(_df)\n    for idx0,row0 in _df.iterrows():\n        for idx1,row1 in _df.iterrows():\n            ret_df.loc[idx1,f\"s{row0.species_id}\"] = 1\n\n    return ret_df        \n\nrenew_species_df = trainu.groupby('recording_id').apply(multi_label_keepindex)\ntrainu = pd.concat([trainu,renew_species_df],axis=1)\ntrainu = trainu.drop_duplicates(subset='recording_id', keep='first', ignore_index=True).copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- CUT8 だと158箇所が重複\n- CUT6 だと73箇所が重複\n- CUT5 だと66箇所が重複\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,2));(traint.t_max-traint.t_min).hist(bins=100);plt.xlabel('sec')\nplt.title(f\"Song Dulation MIN{(traint.t_max-traint.t_min).min():.3f}sec MAX{(traint.t_max-traint.t_min).max():.3f}sec\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"8 秒を切り出してメモリ上に置いておく <strong> traintのindex順に格納するから,　以降のindex操作は慎重にな"},{"metadata":{"trusted":true},"cell_type":"code","source":"orig_meldata_tp = np.load(CNF['meltrainTpPath'])\n\n#\n# メル毎に正規化\n#\n\nfor idx in range(len(orig_meldata_tp)):\n    \n    orig_meldata_tp[idx] = ( orig_meldata_tp[idx] - orig_meldata_tp[idx].mean() )/orig_meldata_tp[idx].std()\n    orig_meldata_tp[idx] = orig_meldata_tp[idx] - orig_meldata_tp[idx].min() \n    \n\n\nif True:\n\n    import skimage.transform\n    from scipy.interpolate import interp1d\n\n    def f2melf(f):\n\n        _f = np.linspace(20, 32000//2, 128) # min,max,nmel\n        _fmel = librosa.core.mel_frequencies(n_mels=128,  fmin = 20,fmax = 32000//2, htk=False)\n        return interp1d(_fmel,_f)(f)\n\n    for ridx in ['0201197ec','009b760e6','0099c367b','01b41f92b','0313e82cf','03b96f209','011f25080','0295e3234']:\n\n        idx = traint[traint.recording_id==ridx].mymelid.values[0]\n\n        fig, ax = plt.subplots(figsize=(15, 5))\n\n        img = librosa.display.specshow(\n            #melspec,\n            orig_meldata_tp[idx], #data\n            sr=32000,\n            x_axis='time', \n            y_axis='linear', \n            fmin = 20,\n            fmax = 32000 // 2,\n            ax=ax)\n\n\n        t_min = traint.iloc[idx].t_min\n        t_max = traint.iloc[idx].t_max\n        t_ctr = (t_min+t_max)/2\n\n        plt.axvline(t_min,c='w')\n        plt.axvline(t_max,c='w')\n\n        plt.axhline(f2melf(traint.iloc[idx].f_min),c='w')\n        plt.axhline(f2melf(traint.iloc[idx].f_max),c='w')\n\n        plt.xlim(t_ctr-4,t_ctr+4)\n        plt.show()\n\n        del fig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MEL_CUT = CNF['trainCUT'] # sec\n\n_, n_mels, n_sample_per_minute = orig_meldata_tp.shape\n\ncut_width = int(n_sample_per_minute*((MEL_CUT)/60)) \n\nmeldata_tp = rfcx_cut_melspec(orig_meldata_tp,cut_width,traint.t_min.values,traint.t_max.values,traint.mymelid.values)\n    \n# del orig_meldata_tp\n\n    \n# 最大値が1になるように正規化 とりあえず全体のMIN,MAXで正規化 \ng_meldata_tp_max = meldata_tp.max()\ng_meldata_tp_min = meldata_tp.min()\n\nmeldata_tp = (meldata_tp-g_meldata_tp_min)/(g_meldata_tp_max-g_meldata_tp_min)\n\n\n# for vgg\nCNF['meldata_std']  = meldata_tp.std()\nCNF['meldata_mean'] = meldata_tp.mean()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(meldata_tp.shape)\nplt.hist(meldata_tp.flatten());","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"from rfcxutil import dup3nmlresizeTrans # 1 channel melspec -> 3 channel \nimport cv2\n\ntrnTransform = A.Compose([  A.ShiftScaleRotate( shift_limit=1, scale_limit=0.1, rotate_limit=0, interpolation=1, border_mode=cv2.BORDER_WRAP, value=None, mask_value=None, shift_limit_x=0.5, shift_limit_y=0, always_apply=False, p=1) , # x よこ\ndup3nmlresizeTrans()] )\n\n\n\nvalTransform = A.Compose([ dup3nmlresizeTrans() ] )\n\n\ndef get_loader(X,Y,batch_size=CNF['batch_size'],transform = None,shuffle=True, drop_last=True, mixup=False):\n\n    ds = rfcx_SimpleDataset(X,Y,transform = transform)\n    \n    if mixup:\n    \n        def collate_mixup_fn(batch):\n\n            if ds.Y is not None:\n\n                Xs = torch.stack([x[0] for x in batch])\n                Ys = torch.stack([x[1] for x in batch])\n                \n                Xmixup = ( (Xs[0]+Xs[1])/2 ).unsqueeze(0)\n                Ymixup = ((Ys[0]+Ys[1]).clip(0,1) ).unsqueeze(0)\n                \n                Xs = torch.cat([Xs,Xmixup],axis=0)\n                Ys = torch.cat([Ys,Ymixup],axis=0)\n\n                return Xs, Ys\n\n            else:\n\n                Xs = torch.stack([x for x in batch])\n\n                return Xs\n            \n        _collate_fn = collate_mixup_fn\n        \n    else:\n        \n        _collate_fn = ds.collate_fn\n    \n    loader = torch.utils.data.DataLoader(ds, batch_size=batch_size, shuffle=shuffle, num_workers=CNF['num_workers'], collate_fn=_collate_fn, drop_last=drop_last)\n    loader.num = len(X)\n    \n    return loader\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fold=0\n# trn_idx,val_idx = traint.query(f\"fold != {fold}\").index.tolist(), traint.query(f\"fold == {fold}\").index.tolist()\n    \n# Xtrn,Xval = meldata_tp[trn_idx],meldata_tp[val_idx] # dfのindexでndarrayから抜き出してるからindexが0～Nって感じでちゃんと並んでる前提\n# Ytrn,Yval = traint[LABEL_COLS].values[trn_idx],traint[LABEL_COLS].values[val_idx]\n    \n# trn_ld = get_loader(Xtrn,Ytrn,transform=trnTransform,shuffle=True,drop_last=True,mixup=CNF['mixup']) # trnTransform is defined outside this class","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for x,y in trn_ld:\n#     #print(x)\n#     plt.imshow( x[0].numpy().transpose(1,2,0) )\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from rfcxutil import rfcx_myGlobalAvgPool2d,rfcx_myGlobalMaxPool2d # 横方向にavg_pool\n\n\nclass GroupFC(nn.Module):\n\n    def __init__(self,n_mel,n_classes,feat_dim):\n        super(GroupFC, self).__init__()\n        self.n_mel = n_mel ; self.n_classes = n_classes ; self.feat_dim = feat_dim # maxpool前のfeat_dim\n        self.dropout0 = nn.Dropout(p=0.2) # 0.0だとちょっとだけ悪くなる\n        \n        self.simplefc =  nn.Sequential(  nn.Linear( 2*self.feat_dim*self.n_mel, self.n_classes ) ) \n        \n    def forward(self, inputs_avg, inputs_max):\n        \n        x_avg = torch.cat([ self.dropout0(inputs_avg[:,self.feat_dim*(_i):self.feat_dim*(_i+1)] ) for _i in range(self.n_mel) ], 1)\n        x_max = torch.cat([ self.dropout0(inputs_max[:,self.feat_dim*(_i):self.feat_dim*(_i+1)] ) for _i in range(self.n_mel) ], 1)\n        \n        x = torch.cat([x_avg,x_max],axis=-1)\n        \n        return self.simplefc( x )\n                                       \n\nclass VggModel(nn.Module):\n\n    def __init__(self, CNF,n_classes=24):\n        \n        super(VggModel, self).__init__()\n        \n        self.n_mel = 4\n        self.CNF = CNF\n        self.n_classes = n_classes\n        self.baseModel = torch.hub.load('pytorch/vision:v0.6.0','vgg11_bn' , pretrained=True if self.CNF['isTrain'] else False)\n        self.baseModel.avgpool = rfcx_myGlobalAvgPool2d(n_mel=self.n_mel)\n        self.baseModel.maxpool = rfcx_myGlobalMaxPool2d(n_mel=self.n_mel)\n        \n        self.baseModel.fc = GroupFC(n_mel=self.n_mel,n_classes=n_classes,feat_dim=512) \n        \n    def vgg_unit(self,x):\n        \n        x = self.baseModel.features(x)\n        x_avgpool = self.baseModel.avgpool(x)\n        x_maxpool = self.baseModel.maxpool(x)\n        \n        x = self.baseModel.fc(x_avgpool,x_maxpool)\n        \n        return x\n        \n    def forward(self, x):\n        \n        xh  = ( x[:,:,0::2]+x[:,:,1::2] )/2 # 64mel -> batch x mel \n        \n        return   ( self.vgg_unit(x) + self.vgg_unit(xh)  )/2\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resnet"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResnetModel(nn.Module):\n\n    def __init__(self, CNF,n_classes=24):\n        \n        super(ResnetModel, self).__init__()\n        \n        self.n_mel = 4\n        self.CNF = CNF\n        self.n_classes = n_classes\n        self.baseModel = torch.hub.load('pytorch/vision:v0.6.0','resnet18' , pretrained=True if self.CNF['isTrain'] else False)\n        \n        self.baseModel.fc = nn.Linear( 512, self.n_classes )\n        \n    def forward(self, x):\n        \n        return self.baseModel(x)\n        \n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitter"},{"metadata":{"trusted":true},"cell_type":"code","source":"from rfcxutil import  SaveFeatures\n    \n    \nclass FitterNN():\n    \n    def __init__(self,CNF,fold_id,prefix=\"resnet\"): # prefixは小文字で\n        \n        self.CNF = CNF ; self.prefix = prefix ; self.fold_id = fold_id\n        self.model = VggModel(self.CNF) if 'vgg' in self.prefix else None\n        self.model = ResnetModel(self.CNF) if 'resnet' in self.prefix else None\n        \n        self.final_layer = self.model.baseModel.layer4 if 'resnet' in self.prefix else None\n        \n        \n    def fit(self,Xtrn,Xval,Ytrn,Yval,Wtrn,Wval):\n        \n        seed_everything(42)\n        device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n        \n        optimizer = optim.Adam( [{'params': self.model.parameters(), 'lr': self.CNF['lr']}] , lr=self.CNF['lr'] ) \n        \n        scheduler = LambdaLR(optimizer, lr_lambda=lambda ep: 0.01+(1-0.01)*np.cos(0.5*np.pi*ep/self.CNF['epoch'])**2, verbose=True )\n        \n        trn_criterion, val_criterion = [ torch.nn.BCEWithLogitsLoss( pos_weight=torch.FloatTensor( w ).to(device) ) for w in [Wtrn,Wval] ]\n        \n        trn_ld = get_loader(Xtrn,Ytrn,transform=trnTransform,shuffle=True,drop_last=True,mixup=self.CNF['mixup']) # trnTransform is defined outside this class\n        \n        isSeqVal = Xval.shape[-1] != Xtrn.shape[-1] # 自動でseqvalかvalを設定\n        print(f\"isSeqVal:{isSeqVal}\")\n        if isSeqVal:\n            Xval = self.split60sec2sequence(Xval)\n            val_batch_size = int(np.ceil(60/self.CNF['trainCUT'])) # 60秒が何分割されるか\n            Yval = np.repeat(Yval, val_batch_size, axis=0) # 無駄だけど、データローディング順を合わせるためだけ\n            \n        else:\n            val_batch_size = CNF['batch_size']\n            \n        print(f\"VAL_BATCH_SIZE:{val_batch_size}\")\n            \n        val_ld = get_loader(Xval,Yval,batch_size=val_batch_size,transform=valTransform,shuffle=False,drop_last=False)\n        \n        self.model.to(device)\n        ema = ModelEMA(self.model) if self.CNF['EMA'] else None\n        trn_losses,val_losses,scores = [],[],[] ; best_score = -np.inf\n            \n        for epoch in range(self.CNF['epoch']):\n\n            self.model.train() ;  tr_loss = 0\n            for _X, _Y in trn_ld: \n                \n                outputs = self.model(_X.to(device))\n                loss = trn_criterion(outputs, _Y.to(device))\n\n                loss.backward(); optimizer.step();optimizer.zero_grad()\n                _ = ema.update(self.model) if self.CNF['EMA'] else None\n                tr_loss+=loss.detach().cpu().numpy().sum()\n            scheduler.step()\n            tr_loss = tr_loss/len(Xtrn)\n\n            \n            #\n            # seq pred の時は batchは1sequenceを表す\n            #\n            with torch.no_grad():\n\n                self.model.eval() ; vl_loss = 0 ; gt = [] ; preds = []\n                for _X, _Y in val_ld:\n\n                    outputs = self.model(_X.to(device))\n                    \n                    if isSeqVal:\n                        outputs = self.getPredictionFromSeqwisePred(outputs)\n                        outputs,_ = torch.max(outputs,axis=1)\n                        vl_loss+=val_criterion(outputs[0],_Y[0].to(device)) # 1 つのレコードに対するval_loss\n                    \n                    else:\n                        vl_loss+=val_criterion(outputs,_Y.to(device)).sum()\n                    \n                    preds +=  [ torch.sigmoid(outputs).cpu().numpy() ]\n\n                preds=np.vstack(preds) \n                vl_loss = vl_loss/len(Xval)\n            \n            \n            if isSeqVal:\n                _score_class, _weight = lwlrap(Yval[::val_batch_size,:],preds); score = (_score_class * _weight).sum()\n            else:\n                _score_class, _weight = lwlrap(Yval,preds); score = (_score_class * _weight).sum()\n\n            if score > best_score:\n\n                best_score = score \n                os.makedirs(f\"{self.prefix}\", exist_ok=True); torch.save(self.model.state_dict(), f\"{self.prefix}/{self.prefix}_model_fold{self.fold_id}.pth\")\n\n            print(f\"EP{epoch:02d}:TRN_LOSS {tr_loss:.5f} VAL_LOSS {vl_loss:.5f} SCORE {score:.5f}\")\n            trn_losses.append(tr_loss);val_losses.append(vl_loss);scores.append( score );torch.cuda.empty_cache()\n        \n        history = {'trn_loss':trn_losses,'val_loss':val_losses,'score':scores,'best_score':best_score}\n        pickle.dump(history, open(f\"{self.prefix}/{self.prefix}_history_fold{self.fold_id}.pkl\", 'wb'))\n        \n        return history\n        \n    def predict(self,X,returnCam=False):\n        \n        device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n        self.model.to(device)\n        self.model.load_state_dict(torch.load(f\"{self.prefix}/{self.prefix}_model_fold{self.fold_id}.pth\"))\n        \n        ld = get_loader(X,None,transform=valTransform,shuffle=False,drop_last=False)\n        with torch.no_grad():\n\n            self.model.eval() ; preds = [] ; cam = []\n            for _X in ld:\n                \n                if returnCam:\n                    activated_features = SaveFeatures(self.final_layer)  # 登録\n\n                pred = torch.sigmoid(self.model(_X.to(device)))\n                \n                if returnCam:\n                    \n                    activated_features.remove() # 登録解除\n                    # fc weight\n                    weight_fc_params = list(self.model.baseModel.fc.parameters())\n                    weight_fc = np.squeeze(weight_fc_params[0].data.cpu().numpy())\n                    # top 1 class id\n                    class_idx = torch.topk(pred, 1)[1].int().squeeze() # 最新predsのtop1_cls_id\n                    \n                    for _id in range(len(class_idx)):\n                        \n                        _w  = weight_fc[class_idx[_id],:] # fc重み 1x512\n                        _ft = activated_features.features[_id,:,:,:] # seq_id,_bandのlast conv出力 512x7x7\n                        \n                        cam.append( _w.dot(_ft.reshape((512, 7*7 ))).reshape(7,7) ) # 1x512 @ 512x7x7 -> 7x7 \n                        \n                        \n                preds +=  [ pred.cpu().numpy() ]        \n                    \n\n            preds=np.vstack(preds);torch.cuda.empty_cache()\n            if returnCam:\n                cam = np.stack(cam)\n            \n        if returnCam:\n            return preds,cam\n        else:\n            return preds\n        \n        \n    def predictSeq(self,X):\n        \n        device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n        self.model.to(device)\n        self.model.load_state_dict(torch.load(f\"{self.prefix}/{self.prefix}_model_fold{self.fold_id}.pth\"))\n        \n        X = self.split60sec2sequence(X)\n        val_batch_size = int(np.ceil(60/self.CNF['trainCUT'])) # 60秒が何分割されるか\n            \n        ld = get_loader(X,None,batch_size=val_batch_size,transform=valTransform,shuffle=False,drop_last=False)\n            \n        with torch.no_grad():\n\n            self.model.eval() ; preds = [] ; cam = []\n            for _X in ld:\n                \n                outputs = self.model(_X.to(device))\n                outputs = self.getPredictionFromSeqwisePred(outputs)\n                outputs,_ = torch.max(outputs,axis=1)\n                \n                preds +=  [ torch.sigmoid(outputs).cpu().numpy() ]\n                    \n\n            preds=np.vstack(preds);torch.cuda.empty_cache()\n        \n        return preds\n    \n    \n    def split60sec2sequence(self,X):\n        \n        # 60秒のメルデータXを小分けにする\n        n_seq = int(np.ceil(60/self.CNF['trainCUT'])) # 60秒が何分割されるか\n        n_data_in_60sec = X.shape[-1] # 60秒melの幅\n        n_data_per_seq = int(np.round( self.CNF['trainCUT']*(n_data_in_60sec/60) )) # CNF['trainCUT']秒はデータにしていくつか\n        n_seq_start_idx = np.floor(np.linspace(0, n_data_in_60sec-1-n_data_per_seq, num=n_seq, endpoint=True)).astype('int') # 60秒をCNF['trainCUT']秒に分割したそれぞれの区間に対応するオリジナルからの切り取り開始位置\n\n        return np.stack([ X[:,:,idx:(idx+n_data_per_seq)] for idx in n_seq_start_idx ] ,axis=1).reshape(-1,128,n_data_per_seq) # batch_id x seq_id x 128(mel) x 375 -> ( batch_id x seq_id ) x 128 x 375\n        \n    def getPredictionFromSeqwisePred(self,P):\n        \n        n_seq = int(np.ceil(60/self.CNF['trainCUT'])) # 60秒が何分割されるか\n        return P.reshape((-1,n_seq,24))\n    \n        \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Pseudo"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !cp -r ../input/pthrfcx1221/vgg .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['pseudo']:\n    \n    sub_df = pd.read_csv(\"../input/rfcx-species-audio-detection/sample_submission.csv\")\n    meldata_tst = np.load(CNF['meltestPath']) ; \n    meldata_tst = meldata_tst[:,:,:-1] # CUT6できりが良くなるように端数を切り捨て（大雑把wwwww）\n    print(meldata_tst.shape)\n    meldata_tst = (meldata_tst-g_meldata_tp_min)/(g_meldata_tp_max-g_meldata_tp_min) # 正規化 # plt.hist(meldata_tst.flatten());\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['pseudo']:\n    \n    fitter =  FitterNN(CNF, fold_id=0, prefix=\"vgg\")\n    seg_preds = []\n    for i in range(3750//375):\n        seg_preds.append( fitter.predict(meldata_tst[:,:,(375*i):(375*(i+1))]) )\n        \n    seg_preds = np.stack(seg_preds,axis=1)\n    \n    tstPred_segid_spid_confidence = np.array([ [ np.where(seg_pred==np.max(seg_pred))[0][0], np.where(seg_pred==np.max(seg_pred))[1][0], np.max(seg_pred) ] for seg_pred in seg_preds ])\n    \n    is_pseudo = tstPred_segid_spid_confidence[:,2]>0.5 # segid0,spid1,confidence2\n    \n    Xp = np.array([ mel_seg[:,(375*seg_id):(375*(seg_id+1))] for mel_seg, seg_id in zip(meldata_tst[is_pseudo], tstPred_segid_spid_confidence[:,0][is_pseudo].astype('int')) ])\n    Yp = np.eye(24)[[ a for a in tstPred_segid_spid_confidence[:,1][is_pseudo].astype('int') ]]\n    \n    del meldata_tst\n    gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['isTrain']:\n    \n    def getWeight(df):\n        _tmp = 1/df.species_id.value_counts().sort_index().values/len(df)\n        return 24*_tmp/sum(_tmp)\n\n    #fold = 0\n    \n    for fold in CNF['targetFold']:\n    \n        trn_idx = traint.query(f\"fold != {fold}\").index.tolist() #; val_idx = traint.query(f\"fold == {fold}\").index.tolist()\n        Xtrn = meldata_tp[trn_idx] # dfのindexでndarrayから抜き出してるからindexが0～Nって感じでちゃんと並んでる前提\n        Ytrn = traint[LABEL_COLS].values[trn_idx]\n        \n        if CNF['validateSequence']:\n            \n            val_idx = trainu.query(f\"fold == {fold}\").index.tolist() # trainu に注意\n        \n            Xval = orig_meldata_tp[val_idx]  \n            Yval = trainu.loc[val_idx,LABEL_COLS].values\n            Wtrn,Wval = getWeight(traint[traint.fold!=fold]), getWeight(trainu[trainu.fold==fold])\n            \n        else:\n            \n            val_idx = traint.query(f\"fold == {fold}\").index.tolist() \n            \n            Xval = meldata_tp[val_idx]\n            Yval = traint.loc[val_idx,LABEL_COLS].values\n            Wtrn,Wval = getWeight(traint[traint.fold!=fold]), getWeight(traint[traint.fold==fold])\n\n        fitter =  FitterNN(CNF, fold, prefix=\"resnet\")\n\n        if CNF['pseudo']:\n            Xtrn = np.vstack([Xtrn,Xp])\n            Ytrn = np.vstack([Ytrn,Yp])\n            #del Xp,Yp\n\n        history = fitter.fit(Xtrn,Xval,Ytrn,Yval,Wtrn,Wval)\n\n        plt.figure(figsize=(12,4))\n        plt.subplot(121);plt.plot(history['trn_loss'],label='trn');plt.plot(history['val_loss'],label='val');plt.legend()\n        plt.subplot(122);plt.plot(history['score'],label='score');plt.legend()\n        plt.show();\n        \n        \n        \n        del Xtrn,Xval,Ytrn,Yval,Wtrn,Wval,fitter\n        gc.collect()\n        torch.cuda.empty_cache()\n\n        print(f\"FOLD{fold}:{history['best_score']}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- CUT6 MultiLabel FOLD0:0.793 安定する\n- CUT4 FOLD0:0.7904\n- CUT6 FOLD0:0.7976\n- CUT8 FOLD0:0.7882"},{"metadata":{},"cell_type":"markdown","source":"# Check Validation for Seq"},{"metadata":{"trusted":true},"cell_type":"code","source":"if False:\n    \n    fold = 0\n    fitter =  FitterNN(CNF, fold_id=fold, prefix=\"resnet\")\n\n    val_idx = trainu.query(f\"fold == {fold}\").index.tolist() # trainu に注意\n\n    Xval = orig_meldata_tp[val_idx]  \n    Yval = trainu.loc[val_idx,LABEL_COLS].values\n\n\n    Pval = fitter.predictSeq(Xval)\n\n    _score_class, _weight = lwlrap(Yval,Pval); score = (_score_class * _weight).sum()\n\n    print(f\"FOLD{fold}:{score:.4f}\")\n\n    del fitter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['isTrain']==False:\n    !cp -r {CNF['loadPthPath']} .","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CAM"},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['plotCAM']:\n    \n    fold = 0\n    val_idx = traint.query(f\"fold == {fold}\").index.tolist()\n\n    fitter =  FitterNN(CNF, fold_id=fold, prefix=\"resnet\")\n    _Pval,cam = fitter.predict(meldata_tp[val_idx],returnCam=True) ; del fitter\n    _score_class, _weight = lwlrap(traint.loc[val_idx,LABEL_COLS].values,_Pval); score = (_score_class * _weight).sum()\n    print(f\"FOLD{fold}:{score}\")\n\n    # _save(cam,f\"cam_fold{fold}.pkl\")\n    # cam = _load('../input/cam1227fold0/cam_fold0.pkl')\n\n    for cnt,idx in enumerate(val_idx):\n\n        plt.figure(figsize=(15, 5))\n        plt.imshow(meldata_tp[val_idx[cnt]][::-1],cmap='magma');\n        plt.imshow(skimage.transform.resize(cam[cnt][::-1], (meldata_tp.shape[1], meldata_tp.shape[2])), alpha=0.25, cmap='jet');\n\n        t_min = traint.iloc[idx].t_min\n        t_max = traint.iloc[idx].t_max\n        f_min = f2melf(traint.iloc[idx].f_min);f_max = f2melf(traint.iloc[idx].f_max)\n        t_ctr = (t_min+t_max)/2\n        t_start = t_ctr - CNF['trainCUT']/2\n        im_t_min = int( (t_min - t_start)*meldata_tp.shape[-1]/CNF['trainCUT'] )\n        im_t_max = int( (t_max - t_start)*meldata_tp.shape[-1]/CNF['trainCUT'] )\n\n        im_f_min = meldata_tp.shape[-2] - f_min*meldata_tp.shape[-2]/16000\n        im_f_max = meldata_tp.shape[-2] - f_max*meldata_tp.shape[-2]/16000\n\n        plt.axvline(im_t_min,c='w'); plt.axvline(im_t_max,c='w')\n        plt.axhline(im_f_max,c='w'); plt.axhline(im_f_min,c='w')\n\n        plt.show();plt.close()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import seaborn as sns\n\n# for cnt,idx in enumerate(val_idx):\n\n#     plt.figure(figsize=(10,10))\n#     cmap = sns.diverging_palette(220, 10, as_cmap=True)\n#     sns.heatmap(valPredSeq[cnt].T,  cmap=cmap, vmin=0,vmax=1, center=0,square=True, linewidths=.5, cbar_kws={\"shrink\": .3})\n\n#     true_ids = list(np.where(trainu.loc[idx,LABEL_COLS]==1)[0])\n#     true_ids_str = \" \".join([ str(i) for i in true_ids])\n    \n#     pred_ids = np.argsort(np.max(valPredSeq[cnt],axis=0))[::-1][:5]\n#     pred_ids_str = \" \".join([ str(i) for i in pred_ids])\n\n#     plt.title(f\"{true_ids_str}\\nPred {pred_ids_str}\",color=\"red\" if pred_ids[0]!=true_ids[0] else \"blue\")\n    \n#     plt.show()\n    \n#     plt.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TEST"},{"metadata":{"trusted":true},"cell_type":"code","source":"del orig_meldata_tp,meldata_tp; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(\"../input/rfcx-species-audio-detection/sample_submission.csv\")\n\n\nmeldata_tst = np.load(CNF['meltestPath'])\nprint(meldata_tst.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 最大値が1になるように正規化 とりあえずtrain_tp全体のMIN,MAXで正規化 個々に正規化するから多分意味ない\nmeldata_tst = (meldata_tst-g_meldata_tp_min)/(g_meldata_tp_max-g_meldata_tp_min)\nplt.hist(meldata_tst.flatten());","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.figure(figsize=(16,4))\n# plt.subplot(121);plt.hist(meldata_tp.flatten(),label='trn');plt.title('trn');\n# plt.subplot(122);plt.hist(meldata_tst.flatten(),label='tst');plt.title('tst');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtst = meldata_tst\n\nfoldPred = []\nfor fold in CNF['targetFold']:\n    \n    fitter =  FitterNN(CNF, fold_id=fold, prefix=\"resnet\")\n    foldPred.append( fitter.predictSeq(Xtst) )\n\n    del fitter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tstPred_avg = np.mean(np.stack(foldPred),axis=0)\ntstPred_max = np.max(np.stack(foldPred),axis=0)\n_save(tstPred_avg,'tstPred_avg.pkl')\n_save(tstPred_max,'tstPred_max.pkl')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## fold max"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df[[ f\"s{i}\" for i in range(24)]] = tstPred_max\nsub_df.to_csv(\"submission_max.csv\", index=False)\nsub_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.bar(x=np.arange(24),height=sub_df[LABEL_COLS].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## fold average"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df[[ f\"s{i}\" for i in range(24)]] = tstPred_avg\nsub_df.to_csv(\"submission_avg.csv\", index=False)\nsub_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.bar(x=np.arange(24),height=sub_df[LABEL_COLS].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}