{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install pytorch-lightning==1.18\n!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version 1.7 --apt-packages libomp5 libopenblas-dev\n!pip install -q nnAudio\n!pip install -q ttach\n!pip install fsspec\n!pip install gcsfs\n!pip install albumentations --upgrade\n!pip install timm","metadata":{"id":"Bgv0tU85MPO1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport pytorch_lightning as pl\nfrom torchmetrics.functional import auroc\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"id":"_Bu3CDQN3zsF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\ntrain = pd.read_csv('/content/training_labels.csv')\ntest = pd.read_csv('/content/sample_submission.csv')\n\ndef get_train_file_path(image_id):\n    return \"/content/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"/content/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ntrain['file_path'] = train['id'].apply(get_train_file_path)\ntest['file_path'] = test['id'].apply(get_test_file_path)\n\ndisplay(train.head())\ndisplay(test.head())","metadata":{"id":"1dkG5NfV3Rww"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_folds(df,folds):\n    x = df.id.values\n    y = df[\"target\"].values    \n    mskf = StratifiedKFold(n_splits = folds)    \n    for fold, (trn_, val_) in enumerate(mskf.split(x, y)):\n        print(\"TRAIN: \", trn_, \"VAL: \",  val_)\n        df.loc[val_, \"kfold\"] = fold\n    return df","metadata":{"id":"bQ7nUYDsMPO8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=get_folds(train,5)","metadata":{"id":"yQPIgzxKMPO9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"id":"PAQQv9ubMPO9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    apex=True\n    debug=False\n    num_workers=40\n    learning_rate=0.001\n    model_name='efficientnet_b3a'\n    scheduler='CosineAnnealingLR' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    epochs=6\n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    T_max=3 # CosineAnnealingLR\n    #T_0=3 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    img_size=340\n    min_lr=1e-6\n    batch_size=16\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    qtransform_params={\"sr\": 2048, \"fmin\": 20, \"fmax\": 1024, \"hop_length\": 32, \"bins_per_octave\": 8, \"verbose\": False}\n    seed=2021\n    target_size=2\n    target_col='target'\n    n_fold=5\n    fold=0 # [0, 1, 2, 3, 4]\n    train=True\n    \nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=10000, random_state=CFG.seed).reset_index(drop=True)","metadata":{"id":"uRey2gQjcLaq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom nnAudio.Spectrogram import CQT1992v2\n\ndef apply_qtransform(waves, transform=CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=64)):\n    waves = np.hstack(waves)\n    waves = waves / np.max(waves)\n    waves = torch.from_numpy(waves).float()\n    image = transform(waves)\n    return image\n\nfor i in range(5):\n    waves = np.load(train.loc[i, 'file_path'])\n    image = apply_qtransform(waves)\n    print(image.shape)\n    target = train.loc[i, 'target']\n    plt.imshow(image[0])\n    plt.title(f\"target: {target}\")\n    plt.show()","metadata":{"id":"G7prfb1ZcSh4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","metadata":{"id":"7iCdkMsTUIwV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return A.Compose([\n            A.augmentations.geometric.resize.Resize(CFG.img_size,CFG.img_size),\n            ToTensorV2()\n        ])\n\n    elif data == 'valid':\n        return A.Compose([\n            A.augmentations.geometric.resize.Resize(CFG.img_size,CFG.img_size),\n            ToTensorV2()\n        ])","metadata":{"id":"SQcCvHhCci98"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass g2netDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        self.df = df\n        self.file_names = df['file_path'].values\n        self.mode=mode\n        if self.mode==\"train\":\n            self.labels = df[CFG.target_col].values\n        self.wave_transform = CQT1992v2(**CFG.qtransform_params)\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def apply_qtransform(self, waves, transform):\n        waves = np.hstack(waves)\n        waves = waves / np.max(waves)\n        waves = torch.from_numpy(waves).float()\n        image = transform(waves)\n        return image\n\n    def __getitem__(self, idx):\n        file_path = self.file_names[idx]\n        waves = np.load(file_path)\n        image = self.apply_qtransform(waves, self.wave_transform)\n        image = image.squeeze().numpy()\n        if self.transform:\n            image = self.transform(image=image)['image']\n        if self.mode==\"train\":\n            label = torch.tensor(self.labels[idx], dtype=torch.long)\n            return image, label\n        else:\n            return image","metadata":{"id":"y0J4wLSdcFuP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class g2netDataModule(pl.LightningDataModule):\n    def __init__(self, df,test_df,cfg=CFG):\n        super().__init__()\n        self.df=df\n        self.test_df=test_df\n        self.cfg=cfg\n        self.fold=self.cfg.fold\n\n    def setup(self, stage= None):\n        if stage==\"fit\" or stage is None:\n            train_df=self.df.loc[self.df.kfold!=self.fold]\n            val_df=self.df.loc[self.df.kfold==self.fold]\n            train_df=train_df.reset_index(drop=True)\n            val_df=val_df.reset_index(drop=True)\n            self.train_dataset=g2netDataset(train_df,transform=get_transforms(data=\"train\"),mode=\"train\")\n            self.val_dataset=g2netDataset(val_df,transform=get_transforms(data=\"valid\"),mode=\"train\")\n            \n        if stage==\"predict\" or stage is None:\n            self.test_dataset=g2netDataset(self.test_df,transform=get_transforms(data=\"valid\"),mode=\"test\")\n        \n    def train_dataloader(self):\n        return torch.utils.data.DataLoader(self.train_dataset,\n                                           batch_size=self.cfg.batch_size,\n                                           num_workers=self.cfg.num_workers,\n                                           shuffle=True)\n        \n    def val_dataloader(self):\n        return torch.utils.data.DataLoader(self.val_dataset,\n                                           batch_size=self.cfg.batch_size,\n                                           num_workers=self.cfg.num_workers,\n                                           shuffle=False) \n    \n    def prdict_dataloader(self):\n        return torch.utils.data.DataLoader(self.test_dataset,\n                                           batch_size=self.cfg.batch_size,\n                                           num_workers=self.cfg.num_workers,\n                                           shuffle=False) ","metadata":{"id":"QNNmtN26MPPB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def epoch_update_gamma(y_true,y_pred, epoch=-1,delta=1):\n        \"\"\"\n        Calculate gamma from last epoch's targets and predictions.\n        Gamma is updated at the end of each epoch.\n        y_true: `Tensor`. Targets (labels).  Float either 0.0 or 1.0 .\n        y_pred: `Tensor` . Predictions.\n        \"\"\"\n        DELTA = delta+1\n        SUB_SAMPLE_SIZE = 2000.0\n        pos = y_pred[y_true==1]\n        neg = y_pred[y_true==0] # yo pytorch, no boolean tensors or operators?  Wassap?\n        # subsample the training set for performance\n        cap_pos = pos.shape[0]\n        cap_neg = neg.shape[0]\n        pos = pos[torch.rand_like(pos) < SUB_SAMPLE_SIZE/cap_pos]\n        neg = neg[torch.rand_like(neg) < SUB_SAMPLE_SIZE/cap_neg]\n        ln_pos = pos.shape[0]\n        ln_neg = neg.shape[0]\n        pos_expand = pos.view(-1,1).expand(-1,ln_neg).reshape(-1)\n        neg_expand = neg.repeat(ln_pos)\n        diff = neg_expand - pos_expand\n        ln_All = diff.shape[0]\n        Lp = diff[diff>0] # because we're taking positive diffs, we got pos and neg flipped.\n        ln_Lp = Lp.shape[0]-1\n        diff_neg = -1.0 * diff[diff<0]\n        diff_neg = diff_neg.sort()[0]\n        ln_neg = diff_neg.shape[0]-1\n        ln_neg = max([ln_neg, 0])\n        left_wing = int(ln_Lp*DELTA)\n        left_wing = max([0,left_wing])\n        left_wing = min([ln_neg,left_wing])\n        default_gamma=torch.tensor(0.2, dtype=torch.float).cuda()\n        if diff_neg.shape[0] > 0 :\n           gamma = diff_neg[left_wing]\n        else:\n           gamma = default_gamma # default=torch.tensor(0.2, dtype=torch.float).cuda() #zoink\n        L1 = diff[diff>-1.0*gamma]\n        ln_L1 = L1.shape[0]\n        if epoch > -1 :\n            return gamma\n        else :\n            return default_gamma\n\n\n\ndef roc_star_loss( _y_true, y_pred, gamma, _epoch_true, epoch_pred):\n        \"\"\"\n        Nearly direct loss function for AUC.\n        See article,\n        C. Reiss, \"Roc-star : An objective function for ROC-AUC that actually works.\"\n        https://github.com/iridiumblue/articles/blob/master/roc_star.md\n            _y_true: `Tensor`. Targets (labels).  Float either 0.0 or 1.0 .\n            y_pred: `Tensor` . Predictions.\n            gamma  : `Float` Gamma, as derived from last epoch.\n            _epoch_true: `Tensor`.  Targets (labels) from last epoch.\n            epoch_pred : `Tensor`.  Predicions from last epoch.\n        \"\"\"\n        #convert labels to boolean\n        y_true = (_y_true>=0.50)\n        epoch_true = (_epoch_true>=0.50)\n\n        # if batch is either all true or false return small random stub value.\n        if torch.sum(y_true)==0 or torch.sum(y_true) == y_true.shape[0]: return torch.sum(y_pred)*1e-8\n\n        pos = y_pred[y_true]\n        neg = y_pred[~y_true]\n\n        epoch_pos = epoch_pred[epoch_true]\n        epoch_neg = epoch_pred[~epoch_true]\n\n        # Take random subsamples of the training set, both positive and negative.\n        max_pos = 1000 # Max number of positive training samples\n        max_neg = 1000 # Max number of positive training samples\n        cap_pos = epoch_pos.shape[0]\n        cap_neg = epoch_neg.shape[0]\n        epoch_pos = epoch_pos[torch.rand_like(epoch_pos) < max_pos/cap_pos]\n        epoch_neg = epoch_neg[torch.rand_like(epoch_neg) < max_neg/cap_pos]\n\n        ln_pos = pos.shape[0]\n        ln_neg = neg.shape[0]\n\n        # sum positive batch elements agaionst (subsampled) negative elements\n        if ln_pos>0 :\n            pos_expand = pos.view(-1,1).expand(-1,epoch_neg.shape[0]).reshape(-1)\n            neg_expand = epoch_neg.repeat(ln_pos)\n\n            diff2 = neg_expand - pos_expand + gamma\n            l2 = diff2[diff2>0]\n            m2 = l2 * l2\n            len2 = l2.shape[0]\n        else:\n            m2 = torch.tensor([0], dtype=torch.float).cuda()\n            len2 = 0\n\n        # Similarly, compare negative batch elements against (subsampled) positive elements\n        if ln_neg>0 :\n            pos_expand = epoch_pos.view(-1,1).expand(-1, ln_neg).reshape(-1)\n            neg_expand = neg.repeat(epoch_pos.shape[0])\n\n            diff3 = neg_expand - pos_expand + gamma\n            l3 = diff3[diff3>0]\n            m3 = l3*l3\n            len3 = l3.shape[0]\n        else:\n            m3 = torch.tensor([0], dtype=torch.float).cuda()\n            len3=0\n\n        if (torch.sum(m2)+torch.sum(m3))!=0 :\n           res2 = torch.sum(m2)/max_pos+torch.sum(m3)/max_neg\n           #code.interact(local=dict(globals(), **locals()))\n        else:\n           res2 = torch.sum(m2)+torch.sum(m3)\n\n        res2 = torch.where(torch.isnan(res2), torch.zeros_like(res2), res2)\n\n        return res2\n    \nclass ROC_Star(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, y_pred,_y_true,i):    #_epoch_true, epoch_pred):\n        return roc_star_loss( _y_true, y_pred, CFG.gamma, torch.from_numpy(CFG.last_epoch_true[i]).cuda(), torch.from_numpy(CFG.last_epoch_pred[i]).cuda())","metadata":{"id":"mliIf56qHPWq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn=nn.CrossEntropyLoss()","metadata":{"id":"SZ5zFzEoHQ2s"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class g2netModel(pl.LightningModule):\n    def __init__(self,cfg=CFG, pretrained=True):\n        super(g2netModel,self).__init__()\n        self.cfg = cfg\n        self.lr=self.cfg.learning_rate\n        self.model = timm.create_model(self.cfg.model_name, pretrained=pretrained,in_chans=1)\n        self.n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(self.n_features, self.cfg.target_size)\n\n    def forward(self, x):\n        output = self.model(x)\n        return output\n\n    def training_step(self,batch,batch_nb):\n        x, y= batch\n        logits=nn.Sigmoid()(self(x))\n        loss=loss_fn(logits,y)\n        acc=auroc(logits,y,num_classes=self.cfg.target_size)\n        self.log(\"train auroc\", acc, prog_bar=True)\n        return loss\n    \n    def validation_step(self,batch,batch_nb):\n        x, y= batch\n        logits=nn.Sigmoid()(self(x))\n        loss=loss_fn(logits,y)\n        acc=auroc(logits,y,num_classes=self.cfg.target_size)\n        self.log(\"val loss\", loss, prog_bar=True)\n        self.log(\"val auroc\", acc, prog_bar=True)\n        return loss\n    \n    def predict_step(self,batch,batch_nb):\n        x= batch\n        logits=nn.Sigmoid()(self(x))\n        x=x[:,1].view(x.size(0),-1)\n        return x\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, \"min\")\n        return {\n            \"optimizer\":optimizer,\n            \"lr_scheduler\":{\n                \"scheduler\":scheduler,\n                \"monitor\":\"val loss\"\n            }\n        }","metadata":{"id":"9Rmib-nEUKm0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dm=g2netDataModule(train,test,CFG)","metadata":{"id":"Kg_AhGQZMPPD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2netModel=g2netModel(CFG)\n\ntrainer = pl.Trainer(max_epochs = CFG.epochs,\n                    gpus=1,\n                    progress_bar_refresh_rate=1,\n                   precision=16,\n                   default_root_dir=\"./\")\ntrainer.fit(g2netModel,dm)    ","metadata":{"id":"NznD46L-wWNT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"nFdyklEXMPPD"},"execution_count":null,"outputs":[]}]}