{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-23T14:21:34.70382Z","iopub.execute_input":"2021-08-23T14:21:34.704193Z","iopub.status.idle":"2021-08-23T14:21:34.711063Z","shell.execute_reply.started":"2021-08-23T14:21:34.704162Z","shell.execute_reply":"2021-08-23T14:21:34.71023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Things to try:\n# GeM pooling\n# soften labels during mixup\n# change stride of first conv layer\n# increasing no. of channels with a pre BB convolution layer","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:34.731763Z","iopub.execute_input":"2021-08-23T14:21:34.732022Z","iopub.status.idle":"2021-08-23T14:21:34.735699Z","shell.execute_reply.started":"2021-08-23T14:21:34.731997Z","shell.execute_reply":"2021-08-23T14:21:34.734516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet_pytorch -qq\n\n!pip install -q nnAudio -qq\nimport torch\nfrom nnAudio.Spectrogram import CQT1992v2, CQT2010v2\n\nimport time\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nfrom torch.autograd import Variable\nimport efficientnet_pytorch\nfrom tqdm.auto import tqdm\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torchaudio.functional import lfilter\nfrom torch.fft import fft, rfft, ifft\nimport numpy as np\n\n\n\nfrom sklearn.metrics import roc_auc_score\n\nfrom sklearn.model_selection import StratifiedKFold\n\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:34.762419Z","iopub.execute_input":"2021-08-23T14:21:34.762685Z","iopub.status.idle":"2021-08-23T14:21:47.103468Z","shell.execute_reply.started":"2021-08-23T14:21:34.762659Z","shell.execute_reply":"2021-08-23T14:21:47.10233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sys.path.append('../input/pytorch-swa')\n#import swa","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:47.107134Z","iopub.execute_input":"2021-08-23T14:21:47.107453Z","iopub.status.idle":"2021-08-23T14:21:47.111549Z","shell.execute_reply.started":"2021-08-23T14:21:47.107422Z","shell.execute_reply":"2021-08-23T14:21:47.110301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/g2net-gravitational-wave-detection/sample_submission.csv\")\ntrain_df = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\ntrain_df_pred = pd.read_csv(\"../input/train-pred-cqt-v10/train_preds_CQT_V10.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:47.113469Z","iopub.execute_input":"2021-08-23T14:21:47.113752Z","iopub.status.idle":"2021-08-23T14:21:49.337415Z","shell.execute_reply.started":"2021-08-23T14:21:47.113726Z","shell.execute_reply":"2021-08-23T14:21:49.336481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['preds'] = train_df_pred['target']\nweight = 0.5\ntrain_df['soft_target'] = train_df['preds']*weight + train_df['target']*(1-weight)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.339053Z","iopub.execute_input":"2021-08-23T14:21:49.339405Z","iopub.status.idle":"2021-08-23T14:21:49.369917Z","shell.execute_reply.started":"2021-08-23T14:21:49.339367Z","shell.execute_reply":"2021-08-23T14:21:49.368852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_pred","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.371469Z","iopub.execute_input":"2021-08-23T14:21:49.37183Z","iopub.status.idle":"2021-08-23T14:21:49.387172Z","shell.execute_reply.started":"2021-08-23T14:21:49.371794Z","shell.execute_reply":"2021-08-23T14:21:49.385996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define config","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nclass CFG:\n    \n    TRAIN = True\n    \n    WARM_START = True\n    \n    EPOCHS = 7\n    \n    lr = 1e-2\n    \n    n_fold = 5\n    fold = 0\n    \n    # scheduler_params\n    scheduler='CosineAnnealingLR'\n    T_max=3 # CosineAnnealingLR\n    T_0=3 # CosineAnnealingWarmRestarts\n    min_lr=1e-6\n    schedulerStepFreq = 400\n    \n    valStepFreq= 1600\n    \n    \n    # Parameters CWT\n    cwt_params = {'fs':2048, 'lower_freq':10, 'upper_freq': 500, \n                  'n_scales':81, 'wavelet_width':1, 'stride':12, 'border_crop':0, 'train_width':True}\n    \n    cqt_params = {'sr':2048, 'fmin':20, 'fmax':512, 'hop_length':32, 'n_bins':69}\n    # cqt_params = {'sr':2048, 'fmin':20, 'fmax':512, 'hop_length':32, 'bins_per_octave': 25, 'norm':1}\n    \n    BPfilter = True\n    \n    \n    \n    \n    # Post Proc Option\n    PREPROC = 'Q_transform'\n    \n    # batch size\n    BATCH = 64\n    BATCH_VAL = 128*4\n    \n    # scale:linear or log\n    SCALE = 'linear'\n    \n    DEBUG = False\n    \n    SMALL_TRAIN_SET = True\n    \n    VISUALIZE = True\n    \n    seed = 42\n    \n    model_name = 'tf_efficientnet_b0' #'tf_efficientnet_b4' #'efficientnet-b7'\n    pretrained = False\n    unfreezeStep = 100 # set 0 for no freezing\n    \n    useSoftLabels = True\n    useTestLabels = True\n    \n\nif not CFG.pretrained:\n    CFG.unfreezeStep = 10\n    \nif CFG.DEBUG:\n    CFG.EPOCHS = 2\n    train_df = train_df.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)\nelif CFG.SMALL_TRAIN_SET:\n    CFG.EPOCHS = 4\n    train_df = train_df.sample(n=CFG.BATCH*500, random_state=CFG.seed).reset_index(drop=True)\n    CFG.valStepFreq = 100\nelif CFG.WARM_START:\n    CFG.EPOCHS = 3\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.389325Z","iopub.execute_input":"2021-08-23T14:21:49.389824Z","iopub.status.idle":"2021-08-23T14:21:49.43387Z","shell.execute_reply.started":"2021-08-23T14:21:49.389748Z","shell.execute_reply":"2021-08-23T14:21:49.433005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = timm.create_model(CFG.model_name, pretrained=CFG.pretrained)\nmodel.global_pool\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.435303Z","iopub.execute_input":"2021-08-23T14:21:49.435788Z","iopub.status.idle":"2021-08-23T14:21:49.555162Z","shell.execute_reply.started":"2021-08-23T14:21:49.435748Z","shell.execute_reply":"2021-08-23T14:21:49.554083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\ndef set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.55819Z","iopub.execute_input":"2021-08-23T14:21:49.558541Z","iopub.status.idle":"2021-08-23T14:21:49.565253Z","shell.execute_reply.started":"2021-08-23T14:21:49.558508Z","shell.execute_reply":"2021-08-23T14:21:49.564023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data retrieving and related functions","metadata":{}},{"cell_type":"code","source":"from scipy import signal \n\nfrom scipy import signal\n\nbHP, aHP = signal.butter(8, (25, 500), btype='bandpass', fs= 2048)\ndef filterSig(waves, a=aHP, b=bHP, axis = 1):\n    '''Apply a 20Hz high pass filter to the three events'''\n    if not CFG.BPfilter:\n        return waves\n    return signal.filtfilt(b, a, waves, axis = axis) #lfilter introduces a larger spike around 20hz\n\nclass DataRetriever(torch_data.Dataset):\n    def __init__(self, paths, targets):\n        self.paths = paths\n        self.targets = targets\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __get_qtransform(self, x):\n        image = x / np.max(x,axis=1,keepdims = True)\n        if CFG.DEBUG:\n            image = filterSig(image).copy()\n        # image = image / np.max(np.abs(image),axis=1,keepdims = True)\n        # image is [chan x time]\n        image = torch.tensor(image).float()\n        return image\n\n    \n    def __getitem__(self, index):\n        #file_path = convert_image_id_2_path(self.paths[index])\n        file_path = self.paths[index]\n        x = np.load(file_path)\n        image = self.__get_qtransform(x)\n        \n        y = torch.tensor(self.targets[index], dtype=torch.float)\n            \n        return {\"X\": image, \"y\": y}\n    \n    \nclass TestDataRetriever(torch_data.Dataset):\n    def __init__(self, paths):\n        self.paths = paths\n        \n        self.q_transform = CQT1992v2(\n            sr=2048, fmin=20, fmax=1024, hop_length=32\n        ) if CFG.PREPROC == 'Q_transform' else None\n        \n          \n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __get_qtransform(self, x):\n        image = x / np.max(x,axis=1,keepdims = True)\n        if CFG.DEBUG:\n            image = filterSig(image).copy()\n        # image = image / np.max(np.abs(image),axis=1,keepdims = True)\n        # image is [chan x time]\n        image = torch.tensor(image).float()\n        return image\n    \n    def __getitem__(self, index):\n        # file_path = convert_image_id_2_path(self.paths[index], is_train=False)\n        file_path = self.paths[index]\n        x = np.load(file_path)\n        image = self.__get_qtransform(x)\n            \n        return {\"X\": image, \"id\": self.paths[index]}","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.56783Z","iopub.execute_input":"2021-08-23T14:21:49.56824Z","iopub.status.idle":"2021-08-23T14:21:49.586015Z","shell.execute_reply.started":"2021-08-23T14:21:49.568178Z","shell.execute_reply":"2021-08-23T14:21:49.584992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    Fold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\n    for n, (train_index, val_index) in enumerate(Fold.split(train_df, train_df['target'])):\n        train_df.loc[val_index, 'fold'] = int(n)\n    train_df['fold'] = train_df['fold'].astype(int)\n    display(train_df.groupby(['fold', 'target']).size())\n\n    df_train = train_df.loc[train_df['fold'] != CFG.fold,:]\n    df_valid= train_df.loc[train_df['fold'] == CFG.fold,:]\nelse:\n    df_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=42, \n    stratify=train_df[\"target\"],\n    )\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.587635Z","iopub.execute_input":"2021-08-23T14:21:49.588229Z","iopub.status.idle":"2021-08-23T14:21:49.624477Z","shell.execute_reply.started":"2021-08-23T14:21:49.588153Z","shell.execute_reply":"2021-08-23T14:21:49.623705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndf_train['file_path'] = df_train['id'].apply(get_train_file_path)\ndf_valid['file_path'] = df_valid['id'].apply(get_train_file_path)\n\nsubmission['file_path'] = submission['id'].apply(get_test_file_path)\nsubmission_869 = pd.read_csv('../input/grnet869/model_submission_B0_no_pretrain.csv')\nsubmission_869['file_path'] = submission_869['id'].apply(get_test_file_path)\n\nif CFG.useTestLabels:\n    submission_869['soft_target'] = 0.5*(submission_869['target']+np.round(submission_869['target']))\n    tmp_df_0 = submission_869.loc[submission_869['target']<0.2,:]\n    tmp_df_1 = submission_869.loc[submission_869['target']>0.9,:]\n    print('No. test negatives selected: '+str(len(tmp_df_0)))\n    print('No. test positives selected: '+str(len(tmp_df_1)))\n    \n    print('Train size: '+str(len(df_train)))\n    if CFG.SMALL_TRAIN_SET:\n        tmp_df_0 = tmp_df_0.head(CFG.BATCH*50)\n        tmp_df_1 = tmp_df_1.head(CFG.BATCH*50)\n    else:\n        tmp_df_0 = tmp_df_0.head(38400) # 128*300\n        tmp_df_1 = tmp_df_1.head(38400)\n    print(tmp_df_0.head())\n    df_train = df_train.append(tmp_df_0)\n    df_train = df_train.append(tmp_df_1).sample(frac=1)\n    df_train['target'] = df_train['target'].clip(lower = 0., upper = 1.)\n    print('Train size with test: '+str(len(df_train)))\n\n\ntrain_data_retriever = DataRetriever(\n    df_train['file_path'].values, \n    df_train[\"target\"].values, \n)\n\ntrain_data_retriever_soft = DataRetriever(\n    df_train['file_path'].values, \n    df_train[\"soft_target\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n    df_valid['file_path'].values, \n    df_valid[\"target\"].values,\n)\n\ntest_data_retriever = TestDataRetriever(\n    submission[\"file_path\"].values, \n)\n\ntest_data_retriever_withlabels = DataRetriever(\n    submission_869['file_path'].values, \n    submission_869[\"target\"].values,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:49.62578Z","iopub.execute_input":"2021-08-23T14:21:49.626121Z","iopub.status.idle":"2021-08-23T14:21:50.365439Z","shell.execute_reply.started":"2021-08-23T14:21:49.626085Z","shell.execute_reply":"2021-08-23T14:21:50.364035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch_data.DataLoader(\n    train_data_retriever,\n    batch_size=CFG.BATCH,\n    shuffle=True,\n    num_workers=12,\n)\n\ntrain_loader_soft = torch_data.DataLoader(\n    train_data_retriever_soft,\n    batch_size=CFG.BATCH,\n    shuffle=True,\n    num_workers=12,\n)\n\nvalid_loader = torch_data.DataLoader(\n    valid_data_retriever, \n    batch_size=CFG.BATCH_VAL,\n    shuffle=False,\n    num_workers=8,\n)\n\ntest_loader = torch_data.DataLoader(\n    test_data_retriever,\n    batch_size=CFG.BATCH_VAL,\n    shuffle=False,\n    num_workers=8,\n)\n\ntest_loader_withlabels = torch_data.DataLoader(\n    test_data_retriever_withlabels,\n    batch_size=CFG.BATCH,\n    shuffle=True,\n    num_workers=12,\n)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:50.366833Z","iopub.execute_input":"2021-08-23T14:21:50.3672Z","iopub.status.idle":"2021-08-23T14:21:50.419885Z","shell.execute_reply.started":"2021-08-23T14:21:50.367163Z","shell.execute_reply":"2021-08-23T14:21:50.417517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# Different heads\n\nclass BasicHead(nn.Module):   \n    def __init__(self,n_features):\n        super().__init__()\n        self.classifier = nn.Sequential(\n          nn.Dropout(0.5),\n          nn.Linear(in_features=n_features, out_features=256, bias=True),\n          nn.ReLU(),\n          # nn.Dropout(0.5), # p is probability of zeroing\n          nn.Linear(in_features=256, out_features=1, bias=True),\n        )\n        \n    def forward(self,x):\n        return self.classifier(x)\n    \nclass MultiDropoutHead(nn.Module):\n    def __init__(self,n_features):\n        super().__init__()\n        self.classifier = nn.Sequential(\n          nn.Linear(in_features=n_features, out_features=256, bias=True),\n          nn.ReLU(),\n          # nn.Dropout(0.5), # p is probability of zeroing\n          nn.Linear(in_features=256, out_features=1, bias=True),\n        )\n        self.dropout = lambda p: nn.Dropout(p)\n        \n    def forward(self,x):\n        return torch.mean(torch.stack([\n            self.classifier(self.dropout(p)(x))\n            for p in np.linspace(0.3,0.7, 5)\n        ], dim=0), dim=0)\n    \nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        x = self.gem(x, p=self.p, eps=self.eps)\n        x = torch.flatten(x,start_dim=1,end_dim=-1)\n        return x\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return torch.nn.functional.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:50.421557Z","iopub.execute_input":"2021-08-23T14:21:50.421951Z","iopub.status.idle":"2021-08-23T14:21:50.437604Z","shell.execute_reply.started":"2021-08-23T14:21:50.421913Z","shell.execute_reply":"2021-08-23T14:21:50.436558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef whiten(signal):\n    # From here: https://www.kaggle.com/kevinmcisaac/g2net-spectral-whitening\n    length = signal.size(2)\n    hann = torch.hann_window(length, periodic=True, dtype=float).view(1,1,-1)\n    spec = fft(signal* hann, dim = 2)\n    mag = torch.sqrt(torch.real(spec*torch.conj(spec))) \n\n    return torch.real(ifft(spec/mag)) * np.sqrt(length/2)\n\ndef batch_preprocessing(X):\n    # X = whiten(X)\n    X = X.numpy()\n    if CFG.BPfilter:        \n        X = filterSig(X,axis=2).copy()\n    X = torch.tensor(X).float()\n    return X      \n        \n\nHead = MultiDropoutHead#BasicHead\n\nmodel_no = 1\n\ndef Backbone():\n    if 'tf_efficientnet' in CFG.model_name:\n        model = timm.create_model(CFG.model_name, pretrained=CFG.pretrained)\n        n_features = model.classifier.in_features\n        model.classifier = nn.Identity()\n        if True:\n            model.global_pool = GeM()\n    elif 'efficientnet-' in CFG.model_name:\n        model = efficientnet_pytorch.EfficientNet.from_pretrained(CFG.model_name)\n        n_features = model._fc.in_features\n        model._fc = nn.Identity()\n    elif 'rexnet_' in CFG.model_name:\n        model = timm.create_model(CFG.model_name, pretrained=CFG.pretrained)\n        n_features = model.head.fc.in_features\n        model.head.fc = nn.Identity()\n        \n    return model, n_features\n\ndef batch_postCQTprocessing(x):\n    if CFG.SCALE == 'log':\n        x = (torch.log10(x) + 1.)/1.5\n    else:\n        x = (torch.clamp(x,max=2.5)-1)\n        #x = torch.divide(x,torch.mean(x,dim=2,keepdims = True))\n    return x\n\ndef batch_postCQTprocessing_experimental(x):\n    x = torch.unsqueeze(x,1)\n    row_mean = torch.mean(x,dim=3,keepdim = True)\n    x = x/row_mean\n    # x = torch.nn.functional.conv2d(x,torch.ones(1,1,5,1).to(device),padding = (2,1))\n    x_std,x_mean = torch.std_mean(x,dim=(2,3),keepdim = True)\n    x = (x-x_mean)/x_std\n    # x = (torch.clamp(x, max=2.5)-1)\n    x = torch.squeeze(x,1)\n    return x\n\n\nif model_no == 1:\n    print('Selecting multi channel model ... ')\n    class Model(nn.Module):\n        def __init__(self, get_spectrogram = False):\n            self.get_spectrogram = get_spectrogram\n            super().__init__()\n            self.q_transform = CQT1992v2(\n                **CFG.cqt_params\n            )\n            if not self.get_spectrogram:\n                \n                self.model, n_features = Backbone()\n                self.head = Head(n_features)\n\n        def freezeModel(self):\n            for param in self.model.parameters():\n                param.requires_grad = False\n\n        def unfreezeModel(self):\n            for param in self.model.parameters():\n                param.requires_grad = True\n\n        def forward(self, x):\n            # reshape from [batch by chan by time] [(batch x chan) by time]\n            batch_size = x.size(0)\n\n            x = torch.divide(x,torch.max(torch.abs(x),dim=2,keepdims = True)[0])\n            x = torch.reshape(x,(batch_size*3,-1))\n            x = self.q_transform(x)\n            x = x[:,0:-1,0:-1]\n            if self.get_spectrogram:\n                x_expt = batch_postCQTprocessing_experimental(x)\n                size = list(x_expt.size())\n                x_expt = torch.reshape(x_expt,(batch_size,3,size[1],size[2]))\n                x = batch_postCQTprocessing(x)\n                size = list(x.size())\n                x = torch.reshape(x,(batch_size,3,size[1],size[2]))\n                return x, x_expt\n            x = batch_postCQTprocessing(x)\n            size = list(x.size())\n            x = torch.reshape(x,(batch_size,3,size[1],size[2]))\n            x = self.model(x)\n            out = self.head(x)\n            return out\nelif model_no == 2:        \n    print('Selecting single channel model ... ')\n    \n    class Model(nn.Module):\n        def __init__(self, get_spectrogram = False):\n            self.get_spectrogram = get_spectrogram\n            super().__init__()\n            self.q_transform = CQT1992v2(\n                **CFG.cqt_params\n            )\n            if not self.get_spectrogram:\n                self.model, n_features = Backbone()\n                self.head = Head(n_features)\n\n        def freezeModel(self):\n            for param in self.model.parameters():\n                param.requires_grad = False\n\n        def unfreezeModel(self):\n            for param in self.model.parameters():\n                param.requires_grad = True\n\n        def forward(self, x):\n            # reshape from [batch by chan by time] [(batch x chan) by time]\n            batch_size = x.size(0)\n\n            x = torch.divide(x,torch.max(torch.abs(x),dim=2,keepdims = True)[0])\n            x = torch.reshape(x,(batch_size*3,-1))\n            x = self.q_transform(x)\n            x = x[:,0:-1,0:-1]\n            x = batch_postCQTprocessing(x)\n            if self.get_spectrogram:\n                size = list(x.size())\n                x = torch.reshape(x,(batch_size,3,size[1],size[2]))\n                return x\n            \n            x = torch.unsqueeze(x,1)\n            \n            # x_mean = torch.mean(x,dim=1,keepdims = True)\n            # x = torch.stack([x,x_mean],dim=1)\n            \n            x = self.model(x)\n            size = list(x.size())\n            x = torch.reshape(x,(batch_size,-1,size[1]))\n            x = torch.max(x,dim=1,keepdims = False)[0]\n            out = self.head(x)\n            out = out\n            return out\n        \nelif model_no == 3:\n    print('Selecting experimental model ... ')\n    class Model(nn.Module):\n        def __init__(self, get_spectrogram = False):\n            self.get_spectrogram = get_spectrogram\n            super().__init__()\n            self.q_transform = CQT1992v2(\n                **CFG.cqt_params\n            )\n            if not self.get_spectrogram:\n                self.model, n_features = Backbone()\n                self.head = Head(2*n_features)\n\n        def freezeModel(self):\n            for param in self.model.parameters():\n                param.requires_grad = False\n\n        def unfreezeModel(self):\n            for param in self.model.parameters():\n                param.requires_grad = True\n\n        def forward(self, x):\n            # reshape from [batch by chan by time] [(batch x chan) by time]\n            batch_size = x.size(0)\n\n            x = torch.divide(x,torch.max(torch.abs(x),dim=2,keepdims = True)[0])\n            x = torch.reshape(x,(batch_size*3,-1))\n            x = self.q_transform(x)\n            x = x[:,0:-1,0:-1]\n            x = batch_postCQTprocessing(x)\n            if self.get_spectrogram:\n                size = list(x.size())\n                x = torch.reshape(x,(batch_size,3,size[1],size[2]))\n                return x\n            \n            x = torch.unsqueeze(x,1)\n            \n            size = list(x.size())\n            x = torch.reshape(x,(batch_size,3,size[2],size[3]))\n            x_mean = torch.mean(x,dim=1,keepdims = True)\n            x = torch.cat([x,x_mean],dim=1)\n            x = torch.reshape(x,(batch_size*4,1,size[2],size[3]))\n            \n            x = self.model(x)\n            size = list(x.size())\n            x = torch.reshape(x,(batch_size,-1,size[1]))\n            x_mean = x[:,3,:].squeeze(1)\n            x = torch.max(x,dim=1,keepdims = False)[0]\n            # this will be [batch by (2 x n_features)]\n            x = torch.cat([x,x_mean],dim=1)\n            \n            out = self.head(x)\n            out = out\n            return out\n    \n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:50.438972Z","iopub.execute_input":"2021-08-23T14:21:50.439357Z","iopub.status.idle":"2021-08-23T14:21:50.48273Z","shell.execute_reply.started":"2021-08-23T14:21:50.43932Z","shell.execute_reply":"2021-08-23T14:21:50.481843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nif CFG.VISUALIZE:\n    import matplotlib.pyplot as plt\n    modelTmp = Model(get_spectrogram = True)\n    for step, batch in enumerate(train_loader_soft,1):\n        X = batch[\"X\"]\n        X = batch_preprocessing(X)\n        modelTmp.to(device)\n        X = X.to(device)\n        targets = batch[\"y\"].to(device)\n        outputs, outputs_expt = modelTmp(X)\n        n = np.random.randint(32)\n        tmp = outputs[n].cpu().numpy()\n        tmp_expt = outputs_expt[n].cpu().numpy()\n        for i in range(3):\n            fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (13,4))\n            fig.suptitle('Target = '+ str(targets[n]))\n            plt1 = ax1.imshow((tmp[i,:,:]).squeeze())\n            #plt.colorbar(plt1)\n            plt2 = ax2.imshow((tmp_expt[i,:,:]).squeeze())\n            #plt.colorbar(plt2)\n        plt.figure()\n        plt.title('Target = '+ str(targets[n]))\n        plt.imshow(np.mean(tmp[:,:,:],axis=0).squeeze())\n        plt.colorbar()\n        print(tmp.shape)\n        break","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:50.485868Z","iopub.execute_input":"2021-08-23T14:21:50.486181Z","iopub.status.idle":"2021-08-23T14:21:53.249123Z","shell.execute_reply.started":"2021-08-23T14:21:50.486154Z","shell.execute_reply":"2021-08-23T14:21:53.248074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.VISUALIZE:\n    for step, batch in enumerate(test_loader_withlabels,1):\n        X = batch[\"X\"]\n        X = batch_preprocessing(X)\n        modelTmp.to(device)\n        X = X.to(device)\n        targets = batch[\"y\"].to(device)\n        outputs, outputs_expt = modelTmp(X)\n        n = np.random.randint(32)\n        tmp = outputs[n].cpu().numpy()\n        tmp_expt = outputs_expt[n].cpu().numpy()\n        for i in range(3):\n            fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (13,4))\n            fig.suptitle('Target = '+ str(targets[n]))\n            plt1 = ax1.imshow((tmp[i,:,:]).squeeze())\n            #plt.colorbar(plt1)\n            plt2 = ax2.imshow((tmp_expt[i,:,:]).squeeze())\n            #plt.colorbar(plt2)\n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (13,4))\n        fig.suptitle('Target = '+ str(targets[n]))\n        plt1 = ax1.imshow(np.transpose(tmp_expt[:,:,:]+1,(1,2,0)).squeeze())\n        #plt.colorbar(plt1)\n        plt2 = ax2.imshow(np.transpose(tmp[:,:,:],(1,2,0)).squeeze())\n        plt.title('Target = '+ str(targets[n]))\n        print(tmp.shape)\n        break","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:53.251064Z","iopub.execute_input":"2021-08-23T14:21:53.25146Z","iopub.status.idle":"2021-08-23T14:21:56.385344Z","shell.execute_reply.started":"2021-08-23T14:21:53.25142Z","shell.execute_reply":"2021-08-23T14:21:56.384303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss related functions","metadata":{}},{"cell_type":"code","source":"class LossMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n\n    def update(self, val):\n        self.n += 1\n        # incremental update\n        self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n\n        \nclass AccMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n        \n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().round().numpy().astype(int)\n        y_pred = y_pred.cpu().numpy() >= 0\n        last_n = self.n\n        self.n += len(y_true)\n        true_count = np.sum(y_true == y_pred)\n        # incremental update\n        self.avg = true_count / self.n + last_n / self.n * self.avg\n        \n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:56.386874Z","iopub.execute_input":"2021-08-23T14:21:56.387244Z","iopub.status.idle":"2021-08-23T14:21:56.396183Z","shell.execute_reply.started":"2021-08-23T14:21:56.387194Z","shell.execute_reply":"2021-08-23T14:21:56.395239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trainer related functions","metadata":{}},{"cell_type":"code","source":"def get_scheduler(optimizer):\n    if CFG.scheduler=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n    elif CFG.scheduler=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n    elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n    return scheduler\n\nclass Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion, \n        loss_meter, \n        score_meter,\n        use_swa = False\n    ):\n        self.model = model\n        # freeze model by default\n        self.model.freezeModel()\n        \n        self.device = device\n        self.use_swa = use_swa\n        self.optimizer = swa.SWA(optimizer) if self.use_swa else optimizer\n        self.criterion = criterion\n        self.loss_meter = loss_meter\n        self.score_meter = score_meter\n        self.scheduler = get_scheduler(optimizer)\n        self.learning_rate = self.scheduler.get_lr()\n        \n        self.best_valid_score = -np.inf\n        self.n_patience = 0\n        \n        self.messages = {\n            \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, auc_score: {:.5f}, time: {} s\",\n            \"checkpoint\": \"The score improved from {:.5f} to {:.5f}. Save model to '{}'\",\n            \"patience\": \"\\nValid score didn't improve last {} epochs.\"\n        }\n        self.training_step = 0\n        self.prevbatch = []\n        self.epoch = -1 \n    \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience,train_loader_soft = False):        \n        for n_epoch in range(1, epochs + 1):\n            \n            self.epoch = n_epoch\n            \n            self.save_path = save_path\n            \n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            if self.epoch==1 or (not train_loader_soft):\n                train_loss, train_score, train_time = self.train_epoch(train_loader)\n            else:\n                print('Using soft labels for this epoch... ')\n                train_loss, train_score, train_time = self.train_epoch(train_loader_soft)\n                \n            valid_loss, valid_score, valid_time, valid_rocauc = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Train\", n_epoch, train_loss, train_score, 0, train_time\n            )\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Valid\", n_epoch, valid_loss, valid_score, valid_rocauc, valid_time\n            )\n            \n\n            if self.best_valid_score < valid_score:\n                self.info_message(\n                    self.messages[\"checkpoint\"], self.best_valid_score, valid_score, save_path\n                )\n                self.best_valid_score = valid_score\n                self.save_model(n_epoch, save_path)\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(self.messages[\"patience\"], patience)\n                break\n        if self.use_swa:\n            self.optimizer.bn_update(train_loader, self.model)\n            self.optimizer.swap_swa_sgd()\n        \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        train_loss = self.loss_meter()\n        train_score = self.score_meter()\n        \n        for step, batch in enumerate(tqdm(train_loader),1):\n            \n            if self.training_step == CFG.unfreezeStep:\n                self.model.unfreezeModel()\n            \n            X = batch[\"X\"]\n            targets = batch[\"y\"]\n            if self.prevbatch:\n                prevX = self.prevbatch[\"X\"]\n                prevY = self.prevbatch['y']\n                rndNum = np.random.rand()\n                if rndNum<0.5:\n                    # only keep prevX where there is no wave\n                    prevX = torch.where(prevY.view(-1,1,1)>0.5,X,prevX)\n                    prevY = torch.where(prevY>0.5,targets,prevY)\n                    \n                    # weight for prevX is at most 0.5, and not replaced\n                    # when there is a wave\n                    X = (1-rndNum)*X + rndNum*prevX \n                    # /2 to prevent target 0 from exceeding 0.5\n                    # targets = torch.clamp(((1-rndNum)*targets + rndNum*prevY/2)*1.5,max = 1.)\n\n            self.prevbatch = batch.copy()  \n            X = batch_preprocessing(X)\n            X = X.to(self.device)\n            targets = targets.to(self.device)\n            \n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            train_loss.update(loss.detach().item())\n            train_score.update(targets, outputs.detach())\n\n            self.optimizer.step()\n            \n            _loss, _score = train_loss.avg, train_score.avg\n            \n            message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}, learning_rate: {:.5f}/{:.5f}'\n            self.info_message(message, step, len(train_loader), _loss, _score, self.learning_rate[0], self.learning_rate[1],end=\"\\r\")\n            self.training_step += 1\n            \n            if self.training_step%CFG.schedulerStepFreq==0:\n                if isinstance(self.scheduler, CosineAnnealingLR):\n                    self.scheduler.step()\n                elif isinstance(self.scheduler, CosineAnnealingWarmRestarts):\n                    self.scheduler.step()\n                self.learning_rate = self.scheduler.get_lr()\n        # print('\\n Updated learning rate: '+ str(self.scheduler.get_lr()))\n            if self.training_step%CFG.valStepFreq==0 and self.epoch>2:\n                valid_loss, valid_score, valid_time, valid_rocauc = self.valid_epoch(valid_loader)\n                self.info_message(\n                    self.messages[\"epoch\"], \"Valid\", self.epoch, valid_loss, valid_score, valid_rocauc, valid_time\n                )\n\n                if self.best_valid_score < valid_score:\n                    self.info_message(\n                        self.messages[\"checkpoint\"], self.best_valid_score, valid_score, self.save_path\n                    )\n                    self.best_valid_score = valid_score\n                    self.save_model(n_epoch, self.save_path)\n                    self.n_patience = 0\n                else:\n                    self.n_patience += 1\n        \n        return train_loss.avg, train_score.avg, int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader,returnPred = False):\n        self.model.eval()\n        t = time.time()\n        valid_loss = self.loss_meter()\n        valid_score = self.score_meter()\n        \n        for step, batch in enumerate(valid_loader, 1):\n            y_pred = []\n            tgts = []\n            with torch.no_grad():\n                X = batch[\"X\"]  \n                X = batch_preprocessing(X)\n                X = X.to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                valid_loss.update(loss.detach().item())\n                valid_score.update(targets, outputs)\n                outputs = outputs\n                y_pred.extend(torch.sigmoid(outputs).cpu().numpy().squeeze())\n                tgts.extend(batch[\"y\"].numpy())\n                    \n            rocauc = roc_auc_score(tgts,y_pred)\n            _loss, _score = valid_loss.avg, valid_score.avg\n            message = 'Valid Step {}/{}, valid_loss: {:.5f}, valid_score: {:.5f},valid_roc_auc: {:.5f}'\n            self.info_message(message, step, len(valid_loader), _loss, _score, rocauc, end=\"\\r\")\n        if not returnPred:\n            return valid_loss.avg, valid_score.avg, int(time.time() - t), rocauc\n        else:\n            return y_pred, tgts\n    \n    def test_eval(self,test_loader):\n        y_pred = []\n        ids = []\n        for e, batch in enumerate(test_loader):\n            print(f\"{e}/{len(test_loader)}\", end=\"\\r\")\n            with torch.no_grad():\n                X = batch[\"X\"]\n                X = batch_preprocessing(X)\n                X = X.to(self.device)\n                outputs = self.model(X)\n                y_pred.extend(torch.sigmoid(outputs).cpu().numpy().squeeze())\n                ids.extend(batch[\"id\"])\n        return y_pred, ids\n    \n    def save_model(self, n_epoch, save_path):\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            save_path,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:56.397972Z","iopub.execute_input":"2021-08-23T14:21:56.398605Z","iopub.status.idle":"2021-08-23T14:21:56.436525Z","shell.execute_reply.started":"2021-08-23T14:21:56.398566Z","shell.execute_reply":"2021-08-23T14:21:56.435669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel = Model()\nmodel.to(device)\nif (not CFG.TRAIN) or CFG.WARM_START:\n    startCheckpoint = torch.load(\"../input/exptv2v4-best-model-fold0b0/best-model.pth\")\n    checkpoint = torch.load(\"../input/exptv2v4-best-model-fold0b0/best-model.pth\")\n    model.load_state_dict(startCheckpoint[\"model_state_dict\"])\n\noptimizer = torch.optim.Adam([{\"params\": model.model.parameters(), \"lr\": CFG.lr},\n                              {\"params\": model.head.parameters(), \"lr\": CFG.lr/10}], \n                             lr=CFG.lr)\ncriterion = torch_functional.binary_cross_entropy_with_logits\n\ntrainer = Trainer(\n    model, \n    device, \n    optimizer, \n    criterion, \n    LossMeter, \n    AccMeter\n)\n\nif CFG.TRAIN:\n    history = trainer.fit(\n        CFG.EPOCHS, \n        train_loader, \n        valid_loader, \n        \"best-model.pth\", \n        400,\n        train_loader_soft = train_loader_soft if CFG.useSoftLabels else False\n    )\n    \n    y_pred_val,tgts = trainer.valid_epoch(valid_loader,returnPred = True)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:21:56.437852Z","iopub.execute_input":"2021-08-23T14:21:56.438302Z","iopub.status.idle":"2021-08-23T14:27:23.216718Z","shell.execute_reply.started":"2021-08-23T14:21:56.438265Z","shell.execute_reply":"2021-08-23T14:27:23.214868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(tgts,y_pred_val,1)\nplt.xlabel('targets')\nplt.ylabel('predictions')","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:27:23.218137Z","iopub.status.idle":"2021-08-23T14:27:23.218569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.TRAIN:\n    checkpoint = torch.load(\"best-model.pth\")\n\nmodel.load_state_dict(checkpoint[\"model_state_dict\"])\nmodel.eval();","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:27:23.219677Z","iopub.status.idle":"2021-08-23T14:27:23.220333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect()\n\n\n\ny_pred, ids = trainer.test_eval(test_loader)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:27:23.221577Z","iopub.status.idle":"2021-08-23T14:27:23.222201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/g2net-gravitational-wave-detection/sample_submission.csv\")\nsubmission = pd.DataFrame({\"id\": submission['id'].values, \"target\": y_pred})\nsubmission.to_csv(\"model_submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:27:23.223508Z","iopub.status.idle":"2021-08-23T14:27:23.224255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:27:23.225389Z","iopub.status.idle":"2021-08-23T14:27:23.226019Z"},"trusted":true},"execution_count":null,"outputs":[]}]}