{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from IPython.display import clear_output\n!pip install efficientnet_pytorch\n!pip install timm\n!python -m pip install gwpy\nclear_output()","metadata":{"id":"wX0WZIB9eXIx","executionInfo":{"status":"ok","timestamp":1632904004272,"user_tz":-330,"elapsed":9335,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys \nimport json\nfrom glob import glob\nimport random\nimport collections\nimport time\nimport re\nimport warnings\nfrom IPython.display import clear_output\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nfrom efficientnet_pytorch import EfficientNet\nimport timm\n\nfrom gwpy.timeseries import TimeSeries\nfrom gwpy.plot import Plot\nfrom scipy import signal\nfrom sklearn.preprocessing import MinMaxScaler\nfrom matplotlib import pyplot as plt\nimport joblib\nfrom tqdm import tqdm\nimport shutil\nimport librosa\n\ndef set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(1)\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) ","metadata":{"id":"ABMUd569Qp5Z","executionInfo":{"status":"ok","timestamp":1632923965006,"user_tz":-330,"elapsed":2747,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-22T17:56:36.630220Z","iopub.execute_input":"2022-06-22T17:56:36.630769Z","iopub.status.idle":"2022-06-22T17:56:46.662090Z","shell.execute_reply.started":"2022-06-22T17:56:36.630663Z","shell.execute_reply":"2022-06-22T17:56:46.660543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Q_RANGE = (16,32)\nF_RANGE = (30,400)\nSIZE = 128\n\ndef read_file(fname):\n    data = np.load(fname)\n    d1 = TimeSeries(data[0,:], sample_rate=2048)\n    d2 = TimeSeries(data[1,:], sample_rate=2048)\n    d3 = TimeSeries(data[2,:], sample_rate=2048)\n        \n    return d1, d2, d3\n\ndef plot_time_data(d1, d2, d3):\n    plot = Plot(d1, d2, d3, separate=True, sharex=True, figsize=[12, 8])\n    ax = plot.gca()\n    ax.set_xlim(0,2)\n    ax.set_xlabel('Time [s]')\n    plot.show()\n    \ndef preprocess(d1, d2, d3, bandpass=False, lf=35, hf=350):\n    white_d1 = d1.whiten(window=(\"tukey\",0.2))\n    white_d2 = d2.whiten(window=(\"tukey\",0.2))\n    white_d3 = d3.whiten(window=(\"tukey\",0.2))\n    \n    if bandpass: # bandpass filter\n        bp_d1 = white_d1.bandpass(lf, hf) \n        bp_d2 = white_d2.bandpass(lf, hf)\n        bp_d3 = white_d3.bandpass(lf, hf)\n        return bp_d1, bp_d2, bp_d3\n    else: # only whiten\n        return white_d1, white_d2, white_d3    \n    return d1, d2, d3        \n    \ndef create_rgb(fname):\n    r1, r2, r3 = read_file(fname)\n    p1, p2, p3 = preprocess(r1, r2, r3)\n    hq1 = p1.q_transform(qrange=Q_RANGE, frange=F_RANGE, logf=True, whiten=False)\n    hq2 = p2.q_transform(qrange=Q_RANGE, frange=F_RANGE, logf=True, whiten=False)\n    hq3 = p3.q_transform(qrange=Q_RANGE, frange=F_RANGE, logf=True, whiten=False)\n\n    img = np.zeros([hq1.shape[0], hq1.shape[1], 3])\n    \n    img[:,:,0] = hq1\n    img[:,:,1] = hq2\n    img[:,:,2] = hq3\n    \n    img = img - np.min(img)\n    img = img / (np.max(img) - np.min(img))\n    img = (img * 255).astype(np.uint8)\n    \n    return cv2.resize(img, (SIZE, SIZE), interpolation = cv2.INTER_LANCZOS4)\n\ndef id2path(img_id, is_test):\n    a, b, c = img_id[0], img_id[1], img_id[2]\n    if is_test: \n        return f'../input/g2net-gravitational-wave-detection/test/{a}/{b}/{c}/{img_id}.npy'\n    else: \n        return f'../input/g2net-gravitational-wave-detection/train/{a}/{b}/{c}/{img_id}.npy'\n    \ndef save_train_img(_id):\n    fname = id2path(_id, False)\n    im = create_rgb(fname)\n    cv2.imwrite(f'./train_images/{_id}.png', im)    \n    \ndf = pd.read_csv('../input/train-files/train.csv').sample(n=500, random_state=1)\n\nif not os.path.exists('train_images'):\n    os.makedirs('train_images', exist_ok=True)    \n    _ = joblib.Parallel(n_jobs=-1)(joblib.delayed(save_train_img)(_id) for _id in tqdm(df['id']))    ","metadata":{"execution":{"iopub.status.busy":"2022-06-22T17:56:46.665105Z","iopub.execute_input":"2022-06-22T17:56:46.665912Z","iopub.status.idle":"2022-06-22T17:56:46.797715Z","shell.execute_reply.started":"2022-06-22T17:56:46.665865Z","shell.execute_reply":"2022-06-22T17:56:46.796421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_directory = '../input/g2net-gravitational-wave-detection'\nSIZE = (32,32)\nbatch_size = 128\nin_channels = 3\nlr = 0.00002","metadata":{"id":"Gzyxpo8uQp2Z","executionInfo":{"status":"ok","timestamp":1632923966479,"user_tz":-330,"elapsed":22,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-22T17:56:46.799162Z","iopub.execute_input":"2022-06-22T17:56:46.799614Z","iopub.status.idle":"2022-06-22T17:56:46.806392Z","shell.execute_reply.started":"2022-06-22T17:56:46.799557Z","shell.execute_reply":"2022-06-22T17:56:46.805080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms():\n    return transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.RandomOrder([                               \n        transforms.RandomAffine(degrees=15),\n        transforms.RandomAutocontrast(p=0.05),\n        transforms.RandomEqualize(p=0.05),\n        transforms.RandomHorizontalFlip(p=0.05),\n        transforms.RandomInvert(p=0.05),     \n        transforms.RandomSolarize(threshold=15, p=0.05),\n        ]),\n        transforms.ToTensor()\n    ])\n\ndef get_valid_transforms():\n    return transforms.Compose([\n        transforms.ToPILImage(),                               \n        transforms.ToTensor()\n    ])     ","metadata":{"id":"C7xAHxK3QpzJ","executionInfo":{"status":"ok","timestamp":1632923966480,"user_tz":-330,"elapsed":21,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-22T17:56:46.810069Z","iopub.execute_input":"2022-06-22T17:56:46.811310Z","iopub.status.idle":"2022-06-22T17:56:46.821977Z","shell.execute_reply.started":"2022-06-22T17:56:46.811267Z","shell.execute_reply":"2022-06-22T17:56:46.820637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float64)\n    vec[target] = 1.\n    return vec  \n\nclass Dataset(torch_data.Dataset):\n    def __init__(self, id, targets=None, split=\"train_images\", augment=False):\n        self.targets = targets\n        self.id = id\n        self.split = split\n        self.augment = augment\n          \n    def __len__(self):\n        return len(self.id)\n    \n    def __getitem__(self, index):\n        id = self.id[index]\n        if self.targets is None:\n            data = cv2.imread(f\"{self.split}/{id}.png\")\n            data = cv2.resize(data, SIZE)\n            data = get_valid_transforms()(data)\n        else:\n            data = cv2.imread(f\"{self.split}/{id}.png\")\n            data = cv2.resize(data, SIZE)\n            if self.augment:\n              data = get_transforms()(data)\n            else:\n              data = get_valid_transforms()(data)\n            \n        if self.targets is None:\n            return {\"X\": data, \"id\": id}\n        else:\n            y = torch.tensor(abs(self.targets[index]- 0.01), dtype=torch.float)\n            return {\"X\": data, \"y\": y}\n\ndef Model():\n\n    net = timm.create_model('resnetv2_50x3_bitm_in21k', pretrained=True, num_classes=1)\n    net.drop_rate = 0.2\n    return net    \n\nclass Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion,\n        scheduler\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.scheduler = scheduler\n\n        self.best_valid_auc = 0\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}, \\nLR: {}\", n_epoch, self.optimizer.param_groups[0]['lr'])\n            \n            train_loss, train_time = self.train_epoch(train_loader)\n\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_loader)\n            \n            self.scheduler.step(valid_auc)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            if self.best_valid_auc < valid_auc: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_auc, valid_auc, self.lastmodel\n                )\n                self.best_valid_auc = valid_auc\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        for step, batch in enumerate(tqdm(train_loader),1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n\n        return sum_loss/len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n        auc = RocAucMeter()\n\n        for step, batch in enumerate(tqdm(valid_loader),1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n                outputs = self.model(X).squeeze(1)\n\n                loss = self.criterion(outputs, targets)\n                sum_loss += loss.detach().item()\n                \n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(torch.sigmoid(outputs).tolist())\n        \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n\n        return sum_loss/len(valid_loader), auc, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_auc,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"id":"FbYXcOBmQpcC","executionInfo":{"status":"ok","timestamp":1632923967245,"user_tz":-330,"elapsed":779,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-22T17:56:46.824288Z","iopub.execute_input":"2022-06-22T17:56:46.824992Z","iopub.status.idle":"2022-06-22T17:56:46.863850Z","shell.execute_reply.started":"2022-06-22T17:56:46.824948Z","shell.execute_reply":"2022-06-22T17:56:46.862408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.array([0,1])\n        self.y_pred = np.array([0.5,0.5])\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = roc_auc_score(self.y_true, self.y_pred)\n\n    @property\n    def avg(self):\n        return self.score","metadata":{"id":"kJPzsnIE4WHo","executionInfo":{"status":"ok","timestamp":1632923967247,"user_tz":-330,"elapsed":6,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-22T17:56:46.866225Z","iopub.execute_input":"2022-06-22T17:56:46.866790Z","iopub.status.idle":"2022-06-22T17:56:46.879870Z","shell.execute_reply.started":"2022-06-22T17:56:46.866746Z","shell.execute_reply":"2022-06-22T17:56:46.878090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef train_type(df_train, df_valid):\n    train_data_retriever = Dataset(\n        df_train[\"id\"].values, \n        df_train[\"target\"].values, \n        augment=True\n    )\n\n    valid_data_retriever = Dataset(\n        df_valid[\"id\"].values, \n        df_valid[\"target\"].values,\n    )\n\n    train_loader = torch_data.DataLoader(\n        train_data_retriever,\n        batch_size=batch_size,\n        shuffle=True,\n        num_workers=2,pin_memory = False\n    )\n\n    valid_loader = torch_data.DataLoader(\n        valid_data_retriever, \n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=2,pin_memory = False\n    )\n\n    model = Model()\n    model.to(device)\n\n    # checkpoint = torch.load(\"model.pth\")\n    # model.load_state_dict(checkpoint[\"model_state_dict\"])\n\n    param_optimizer = list(model.named_parameters())\n    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n    optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ]\n    optimizer = torch.optim.Adam(optimizer_grouped_parameters, lr=lr)\n    criterion = torch_functional.binary_cross_entropy_with_logits\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2, \n                                                           threshold=0.0001, threshold_mode='abs', cooldown=0, \n                                                           min_lr=0, eps=1e-08, verbose=False)\n\n    trainer = Trainer(\n        model, \n        device, \n        optimizer, \n        criterion,\n        scheduler\n    )\n\n    history = trainer.fit( \n        5,  \n        train_loader, \n        valid_loader, \n        'model',  \n        2,\n    )\n    \n    return trainer.lastmodel\n","metadata":{"id":"Z1tb5GIbnwo_","executionInfo":{"status":"ok","timestamp":1632923967980,"user_tz":-330,"elapsed":10,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-22T17:56:46.882480Z","iopub.execute_input":"2022-06-22T17:56:46.884452Z","iopub.status.idle":"2022-06-22T17:56:46.900116Z","shell.execute_reply.started":"2022-06-22T17:56:46.884411Z","shell.execute_reply":"2022-06-22T17:56:46.898531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_type(df.iloc[0:450], df.iloc[450::])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T17:56:46.902065Z","iopub.execute_input":"2022-06-22T17:56:46.903343Z","iopub.status.idle":"2022-06-22T17:57:25.853035Z","shell.execute_reply.started":"2022-06-22T17:56:46.903259Z","shell.execute_reply":"2022-06-22T17:57:25.849467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}