{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\nThis notebook is the inference notebook for [G2Net: TF On-the-fly CQT TPU Training](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-training).\n\nOn the fly CQT computation achieves better result compared to [Welf's Notebook](https://www.kaggle.com/miklgr500/g2net-efficientnetb1-tpu-evaluate) given the same image size and EfficientNet size, which means if you scale up the model or scale up the image size, you'll possibly get the best single model compared to publicly shared models.\nIt also allows you to make more variations for the input, which gives you a great advantage.\n\n### Updates\n\n* V3: Use the weights of V2 of the Training Notebook\n    * EfficientNetB0 -> EfficientNetB7","metadata":{}},{"cell_type":"markdown","source":"Reference\nhttps://www.kaggle.com/hidehisaarai1213/g2net-read-from-tfrecord-train-with-pytorch\n","metadata":{}},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null\n!pip install -q nnAudio\n!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:19.52512Z","iopub.execute_input":"2021-08-30T11:46:19.525737Z","iopub.status.idle":"2021-08-30T11:46:44.031552Z","shell.execute_reply.started":"2021-08-30T11:46:19.525636Z","shell.execute_reply":"2021-08-30T11:46:44.030583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window\nfrom matplotlib import pyplot as plt\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:44.033685Z","iopub.execute_input":"2021-08-30T11:46:44.034053Z","iopub.status.idle":"2021-08-30T11:46:51.424348Z","shell.execute_reply.started":"2021-08-30T11:46:44.034015Z","shell.execute_reply":"2021-08-30T11:46:51.423339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:51.426085Z","iopub.execute_input":"2021-08-30T11:46:51.426449Z","iopub.status.idle":"2021-08-30T11:46:51.43523Z","shell.execute_reply.started":"2021-08-30T11:46:51.42642Z","shell.execute_reply":"2021-08-30T11:46:51.43431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = 256 #for b0\nBATCH_SIZE = 64\nEFFICIENTNET_SIZE = 5\nWEIGHTS = \"imagenet\"\nclass CFG:\n    debug=False\n    num_workers=4\n    model_name='tf_efficientnet_b0_ns'\n    model_dir='../input/' \n    batch_size=512 \n    \n    qtransform_params={\"sr\": 2048, \"fmin\": 20, \"fmax\": 1024, \"hop_length\": 64,\n                       \"bins_per_octave\": 24 }\n    seed=42\n    target_size=1\n    target_col='target'\n    n_fold=5\n    trn_fold=[1] # [0, 1, 2, 3, 4]","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:59.733655Z","iopub.execute_input":"2021-08-30T11:47:59.733999Z","iopub.status.idle":"2021-08-30T11:47:59.739677Z","shell.execute_reply.started":"2021-08-30T11:47:59.733955Z","shell.execute_reply":"2021-08-30T11:47:59.738776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(1213)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:51.445881Z","iopub.execute_input":"2021-08-30T11:46:51.446275Z","iopub.status.idle":"2021-08-30T11:46:51.453873Z","shell.execute_reply.started":"2021-08-30T11:46:51.446242Z","shell.execute_reply":"2021-08-30T11:46:51.453051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:51.45504Z","iopub.execute_input":"2021-08-30T11:46:51.455695Z","iopub.status.idle":"2021-08-30T11:46:51.463617Z","shell.execute_reply.started":"2021-08-30T11:46:51.455599Z","shell.execute_reply":"2021-08-30T11:46:51.462766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:51.466296Z","iopub.execute_input":"2021-08-30T11:46:51.466764Z","iopub.status.idle":"2021-08-30T11:46:51.47871Z","shell.execute_reply.started":"2021-08-30T11:46:51.466725Z","shell.execute_reply":"2021-08-30T11:46:51.477669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9)]:\n    GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-test-{i}-{j}\")\n    gcs_paths.append(GCS_path)\n    print(GCS_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:51.481791Z","iopub.execute_input":"2021-08-30T11:46:51.482048Z","iopub.status.idle":"2021-08-30T11:46:52.807749Z","shell.execute_reply.started":"2021-08-30T11:46:51.482025Z","shell.execute_reply":"2021-08-30T11:46:52.806845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/test*.tfrecords\"))))\n\nprint(\"test_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:46:52.808968Z","iopub.execute_input":"2021-08-30T11:46:52.809323Z","iopub.status.idle":"2021-08-30T11:46:54.319789Z","shell.execute_reply.started":"2021-08-30T11:46:52.809286Z","shell.execute_reply":"2021-08-30T11:46:54.318812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Preparation","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.394706Z","iopub.execute_input":"2021-08-30T11:47:00.3953Z","iopub.status.idle":"2021-08-30T11:47:00.404116Z","shell.execute_reply.started":"2021-08-30T11:47:00.395265Z","shell.execute_reply":"2021-08-30T11:47:00.403129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_wave(wave):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves, axis=0)\n    wave = tf.cast(wave, tf.float32)\n    return wave\n\n\ndef read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\ndef read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_wave(example[\"wave\"]), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1]), example[\"wave_id\"]\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_wave(example[\"wave\"]), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    #ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size )\n    #if aug:\n    #    ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return tfds.as_numpy(ds)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.405647Z","iopub.execute_input":"2021-08-30T11:47:00.406107Z","iopub.status.idle":"2021-08-30T11:47:00.425532Z","shell.execute_reply.started":"2021-08-30T11:47:00.406073Z","shell.execute_reply":"2021-08-30T11:47:00.424704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class TFRecordDataLoader:\n    def __init__(self, files, batch_size=32, cache=False, train=False, repeat=False, \n                 shuffle=False, labeled=False, return_image_ids=True):\n        self.ds = get_dataset(\n            files, \n            batch_size=batch_size ,\n            repeat=repeat,\n            shuffle=shuffle,\n            labeled=labeled,\n            return_image_ids=return_image_ids)\n        \n        if train:\n            self.num_examples = count_data_items(files)\n        else:\n            self.num_examples = count_data_items_test(files)\n\n        self.batch_size = batch_size\n        self.labeled = labeled\n        self.return_image_ids = return_image_ids\n        self._iterator = None\n    \n    def __iter__(self):\n        if self._iterator is None:\n            self._iterator = iter(self.ds)\n        else:\n            self._reset()\n        return self._iterator\n\n    def _reset(self):\n        self._iterator = iter(self.ds)\n\n    def __next__(self):\n        batch = next(self._iterator)\n        return batch\n\n    def __len__(self):\n        n_batches = self.num_examples // self.batch_size\n        if self.num_examples % self.batch_size == 0:\n            return n_batches\n        else:\n            return n_batches + 1","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.428341Z","iopub.execute_input":"2021-08-30T11:47:00.428591Z","iopub.status.idle":"2021-08-30T11:47:00.439363Z","shell.execute_reply.started":"2021-08-30T11:47:00.428563Z","shell.execute_reply":"2021-08-30T11:47:00.43855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom nnAudio.Spectrogram import CQT1992v2\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        self.wave_transform = CQT1992v2(**CFG.qtransform_params)\n        self.model = timm.create_model(self.cfg.model_name, pretrained=pretrained, in_chans=3)\n        self.n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(self.n_features, self.cfg.target_size)\n\n    def forward(self, x):\n        waves = []\n        for i in range(3):\n            waves.append(self.wave_transform(x[:, i]))\n        x = torch.stack(waves, dim=1)\n        output = self.model(x)\n        return output","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.440969Z","iopub.execute_input":"2021-08-30T11:47:00.441408Z","iopub.status.idle":"2021-08-30T11:47:00.466339Z","shell.execute_reply.started":"2021-08-30T11:47:00.441373Z","shell.execute_reply":"2021-08-30T11:47:00.465416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"files_test_all = np.array(all_files)\nall_test_preds = []","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.467601Z","iopub.execute_input":"2021-08-30T11:47:00.467997Z","iopub.status.idle":"2021-08-30T11:47:00.473785Z","shell.execute_reply.started":"2021-08-30T11:47:00.467941Z","shell.execute_reply":"2021-08-30T11:47:00.472786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.475192Z","iopub.execute_input":"2021-08-30T11:47:00.475663Z","iopub.status.idle":"2021-08-30T11:47:00.483685Z","shell.execute_reply.started":"2021-08-30T11:47:00.475627Z","shell.execute_reply":"2021-08-30T11:47:00.482818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    model = CustomModel(CFG, pretrained=False)\n    model.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:00.484886Z","iopub.execute_input":"2021-08-30T11:47:00.485436Z","iopub.status.idle":"2021-08-30T11:47:01.170108Z","shell.execute_reply.started":"2021-08-30T11:47:00.4854Z","shell.execute_reply":"2021-08-30T11:47:01.169066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm as tqdm\nimport tensorflow_datasets as tfds\nimport gc ","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:01.17143Z","iopub.execute_input":"2021-08-30T11:47:01.171775Z","iopub.status.idle":"2021-08-30T11:47:01.58086Z","shell.execute_reply.started":"2021-08-30T11:47:01.171735Z","shell.execute_reply":"2021-08-30T11:47:01.580054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nstates = [torch.load(CFG.model_dir+f'{CFG.model_name}_fold{fold}_best_score.pth',\n                     map_location=device) \n          for fold in CFG.trn_fold]\nnum_folds=4\nfilenames = []\n\ntest_loader = TFRecordDataLoader(\n        all_files, batch_size=CFG.batch_size  , shuffle=False)\n\nwith torch.no_grad():\n        \n    for step, d in enumerate(tqdm(test_loader)):\n        avg_preds=[]\n            #targets.extend(d[1].reshape(-1).tolist())\n        filenames.extend([f.decode(\"UTF-8\") for f in d[1]])\n\n        images = torch.from_numpy(d[0]).to(device)\n        del d \n        gc.collect()\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n               \n            y_preds = model(images)\n            avg_preds.append(y_preds.sigmoid().to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            #labels = torch.from_numpy(d[1]).to(device)\n\n            #batch_size = labels.size(0)\n            # compute loss\n\n\n        \n        \n\n        all_test_preds.append(avg_preds )\n    probs = np.concatenate(all_test_preds)\n    #file_names=np.concatenate(file_names)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:48:26.00592Z","iopub.execute_input":"2021-08-30T11:48:26.006276Z","iopub.status.idle":"2021-08-30T12:01:41.315997Z","shell.execute_reply.started":"2021-08-30T11:48:26.006245Z","shell.execute_reply":"2021-08-30T12:01:41.314582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs = np.concatenate(all_test_preds)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-30T12:03:47.842833Z","iopub.execute_input":"2021-08-30T12:03:47.843164Z","iopub.status.idle":"2021-08-30T12:03:47.84954Z","shell.execute_reply.started":"2021-08-30T12:03:47.843135Z","shell.execute_reply":"2021-08-30T12:03:47.848581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#all_test_preds\ntest_df = pd.DataFrame({\n    \"id\": filenames ,\n    \"target\": probs.reshape(-1)\n})","metadata":{"execution":{"iopub.status.busy":"2021-08-30T12:03:56.567526Z","iopub.execute_input":"2021-08-30T12:03:56.567858Z","iopub.status.idle":"2021-08-30T12:03:56.598821Z","shell.execute_reply.started":"2021-08-30T12:03:56.567814Z","shell.execute_reply":"2021-08-30T12:03:56.598029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=False, shuffle=False, aug=False, labeled=False, return_image_ids=True)\n#file_ids = np.array([target.numpy() for img, target in iter(ds_test.unbatch())])","metadata":{"execution":{"iopub.status.busy":"2021-08-30T11:47:46.90312Z","iopub.status.idle":"2021-08-30T11:47:46.903747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T12:04:00.387813Z","iopub.execute_input":"2021-08-30T12:04:00.38815Z","iopub.status.idle":"2021-08-30T12:04:00.410002Z","shell.execute_reply.started":"2021-08-30T12:04:00.38812Z","shell.execute_reply":"2021-08-30T12:04:00.409006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(test_df.target.values)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T12:04:06.454179Z","iopub.execute_input":"2021-08-30T12:04:06.454496Z","iopub.status.idle":"2021-08-30T12:04:06.623696Z","shell.execute_reply.started":"2021-08-30T12:04:06.454467Z","shell.execute_reply":"2021-08-30T12:04:06.62294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T12:04:25.07996Z","iopub.execute_input":"2021-08-30T12:04:25.080436Z","iopub.status.idle":"2021-08-30T12:04:25.940517Z","shell.execute_reply.started":"2021-08-30T12:04:25.080392Z","shell.execute_reply":"2021-08-30T12:04:25.939652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}