{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet\n\nimport os\nimport re\nimport warnings\nimport random\nimport sklearn.exceptions\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import roc_auc_score\nfrom time import perf_counter\nfrom tensorflow.keras import backend as K\nfrom tqdm.notebook import tqdm\nfrom kaggle_datasets import KaggleDatasets\nfrom glob import glob\n\n\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=sklearn.exceptions.UndefinedMetricWarning)\n\nRANDOM_SEED = 42\nCOMPETITION_DATASET_PATH = \"../input/g2net-gravitational-wave-detection\"\nPRETRAINED_MODEL_PATH = \"../input/g2net-effnetb7v2\"\nQUANTILE = 0.7\nFOLDS = (0, 1, 2, 3)\nIMG_SIZES = 256\nBATCH_SIZES = 32\n\ndef seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_everything()","metadata":{"papermill":{"duration":10.406747,"end_time":"2021-06-12T15:10:18.648284","exception":false,"start_time":"2021-06-12T15:10:08.241537","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T06:07:27.512722Z","iopub.execute_input":"2021-08-07T06:07:27.513246Z","iopub.status.idle":"2021-08-07T06:07:34.720321Z","shell.execute_reply.started":"2021-08-07T06:07:27.513214Z","shell.execute_reply":"2021-08-07T06:07:34.719121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/xhlulu/ranzcr-efficientnet-tpu-training\ndef auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED =True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy, TPU_DETECTED","metadata":{"papermill":{"duration":0.02909,"end_time":"2021-06-12T15:10:18.696808","exception":false,"start_time":"2021-06-12T15:10:18.667718","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:08:14.836395Z","iopub.execute_input":"2021-08-07T05:08:14.83669Z","iopub.status.idle":"2021-08-07T05:08:14.843948Z","shell.execute_reply.started":"2021-08-07T05:08:14.836663Z","shell.execute_reply":"2021-08-07T05:08:14.842959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"top\"></a>\n<img src='https://storage.googleapis.com/kaggle-competitions/kaggle/23249/logos/header.png?t=2021-05-26-16-18-03'></img>\n\n* [1. Train Notebook](#1)\n* [2. Train Dataset](#2)\n* [3. Test Dataset](#3)\n* [4. TPU Configuration & Utils scripts](#4)\n* [5. Search best quantile](#5)\n* [6. Inference](#6)\n* [7. Next steps](#7)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n# Train Noteboook\n* [CQT G2Net EfficientNetB1[TPU Training]](https://www.kaggle.com/miklgr500/cqt-g2net-efficientnetb1-tpu-training)\n\n<a id=\"2\"></a>\n# Train Datasets\n* [Q-Transform TFRecords](https://www.kaggle.com/miklgr500/q-transform-tfrecords)\n    * [CQT G2Net V2 [0 - 1]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-0-1)\n    * [CQT G2Net V2 [2 - 3]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-2-3)\n    * [CQT G2Net V2 [4 - 5]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-4-5)\n    * [CQT G2Net V2 [6 - 7]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-6-7)\n    * [CQT G2Net V2 [8 - 9]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-8-9)\n    * [CQT G2Net V2 [10 - 11]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-10-11)\n    * [CQT G2Net V2 [12 - 13]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-12-13)\n    * [CQT G2Net V2 [14 - 15]](https://www.kaggle.com/miklgr500/cqt-g2net-v2-14-15)\n  \n<a id=\"3\"></a>\n# Test Datasets\n* [CQT G2Net Test [0 - 1]](https://www.kaggle.com/miklgr500/cqt-g2net-test-0-1)\n* [CQT G2Net Test [2 - 3]](https://www.kaggle.com/miklgr500/cqt-g2net-test-2-3)\n* [CQT G2Net Test [4 - 5]](https://www.kaggle.com/miklgr500/cqt-g2net-test-4-5)\n* [CQT G2Net Test [6 - 7]](https://www.kaggle.com/miklgr500/cqt-g2net-test-6-7)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"4\"></a>\n# TPU Configuration & Utils scripts","metadata":{"papermill":{"duration":0.017986,"end_time":"2021-06-12T15:10:18.733021","exception":false,"start_time":"2021-06-12T15:10:18.715035","status":"completed"},"tags":[]}},{"cell_type":"code","source":"strategy, TPU_DETECTED = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"papermill":{"duration":5.397929,"end_time":"2021-06-12T15:10:24.197997","exception":false,"start_time":"2021-06-12T15:10:18.800068","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:08:14.846024Z","iopub.execute_input":"2021-08-07T05:08:14.846355Z","iopub.status.idle":"2021-08-07T05:08:20.34122Z","shell.execute_reply.started":"2021-08-07T05:08:14.846324Z","shell.execute_reply":"2021-08-07T05:08:20.339573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_set_paths(idxs, path_prefix: str ='cqt-g2net-test', file_prefix: str = 'test', sep: str='-'):\n    files = []\n    for i,k in tqdm(idxs):\n        GCS_PATH = KaggleDatasets().get_gcs_path(f'{path_prefix}-{i}{sep}{k}')\n        files.extend(np.sort(np.array(tf.io.gfile.glob(GCS_PATH + f'/{file_prefix}*.tfrec'))).tolist())\n    print('Detected', len(files), file_prefix, 'files')\n    return files","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:08:20.342736Z","iopub.execute_input":"2021-08-07T05:08:20.343042Z","iopub.status.idle":"2021-08-07T05:08:20.351952Z","shell.execute_reply.started":"2021-08-07T05:08:20.343013Z","shell.execute_reply":"2021-08-07T05:08:20.349613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_train_g = get_set_paths([(0, 1), (2, 3), (4, 5), (6, 7), \n                               (8, 9), (10, 11), (12, 13), (14, 15)], \n                              path_prefix='cqt-g2net-v2', file_prefix='train', sep='-')\nfiles_test_g = get_set_paths([(0, 1), (2, 3), (4, 5), (6, 7)], file_prefix='test')","metadata":{"papermill":{"duration":1.815172,"end_time":"2021-06-12T15:10:26.070321","exception":false,"start_time":"2021-06-12T15:10:24.255149","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:09:35.644543Z","iopub.execute_input":"2021-08-07T05:09:35.645089Z","iopub.status.idle":"2021-08-07T05:09:40.632999Z","shell.execute_reply.started":"2021-08-07T05:09:35.645055Z","shell.execute_reply":"2021-08-07T05:09:40.63197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading Tfrecords","metadata":{"papermill":{"duration":0.020506,"end_time":"2021-06-12T15:10:26.31495","exception":false,"start_time":"2021-06-12T15:10:26.294444","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_id'                     : tf.io.FixedLenFeature([], tf.string),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example['image']), tf.reshape(tf.cast(example['target'], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_id'                     : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example['image']), example['image_id'] if return_image_id else 0\n\n \ndef prepare_image(img, dim=IMG_SIZES):    \n    img = tf.image.resize(tf.image.decode_png(img, channels=3), size=(dim, dim))\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [dim,dim, 3])\n            \n    return img\n\ndef count_data_items(fileids):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(fileid).group(1)) \n         for fileid in fileids]\n    return np.sum(n)","metadata":{"papermill":{"duration":0.046323,"end_time":"2021-06-12T15:10:26.380685","exception":false,"start_time":"2021-06-12T15:10:26.334362","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:09:43.105539Z","iopub.execute_input":"2021-08-07T05:09:43.105924Z","iopub.status.idle":"2021-08-07T05:09:43.117424Z","shell.execute_reply.started":"2021-08-07T05:09:43.105875Z","shell.execute_reply":"2021-08-07T05:09:43.116242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Creation","metadata":{"papermill":{"duration":0.01899,"end_time":"2021-06-12T15:10:26.419124","exception":false,"start_time":"2021-06-12T15:10:26.400134","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_dataset(files, shuffle = False, repeat = False, \n                labeled=True, return_image_ids=True, batch_size=16, dim=IMG_SIZES):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:09:49.500119Z","iopub.execute_input":"2021-08-07T05:09:49.500545Z","iopub.status.idle":"2021-08-07T05:09:49.510269Z","shell.execute_reply.started":"2021-08-07T05:09:49.500514Z","shell.execute_reply":"2021-08-07T05:09:49.508762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Build Model","metadata":{}},{"cell_type":"code","source":"def build_model(size, path):\n    inp = tf.keras.layers.Input(shape=(size, size,3))\n    base_net = efn.EfficientNetB7(input_shape=(size,size,3),weights=path,include_top=False)\n\n    x = base_net(inp)    \n    x = tf.keras.layers.GlobalAvgPool2D()(x)\n    x = tf.keras.layers.Dropout(0.)(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    \n    model = tf.keras.Model(inputs=inp, outputs=x)\n    loss = tf.keras.losses.BinaryCrossentropy() \n    model.compile(optimizer='adam',loss=loss,metrics=['AUC'])\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T05:09:51.438369Z","iopub.execute_input":"2021-08-07T05:09:51.438726Z","iopub.status.idle":"2021-08-07T05:09:51.446842Z","shell.execute_reply.started":"2021-08-07T05:09:51.438697Z","shell.execute_reply":"2021-08-07T05:09:51.445567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"5\"></a>\n# Search best quantile","metadata":{"papermill":{"duration":0.019514,"end_time":"2021-06-12T15:10:26.55885","exception":false,"start_time":"2021-06-12T15:10:26.539336","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict(paths, is_label=False):\n    pred = []; ids = []\n\n    ds = get_dataset(paths,labeled=False,return_image_ids=False,\n                repeat=False,shuffle=False,dim=IMG_SIZES,batch_size=BATCH_SIZES*2)\n        \n    for fold in FOLDS:\n    \n        print('#'*50); print('--> FOLD',fold+1);\n        start_time = perf_counter()\n    \n        K.clear_session()\n    \n        with strategy.scope():\n            model = build_model(IMG_SIZES, None)\n            print('\\t-->Loading model...')\n            model.load_weights(f'{PRETRAINED_MODEL_PATH}/fold-{fold}.h5')\n            print('\\t<--Model loaded.')\n    \n        print('\\t-->Start Predict...')\n    \n        pred.append(model.predict(ds, verbose=0).flatten())      \n        print('\\t<--Predict finished.')\n        print('<-- FOLD',fold+1, f'finished; duration = {perf_counter() - start_time} s')\n    \n    if is_label:\n        ds = get_dataset(paths,labeled=True,return_image_ids=False,\n                repeat=False,shuffle=False,dim=IMG_SIZES,batch_size=BATCH_SIZES*2)\n        ids = np.array([target.numpy() for _, target in tqdm(ds.unbatch())]).flatten()\n    else:\n        ds = get_dataset(paths,labeled=False,return_image_ids=True,\n                repeat=False,shuffle=False,dim=IMG_SIZES,batch_size=BATCH_SIZES*2)\n        ids = np.array([target.numpy().decode(\"utf-8\") for _, target in tqdm(ds.unbatch())]).flatten()\n    return pred, ids","metadata":{"execution":{"iopub.status.busy":"2021-08-07T05:13:12.456997Z","iopub.execute_input":"2021-08-07T05:13:12.457552Z","iopub.status.idle":"2021-08-07T05:13:12.470382Z","shell.execute_reply.started":"2021-08-07T05:13:12.457516Z","shell.execute_reply":"2021-08-07T05:13:12.469165Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred, target = predict(np.array(files_train_g), True)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T05:13:12.729599Z","iopub.execute_input":"2021-08-07T05:13:12.730066Z","iopub.status.idle":"2021-08-07T05:50:46.356113Z","shell.execute_reply.started":"2021-08-07T05:13:12.730029Z","shell.execute_reply":"2021-08-07T05:50:46.354865Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_labels = [f'Fold {i}' for i in range(1, 5)]\n\nplt.figure(figsize=(14, 9))\nfor p, fold in zip(pred, fold_labels):\n    sns.distplot(p, label=fold, rug=False, hist=False)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-07T06:03:38.470771Z","iopub.execute_input":"2021-08-07T06:03:38.471257Z","iopub.status.idle":"2021-08-07T06:03:49.906944Z","shell.execute_reply.started":"2021-08-07T06:03:38.471218Z","shell.execute_reply":"2021-08-07T06:03:49.905891Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_best_quantile(pred, target, verbose=True):\n    quantiles = np.arange(0, 1, 0.001)\n    scores = []\n    for q in tqdm(quantiles, total=len(quantiles)):\n        scores.append(roc_auc_score(target, np.quantile(pred, q, axis=0)))\n    best_idx = np.argmax(scores)\n    if verbose:\n        plt.figure(figsize=(14, 9))\n        plt.plot(quantiles, scores, '--')\n        plt.plot(quantiles[best_idx], scores[best_idx], '*r')\n        plt.text(quantiles[best_idx], scores[best_idx], str(scores[best_idx]))\n        plt.xlabel('Quantile')\n        plt.ylabel('ROC AUC')\n    return quantiles[best_idx]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T06:16:22.678046Z","iopub.execute_input":"2021-08-07T06:16:22.678464Z","iopub.status.idle":"2021-08-07T06:16:22.687938Z","shell.execute_reply.started":"2021-08-07T06:16:22.678430Z","shell.execute_reply":"2021-08-07T06:16:22.686782Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"QUANTILE = get_best_quantile(pred, target)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T06:16:23.662411Z","iopub.execute_input":"2021-08-07T06:16:23.662781Z","iopub.status.idle":"2021-08-07T06:21:40.200810Z","shell.execute_reply.started":"2021-08-07T06:16:23.662751Z","shell.execute_reply":"2021-08-07T06:21:40.199970Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"6\"></a>\n# Inference","metadata":{}},{"cell_type":"code","source":"pred, ids = predict(np.array(files_test_g), False)","metadata":{"papermill":{"duration":21.234649,"end_time":"2021-06-12T18:39:46.151391","exception":false,"start_time":"2021-06-12T18:39:24.916742","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T06:27:38.699928Z","iopub.execute_input":"2021-08-07T06:27:38.701053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(f'{COMPETITION_DATASET_PATH}/sample_submission.csv')\nsub['id'] = ids\nsub['target'] = np.quantile(pred, QUANTILE, axis=0)\nsub = sub.sort_values('id') \nsub.head()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"7\"></a>\n# Next steps\n* Add TTA Inference","metadata":{}}]}