{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.python as tfp\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-12T01:37:55.722961Z","iopub.execute_input":"2021-09-12T01:37:55.723401Z","iopub.status.idle":"2021-09-12T01:38:00.780362Z","shell.execute_reply.started":"2021-09-12T01:37:55.723315Z","shell.execute_reply":"2021-09-12T01:38:00.779514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\n#test_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\ntest_df = pd.read_csv('../input/g2net-efficientnetb3-b7-ensemble/submission.csv')\n\ndef get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ntrain_df['image_path'] = train_df['id'].apply(get_train_file_path)\ntest_df['image_path'] = test_df['id'].apply(get_test_file_path)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:00.781527Z","iopub.execute_input":"2021-09-12T01:38:00.781926Z","iopub.status.idle":"2021-09-12T01:38:02.249784Z","shell.execute_reply.started":"2021-09-12T01:38:00.781898Z","shell.execute_reply":"2021-09-12T01:38:02.249008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = pd.read_csv('../input/g2net-tf-on-the-fly-cqt-tpu-training/oof/oof.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:02.251321Z","iopub.execute_input":"2021-09-12T01:38:02.251662Z","iopub.status.idle":"2021-09-12T01:38:02.525469Z","shell.execute_reply.started":"2021-09-12T01:38:02.25163Z","shell.execute_reply":"2021-09-12T01:38:02.524448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:02.527177Z","iopub.execute_input":"2021-09-12T01:38:02.527488Z","iopub.status.idle":"2021-09-12T01:38:02.548102Z","shell.execute_reply.started":"2021-09-12T01:38:02.52746Z","shell.execute_reply":"2021-09-12T01:38:02.547019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:02.549318Z","iopub.execute_input":"2021-09-12T01:38:02.54961Z","iopub.status.idle":"2021-09-12T01:38:03.137625Z","shell.execute_reply.started":"2021-09-12T01:38:02.549584Z","shell.execute_reply":"2021-09-12T01:38:03.136593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = 0.5\nN=100\ndx=1/N\nbest_score = -2\nfor i in range(N):\n    p = i*dx\n    score = roc_auc_score(oof.y_true, oof.y_pred>p)\n    if best_score < score:\n        best_score = score\n        best_p = p","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:03.139094Z","iopub.execute_input":"2021-09-12T01:38:03.139392Z","iopub.status.idle":"2021-09-12T01:38:17.930453Z","shell.execute_reply.started":"2021-09-12T01:38:03.139363Z","shell.execute_reply":"2021-09-12T01:38:17.929469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (best_p, best_score)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:17.93178Z","iopub.execute_input":"2021-09-12T01:38:17.932102Z","iopub.status.idle":"2021-09-12T01:38:17.937136Z","shell.execute_reply.started":"2021-09-12T01:38:17.93207Z","shell.execute_reply":"2021-09-12T01:38:17.936048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.target = (test_df.target > best_p).astype(int)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:17.940089Z","iopub.execute_input":"2021-09-12T01:38:17.940531Z","iopub.status.idle":"2021-09-12T01:38:17.960078Z","shell.execute_reply.started":"2021-09-12T01:38:17.940489Z","shell.execute_reply":"2021-09-12T01:38:17.95887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    if isinstance(value, tfp.framework.ops.EagerTensor):\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:17.961365Z","iopub.execute_input":"2021-09-12T01:38:17.961653Z","iopub.status.idle":"2021-09-12T01:38:17.968407Z","shell.execute_reply.started":"2021-09-12T01:38:17.961622Z","shell.execute_reply":"2021-09-12T01:38:17.967383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_tf_example(wave_id: str, wave: bytes) -> tf.train.Example:\n#     feature = {\n#         \"wave_id\": _bytes_feature(wave_id),\n#         \"wave\": _bytes_feature(wave)\n#     }\n#     return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\n# def write_tfrecord(df: pd.DataFrame, filename: str):\n#     options = tf.io.TFRecordOptions(\"GZIP\")\n#     with tf.io.TFRecordWriter(filename, options=options) as writer:\n#         for i in tqdm(range(len(df))):\n#             wave_id = str.encode(df.iloc[i][\"id\"])\n#             wave_dir = df.iloc[i][\"image_path\"]\n#             wave = np.load(wave_dir).tobytes()\n#             tf_example = create_tf_example(wave_id, wave)\n#             writer.write(tf_example.SerializeToString())\n            \ndef create_tf_example(wave_id: str, wave: bytes, target: int) -> tf.train.Example:\n    feature = {\n        \"wave_id\": _bytes_feature(wave_id),\n        \"wave\": _bytes_feature(wave),\n        \"target\": _int64_feature(target)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef write_tfrecord(df: pd.DataFrame, filename: str):\n    options = tf.io.TFRecordOptions(\"GZIP\")\n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        for i in tqdm(range(len(df))):\n            wave_id = str.encode(df.iloc[i][\"id\"])\n            wave_dir = df.iloc[i][\"image_path\"]\n            wave = np.load(wave_dir).tobytes()\n            target = df.iloc[i][\"target\"]\n            tf_example = create_tf_example(wave_id, wave, target)\n            writer.write(tf_example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:17.969594Z","iopub.execute_input":"2021-09-12T01:38:17.970089Z","iopub.status.idle":"2021-09-12T01:38:17.979658Z","shell.execute_reply.started":"2021-09-12T01:38:17.970051Z","shell.execute_reply":"2021-09-12T01:38:17.978496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_samples_per_file = 22600\ntest_number_of_files = len(test_df) // test_samples_per_file\n\nfor i in range(0, 5):\n    start = i * test_samples_per_file\n    end = (i + 1) * test_samples_per_file\n    df = test_df.iloc[start:end].reset_index(drop=True)\n    filename = f\"test{i}-{test_samples_per_file}.tfrecords\"\n    write_tfrecord(df, filename)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T01:38:17.981237Z","iopub.execute_input":"2021-09-12T01:38:17.981739Z","iopub.status.idle":"2021-09-12T01:41:59.533425Z","shell.execute_reply.started":"2021-09-12T01:38:17.981696Z","shell.execute_reply":"2021-09-12T01:41:59.527593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}