{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🐋🐬 Convert backfintfrecords\n\nConvert Jan Bre's [backfintfrecords dataset](https://www.kaggle.com/datasets/jpbremer/backfintfrecords) to the competitions dataset format.","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import re\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom PIL import Image\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-17T09:34:39.623103Z","iopub.execute_input":"2022-03-17T09:34:39.623979Z","iopub.status.idle":"2022-03-17T09:34:45.211301Z","shell.execute_reply.started":"2022-03-17T09:34:39.623883Z","shell.execute_reply":"2022-03-17T09:34:45.210347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paths & Settings","metadata":{}},{"cell_type":"code","source":"KAGGLE_DIR = Path(\"/\") / \"kaggle\"\nHAPPY_WHALE_AND_DOLPHIN_DIR = KAGGLE_DIR / \"input\" / \"happy-whale-and-dolphin\"\nBACKFINTFRECORDS_DIR = KAGGLE_DIR / \"input\" / \"backfintfrecords\"\nHAPPY_WHALE_AND_DOLPHIN_BACKFIN_DIR = KAGGLE_DIR / \"working\" / \"happy-whale-and-dolphin-backfin\"\n\nIMAGE_SIZE = (512, 512)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T09:34:45.212965Z","iopub.execute_input":"2022-03-17T09:34:45.213228Z","iopub.status.idle":"2022-03-17T09:34:45.22068Z","shell.execute_reply.started":"2022-03-17T09:34:45.213199Z","shell.execute_reply":"2022-03-17T09:34:45.217481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndef load_dataset(filenames, image_size):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(lambda x: read_labeled_tfrecord(x, image_size))\n    return dataset\n\n\ndef read_labeled_tfrecord(example, image_size):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"], image_size)\n    image_name = example[\"image_name\"]\n    target = example[\"target\"]\n\n    return image, image_name, target\n\n\ndef decode_image(image_data, image_size):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-03-17T09:34:45.221746Z","iopub.execute_input":"2022-03-17T09:34:45.222173Z","iopub.status.idle":"2022-03-17T09:34:45.241149Z","shell.execute_reply.started":"2022-03-17T09:34:45.222124Z","shell.execute_reply":"2022-03-17T09:34:45.240477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Conversion","metadata":{}},{"cell_type":"code","source":"def convert_dataset(step, image_size):\n    print(f\"Converting {step} backfintfrecords with image size {image_size}\")\n\n    backfin_images_dir = HAPPY_WHALE_AND_DOLPHIN_BACKFIN_DIR / f\"{step}_images\"\n    backfin_images_dir.mkdir(parents=True, exist_ok=True)\n    print(f\"Created {backfin_images_dir}\")\n\n    filenames = tf.io.gfile.glob(f\"{BACKFINTFRECORDS_DIR}/happywhale-2022-{step}*.tfrec\")\n    print(f\"Number of {step} tfrecords: {len(filenames)}\")\n\n    num_items = count_data_items(filenames)\n    print(f\"Number of {step} images: {num_items}\")\n\n    dataset = load_dataset(filenames, image_size)\n\n    image_names = []\n    for sample in tqdm(dataset, total=num_items, desc=f\"Saving {step} images\"):\n        image, image_name, _ = sample\n        image, image_name = image.numpy(), image_name.numpy().decode(\"utf-8\")\n\n        image = (image * 255.0).astype(np.uint8)\n        image = Image.fromarray(image)\n        image.save(backfin_images_dir / image_name)\n\n        image_names.append(image_name)\n\n    df_backfin = pd.DataFrame({\"image\": image_names})\n    \n    filename = f\"{step}.csv\" if step == \"train\" else \"sample_submission.csv\"\n\n    # Remove missing images from original df\n    df_original = pd.read_csv(HAPPY_WHALE_AND_DOLPHIN_DIR / filename)\n    df_backfin = pd.merge(df_original, df_backfin, how=\"inner\", on=\"image\")\n\n    df_backfin.to_csv(HAPPY_WHALE_AND_DOLPHIN_BACKFIN_DIR / filename, index=False)\n    \n    print(f\"Created {HAPPY_WHALE_AND_DOLPHIN_BACKFIN_DIR / filename}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-17T09:34:45.242536Z","iopub.execute_input":"2022-03-17T09:34:45.243014Z","iopub.status.idle":"2022-03-17T09:34:45.255751Z","shell.execute_reply.started":"2022-03-17T09:34:45.242979Z","shell.execute_reply":"2022-03-17T09:34:45.255052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_dataset(\"train\", IMAGE_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T09:34:45.25761Z","iopub.execute_input":"2022-03-17T09:34:45.257962Z","iopub.status.idle":"2022-03-17T09:35:58.459473Z","shell.execute_reply.started":"2022-03-17T09:34:45.257933Z","shell.execute_reply":"2022-03-17T09:35:58.458739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_dataset(\"test\", IMAGE_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T09:35:58.460688Z","iopub.execute_input":"2022-03-17T09:35:58.461309Z","iopub.status.idle":"2022-03-17T09:36:47.209553Z","shell.execute_reply.started":"2022-03-17T09:35:58.461269Z","shell.execute_reply":"2022-03-17T09:36:47.208648Z"},"trusted":true},"execution_count":null,"outputs":[]}]}