{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":21154,"databundleVersionId":1243559}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"358574cc","cell_type":"markdown","source":"# Petals to the Metal — EfficientNet (GPU)\n\nКлассификация **104** классов цветов для соревнования [tpu-getting-started](https://www.kaggle.com/competitions/tpu-getting-started). Обучение на **GPU** (`MirroredStrategy`), данные из **TFRecord** (разрешение **224×224**). Сабмит: `submission.csv` (`id`, `label`).","metadata":{}},{"id":"de30e71c","cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.utils.class_weight import compute_class_weight\n\nprint(\"TensorFlow:\", tf.__version__)","metadata":{},"outputs":[],"execution_count":null},{"id":"877efa4e","cell_type":"markdown","source":"## Распределённая стратегия (GPU)\n\nНа Kaggle: **Settings → Accelerator → GPU**. `MirroredStrategy` при нескольких GPU, иначе одно устройство.","metadata":{}},{"id":"0729a914","cell_type":"code","source":"gpus = tf.config.list_physical_devices(\"GPU\")\nprint(\"TF devices:\", gpus)\n\nif gpus:\n    strategy = tf.distribute.MirroredStrategy()\n    print(\"Strategy: MirroredStrategy,\", len(gpus), \"GPU(s)\")\nelse:\n    strategy = tf.distribute.get_strategy()\n    print(\"Strategy: default (CPU)\")\n\nprint(\"REPLICAS:\", strategy.num_replicas_in_sync)","metadata":{},"outputs":[],"execution_count":null},{"id":"f456905b","cell_type":"markdown","source":"## Путь к данным","metadata":{}},{"id":"759df689","cell_type":"code","source":"import os\n\nCOMPETITION = \"tpu-getting-started\"\n\n\ndef _has_tfrec_shards(base):\n    root224 = f\"{base.rstrip('/')}/tfrecords-jpeg-224x224\"\n    return os.path.isdir(root224) and (\n        os.path.isdir(os.path.join(root224, \"train\"))\n        or bool(tf.io.gfile.glob(root224 + \"/*.tfrec\"))\n    )\n\n\ndef _resolve_competition_path(competition=COMPETITION):\n    candidates = [\n        f\"/kaggle/input/competitions/{competition}\",\n        f\"/kaggle/input/{competition}\",\n    ]\n    comp_root = \"/kaggle/input/competitions\"\n    if os.path.isdir(comp_root):\n        for name in sorted(os.listdir(comp_root)):\n            candidates.append(f\"{comp_root}/{name}\")\n\n    seen = set()\n    for cand in candidates:\n        if not cand or cand in seen:\n            continue\n        seen.add(cand)\n        if _has_tfrec_shards(cand):\n            return cand.rstrip(\"/\")\n\n    return candidates[0].rstrip(\"/\")\n\n\nGCS_DS_PATH = _resolve_competition_path()\nprint(\"GCS_DS_PATH:\", GCS_DS_PATH)\n\n_root224 = f\"{GCS_DS_PATH}/tfrecords-jpeg-224x224\"\n_has = _has_tfrec_shards(GCS_DS_PATH)\nprint(\"TFRecord 224 root exists:\", os.path.isdir(_root224), \"has shards:\", _has)\n\nif not _has:\n    print(\n        \"Подсказка: TFRecord 224×224 не найдены. Проверьте Input → competition data \"\n        \"и Restart Session.\"\n    )","metadata":{},"outputs":[],"execution_count":null},{"id":"8a9dee64-eb6d-4edf-828e-dfe8ff7e00cd","cell_type":"markdown","source":"## Гиперпараметры\n\nEfficientNet ожидает вход **224×224** (после `preprocess_input` из `keras.applications.efficientnet`). Числа примеров — как в описании датасета.","metadata":{}},{"id":"4540732d","cell_type":"code","source":"IMAGE_SIZE = [224, 224]\nNUM_CLASSES = 104\n\nEPOCHS = 40\nGLOBAL_BATCH_SIZE = 16 * strategy.num_replicas_in_sync\nPER_REPLICA_BATCH_SIZE = max(GLOBAL_BATCH_SIZE // strategy.num_replicas_in_sync, 1)\n\nNUM_TRAINING_IMAGES = 12753\nNUM_VALIDATION_IMAGES = 3712\nNUM_TEST_IMAGES = 7382\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // GLOBAL_BATCH_SIZE\nVALIDATION_STEPS = (NUM_VALIDATION_IMAGES + GLOBAL_BATCH_SIZE - 1) // GLOBAL_BATCH_SIZE\n\n# Путь к *.tfrec строится в ячейке «Загрузка TFRecord» из GCS_DS_PATH — одна точка правды.\n\nprint(\n    \"GLOBAL_BATCH_SIZE:\", GLOBAL_BATCH_SIZE,\n    \"PER_REPLICA_BATCH_SIZE:\", PER_REPLICA_BATCH_SIZE,\n    \"STEPS_PER_EPOCH:\", STEPS_PER_EPOCH,\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"2048490b","cell_type":"markdown","source":"## Загрузка TFRecord (`tf.data`)\n\nАугментации только для train. Пиксели в **\\[0, 255\\]** перед `efficientnet.preprocess_input`.","metadata":{}},{"id":"bf51cf80","cell_type":"code","source":"from tensorflow.keras.applications import efficientnet as effnet\n\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.image.resize(image, IMAGE_SIZE, method=\"bilinear\")\n    image = tf.cast(image, tf.float32)\n    return image\n\n\ndef augment_image(image):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_brightness(image, 0.12)\n    image = tf.image.random_contrast(image, 0.85, 1.15)\n    image = tf.image.random_saturation(image, 0.85, 1.15)\n    image = tf.clip_by_value(image, 0.0, 255.0)\n    return image\n\n\ndef preprocess_for_model(image):\n    return effnet.preprocess_input(image)\n\n\ndef read_labeled_tfrecord(example):\n    fmt = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    ex = tf.io.parse_single_example(example, fmt)\n    image = decode_image(ex[\"image\"])\n    label = tf.cast(ex[\"class\"], tf.int32)\n    return image, label\n\n\ndef read_unlabeled_tfrecord(example):\n    fmt = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    ex = tf.io.parse_single_example(example, fmt)\n    image = decode_image(ex[\"image\"])\n    idnum = ex[\"id\"]\n    return image, idnum\n\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    if not filenames:\n        raise ValueError(\"TFRecord filenames list is empty — проверьте GCS_DS_PATH и Input.\")\n    opts = tf.data.Options()\n    if not ordered:\n        opts.experimental_deterministic = False\n    if strategy.num_replicas_in_sync > 1:\n        opts.experimental_distribute.auto_shard_policy = (\n            tf.data.experimental.AutoShardPolicy.DATA\n        )\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=tf.data.AUTOTUNE)\n    dataset = dataset.with_options(opts)\n    dataset = dataset.map(\n        read_labeled_tfrecord if labeled else read_unlabeled_tfrecord,\n        num_parallel_calls=tf.data.AUTOTUNE,\n    )\n    return dataset\n\n\ndef add_train_preprocessing(ds):\n    def _prep(image, label):\n        image = augment_image(image)\n        image = preprocess_for_model(image)\n        return image, label\n\n    return ds.map(_prep, num_parallel_calls=tf.data.AUTOTUNE)\n\n\ndef add_eval_preprocessing(ds):\n    def _prep(image, label):\n        image = preprocess_for_model(image)\n        return image, label\n\n    return ds.map(_prep, num_parallel_calls=tf.data.AUTOTUNE)\n\n\ndef _flowers_tfrec_prefix():\n    \"\"\"Корень папки 224×224: есть либо train/*.tfrec, либо (старый вариант) все шарды *.tfrec в корне.\"\"\"\n    base = GCS_DS_PATH.rstrip(\"/\")\n    standard = f\"{base}/tfrecords-jpeg-224x224\"\n\n    def has_any_shard(root):\n        return bool(\n            tf.io.gfile.glob(root + \"/train/*.tfrec\")\n            or tf.io.gfile.glob(root + \"/*.tfrec\")\n        )\n\n    if has_any_shard(standard):\n        return standard\n    try:\n        names = tf.io.gfile.listdir(base)\n    except tf.errors.OpError:\n        names = []\n    for name in sorted(names):\n        if \"tfrec\" not in name.lower():\n            continue\n        cand = f\"{base}/{name}\"\n        if has_any_shard(cand):\n            if name != \"tfrecords-jpeg-224x224\":\n                print(\"Используется папка TFRecord:\", cand)\n            return cand\n    return standard\n\n\ndef get_train_filenames():\n    root = _flowers_tfrec_prefix()\n    split_files = tf.io.gfile.glob(root + \"/train/*.tfrec\")\n    if split_files:\n        return sorted(split_files)\n    return sorted(tf.io.gfile.glob(root + \"/*.tfrec\"))\n\n\ndef get_val_filenames():\n    root = _flowers_tfrec_prefix()\n    return sorted(tf.io.gfile.glob(root + \"/val/*.tfrec\"))\n\n\ndef get_test_filenames():\n    root = _flowers_tfrec_prefix()\n    split_files = tf.io.gfile.glob(root + \"/test/*.tfrec\")\n    if split_files:\n        return sorted(split_files)\n    return []\n\n\ndef get_training_dataset():\n    ds = load_dataset(get_train_filenames(), labeled=True, ordered=False)\n    ds = add_train_preprocessing(ds)\n    ds = ds.shuffle(4096)\n    ds = ds.repeat()\n    ds = ds.batch(PER_REPLICA_BATCH_SIZE, drop_remainder=True)\n    return ds.prefetch(tf.data.AUTOTUNE)\n\n\ndef get_validation_dataset(ordered=False):\n    ds = load_dataset(get_val_filenames(), labeled=True, ordered=ordered)\n    ds = add_eval_preprocessing(ds)\n    ds = ds.batch(PER_REPLICA_BATCH_SIZE)\n    if strategy.num_replicas_in_sync == 1:\n        ds = ds.cache()\n    return ds.prefetch(tf.data.AUTOTUNE)\n\n\ndef get_test_dataset(ordered=False):\n    ds = load_dataset(get_test_filenames(), labeled=False, ordered=ordered)\n\n    def _prep(image, idnum):\n        return preprocess_for_model(image), idnum\n\n    ds = ds.map(_prep, num_parallel_calls=tf.data.AUTOTUNE)\n    ds = ds.batch(PER_REPLICA_BATCH_SIZE)\n    return ds.prefetch(tf.data.AUTOTUNE)\n\n\n_train_n = len(get_train_filenames())\n_val_n = len(get_val_filenames())\n_test_n = len(get_test_filenames())\nprint(\"TFRecord counts — train:\", _train_n, \"val:\", _val_n, \"test:\", _test_n)\n\nif _train_n == 0:\n    raise ValueError(\"Нет train TFRecord — перезапустите ячейку «Путь к данным» и TFRecord.\")\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset(ordered=False)\n_x, _y = next(iter(training_dataset))\nprint(\n    \"Train batch OK (per replica):\", _x.shape, _y.shape,\n    \"global batch:\", _x.shape[0] * strategy.num_replicas_in_sync,\n    \"REPLICAS:\", strategy.num_replicas_in_sync,\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"cc8f6317-064a-4b53-98ee-7750c43de881","cell_type":"markdown","source":"## Веса классов (дисбаланс → ближе к **macro F1**)\n\nОдин проход по train только для меток (без тяжёлого обучения).","metadata":{}},{"id":"5b7ed4fc","cell_type":"code","source":"_label_files = get_train_filenames()\n_from_val = False\nif not _label_files:\n    _vf = get_val_filenames()\n    if _vf:\n        _label_files = _vf\n        _from_val = True\n        print(\n            \"Веса классов: train TFRecord недоступен — считаю balanced-веса по val \"\n            \"(распределение классов близко к train).\"\n        )\n\nif not _label_files:\n    print(\n        \"Предупреждение: нет ни train, ни val TFRecord — class_weight = 1.0 для всех классов.\"\n    )\n    class_weight = {int(i): 1.0 for i in range(NUM_CLASSES)}\nelse:\n    label_ds = load_dataset(_label_files, labeled=True, ordered=True).map(\n        lambda img, y: y, num_parallel_calls=tf.data.AUTOTUNE\n    )\n    _batches = [b.numpy() for b in label_ds.batch(2048)]\n    if not _batches:\n        raise RuntimeError(\n            \"label_ds пуст после batch — проверьте целостность TFRecord.\"\n        )\n    y_train = np.concatenate(_batches, dtype=np.int64)\n    if _from_val:\n        assert y_train.shape[0] == NUM_VALIDATION_IMAGES\n    else:\n        assert y_train.shape[0] == NUM_TRAINING_IMAGES\n\n    cw = compute_class_weight(\n        \"balanced\", classes=np.arange(NUM_CLASSES), y=y_train\n    )\n    class_weight = {int(i): float(cw[i]) for i in range(NUM_CLASSES)}\n\n\ndef weighted_sparse_ce(class_weight_dict):\n    \"\"\"Веса классов в loss (совместимо с MirroredStrategy / DistributedDataset).\"\"\"\n    w = tf.constant(\n        [float(class_weight_dict[i]) for i in range(NUM_CLASSES)],\n        dtype=tf.float32,\n    )\n\n    def loss(y_true, y_pred):\n        y_true = tf.cast(tf.reshape(y_true, [-1]), tf.int32)\n        ce = tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred)\n        return tf.reduce_mean(ce * tf.gather(w, y_true))\n\n    return loss\n\n\nprint(\"class_weight[0]:\", class_weight[0], \"class_weight[34]:\", class_weight[34])","metadata":{},"outputs":[],"execution_count":null},{"id":"c8e70231-09a9-474d-8a0a-f017720a2f25","cell_type":"markdown","source":"## Модель: EfficientNetB0 + головка (в `strategy.scope`)","metadata":{}},{"id":"bcb3f790","cell_type":"code","source":"with strategy.scope():\n    base = tf.keras.applications.EfficientNetB0(\n        include_top=False,\n        weights=\"imagenet\",\n        input_shape=[*IMAGE_SIZE, 3],\n    )\n    base.trainable = False\n\n    inputs = tf.keras.Input(shape=[*IMAGE_SIZE, 3])\n    x = base(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    outputs = tf.keras.layers.Dense(NUM_CLASSES, activation=\"softmax\")(x)\n    model = tf.keras.Model(inputs, outputs)\n\n    lr = 1e-3 * strategy.num_replicas_in_sync / 8.0\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr),\n        loss=weighted_sparse_ce(class_weight),\n        metrics=[\"sparse_categorical_accuracy\"],\n    )\n\nmodel.summary()","metadata":{},"outputs":[],"execution_count":null},{"id":"fc0ab41f","cell_type":"markdown","source":"## Обучение (EfficientNet backbone заморожен)","metadata":{}},{"id":"2d1fbd42","cell_type":"code","source":"training_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset(ordered=False)\n\nhistory = model.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    validation_steps=VALIDATION_STEPS,\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"96f4c4ba","cell_type":"markdown","source":"## Кривые обучения (loss)","metadata":{}},{"id":"83e7006f","cell_type":"code","source":"import matplotlib.pyplot as plt\n\ntrain_loss = history.history.get(\"loss\", [])\nval_loss = history.history.get(\"val_loss\", [])\nepochs = range(1, len(train_loss) + 1)\n\nplt.figure(figsize=(10, 5))\nplt.plot(epochs, train_loss, label=\"train loss\", marker=\"o\", markersize=4)\nif val_loss:\n    plt.plot(epochs, val_loss, label=\"val loss\", marker=\"o\", markersize=4)\nplt.xlabel(\"epoch\")\nplt.ylabel(\"loss\")\nplt.title(\"EfficientNet — кривые loss\")\nplt.legend()\nplt.grid(True, alpha=0.3)\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"d12eee87","cell_type":"markdown","source":"## **Macro F1**\n\nЕсли доступен split `val`, считаем validation F1 как на лидерборде. Если `val` не найден, ячейка пробует диагностический F1 на train-файлах. Если данных для метрики нет, ячейка пропускает F1 без ошибки.","metadata":{}},{"id":"940527b1","cell_type":"code","source":"val_files = get_val_filenames()\ntrain_files = get_train_filenames()\nprint(\"Validation TFRecord files:\", len(val_files))\n\nmacro_f1 = None\nif val_files:\n    eval_name = \"Validation\"\n    eval_size = NUM_VALIDATION_IMAGES\n    eval_ds = get_validation_dataset(ordered=True)\nelif train_files:\n    eval_name = \"Train diagnostic\"\n    eval_size = NUM_TRAINING_IMAGES\n    print(\n        \"Validation split not found, so macro F1 will be computed on train files only \"\n        \"(this is not a leaderboard-style validation score).\"\n    )\n    eval_ds = load_dataset(train_files, labeled=True, ordered=True)\n    eval_ds = add_eval_preprocessing(eval_ds)\n    eval_ds = eval_ds.batch(PER_REPLICA_BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\nelse:\n    eval_name = None\n    eval_ds = None\n    print(\"No validation or train TFRecord files found for F1; skipping this metric.\")\n\nif eval_ds is not None:\n    y_true = []\n    y_pred = []\n    for batch_x, batch_y in eval_ds:\n        p = model.predict_on_batch(batch_x)\n        y_true.append(batch_y.numpy())\n        y_pred.append(np.argmax(p, axis=-1))\n\n    if y_true:\n        y_true = np.concatenate(y_true)[:eval_size]\n        y_pred = np.concatenate(y_pred)[:eval_size]\n        macro_f1 = f1_score(y_true, y_pred, average=\"macro\")\n        print(f\"{eval_name} macro F1:\", macro_f1)\n    else:\n        print(\"Evaluation dataset produced zero batches; skipping macro F1.\")","metadata":{},"outputs":[],"execution_count":null},{"id":"65920534","cell_type":"markdown","source":"## Тест и `submission.csv`\n\nЕсли test TFRecord не найден или даёт 0 батчей, ячейка пропускает генерацию `submission.csv` без ошибки.","metadata":{}},{"id":"aca9bd1d","cell_type":"code","source":"test_files = get_test_filenames()\nprint(\"Test TFRecord files:\", len(test_files))\n\nif not test_files:\n    print(\"No test TFRecord files found; skipping submission.csv generation.\")\nelse:\n    test_ds = get_test_dataset(ordered=True)\n    test_ids_batches = []\n    probabilities_batches = []\n\n    for batch_x, batch_ids in test_ds:\n        probabilities_batches.append(model.predict_on_batch(batch_x))\n        test_ids_batches.append(batch_ids.numpy().astype(\"U\"))\n\n    if not probabilities_batches:\n        print(\"Test dataset produced zero batches; skipping submission.csv generation.\")\n    else:\n        probabilities = np.concatenate(probabilities_batches, axis=0)\n        predictions = np.argmax(probabilities, axis=-1).reshape(-1)\n        test_ids = np.concatenate(test_ids_batches, axis=0).reshape(-1)\n\n        # MirroredStrategy иногда удваивает число предсказаний относительно id\n        if len(predictions) == 2 * len(test_ids):\n            predictions = predictions[: len(test_ids)]\n        n = min(len(test_ids), len(predictions), NUM_TEST_IMAGES)\n        test_ids = test_ids[:n]\n        predictions = predictions[:n].astype(int)\n\n        if test_ids.dtype.kind in (\"S\", \"O\"):\n            test_ids = np.array(\n                [x.decode() if isinstance(x, (bytes, np.bytes_)) else str(x) for x in test_ids],\n                dtype=\"U\",\n            )\n\n        predictions = predictions.astype(np.int64)\n        print(\"submission rows:\", n)\n\n        with open(\"submission.csv\", \"w\", encoding=\"utf-8\") as f:\n            f.write(\"id,label\\n\")\n            for img_id, label in zip(test_ids, predictions):\n                f.write(f\"{img_id},{int(label)}\\n\")\n\n        print(\"Wrote submission.csv\")","metadata":{},"outputs":[],"execution_count":null}]}