{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Установка и импорт необходимых библиотек. Загрузка датасета","metadata":{}},{"cell_type":"code","source":"import re\nimport random\n\nimport tensorflow as tf\n# последовательная модель (стек слоев)\nfrom tensorflow.keras.models import Sequential, Model\n# полносвязный слой и слой выпрямляющий матрицу в вектор\nfrom tensorflow.keras.layers import Dense, Flatten, Input\n# слой выключения нейронов и слой нормализации выходных данных (нормализует данные в пределах текущей выборки)\nfrom tensorflow.keras.layers import (Dropout, BatchNormalization, SpatialDropout2D, GaussianDropout,\n                                     GlobalAveragePooling2D, Dropout, LeakyReLU)\n# слои свертки и подвыборки\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D\n# работа с обратной связью от обучающейся нейронной сети\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n# вспомогательные инструменты\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.regularizers import *\nimport numpy as np\nimport os\nfrom tensorflow.random import set_seed\ndef seed_everything(seed):\n    np.random.seed(seed)\n    set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nSEED = 42\nseed_everything(SEED)\n\n# работа с изображениями\nfrom tensorflow.keras.preprocessing import image\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\n#  библиотека для работы с наборами данных на Kaggle\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:34:36.17516Z","iopub.execute_input":"2025-12-16T16:34:36.175317Z","iopub.status.idle":"2025-12-16T16:34:52.301911Z","shell.execute_reply.started":"2025-12-16T16:34:36.175301Z","shell.execute_reply":"2025-12-16T16:34:52.301282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T06:25:18.231427Z","iopub.execute_input":"2025-12-16T06:25:18.231894Z","iopub.status.idle":"2025-12-16T06:25:18.235962Z","shell.execute_reply.started":"2025-12-16T06:25:18.23187Z","shell.execute_reply":"2025-12-16T06:25:18.23511Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TPU Setup","metadata":{}},{"cell_type":"code","source":"# Автоматическое определение стратегии: GPU (если есть), иначе CPU\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Включаем память по требованию (важно для Kaggle и Colab)\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        strategy = tf.distribute.MirroredStrategy()\n        print(f\"Running on {len(gpus)} GPU(s)\")\n    except RuntimeError as e:\n        # Должно быть вызвано до инициализации GPU\n        print(e)\n        strategy = tf.distribute.get_strategy()\nelse:\n    strategy = tf.distribute.get_strategy()\n    print(\"Running on CPU\")\n\nprint(\"REPLICAS:\", strategy.num_replicas_in_sync)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:34:52.303398Z","iopub.execute_input":"2025-12-16T16:34:52.303892Z","iopub.status.idle":"2025-12-16T16:34:52.991111Z","shell.execute_reply.started":"2025-12-16T16:34:52.303873Z","shell.execute_reply":"2025-12-16T16:34:52.990427Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Paths ","metadata":{}},{"cell_type":"markdown","source":"/kaggle/input/tpu-getting-started/tfrecords-jpeg-331x331/test/03-331x331-462.tfrec","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [331, 331]\nEPOCHS = 35\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync  # будет 16 на 1 GPU\nNUM_CLASSES = 104\n\nLOCAL_DS_PATH = \"/kaggle/input/tpu-getting-started/tfrecords-jpeg-331x331\"\n\nTRAINING_FILENAMES = tf.io.gfile.glob(os.path.join(LOCAL_DS_PATH, \"train/*.tfrec\"))\nVALIDATION_FILENAMES = tf.io.gfile.glob(os.path.join(LOCAL_DS_PATH, \"val/*.tfrec\"))\nTEST_FILENAMES = tf.io.gfile.glob(os.path.join(LOCAL_DS_PATH, \"test/*.tfrec\"))\n\nprint(\"Train files:\", len(TRAINING_FILENAMES))\nprint(\"Val files:\", len(VALIDATION_FILENAMES))\nprint(\"Test files:\", len(TEST_FILENAMES))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:35:02.971704Z","iopub.execute_input":"2025-12-16T16:35:02.972366Z","iopub.status.idle":"2025-12-16T16:35:02.994521Z","shell.execute_reply.started":"2025-12-16T16:35:02.97233Z","shell.execute_reply":"2025-12-16T16:35:02.993763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# 1. Декодирование и обработка изображений\n\ndef decode_and_resize(image_data):\n    \"\"\"\n    Декодирует JPEG-изображение и приводит его к заданному размеру.\n    Нормализует пиксели в диапазон [0, 1].\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.image.resize(image, IMAGE_SIZE, method='bilinear')\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\ndef augment_image(image, label):\n    # Случайное горизонтальное отражение\n    image = tf.image.random_flip_left_right(image)\n    # Случайные изменения яркости и насыщенности\n    image = tf.image.random_brightness(image, max_delta=0.15)\n    image = tf.image.random_saturation(image, lower=0.8, upper=1.2)\n    return image, label\n\n\n# 2. Чтение данных из TFRecord\n\ndef read_labeled_record(example):\n    \"\"\"Парсит пример с меткой класса (для train/val).\"\"\"\n    features = tf.io.parse_single_example(example, {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    })\n    image = decode_and_resize(features[\"image\"])\n    label = tf.cast(features[\"class\"], tf.int32)\n    return image, label\n\ndef read_unlabeled_record(example):\n    \"\"\"Парсит тестовый пример без метки — только id.\"\"\"\n    features = tf.io.parse_single_example(example, {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    })\n    image = decode_and_resize(features[\"image\"])\n    return image, features[\"id\"]\n\n\n# 3. Формирование датасетов\n\ndef build_dataset(filenames, labeled=True, ordered=False, augment=False):\n    \"\"\"\n    Создаёт tf.data.Dataset из списка TFRecord-файлов.\n    \n    \"\"\"\n    options = tf.data.Options()\n    if not ordered:\n        options.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(options)\n    \n    parse_fn = read_labeled_record if labeled else read_unlabeled_record\n    dataset = dataset.map(parse_fn, num_parallel_calls=AUTO)\n    \n    if augment:\n        dataset = dataset.map(augment_image, num_parallel_calls=AUTO)\n    \n    return dataset\n\ndef get_training_dataset():\n    \"\"\"Обучающий датасет с аугментацией, перемешиванием и повторением.\"\"\"\n    dataset = build_dataset(TRAINING_FILENAMES, labeled=True, augment=True)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(buffer_size=2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset():\n    \"\"\"Валидационный датасет — без аугментации, с кэшированием.\"\"\"\n    dataset = build_dataset(VALIDATION_FILENAMES, labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()  \n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=True):\n    \"\"\"Тестовый датасет — порядок важен для корректного сабмита.\"\"\"\n    dataset = build_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\n# 4. Вспомогательные функции\n\ndef count_samples(filenames):\n    \"\"\"Подсчитывает общее число примеров по именам TFRecord-файлов.\"\"\"\n    counts = [int(re.search(r\"-([0-9]+)\\.\", f).group(1)) for f in filenames]\n    return np.sum(counts)\n\n# Расчёт количества примеров и шагов на эпоху\nNUM_TRAINING_IMAGES = count_samples(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_samples(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_samples(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint(f\"Загружено данных: \"\n      f\"{NUM_TRAINING_IMAGES} обучающих, \"\n      f\"{NUM_VALIDATION_IMAGES} валидационных, \"\n      f\"{NUM_TEST_IMAGES} тестовых изображений.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:35:07.614094Z","iopub.execute_input":"2025-12-16T16:35:07.614385Z","iopub.status.idle":"2025-12-16T16:35:07.627311Z","shell.execute_reply.started":"2025-12-16T16:35:07.614364Z","shell.execute_reply":"2025-12-16T16:35:07.62669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, Model\n\ndef residual_block(x, filters, stride=1, downsample=False):\n    shortcut = x\n    x = layers.Conv2D(filters, 3, strides=stride, padding='same', use_bias=False)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Conv2D(filters, 3, strides=1, padding='same', use_bias=False)(x)\n    x = layers.BatchNormalization()(x)\n\n    if downsample:\n        shortcut = layers.Conv2D(filters, 1, strides=stride, padding='same', use_bias=False)(shortcut)\n        shortcut = layers.BatchNormalization()(shortcut)\n\n    x = layers.Add()([x, shortcut])\n    x = layers.ReLU()(x)\n    return x\n\ndef build_resnet34(input_shape, num_classes):\n    inputs = layers.Input(shape=input_shape)\n    \n    # Начальный слой\n    x = layers.Conv2D(64, 7, strides=2, padding='same', use_bias=False)(inputs)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n\n    # Stage 1 (64)\n    for _ in range(3):\n        x = residual_block(x, 64)\n\n    # Stage 2 (128)\n    x = residual_block(x, 128, stride=2, downsample=True)\n    for _ in range(3):\n        x = residual_block(x, 128)\n\n    # Stage 3 (256)\n    x = residual_block(x, 256, stride=2, downsample=True)\n    for _ in range(5):\n        x = residual_block(x, 256)\n\n    # Stage 4 (512)\n    x = residual_block(x, 512, stride=2, downsample=True)\n    for _ in range(2):\n        x = residual_block(x, 512)\n\n    # Голова классификации\n    x = layers.GlobalAveragePooling2D()(x)\n    outputs = layers.Dense(NUM_CLASSES, activation='softmax')(x)\n\n    return Model(inputs, outputs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:35:11.287773Z","iopub.execute_input":"2025-12-16T16:35:11.288268Z","iopub.status.idle":"2025-12-16T16:35:11.296098Z","shell.execute_reply.started":"2025-12-16T16:35:11.288241Z","shell.execute_reply":"2025-12-16T16:35:11.295434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model():\n    return build_resnet34(input_shape=(*IMAGE_SIZE, 3), num_classes=NUM_CLASSES)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:35:14.89915Z","iopub.execute_input":"2025-12-16T16:35:14.899427Z","iopub.status.idle":"2025-12-16T16:35:14.903465Z","shell.execute_reply.started":"2025-12-16T16:35:14.899405Z","shell.execute_reply":"2025-12-16T16:35:14.902763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with strategy.scope():\n    model = build_resnet34(input_shape=(*IMAGE_SIZE, 3), num_classes=NUM_CLASSES)\n    model.compile(\n        optimizer='adam',\n        loss='sparse_categorical_crossentropy',\n        metrics=['accuracy']\n    )\n\n# Вывод структуры (можно за пределами scope)\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:35:17.588421Z","iopub.execute_input":"2025-12-16T16:35:17.589143Z","iopub.status.idle":"2025-12-16T16:35:19.330067Z","shell.execute_reply.started":"2025-12-16T16:35:17.58912Z","shell.execute_reply":"2025-12-16T16:35:19.329524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Создание и компиляция модели ---\nwith strategy.scope():\n    model = get_model() \n    model.compile(\n        optimizer='nadam',\n        loss='sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n\nprint(\"Начинаем обучение модели...\")\nprint(f\"  • Общее число эпох: {EPOCHS}\")\nprint(f\"  • Batch size: {BATCH_SIZE} (реплик: {strategy.num_replicas_in_sync})\")\nprint(f\"  • Шагов на эпоху: {STEPS_PER_EPOCH}\")\nprint(f\"  • Обучающих изображений: {NUM_TRAINING_IMAGES}\")\nprint(f\"  • Валидационных изображений: {NUM_VALIDATION_IMAGES}\")\nprint(\"-\" * 50)\n\n# --- Запуск обучения ---\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=get_validation_dataset(),\n    #callbacks=callbacks_list,\n    verbose=1\n)\n\nprint(\"\\nОбучение завершено!\")\nprint(f\"  • Пройдено эпох: {len(history.history['loss'])}\")\nif 'lr' in history.history:\n    print(f\"  • Финальная learning rate: {history.history['lr'][-1]:.2e}\")\n\n# --- Загружаем лучшую модель (по val_loss) ---\nprint(\"\\nЗагружаем лучшую модель по валидационной метрике...\")\n#model = tf.keras.models.load_model(checkpoint_path)\nprint(\"Лучшая модель загружена успешно.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T16:35:28.312769Z","iopub.execute_input":"2025-12-16T16:35:28.313544Z","iopub.status.idle":"2025-12-16T16:35:52.373087Z","shell.execute_reply.started":"2025-12-16T16:35:28.31352Z","shell.execute_reply":"2025-12-16T16:35:52.371721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.metrics import classification_report, f1_score, confusion_matrix\nimport tensorflow as tf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T14:20:58.517304Z","iopub.execute_input":"2025-12-16T14:20:58.517657Z","iopub.status.idle":"2025-12-16T14:20:58.880005Z","shell.execute_reply.started":"2025-12-16T14:20:58.517626Z","shell.execute_reply":"2025-12-16T14:20:58.87931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_training_history(history):\n    fig, ax = plt.subplots(1, 2, figsize=(15, 5))\n    \n    # Loss\n    ax[0].plot(history.history['loss'], label='Train Loss')\n    ax[0].plot(history.history['val_loss'], label='Val Loss')\n    ax[0].set_title('Model Loss')\n    ax[0].set_xlabel('Epoch')\n    ax[0].set_ylabel('Loss')\n    ax[0].legend()\n    ax[0].grid(True)\n    \n    # Accuracy\n    if 'sparse_categorical_accuracy' in history.history:\n        acc_key = 'sparse_categorical_accuracy'\n    else:\n        acc_key = 'accuracy'\n    \n    ax[1].plot(history.history[acc_key], label='Train Accuracy')\n    ax[1].plot(history.history[f'val_{acc_key}'], label='Val Accuracy')\n    ax[1].set_title('Model Accuracy')\n    ax[1].set_xlabel('Epoch')\n    ax[1].set_ylabel('Accuracy')\n    ax[1].legend()\n    ax[1].grid(True)\n    \n    plt.tight_layout()\n    plt.show()\n\n# Построить графики\nplot_training_history(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T07:56:07.628615Z","iopub.execute_input":"2025-12-16T07:56:07.629506Z","iopub.status.idle":"2025-12-16T07:56:08.128905Z","shell.execute_reply.started":"2025-12-16T07:56:07.62947Z","shell.execute_reply":"2025-12-16T07:56:08.128097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Загружаем тестовый датасет \ntest_dataset = get_test_dataset(ordered=True)\n\n# 2. Извлекаем ID и предсказания\ntest_ids = []\ntest_labels = []\n\n# Проходим по всем батчам\nfor images, ids in test_dataset:\n    # Предсказание\n    predictions = model.predict(images, verbose=0)\n    # Получаем класс с максимальной вероятностью\n    labels = np.argmax(predictions, axis=1)\n    \n    # Сохраняем\n    test_ids.extend(ids.numpy())\n    test_labels.extend(labels)\n\n# 3. Декодируем ID из байтов в строки\ntest_ids = [id_.decode('utf-8') for id_ in test_ids]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T14:21:04.973433Z","iopub.execute_input":"2025-12-16T14:21:04.974334Z","iopub.status.idle":"2025-12-16T14:22:30.52429Z","shell.execute_reply.started":"2025-12-16T14:21:04.974302Z","shell.execute_reply":"2025-12-16T14:22:30.523098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# 4. Создаём DataFrame\nsubmission_df = pd.DataFrame({\n    'id': test_ids,\n    'label': test_labels\n})\n\n# 5. Сохраняем в файл \nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"Файл submission.csv успешно создан!\")\nprint(\"Пример:\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T14:23:05.606042Z","iopub.execute_input":"2025-12-16T14:23:05.606405Z","iopub.status.idle":"2025-12-16T14:23:05.646043Z","shell.execute_reply.started":"2025-12-16T14:23:05.606366Z","shell.execute_reply":"2025-12-16T14:23:05.645381Z"}},"outputs":[],"execution_count":null}]}