{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport os\nimport pandas as pd\nfrom typing import Tuple, List, Dict, Any\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Проверка доступности GPU\ngpus = tf.config.list_physical_devices('GPU')\n\nclass TrainingConfig:\n    \"\"\"Конфигурация обучения\"\"\"\n    \n    def __init__(self):\n        self.IMAGE_DIMENSIONS = (192, 192)\n        self.EPOCH_COUNT = 100\n        self.VALIDATION_SPLIT = 0.15\n        self.SEED_VALUE = 42\n        self.CLASS_COUNT = 104\n        self.PATIENCE_EARLY_STOP = 7\n        self.INITIAL_LEARNING_RATE = 0.001\n        self.LEARNING_RATE_DECAY = 0.95\n        \n    def setup_seeds(self):\n        \"\"\"Устанавливает сиды для воспроизводимости\"\"\"\n        np.random.seed(self.SEED_VALUE)\n        tf.random.set_seed(self.SEED_VALUE)\n        random.seed(self.SEED_VALUE)\n        os.environ['PYTHONHASHSEED'] = str(self.SEED_VALUE)\n\nconfig = TrainingConfig()\nconfig.setup_seeds()\n\ndef setup_gpu():\n    if gpus:\n        try:\n            for gpu in gpus:\n                tf.config.experimental.set_memory_growth(gpu, True)\n            print(\"✓ Настройки GPU применены\")\n        except RuntimeError as e:\n            print(f\"Предупреждение: {e}\")\n\nsetup_gpu()\nBATCH_SIZE = 32\n\n# ЗАГРУЗКА И ПРЕДОБРАБОТКА ДАННЫХ\nclass ImageProcessor:\n    \"\"\"Обработка изображений\"\"\"\n    \n    @staticmethod\n    def decode_image(encoded_img, target_size=config.IMAGE_DIMENSIONS):\n        \"\"\"Декодирует и подготавливает изображение\"\"\"\n        img = tf.image.decode_jpeg(encoded_img, channels=3)\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n        return img\n    \n    @staticmethod\n    def augment_image(img, label, prob=0.8):\n        \"\"\"Аугментация изображения\"\"\"\n        if tf.random.uniform(()) < prob:\n            # Горизонтальное отражение\n            img = tf.image.random_flip_left_right(img)\n            \n            # Яркость\n            img = tf.image.random_brightness(img, max_delta=0.15)\n            \n            # Контраст\n            img = tf.image.random_contrast(img, lower=0.8, upper=1.2)\n            \n            # Вращение\n            if tf.random.uniform(()) > 0.7:\n                k = tf.random.uniform(shape=[], minval=0, maxval=4, dtype=tf.int32)\n                img = tf.image.rot90(img, k=k)\n            \n            # Кадрирование\n            if tf.random.uniform(()) > 0.5:\n                h = tf.shape(img)[0]\n                w = tf.shape(img)[1]\n                new_h = tf.cast(tf.cast(h, tf.float32) * 0.9, tf.int32)\n                new_w = tf.cast(tf.cast(w, tf.float32) * 0.9, tf.int32)\n                img = tf.image.random_crop(img, size=[new_h, new_w, 3])\n                img = tf.image.resize(img, config.IMAGE_DIMENSIONS)\n        \n        return img, label\n\n# СОЗДАНИЕ ДАТАСЕТОВ\ndef create_dataset(file_patterns, labeled=True, training=False, shuffle_size=2000):\n    \"\"\"Создает датасет из TFRecord файлов\"\"\"\n    \n    def parse_record(record):\n        if labeled:\n            features = {\n                'image': tf.io.FixedLenFeature([], tf.string),\n                'class': tf.io.FixedLenFeature([], tf.int64)\n            }\n            parsed = tf.io.parse_single_example(record, features)\n            img = ImageProcessor.decode_image(parsed['image'])\n            label = tf.cast(parsed['class'], tf.int32)\n            return img, label\n        else:\n            features = {\n                'image': tf.io.FixedLenFeature([], tf.string),\n                'id': tf.io.FixedLenFeature([], tf.string)\n            }\n            parsed = tf.io.parse_single_example(record, features)\n            img = ImageProcessor.decode_image(parsed['image'])\n            return img, parsed['id']\n    \n    # Поиск файлов\n    files = []\n    if isinstance(file_patterns, str):\n        file_patterns = [file_patterns]\n    \n    for pattern in file_patterns:\n        found = tf.io.gfile.glob(pattern)\n        if found:\n            files.extend(found)\n    \n    if not files:\n        raise ValueError(f\"Файлы не найдены: {file_patterns}\")\n    \n    print(f\"  Найдено файлов: {len(files)}\")\n    \n    # Создание датасета\n    dataset = tf.data.TFRecordDataset(files)\n    dataset = dataset.map(parse_record, num_parallel_calls=tf.data.AUTOTUNE)\n    \n    # Аугментация\n    if training and labeled:\n        dataset = dataset.map(\n            lambda img, lbl: ImageProcessor.augment_image(img, lbl),\n            num_parallel_calls=tf.data.AUTOTUNE\n        )\n    \n    # Оптимизация\n    if training:\n        dataset = dataset.shuffle(shuffle_size)\n        dataset = dataset.repeat()\n    \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(tf.data.AUTOTUNE)\n    \n    return dataset\n\n# АРХИТЕКТУРА МОДЕЛИ\ndef build_classifier():\n    \"\"\"Создает модель классификатора\"\"\"\n    \n    def conv_block(x, filters, kernel=3, stride=1, attention=True):\n        \"\"\"Сверточный блок\"\"\"\n        # Основной путь\n        y = tf.keras.layers.Conv2D(filters, kernel, strides=stride, \n                                  padding='same', use_bias=False)(x)\n        y = tf.keras.layers.BatchNormalization()(y)\n        y = tf.keras.layers.Activation('relu')(y)\n        \n        y = tf.keras.layers.Conv2D(filters, kernel, padding='same', \n                                  use_bias=False)(y)\n        y = tf.keras.layers.BatchNormalization()(y)\n        \n        # Механизм внимания\n        if attention:\n            # Squeeze-and-Excitation\n            se = tf.keras.layers.GlobalAveragePooling2D()(y)\n            se = tf.keras.layers.Reshape((1, 1, filters))(se)\n            se = tf.keras.layers.Dense(filters//8, activation='relu')(se)\n            se = tf.keras.layers.Dense(filters, activation='sigmoid')(se)\n            y = tf.keras.layers.Multiply()([y, se])\n        \n        # Остаточная связь\n        if stride == 1 and x.shape[-1] == filters:\n            y = tf.keras.layers.Add()([x, y])\n        \n        y = tf.keras.layers.Activation('relu')(y)\n        return y\n    \n    # Входной слой\n    inputs = tf.keras.layers.Input(shape=(config.IMAGE_DIMENSIONS[0], \n                                         config.IMAGE_DIMENSIONS[1], 3))\n    \n    # Начальный блок\n    x = tf.keras.layers.Conv2D(32, 3, strides=2, padding='same')(inputs)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    \n    # Основные блоки\n    x = conv_block(x, 64, stride=1)\n    x = conv_block(x, 64, stride=1)\n    \n    x = conv_block(x, 128, stride=2)\n    x = conv_block(x, 128, stride=1)\n    x = conv_block(x, 128, stride=1)\n    \n    x = conv_block(x, 256, stride=2)\n    x = conv_block(x, 256, stride=1)\n    x = conv_block(x, 256, stride=1)\n    \n    x = conv_block(x, 512, stride=2)\n    x = conv_block(x, 512, stride=1)\n    \n    # Глобальное усреднение\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n    # Полносвязные слои\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(256, activation='relu')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    \n    # Выходной слой\n    outputs = tf.keras.layers.Dense(config.CLASS_COUNT, activation='softmax')(x)\n    \n    # Создание модели\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    return model\n\n# ОПТИМИЗАЦИЯ И КАЛЛБЭКИ\ndef create_callbacks():\n    \"\"\"Создает каллбеки для обучения\"\"\"\n    callbacks = [\n        # Ранняя остановка\n        tf.keras.callbacks.EarlyStopping(\n            monitor='val_accuracy',\n            patience=config.PATIENCE_EARLY_STOP,\n            restore_best_weights=True,\n            verbose=1\n        ),\n        \n        # Сохранение лучшей модели\n        tf.keras.callbacks.ModelCheckpoint(\n            'best_model.h5',\n            monitor='val_accuracy',\n            save_best_only=True,\n            save_weights_only=False,\n            mode='max',\n            verbose=1\n        ),\n        \n        # Логирование\n        tf.keras.callbacks.CSVLogger('training_log.csv')\n    ]\n    \n    return callbacks\n\n# ОБУЧЕНИЕ МОДЕЛИ\ndef train():\n    \"\"\"Основная функция обучения\"\"\"\n    print(\"ПОДГОТОВКА ДАННЫХ\")\n    \n    # Определение путей\n    try:\n        from kaggle_datasets import KaggleDatasets\n        BASE_PATH = KaggleDatasets().get_gcs_path()\n        print(\"Используется Kaggle dataset\")\n    except:\n        BASE_PATH = \"/kaggle/input/tpu-getting-started\"\n        print(f\"Используется локальный путь: {BASE_PATH}\")\n    \n    # Пути к файлам\n    train_path = f'{BASE_PATH}/tfrecords-jpeg-192x192/train/*.tfrec'\n    val_path = f'{BASE_PATH}/tfrecords-jpeg-192x192/val/*.tfrec'\n    test_path = f'{BASE_PATH}/tfrecords-jpeg-192x192/test/*.tfrec'\n    \n    print(\"\\nСоздание датасетов...\")\n    \n    try:\n        train_ds = create_dataset(train_path, labeled=True, training=True)\n        val_ds = create_dataset(val_path, labeled=True, training=False)\n        test_ds = create_dataset(test_path, labeled=False, training=False)\n    except Exception as e:\n        print(f\"Ошибка: {e}\")\n        # Пробуем другой размер\n        train_path = f'{BASE_PATH}/tfrecords-jpeg-192x192/train/*.tfrec'\n        val_path = f'{BASE_PATH}/tfrecords-jpeg-192x192/val/*.tfrec'\n        test_path = f'{BASE_PATH}/tfrecords-jpeg-192x192/test/*.tfrec'\n        \n        train_ds = create_dataset(train_path, labeled=True, training=True)\n        val_ds = create_dataset(val_path, labeled=True, training=False)\n        test_ds = create_dataset(test_path, labeled=False, training=False)\n    \n    # Параметры обучения\n    TRAIN_SAMPLES = 12753\n    VAL_SAMPLES = 3712\n    TEST_SAMPLES = 7382\n    \n    steps_per_epoch = TRAIN_SAMPLES // BATCH_SIZE\n    val_steps = max(1, VAL_SAMPLES // BATCH_SIZE)\n    \n    print(f\"\\nПараметры:\")\n    print(f\"  Шагов на эпоху: {steps_per_epoch}\")\n    print(f\"  Валидационных шагов: {val_steps}\")\n    print(f\"  Размер изображения: {config.IMAGE_DIMENSIONS}\")\n    \n    # Построение модели    \n    model = build_classifier()\n    \n    # Компиляция с фиксированным learning rate\n    optimizer = tf.keras.optimizers.Adam(\n        learning_rate=config.INITIAL_LEARNING_RATE,\n        beta_1=0.9,\n        beta_2=0.999,\n        epsilon=1e-7\n    )\n    \n    model.compile(\n        optimizer=optimizer,\n        loss='sparse_categorical_crossentropy',\n        metrics=['accuracy']\n    )\n    \n    model.summary()\n    \n    # Создание каллбэков\n    callbacks = create_callbacks()\n    \n    # Обучение\n    history = model.fit(\n        train_ds,\n        steps_per_epoch=steps_per_epoch,\n        epochs=config.EPOCH_COUNT,\n        validation_data=val_ds,\n        validation_steps=val_steps,\n        callbacks=callbacks,\n        verbose=1\n    )\n    \n    return model, test_ds, history\n\n# ВИЗУАЛИЗАЦИЯ\ndef plot_history(history):\n    \"\"\"Визуализация истории обучения\"\"\"\n    fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n    \n    # Точность\n    axes[0].plot(history.history['accuracy'], label='Обучение')\n    axes[0].plot(history.history['val_accuracy'], label='Валидация')\n    axes[0].set_title('Точность')\n    axes[0].set_xlabel('Эпоха')\n    axes[0].set_ylabel('Точность')\n    axes[0].legend()\n    axes[0].grid(True, alpha=0.3)\n    \n    # Потери\n    axes[1].plot(history.history['loss'], label='Обучение')\n    axes[1].plot(history.history['val_loss'], label='Валидация')\n    axes[1].set_title('Потери')\n    axes[1].set_xlabel('Эпоха')\n    axes[1].set_ylabel('Потери')\n    axes[1].legend()\n    axes[1].grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # Итоговые метрики\n    print(\"\\nИТОГОВЫЕ МЕТРИКИ:\")\n    print(f\"Финальная точность обучения: {history.history['accuracy'][-1]:.4f}\")\n    print(f\"Финальная точность валидации: {history.history['val_accuracy'][-1]:.4f}\")\n    \n    best_val = max(history.history['val_accuracy'])\n    best_epoch = history.history['val_accuracy'].index(best_val) + 1\n    print(f\"Лучшая точность валидации: {best_val:.4f} (эпоха {best_epoch})\")\n\n# ГЕНЕРАЦИЯ ПРЕДСКАЗАНИЙ\ndef predict(model, test_ds, num_samples=7382):\n    \"\"\"Генерация предсказаний\"\"\"    \n    # Изображения для предсказания\n    test_images = test_ds.map(lambda img, img_id: img)\n    \n    # Количество шагов\n    predict_steps = (num_samples + BATCH_SIZE - 1) // BATCH_SIZE\n    \n    print(f\"Генерация {num_samples} предсказаний...\")\n    \n    # Предсказания\n    predictions = model.predict(\n        test_images,\n        steps=predict_steps,\n        verbose=1\n    )\n    \n    # Получение ID\n    test_ids = test_ds.map(lambda img, img_id: img_id)\n    all_ids = []\n    \n    for ids_batch in test_ids.take(predict_steps):\n        for id_bytes in ids_batch.numpy():\n            try:\n                all_ids.append(id_bytes.decode('utf-8'))\n            except:\n                all_ids.append(str(id_bytes))\n    \n    # Обрезка\n    all_ids = all_ids[:num_samples]\n    predictions = predictions[:num_samples]\n    \n    # Классы\n    pred_classes = np.argmax(predictions, axis=1)\n    \n    print(f\"\\nСгенерировано предсказаний: {len(pred_classes)}\")\n    print(f\"Средняя уверенность: {np.mean(np.max(predictions, axis=1)):.4f}\")\n    \n    return all_ids, pred_classes\n\n# СОЗДАНИЕ САБМИТА\ndef create_submission(ids, preds, filename='submission.csv'):\n    \"\"\"Создает файл для сабмита\"\"\"\n    # DataFrame\n    df = pd.DataFrame({\n        'id': ids,\n        'label': preds\n    })\n    \n    # Сохранение\n    df.to_csv(filename, index=False)\n    \n    print(f\"\\nФайл создан: {filename}\")\n    print(f\"Размер: {os.path.getsize(filename)} байт\")\n    print(\"\\nПервые 10 строк:\")\n    print(df.head(10))\n    \n    # Статистика\n    print(f\"\\nУникальных классов: {df['label'].nunique()}\")\n    print(\"Распределение классов (топ-10):\")\n    print(df['label'].value_counts().head(10))\n    \n    return df\n\n# ОСНОВНАЯ ФУНКЦИЯ\ndef main():\n    \"\"\"Основная функция\"\"\"    \n    try:\n        # Обучение\n        model, test_ds, history = train()\n        \n        # Визуализация\n        plot_history(history)\n        \n        # Предсказания\n        ids, preds = predict(model, test_ds)\n        \n        # Создание сабмита\n        submission = create_submission(ids, preds)        \n        # Сохранение финальной модели\n        model.save('final_model.h5')\n        print(\"Модель сохранена\")\n        \n        return model, submission\n        \n    except Exception as e:\n        print(f\"ОШИБКА: {e}\")\n        import traceback\n        traceback.print_exc()\n        return None, None\n\n# Запуск\nif __name__ == \"__main__\":\n    model, submission = main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T00:53:57.874635Z","iopub.execute_input":"2025-12-07T00:53:57.87495Z","iopub.status.idle":"2025-12-07T01:30:28.095001Z","shell.execute_reply.started":"2025-12-07T00:53:57.874928Z","shell.execute_reply":"2025-12-07T01:30:28.094387Z"}},"outputs":[],"execution_count":null}]}