{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Предыдущий эксперимент\n\n[Здесь](https://www.kaggle.com/code/seriousdim/notebookfdaedc1ec8) \n\nЗдесь использовлась самописная ResNet34, результат сабмита - 0.63","metadata":{}},{"cell_type":"markdown","source":"# Зависимости","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport warnings\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom sklearn.metrics import classification_report, f1_score\nfrom sklearn.model_selection import train_test_split\n\nimport pandas as pd\nimport numpy as np\n\nwarnings.filterwarnings(\"ignore\")\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:30:12.117925Z","iopub.execute_input":"2025-12-17T19:30:12.118251Z","iopub.status.idle":"2025-12-17T19:30:36.780138Z","shell.execute_reply.started":"2025-12-17T19:30:12.118224Z","shell.execute_reply":"2025-12-17T19:30:36.779493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Num GPUs Available:\", len(tf.config.list_physical_devices(\"GPU\")))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:30:36.781258Z","iopub.execute_input":"2025-12-17T19:30:36.781729Z","iopub.status.idle":"2025-12-17T19:30:38.216113Z","shell.execute_reply.started":"2025-12-17T19:30:36.781706Z","shell.execute_reply":"2025-12-17T19:30:38.215366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMAGE_SIZE = [224, 224]\nprint(f\"Using image size: {IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}\")\n\nGCS_PATH = \"/kaggle/input/tpu-getting-started\"\nTRAIN_PATH = f\"{GCS_PATH}/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}/train\"\nVAL_PATH = f\"{GCS_PATH}/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}/val\"\nTEST_PATH = f\"{GCS_PATH}/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}/test\"\n\nsubmission_df = pd.read_csv(f\"{GCS_PATH}/sample_submission.csv\")\nprint(f\"Test samples to predict: {len(submission_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:30:40.940534Z","iopub.execute_input":"2025-12-17T19:30:40.940875Z","iopub.status.idle":"2025-12-17T19:30:40.971967Z","shell.execute_reply.started":"2025-12-17T19:30:40.940848Z","shell.execute_reply":"2025-12-17T19:30:40.971258Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Подготовка датасета","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, IMAGE_SIZE)\n    return image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:30:47.586096Z","iopub.execute_input":"2025-12-17T19:30:47.586714Z","iopub.status.idle":"2025-12-17T19:30:47.590565Z","shell.execute_reply.started":"2025-12-17T19:30:47.586677Z","shell.execute_reply":"2025-12-17T19:30:47.589932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"])\n    label = tf.cast(example[\"class\"], tf.int32)\n    return image, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:30:52.611259Z","iopub.execute_input":"2025-12-17T19:30:52.611558Z","iopub.status.idle":"2025-12-17T19:30:52.615887Z","shell.execute_reply.started":"2025-12-17T19:30:52.611531Z","shell.execute_reply":"2025-12-17T19:30:52.615228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"])\n    idnum = example[\"id\"]\n    return image, idnum","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:30:58.685723Z","iopub.execute_input":"2025-12-17T19:30:58.686043Z","iopub.status.idle":"2025-12-17T19:30:58.690591Z","shell.execute_reply.started":"2025-12-17T19:30:58.686013Z","shell.execute_reply":"2025-12-17T19:30:58.690048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n\n    if labeled:\n        dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        dataset = dataset.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n\n    return dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:05.013184Z","iopub.execute_input":"2025-12-17T19:31:05.013969Z","iopub.status.idle":"2025-12-17T19:31:05.018442Z","shell.execute_reply.started":"2025-12-17T19:31:05.013943Z","shell.execute_reply":"2025-12-17T19:31:05.017917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_data_items(filenames):\n    pattern = re.compile(r\"-([0-9]*)\\.\")\n    n = [int(pattern.search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:09.530429Z","iopub.execute_input":"2025-12-17T19:31:09.531207Z","iopub.status.idle":"2025-12-17T19:31:09.535168Z","shell.execute_reply.started":"2025-12-17T19:31:09.531177Z","shell.execute_reply":"2025-12-17T19:31:09.53452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.rot90(\n        image,\n        k=tf.random.uniform(shape=[], minval=0, maxval=4, dtype=tf.int32),\n    )\n    image = tf.image.random_brightness(image, max_delta=0.1)\n    image = tf.image.random_contrast(image, lower=0.7, upper=1.3)\n    if tf.random.uniform([]) > 0.5:\n        noise = tf.random.normal(shape=tf.shape(image), mean=0.0, stddev=0.01)\n        image = image + noise\n        image = tf.clip_by_value(image, 0.0, 1.0)\n    return image, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:16.893885Z","iopub.execute_input":"2025-12-17T19:31:16.89462Z","iopub.status.idle":"2025-12-17T19:31:16.899503Z","shell.execute_reply.started":"2025-12-17T19:31:16.894595Z","shell.execute_reply":"2025-12-17T19:31:16.898853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_training_dataset(filenames):\n    dataset = load_dataset(filenames, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(filenames, ordered=False):\n    dataset = load_dataset(filenames, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:22.416836Z","iopub.execute_input":"2025-12-17T19:31:22.417463Z","iopub.status.idle":"2025-12-17T19:31:22.421527Z","shell.execute_reply.started":"2025-12-17T19:31:22.417435Z","shell.execute_reply":"2025-12-17T19:31:22.420811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_weighted_sampler(labels, smoothing_factor=0.3):\n    unique, counts = np.unique(labels, return_counts=True)\n    class_weights = {}\n\n    total_samples = len(labels)\n    class_frequencies = counts / total_samples\n\n    base_weights = 1.0 / (class_frequencies + 1e-8)\n    base_weights = base_weights / np.mean(base_weights)\n\n    smoothed_weights = (\n        1 - smoothing_factor\n    ) * 1.0 + smoothing_factor * base_weights\n\n    for cls, weight in zip(unique, smoothed_weights):\n        class_weights[cls] = weight\n\n    sample_weights = np.array([class_weights[label] for label in labels])\n    sample_weights = sample_weights / np.sum(sample_weights)\n\n    return sample_weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:31.811554Z","iopub.execute_input":"2025-12-17T19:31:31.812233Z","iopub.status.idle":"2025-12-17T19:31:31.817111Z","shell.execute_reply.started":"2025-12-17T19:31:31.812204Z","shell.execute_reply":"2025-12-17T19:31:31.816341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_labels_for_analysis(filenames, num_samples=10000):\n    dataset = load_dataset(filenames, labeled=True, ordered=False)\n    labels = []\n    for _, label in dataset.take(num_samples):\n        labels.append(label.numpy())\n    return np.array(labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:37.829068Z","iopub.execute_input":"2025-12-17T19:31:37.829556Z","iopub.status.idle":"2025-12-17T19:31:37.833326Z","shell.execute_reply.started":"2025-12-17T19:31:37.82953Z","shell.execute_reply":"2025-12-17T19:31:37.832716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_files = tf.io.gfile.glob(TRAIN_PATH + \"/*.tfrec\")\nlabels_sample = load_labels_for_analysis(train_files, num_samples=12753)\n\nsample_weights = create_weighted_sampler(labels_sample, smoothing_factor=0.3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:42.930242Z","iopub.execute_input":"2025-12-17T19:31:42.930527Z","iopub.status.idle":"2025-12-17T19:31:48.793472Z","shell.execute_reply.started":"2025-12-17T19:31:42.930503Z","shell.execute_reply":"2025-12-17T19:31:48.792847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_weighted_training_dataset(filenames, sample_weights, buffer_size=2048):\n    dataset = load_dataset(filenames, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(buffer_size * 2)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:49.279364Z","iopub.execute_input":"2025-12-17T19:31:49.279651Z","iopub.status.idle":"2025-12-17T19:31:49.284148Z","shell.execute_reply.started":"2025-12-17T19:31:49.279627Z","shell.execute_reply":"2025-12-17T19:31:49.283358Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Создание датасета","metadata":{}},{"cell_type":"code","source":"# Загрузка и проверка данных\nBATCH_SIZE = 64\nEPOCHS = 125\n\ntrain_files = tf.io.gfile.glob(TRAIN_PATH + \"/*.tfrec\")\nval_files = tf.io.gfile.glob(VAL_PATH + \"/*.tfrec\")\n\nprint(f\"Train files: {len(train_files)}\")\nprint(f\"Val files: {len(val_files)}\")\n\n# Подсчет количества изображений\nnum_train_images = count_data_items(train_files)\nnum_val_images = count_data_items(val_files)\n\nprint(f\"Train images: {num_train_images}\")\nprint(f\"Val images: {num_val_images}\")\n\n# Создание датасетов\n# train_dataset = get_training_dataset(train_files)]\ntrain_dataset = get_weighted_training_dataset(train_files, sample_weights)\nval_dataset = get_validation_dataset(val_files)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:31:54.892355Z","iopub.execute_input":"2025-12-17T19:31:54.893254Z","iopub.status.idle":"2025-12-17T19:31:55.110359Z","shell.execute_reply.started":"2025-12-17T19:31:54.893217Z","shell.execute_reply":"2025-12-17T19:31:55.109517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Визуализирую, чтобы проверить что данные корректные и аугментации работают\ndef visualize_batch(dataset, title=\"Training Images\"):\n    plt.figure(figsize=(12, 8))\n    for i, (images, labels) in enumerate(dataset.take(1)):\n        for j in range(min(8, BATCH_SIZE)):\n            plt.subplot(2, 4, j + 1)\n            plt.imshow(images[j].numpy())\n            plt.title(f\"Label: {labels[j].numpy()}\")\n            plt.axis(\"off\")\n    plt.suptitle(title, fontsize=16)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:01.183019Z","iopub.execute_input":"2025-12-17T19:32:01.18334Z","iopub.status.idle":"2025-12-17T19:32:01.188485Z","shell.execute_reply.started":"2025-12-17T19:32:01.183315Z","shell.execute_reply":"2025-12-17T19:32:01.187878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"visualize_batch(train_dataset, \"Augmented Training Images\")\nvisualize_batch(val_dataset, \"Validation Images\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:07.906516Z","iopub.execute_input":"2025-12-17T19:32:07.907083Z","iopub.status.idle":"2025-12-17T19:32:15.681901Z","shell.execute_reply.started":"2025-12-17T19:32:07.907057Z","shell.execute_reply":"2025-12-17T19:32:15.680871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Создаю CNN модель\nNUM_CLASSES = 104\nLEARNING_RATE = 0.01","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:25.04967Z","iopub.execute_input":"2025-12-17T19:32:25.049962Z","iopub.status.idle":"2025-12-17T19:32:25.053682Z","shell.execute_reply.started":"2025-12-17T19:32:25.049937Z","shell.execute_reply":"2025-12-17T19:32:25.052971Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Модель\n\nОбычная CNN","metadata":{}},{"cell_type":"code","source":"def cnn():\n    model = tf.keras.Sequential(\n        [\n            # Первый сверточный блок\n            tf.keras.layers.Conv2D(\n                32,\n                (3, 3),\n                padding=\"same\",\n                activation=\"relu\",\n                input_shape=(*IMAGE_SIZE, 3),\n            ),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n            tf.keras.layers.Dropout(0.35),\n            # Второй сверточный блок\n            tf.keras.layers.Conv2D(\n                64, (3, 3), padding=\"same\", activation=\"relu\"\n            ),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n            tf.keras.layers.Dropout(0.3),\n            # Третий сверточный блок\n            tf.keras.layers.Conv2D(\n                128, (3, 3), padding=\"same\", activation=\"relu\"\n            ),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n            tf.keras.layers.Dropout(0.25),\n            # Третий+++ сверточный блок\n            tf.keras.layers.Conv2D(\n                128, (3, 3), padding=\"same\", activation=\"relu\"\n            ),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n            tf.keras.layers.Dropout(0.3),\n            # Четвертый сверточный блок\n            tf.keras.layers.Conv2D(\n                256, (3, 3), padding=\"same\", activation=\"relu\"\n            ),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n            tf.keras.layers.Dropout(0.4),\n            # Пятый сверточный блок\n            tf.keras.layers.Conv2D(\n                512, (3, 3), padding=\"same\", activation=\"relu\"\n            ),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n            tf.keras.layers.Dropout(0.5),\n            # Полносвязные слои\n            tf.keras.layers.Flatten(),\n            tf.keras.layers.Dense(512, activation=\"relu\"),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.5),\n            tf.keras.layers.Dense(256, activation=\"relu\"),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.3),\n            # Выходной слой\n            tf.keras.layers.Dense(NUM_CLASSES, activation=\"softmax\"),\n        ]\n    )\n\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:33.645763Z","iopub.execute_input":"2025-12-17T19:32:33.64648Z","iopub.status.idle":"2025-12-17T19:32:33.655686Z","shell.execute_reply.started":"2025-12-17T19:32:33.64645Z","shell.execute_reply":"2025-12-17T19:32:33.65514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLoss(tf.keras.losses.Loss):\n    def __init__(\n        self,\n        alpha=0.25,\n        gamma=2.0,\n        label_smoothing=0.1,\n        name=\"focal_loss\",\n        **kwargs\n    ):\n        super().__init__(name=name, **kwargs)\n        self.alpha = alpha\n        self.gamma = gamma\n        self.label_smoothing = label_smoothing\n\n    def call(self, y_true, y_pred):\n        y_true = tf.one_hot(tf.cast(y_true, tf.int32), depth=NUM_CLASSES)\n        y_true = y_true * (1.0 - self.label_smoothing) + (\n            self.label_smoothing / NUM_CLASSES\n        )\n        cross_entropy = -y_true * tf.math.log(\n            tf.clip_by_value(y_pred, 1e-7, 1.0)\n        )\n        p_t = y_true * y_pred + (1 - y_true) * (1 - y_pred)\n        modulating_factor = tf.pow(1.0 - p_t, self.gamma)\n        alpha_weight = y_true * self.alpha + (1 - y_true) * (1 - self.alpha)\n        focal_loss = modulating_factor * alpha_weight * cross_entropy\n\n        return tf.reduce_mean(focal_loss)\n\n    def get_config(self):\n        config = super().get_config()\n        config.update(\n            {\n                \"gamma\": self.gamma,\n                \"alpha\": self.alpha,\n                \"label_smoothing\": self.label_smoothing,\n            }\n        )\n        return config","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:43.492134Z","iopub.execute_input":"2025-12-17T19:32:43.492731Z","iopub.status.idle":"2025-12-17T19:32:43.499399Z","shell.execute_reply.started":"2025-12-17T19:32:43.492706Z","shell.execute_reply":"2025-12-17T19:32:43.498624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"callbacks = [\n    # Сохраняю лучшую модель\n    tf.keras.callbacks.ModelCheckpoint(\n        \"best_model.keras\",\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        mode=\"max\",\n        verbose=1,\n    ),\n    # Ранняя остановка\n    tf.keras.callbacks.EarlyStopping(\n        monitor=\"val_accuracy\",\n        patience=15,\n        restore_best_weights=True,\n        mode=\"max\",\n        verbose=1,\n    ),\n    # Уменьшение learning rate\n    tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", factor=0.5, patience=10, min_lr=1e-6, verbose=1\n    ),\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:49.884236Z","iopub.execute_input":"2025-12-17T19:32:49.884523Z","iopub.status.idle":"2025-12-17T19:32:49.889338Z","shell.execute_reply.started":"2025-12-17T19:32:49.884501Z","shell.execute_reply":"2025-12-17T19:32:49.888698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:57.704386Z","iopub.execute_input":"2025-12-17T19:32:57.704679Z","iopub.status.idle":"2025-12-17T19:32:57.719729Z","shell.execute_reply.started":"2025-12-17T19:32:57.704655Z","shell.execute_reply":"2025-12-17T19:32:57.719142Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Обучение","metadata":{}},{"cell_type":"code","source":"model = cnn()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:33:02.034118Z","iopub.execute_input":"2025-12-17T19:33:02.034411Z","iopub.status.idle":"2025-12-17T19:33:04.17609Z","shell.execute_reply.started":"2025-12-17T19:33:02.034388Z","shell.execute_reply":"2025-12-17T19:33:04.175527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(\n    optimizer=optimizer,\n    loss=FocalLoss(alpha=0.75, gamma=1.25, label_smoothing=0.02),\n    metrics=[\"accuracy\"],\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:33:14.550301Z","iopub.execute_input":"2025-12-17T19:33:14.551064Z","iopub.status.idle":"2025-12-17T19:33:14.558735Z","shell.execute_reply.started":"2025-12-17T19:33:14.551026Z","shell.execute_reply":"2025-12-17T19:33:14.558039Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"steps_per_epoch = num_train_images // BATCH_SIZE\nvalidation_steps = num_val_images // BATCH_SIZE\n\nprint(f\"Steps per epoch: {steps_per_epoch}\")\nprint(f\"Validation steps: {validation_steps}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:33:17.148483Z","iopub.execute_input":"2025-12-17T19:33:17.148975Z","iopub.status.idle":"2025-12-17T19:33:17.153344Z","shell.execute_reply.started":"2025-12-17T19:33:17.148949Z","shell.execute_reply":"2025-12-17T19:33:17.152563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    steps_per_epoch=steps_per_epoch,\n    epochs=EPOCHS,\n    validation_data=val_dataset,\n    validation_steps=validation_steps,\n    callbacks=callbacks,\n    verbose=1,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:33:22.953155Z","iopub.execute_input":"2025-12-17T19:33:22.953878Z","iopub.status.idle":"2025-12-17T20:27:15.605036Z","shell.execute_reply.started":"2025-12-17T19:33:22.953852Z","shell.execute_reply":"2025-12-17T20:27:15.604178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = tf.keras.models.load_model(\n    \"best_model.keras\", custom_objects={\"FocalLoss\": FocalLoss}\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:15.606612Z","iopub.execute_input":"2025-12-17T20:27:15.606844Z","iopub.status.idle":"2025-12-17T20:27:16.299685Z","shell.execute_reply.started":"2025-12-17T20:27:15.606824Z","shell.execute_reply":"2025-12-17T20:27:16.299027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Расчет необходимых метрик для валидации (проверка что обучил модель корректно)\nfrom sklearn.metrics import confusion_matrix\n\n\ndef calculate_val_f1(model, val_dataset, num_val_images, batch_size):\n\n    val_dataset_ordered = get_validation_dataset(val_files, ordered=True)\n\n    y_true = []\n    y_pred = []\n\n    for images, labels in val_dataset_ordered.take(\n        num_val_images // batch_size + 1\n    ):\n        preds = model.predict(images, verbose=0)\n        preds = tf.argmax(preds, axis=-1).numpy()\n\n        y_true.extend(labels.numpy())\n        y_pred.extend(preds)\n\n    y_true = y_true[:num_val_images]\n    y_pred = y_pred[:num_val_images]\n\n    macro_f1 = f1_score(y_true, y_pred, average=\"macro\")\n    weighted_f1 = f1_score(y_true, y_pred, average=\"weighted\")\n    accuracy = np.mean(np.array(y_true) == np.array(y_pred))\n    print(confusion_matrix(y_true, y_pred))\n    return y_true, y_pred, macro_f1, weighted_f1, accuracy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:16.300653Z","iopub.execute_input":"2025-12-17T20:27:16.300881Z","iopub.status.idle":"2025-12-17T20:27:16.307042Z","shell.execute_reply.started":"2025-12-17T20:27:16.300861Z","shell.execute_reply":"2025-12-17T20:27:16.306167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Проверка метрик на валидационном датасете\ny_true, y_pred, macro_f1, weighted_f1, accuracy = calculate_val_f1(\n    best_model, val_dataset, num_val_images, BATCH_SIZE\n)\nprint(macro_f1, weighted_f1, accuracy)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:16.308489Z","iopub.execute_input":"2025-12-17T20:27:16.308704Z","iopub.status.idle":"2025-12-17T20:27:26.141643Z","shell.execute_reply.started":"2025-12-17T20:27:16.308686Z","shell.execute_reply":"2025-12-17T20:27:26.140974Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Сабмит","metadata":{}},{"cell_type":"code","source":"test_files = tf.io.gfile.glob(TEST_PATH + \"/*.tfrec\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:26.142572Z","iopub.execute_input":"2025-12-17T20:27:26.142863Z","iopub.status.idle":"2025-12-17T20:27:26.158779Z","shell.execute_reply.started":"2025-12-17T20:27:26.142833Z","shell.execute_reply":"2025-12-17T20:27:26.15826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_submission(model):\n\n    submission_sample = pd.read_csv(f\"{GCS_PATH}/sample_submission.csv\")\n\n    dataset = tf.data.TFRecordDataset(test_files, num_parallel_reads=1)\n    dataset = dataset.map(read_unlabeled_tfrecord, num_parallel_calls=1)\n    dataset = dataset.batch(BATCH_SIZE)\n\n    all_ids = []\n    all_images = []\n\n    for images, ids in dataset:\n        all_images.append(images)\n        all_ids.extend([id.numpy().decode(\"utf-8\") for id in ids])\n\n    all_images = tf.concat(all_images, axis=0)\n\n    predictions = model.predict(all_images, verbose=1)\n    predictions = tf.argmax(predictions, axis=-1).numpy()\n\n    result_df = pd.DataFrame(\n        {\"id\": all_ids[: len(predictions)], \"label\": predictions}\n    )\n\n    result_df = result_df.set_index(\"id\")\n    result_df = result_df.reindex(submission_sample[\"id\"]).reset_index()\n\n    return result_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:26.159614Z","iopub.execute_input":"2025-12-17T20:27:26.159865Z","iopub.status.idle":"2025-12-17T20:27:26.166315Z","shell.execute_reply.started":"2025-12-17T20:27:26.15984Z","shell.execute_reply":"2025-12-17T20:27:26.165546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = create_submission(best_model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:26.167404Z","iopub.execute_input":"2025-12-17T20:27:26.167626Z","iopub.status.idle":"2025-12-17T20:27:52.788138Z","shell.execute_reply.started":"2025-12-17T20:27:26.167605Z","shell.execute_reply":"2025-12-17T20:27:52.787336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T20:27:52.789321Z","iopub.execute_input":"2025-12-17T20:27:52.789909Z","iopub.status.idle":"2025-12-17T20:27:52.80949Z","shell.execute_reply.started":"2025-12-17T20:27:52.789877Z","shell.execute_reply":"2025-12-17T20:27:52.808709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}