{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Установка и импорт необходимых библиотек. Загрузка датасета","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:04.860454Z","iopub.execute_input":"2025-12-29T06:33:04.860764Z","iopub.status.idle":"2025-12-29T06:33:05.239653Z","shell.execute_reply.started":"2025-12-29T06:33:04.860712Z","shell.execute_reply":"2025-12-29T06:33:05.238749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Dense, Flatten, Input\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, SpatialDropout2D, GaussianDropout, Dropout\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, GlobalAveragePooling2D, GlobalMaxPooling2D\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.regularizers import *\nimport numpy as np\nimport os\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom tensorflow.random import set_seed\ndef seed_everything(seed):\n    np.random.seed(seed)\n    set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\n\nfrom tensorflow.keras.preprocessing import image\nimport matplotlib.pyplot as plt\n%matplotlib inline \nfrom tensorflow.keras.regularizers import l2\n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\n%matplotlib inline \nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:05.240903Z","iopub.execute_input":"2025-12-29T06:33:05.241387Z","iopub.status.idle":"2025-12-29T06:33:16.588362Z","shell.execute_reply.started":"2025-12-29T06:33:05.241363Z","shell.execute_reply":"2025-12-29T06:33:16.587644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport tensorflow as tf\n\n# Решение 2: Отключаем XLA для GPU\nos.environ['TF_XLA_FLAGS'] = '--tf_xla_auto_jit=0'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:16.589759Z","iopub.execute_input":"2025-12-29T06:33:16.590178Z","iopub.status.idle":"2025-12-29T06:33:16.593799Z","shell.execute_reply.started":"2025-12-29T06:33:16.590157Z","shell.execute_reply":"2025-12-29T06:33:16.592922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Обнаружение оборудования, возврат соответствующей стратегии распространения: TPU, GPU, CPU\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # Обнаружение TPU. Параметры среды не требуются, если задана переменная среды TPU_NAME. На Kaggle это всегда так.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # стратегия распространения по умолчанию в Tensorflow. Работает на CPU и одном GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:16.594978Z","iopub.execute_input":"2025-12-29T06:33:16.595327Z","iopub.status.idle":"2025-12-29T06:33:16.752824Z","shell.execute_reply.started":"2025-12-29T06:33:16.595298Z","shell.execute_reply":"2025-12-29T06:33:16.752149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"GCS_DS_PATH = '/kaggle/input/tpu-getting-started' #получаем путь к наборам данных","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:16.753677Z","iopub.execute_input":"2025-12-29T06:33:16.753975Z","iopub.status.idle":"2025-12-29T06:33:16.766688Z","shell.execute_reply.started":"2025-12-29T06:33:16.753947Z","shell.execute_reply":"2025-12-29T06:33:16.766072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMAGE_SIZE = [224, 224] \n\nEPOCHS = 80\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:16.767588Z","iopub.execute_input":"2025-12-29T06:33:16.767833Z","iopub.status.idle":"2025-12-29T06:33:16.779945Z","shell.execute_reply.started":"2025-12-29T06:33:16.767815Z","shell.execute_reply":"2025-12-29T06:33:16.779154Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Загружаем данные","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Декодирует изображение. Нормализует данные и преобразовывает изображения к указанному размеру\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) \n    return image\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # [] отдельный элемент\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT) # парсим отдельный пример в указанном формате\n    image = decode_image(example['image']) # преобразуем изображение к нужному формату\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # возвращает набор данных пар (изображение, метка)\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # [] означает отдельный элемент\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) \n    idnum = example['id']\n    return image, idnum \n\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n\n    ignore_order = tf.data.Options() \n    if not ordered:\n        ignore_order.experimental_deterministic = False \n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    return dataset\n\n\ndef get_training_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/train/*.tfrec'), labeled=True)\n    dataset = dataset.repeat() \n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\n\ndef get_validation_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/val/*.tfrec'), labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    return dataset\n\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/test/*.tfrec'), labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:16.780889Z","iopub.execute_input":"2025-12-29T06:33:16.781188Z","iopub.status.idle":"2025-12-29T06:33:19.333667Z","shell.execute_reply.started":"2025-12-29T06:33:16.781135Z","shell.execute_reply":"2025-12-29T06:33:19.333026Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Построим модель","metadata":{}},{"cell_type":"code","source":"def get_model():\n \n    # Создаем последовательную модель\n    model = Sequential()\n\n    ## Первый сверточный блок\n\n    model.add(Conv2D(64, (3, 3), input_shape=(*IMAGE_SIZE, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64, (3, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.3))\n\n    ## Второй сверточный блок\n    model.add(Conv2D(128, (3, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(128, (3, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.4))\n\n    ## Третий сверточный блок\n    model.add(Conv2D(256, (3, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(256, (3, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(256, (3, 3), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.4))\n\n    ## Четвертый сверточный блок\n    model.add(Conv2D(512, (5, 5), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(512, (5, 5), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(512, (5, 5), activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(GlobalAveragePooling2D())\n    model.add(Dropout(0.5))\n\n    \n    # Полносвязный слой для классификации\n    model.add(Dense(512, activation='relu', kernel_regularizer=l2(0.0005)))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.5))\n    model.add(Dense(104, activation='softmax'))\n    return model\n\nwith strategy.scope():    \n    model = get_model()\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:19.334483Z","iopub.execute_input":"2025-12-29T06:33:19.334803Z","iopub.status.idle":"2025-12-29T06:33:20.735821Z","shell.execute_reply.started":"2025-12-29T06:33:19.334775Z","shell.execute_reply":"2025-12-29T06:33:20.734993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"callbacks_list = [EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n                  ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3),\n                  ]\n\noptimizer = tf.keras.optimizers.Nadam(learning_rate=1e-4, beta_1=0.9, beta_2=0.999)\n\nmodel.compile(\n    optimizer=optimizer,\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nhistorical = model.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          callbacks=callbacks_list,\n          validation_data=validation_dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:33:20.737402Z","iopub.execute_input":"2025-12-29T06:33:20.737608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) \n\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # все в одной партии\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}