{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30446,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TF блокнот\n[По сути, это перевод стартового блокнота от команды TensorFlow](https://www.kaggle.com/philculliton/a-simple-petals-tf-2-2-notebook)","metadata":{}},{"cell_type":"code","source":"!pip install tensorflow-addons","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:07:58.167138Z","iopub.execute_input":"2025-01-08T22:07:58.167685Z","iopub.status.idle":"2025-01-08T22:08:10.334463Z","shell.execute_reply.started":"2025-01-08T22:07:58.167631Z","shell.execute_reply":"2025-01-08T22:08:10.332720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport tensorflow as tf\n# последовательная модель (стек слоев)\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import (Dense, Flatten, Input, Reshape, Add, \n    GlobalAveragePooling2D, Multiply, Activation, DepthwiseConv2D, SeparableConv2D,\n    Dropout, BatchNormalization, SpatialDropout2D, GaussianDropout,\n    Conv2D, MaxPooling2D, AveragePooling2D, LeakyReLU, Concatenate)\n\n# работа с обратной связью от обучающейся нейронной сети\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n# вспомогательные инструменты\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.regularizers import *\nfrom tensorflow.keras.mixed_precision import set_global_policy\nimport numpy as np\nimport os\nfrom tensorflow.random import set_seed\n# from tensorflow_addons.optimizers import AdamW\n\n# работа с изображениями\nfrom tensorflow.keras.preprocessing import image\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\n#  библиотека для работы с наборами данных на Kaggle\nfrom kaggle_datasets import KaggleDatasets ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:10.337732Z","iopub.execute_input":"2025-01-08T22:08:10.338152Z","iopub.status.idle":"2025-01-08T22:08:20.105496Z","shell.execute_reply.started":"2025-01-08T22:08:10.338111Z","shell.execute_reply":"2025-01-08T22:08:20.104143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed):\n    np.random.seed(seed)\n    set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\n\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:20.107239Z","iopub.execute_input":"2025-01-08T22:08:20.108171Z","iopub.status.idle":"2025-01-08T22:08:20.118072Z","shell.execute_reply.started":"2025-01-08T22:08:20.108118Z","shell.execute_reply":"2025-01-08T22:08:20.116400Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Определяем, какой ускоритель можем использовать","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Detect TPU, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n# \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:20.119980Z","iopub.execute_input":"2025-01-08T22:08:20.120525Z","iopub.status.idle":"2025-01-08T22:08:20.147009Z","shell.execute_reply.started":"2025-01-08T22:08:20.120470Z","shell.execute_reply":"2025-01-08T22:08:20.145443Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Get my data path","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path() #получаем путь к наборам данных","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:20.150558Z","iopub.execute_input":"2025-01-08T22:08:20.151506Z","iopub.status.idle":"2025-01-08T22:08:20.500520Z","shell.execute_reply.started":"2025-01-08T22:08:20.151459Z","shell.execute_reply":"2025-01-08T22:08:20.499073Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Set some parameters","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [224, 224] # при таком размере графическому процессору не хватит памяти. Используйте TPU\nEPOCHS = 90\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync\n\n# IMAGE_SIZE = [192, 192] # при таком размере графическому процессору не хватит памяти. Используйте TPU\n# EPOCHS = 80\n# BATCH_SIZE = 8 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\n# NUM_TRAINING_IMAGES = 12753\n# NUM_TEST_IMAGES = 7382\n# STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE # находим количество шагов за эпоху\n\nprint('BATCH_SIZE:', BATCH_SIZE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:20.502462Z","iopub.execute_input":"2025-01-08T22:08:20.502997Z","iopub.status.idle":"2025-01-08T22:08:21.353405Z","shell.execute_reply.started":"2025-01-08T22:08:20.502942Z","shell.execute_reply":"2025-01-08T22:08:21.351833Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Загружаем данные\n\nЭти данные загружаются из Kaggle и автоматически сегментируются для максимального распараллеливания.","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Декодирует изображение в vyjujvthye. vfnhbwe (тензор)\n    Нормализует данные и преобразовывает изображения к указанному размеру\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3) # Декодирование изображения в формате JPEG в тензор uint8.\n    image = tf.cast(image, tf.float32) / 255.0  # преобразовать изображение в плавающее в диапазоне [0, 1]\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # явный размер, необходимый для TPU\n#     image = tf.keras.applications.inception_resnet_v2.preprocess_input(image)\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # [] означает отдельный элемент\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT) # парсим отдельный пример в указанном формате\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # возвращает набор данных пар (изображение, метка)\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # [] означает отдельный элемент\n        # класс отсутствует, задача этого конкурса - предсказать классы цветов для тестового набора данных\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"Читает из TFRecords. Для оптимальной производительности одновременное чтение из нескольких\n    файлов без учета порядка данных. Порядок не имеет значения, поскольку мы все равно будем перетасовывать данные\"\"\"\n\n    ignore_order = tf.data.Options() # Представляет параметры для tf.data.Dataset.\n    if not ordered:\n        ignore_order.experimental_deterministic = False # отключить порядок, увеличить скорость\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # автоматически чередует чтение из нескольких файлов\n    dataset = dataset.with_options(ignore_order) # использует данные сразу после их поступления, а не в исходном порядке\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # возвращает набор данных пар (изображение, метка), если метка = Истина, или пар (изображение, идентификатор), если метка = Ложь\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # набор обучающих данных должен повторяться в течение нескольких эпох\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) #готовим следующий набор, пока предыдущий обучается\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache() # кешируем набор\n    dataset = dataset.prefetch(AUTO) #готовим следующий набор, пока предыдущий обучается\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) #готовим следующий набор, пока предыдущий обучается\n    return dataset\n                               \ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n# training_dataset = get_training_dataset()\n# validation_dataset = get_validation_dataset()\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))\nprint('STEPS_PER_EPOCH:',STEPS_PER_EPOCH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:21.355652Z","iopub.execute_input":"2025-01-08T22:08:21.356066Z","iopub.status.idle":"2025-01-08T22:08:21.380401Z","shell.execute_reply.started":"2025-01-08T22:08:21.356030Z","shell.execute_reply":"2025-01-08T22:08:21.378504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Построение модели","metadata":{}},{"cell_type":"code","source":"# def get_model():\n#     def residual_block(x, filters, kernel_size, regularizer):\n#         shortcut = x\n#         # Если число каналов изменилось, то приводим shortcut к нужному размеру\n#         if x.shape[-1] != filters:\n#             shortcut = Conv2D(filters, (1, 1), padding='same', activation=None, kernel_regularizer=regularizer)(shortcut)\n#             shortcut = BatchNormalization()(shortcut)\n    \n#         x = Conv2D(filters, kernel_size, padding='same', activation=None, kernel_regularizer=regularizer)(x)\n#         x = BatchNormalization()(x)\n#         x = Add()([shortcut, x])\n#         x = tf.keras.layers.Activation('relu')(x)\n#         return x\n        \n#     input_layer = Input(shape=(*IMAGE_SIZE, 3))\n\n#     # Входной блок с улучшенной регуляризацией и параметрами\n#     x = Conv2D(64, (5, 5), activation='relu', kernel_regularizer=l2(1e-4), padding='same')(input_layer)\n#     x = BatchNormalization()(x)\n#     x = AveragePooling2D(pool_size=(2, 2))(x)\n#     x = Dropout(0.3)(x)\n\n#     # Сверточные блоки с резидуальными соединениями\n#     x = residual_block(x, 64, (3, 3), l2(1e-4))\n#     x = AveragePooling2D(pool_size=(2, 2))(x)\n#     x = residual_block(x, 128, (3, 3), l2(1e-4))\n#     x = AveragePooling2D(pool_size=(2, 2))(x)\n#     x = residual_block(x, 256, (3, 3), l2(1e-4))\n#     x = AveragePooling2D(pool_size=(2, 2))(x)\n\n#     # Глобальное усреднение и полносвязный блок\n#     x = GlobalAveragePooling2D()(x)\n#     x = Dense(512, activation='relu', kernel_regularizer=l2(1e-4))(x)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.5)(x)\n#     output_layer = Dense(104, activation='softmax')(x)\n\n#     model = Model(inputs=input_layer, outputs=output_layer)\n#     return model\n\ndef get_model():\n    model = Sequential()\n\n    ## Первый сверточный блок\n    model.add(Conv2D(64, (5, 5), activation='relu', kernel_regularizer=l2(1e-5), input_shape=(*IMAGE_SIZE, 3)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64, (3, 3), activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.35))\n    \n\n    ## Второй сверточный блок\n    model.add(Conv2D(128, (3, 3), activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(128, (3, 3), activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.25))\n    \n\n    ## Третий сверточный блок\n    model.add(Conv2D(256, (5, 5), activation='relu', kernel_regularizer=l2(1e-5)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(256, (3, 3), activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(256, (1, 1), activation='relu', kernel_regularizer=l2(1e-3)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.45))\n    \n\n    ## Четвертый сверточный блок\n    model.add(Conv2D(512, (3, 3), activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(512, (1, 1), activation='relu', kernel_regularizer=l2(1e-3)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.5))\n\n    ## Пятый сверточный блок\n    model.add(Conv2D(1024, (3, 3), activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(1024, (1, 1), activation='relu', kernel_regularizer=l2(1e-3)))\n    model.add(BatchNormalization())\n    model.add(AveragePooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.5))\n    \n\n    ## Global Average Pooling\n    model.add(GlobalAveragePooling2D())\n\n    ## Полносвязный блок\n    model.add(Dense(1024, activation='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.5))\n    model.add(Dense(104, activation='softmax'))\n\n    return model\n\n\n\n# Модель внутри TPU-стратегии\nwith strategy.scope():\n    model = get_model()\n\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:21.383182Z","iopub.execute_input":"2025-01-08T22:08:21.383754Z","iopub.status.idle":"2025-01-08T22:08:22.243614Z","shell.execute_reply.started":"2025-01-08T22:08:21.383701Z","shell.execute_reply":"2025-01-08T22:08:22.242163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Запуск","metadata":{}},{"cell_type":"code","source":"callbacks_list = [EarlyStopping(monitor='val_sparse_categorical_accuracy', patience=15, restore_best_weights=True),\n                  ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy', factor=0.5, patience=5),\n                  ]\nbase_lr = 1e-4\nadjusted_lr = base_lr * (BATCH_SIZE / 256)\n\nwith strategy.scope():\n    model = get_model()\n    model.compile(\n        # optimizer='nadam',\n        # optimizer=AdamW(learning_rate=adjusted_lr, weight_decay=1e-4),\n        # optimizer=AdamW(learning_rate=1e-4, weight_decay=1e-4),\n        optimizer='adam',\n        loss='sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    \n    historical = model.fit(\n        get_training_dataset(),\n        steps_per_epoch=STEPS_PER_EPOCH,\n        epochs=EPOCHS,\n        validation_data=get_validation_dataset(),\n        callbacks=callbacks_list,\n        # verbose=2\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:08:22.245570Z","iopub.execute_input":"2025-01-08T22:08:22.245970Z","iopub.status.idle":"2025-01-08T22:12:48.116418Z","shell.execute_reply.started":"2025-01-08T22:08:22.245932Z","shell.execute_reply":"2025-01-08T22:12:48.114075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# with strategy.scope(): \n#     historical = model.fit(\n#         get_training_dataset(),\n#         steps_per_epoch=STEPS_PER_EPOCH,\n#         epochs=EPOCHS,\n#         validation_data=get_validation_dataset(),\n#         callbacks=callbacks_list,\n#         # verbose=2\n#     )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:12:48.117982Z","iopub.status.idle":"2025-01-08T22:12:48.118512Z","shell.execute_reply.started":"2025-01-08T22:12:48.118259Z","shell.execute_reply":"2025-01-08T22:12:48.118285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# with strategy.scope(): \n#     historical = model.fit(\n#         get_training_dataset(),\n#         steps_per_epoch=STEPS_PER_EPOCH,\n#         epochs=EPOCHS,\n#         validation_data=get_validation_dataset(),\n#         callbacks=callbacks_list,\n#         verbose=2\n#     )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:12:48.120479Z","iopub.status.idle":"2025-01-08T22:12:48.120940Z","shell.execute_reply.started":"2025-01-08T22:12:48.120733Z","shell.execute_reply":"2025-01-08T22:12:48.120759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# with strategy.scope(): \n#     historical = model.fit(\n#         get_training_dataset(),\n#         steps_per_epoch=STEPS_PER_EPOCH,\n#         epochs=EPOCHS,\n#         validation_data=get_validation_dataset(),\n#         callbacks=callbacks_list,\n#         verbose=2\n#     )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:12:48.121865Z","iopub.status.idle":"2025-01-08T22:12:48.122299Z","shell.execute_reply.started":"2025-01-08T22:12:48.122100Z","shell.execute_reply":"2025-01-08T22:12:48.122123Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Вычислите свои прогнозы на тестовом наборе!\n\nCоздадим файл, который можно будет отправить на конкурс.","metadata":{}},{"cell_type":"code","source":"# Поскольку мы разделяем набор данных и выполняем итерацию отдельно для изображений и идентификаторов, порядок имеет значение.\ntest_ds = get_test_dataset(ordered=True) \n\nprint('Вычисляем предсказания...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Создание файла submission.csv...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # все в одной партии\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T22:12:48.124477Z","iopub.status.idle":"2025-01-08T22:12:48.124919Z","shell.execute_reply.started":"2025-01-08T22:12:48.124707Z","shell.execute_reply":"2025-01-08T22:12:48.124737Z"}},"outputs":[],"execution_count":null}]}