{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30628,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-19T06:02:48.489449Z","iopub.execute_input":"2023-12-19T06:02:48.489797Z","iopub.status.idle":"2023-12-19T06:02:55.201622Z","shell.execute_reply.started":"2023-12-19T06:02:48.489768Z","shell.execute_reply":"2023-12-19T06:02:55.200629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Импорт библиотек","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n#предварительно обученные модели\nfrom tensorflow.keras.applications import DenseNet201 \nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Dense, Flatten\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, GaussianDropout\nfrom tensorflow.keras.layers import GlobalAveragePooling2D\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.applications.vgg19 import VGG19\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2\nfrom tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201 \nfrom tensorflow.keras.applications.xception import Xception\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2\nfrom tensorflow.keras.applications.nasnet import NASNetLarge\nfrom efficientnet.tfkeras import EfficientNetB7, EfficientNetL2, EfficientNetB0, EfficientNetB1\n#  библиотека для работы с наборами данных на Kaggle\nfrom kaggle_datasets import KaggleDatasets\nimport re\nimport numpy as np\nimport random\nimport matplotlib.pyplot as plt\n%matplotlib inline \nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:02:55.203245Z","iopub.execute_input":"2023-12-19T06:02:55.20372Z","iopub.status.idle":"2023-12-19T06:03:10.363875Z","shell.execute_reply.started":"2023-12-19T06:02:55.203685Z","shell.execute_reply":"2023-12-19T06:03:10.363178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ускоритель\n","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Обнаружение оборудования, возврат соответствующей стратегии распространения: TPU, GPU, CPU\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # Обнаружение TPU. Параметры среды не требуются, если задана переменная среды TPU_NAME. На Kaggle это всегда так.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # стратегия распространения по умолчанию в Tensorflow. Работает на CPU и одном GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:10.364704Z","iopub.execute_input":"2023-12-19T06:03:10.365109Z","iopub.status.idle":"2023-12-19T06:03:18.299155Z","shell.execute_reply.started":"2023-12-19T06:03:10.365081Z","shell.execute_reply":"2023-12-19T06:03:18.298377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Загрузка данных","metadata":{}},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started') #получаем путь к наборам данных\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:18.300732Z","iopub.execute_input":"2023-12-19T06:03:18.301008Z","iopub.status.idle":"2023-12-19T06:03:18.305168Z","shell.execute_reply.started":"2023-12-19T06:03:18.300978Z","shell.execute_reply":"2023-12-19T06:03:18.304509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [512, 512]\nEPOCHS = 13\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\n\nSEED = 2020","metadata":{"execution":{"iopub.status.busy":"2023-12-19T07:18:36.870911Z","iopub.execute_input":"2023-12-19T07:18:36.871742Z","iopub.status.idle":"2023-12-19T07:18:36.896964Z","shell.execute_reply.started":"2023-12-19T07:18:36.871705Z","shell.execute_reply":"2023-12-19T07:18:36.895905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Декодирует изображение в vyjujvthye. vfnhbwe (тензор)\n    Нормализует данные и преобразовывает изображения к указанному размеру\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3) # Декодирование изображения в формате JPEG в тензор uint8.\n    image = tf.cast(image, tf.float32) / 255.0  # преобразовать изображение в плавающее в диапазоне [0, 1]\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # явный размер, необходимый для TPU\n#     image = tf.keras.applications.inception_resnet_v2.preprocess_input(image)\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # [] означает отдельный элемент\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT) # парсим отдельный пример в указанном формате\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # возвращает набор данных пар (изображение, метка)\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # [] означает отдельный элемент\n        # класс отсутствует, задача этого конкурса - предсказать классы цветов для тестового набора данных\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"Читает из TFRecords. Для оптимальной производительности одновременное чтение из нескольких\n    файлов без учета порядка данных. Порядок не имеет значения, поскольку мы все равно будем перетасовывать данные\"\"\"\n\n    ignore_order = tf.data.Options() # Представляет параметры для tf.data.Dataset.\n    if not ordered:\n        ignore_order.experimental_deterministic = False # отключить порядок, увеличить скорость\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # автоматически чередует чтение из нескольких файлов\n    dataset = dataset.with_options(ignore_order) # использует данные сразу после их поступления, а не в исходном порядке\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # возвращает набор данных пар (изображение, метка), если метка = Истина, или пар (изображение, идентификатор), если метка = Ложь\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    flag = random.randint(1,3)\n    if flag == 1:\n        image = tf.image.random_flip_left_right(image, seed=SEED)\n    elif flag == 2:\n        image = tf.image.random_flip_up_down(image, seed=SEED)\n    else:\n        pass\n    return image, label   \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # набор обучающих данных должен повторяться в течение нескольких эпох\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) #готовим следующий набор, пока предыдущий обучается\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache() # кешируем набор\n    dataset = dataset.prefetch(AUTO) #готовим следующий набор, пока предыдущий обучается\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) #готовим следующий набор, пока предыдущий обучается\n    return dataset\n                               \ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:18.358656Z","iopub.execute_input":"2023-12-19T06:03:18.358994Z","iopub.status.idle":"2023-12-19T06:03:18.375959Z","shell.execute_reply.started":"2023-12-19T06:03:18.358966Z","shell.execute_reply":"2023-12-19T06:03:18.375232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# функция управляющая изменениями шага обучения в процессе тренировки нейронной сети\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync#0.0001\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\n# построим график изменения шага обучение в зависимости от эпох\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:18.376951Z","iopub.execute_input":"2023-12-19T06:03:18.377205Z","iopub.status.idle":"2023-12-19T06:03:18.559362Z","shell.execute_reply.started":"2023-12-19T06:03:18.377179Z","shell.execute_reply":"2023-12-19T06:03:18.558693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef get_model(use_model, weights):\n    # noisy-student\n    base_model = use_model(weights=weights, \n                      include_top=False, pooling='avg',\n                      input_shape=(*IMAGE_SIZE, 3))\n    x = base_model.output\n    predictions = Dense(104, activation='softmax')(x)\n    return Model(inputs=base_model.input, outputs=predictions)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:18.560283Z","iopub.execute_input":"2023-12-19T06:03:18.560573Z","iopub.status.idle":"2023-12-19T06:03:18.564904Z","shell.execute_reply.started":"2023-12-19T06:03:18.560544Z","shell.execute_reply":"2023-12-19T06:03:18.564195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DenseNet121","metadata":{}},{"cell_type":"code","source":"with strategy.scope():    \n    dense = get_model(DenseNet121, 'imagenet')\n        \ndense.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:18.566792Z","iopub.execute_input":"2023-12-19T06:03:18.567029Z","iopub.status.idle":"2023-12-19T06:03:51.136422Z","shell.execute_reply.started":"2023-12-19T06:03:18.567004Z","shell.execute_reply":"2023-12-19T06:03:51.135485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = dense.fit(get_training_dataset(), \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          callbacks=[lr_callback, ModelCheckpoint(filepath='dense.h5', monitor='val_loss',\n                                  save_best_only=True)],workers = 3)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:03:51.137544Z","iopub.execute_input":"2023-12-19T06:03:51.137839Z","iopub.status.idle":"2023-12-19T06:23:10.049231Z","shell.execute_reply.started":"2023-12-19T06:03:51.137813Z","shell.execute_reply":"2023-12-19T06:23:10.047851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# InceptoinV3","metadata":{}},{"cell_type":"code","source":"with strategy.scope():    \n    inception = get_model(InceptionV3, 'imagenet')\n        \ninception.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:23:10.052943Z","iopub.execute_input":"2023-12-19T06:23:10.053226Z","iopub.status.idle":"2023-12-19T06:23:39.408407Z","shell.execute_reply.started":"2023-12-19T06:23:10.053198Z","shell.execute_reply":"2023-12-19T06:23:39.407412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = inception.fit(get_training_dataset(), \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          callbacks=[lr_callback, ModelCheckpoint(filepath='inceptionV3.h5', monitor='val_loss',\n                                  save_best_only=True)],workers = 3)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T06:23:39.409467Z","iopub.execute_input":"2023-12-19T06:23:39.409709Z","iopub.status.idle":"2023-12-19T06:39:53.406974Z","shell.execute_reply.started":"2023-12-19T06:23:39.409682Z","shell.execute_reply":"2023-12-19T06:39:53.405897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EfficientNetB7","metadata":{}},{"cell_type":"code","source":"with strategy.scope():    \n    efficientnet = get_model(EfficientNetB7, 'noisy-student')\n        \nefficientnet.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-12-19T07:18:46.643265Z","iopub.execute_input":"2023-12-19T07:18:46.644129Z","iopub.status.idle":"2023-12-19T07:19:21.913179Z","shell.execute_reply.started":"2023-12-19T07:18:46.644097Z","shell.execute_reply":"2023-12-19T07:19:21.91203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = efficientnet.fit(get_training_dataset(), \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          callbacks=[lr_callback, ModelCheckpoint(filepath='/kaggle/working/efnetb7.h5', monitor='val_loss',\n                                  save_best_only=True)],workers = 3)","metadata":{"execution":{"iopub.status.busy":"2023-12-19T07:19:21.914695Z","iopub.execute_input":"2023-12-19T07:19:21.914977Z","iopub.status.idle":"2023-12-19T07:49:44.2703Z","shell.execute_reply.started":"2023-12-19T07:19:21.914948Z","shell.execute_reply":"2023-12-19T07:49:44.26928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Лучшая модель","metadata":{}},{"cell_type":"code","source":"eval_dataset = get_validation_dataset()\n\neval_results = {\n    'DenseNet121': dense.evaluate(eval_dataset)[1],\n    'InceptionV3': inception.evaluate(eval_dataset)[1],\n    'EfficientNetB7': efficientnet.evaluate(eval_dataset)[1],\n}\n\nfig, ax = plt.subplots()\nax.bar(list(eval_results.keys()), list(eval_results.values()))\nax.set_ylabel('sparse_categorical_accuracy')\nax.set_title('Метрика моделей на валидационных данных')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-19T07:50:18.733127Z","iopub.execute_input":"2023-12-19T07:50:18.733984Z","iopub.status.idle":"2023-12-19T07:51:21.952244Z","shell.execute_reply.started":"2023-12-19T07:50:18.733947Z","shell.execute_reply":"2023-12-19T07:51:21.951199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) \n\nprint('Вычисляем предсказания...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = efficientnet.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Создание файла submission.csv...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # все в одной партии\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\nprint('Создан')","metadata":{"execution":{"iopub.status.busy":"2023-12-19T07:54:54.055068Z","iopub.execute_input":"2023-12-19T07:54:54.055923Z","iopub.status.idle":"2023-12-19T07:55:27.744062Z","shell.execute_reply.started":"2023-12-19T07:54:54.055884Z","shell.execute_reply":"2023-12-19T07:55:27.743149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}