{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"},{"sourceId":37130068,"sourceType":"kernelVersion"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Step 1: Imports #\n\nWe begin by importing several Python packages.","metadata":{}},{"cell_type":"code","source":"import math, re, os\nimport numpy as np\nimport tensorflow as tf\n\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-11-29T08:04:15.625777Z","iopub.execute_input":"2023-11-29T08:04:15.626183Z","iopub.status.idle":"2023-11-29T08:04:30.258900Z","shell.execute_reply.started":"2023-11-29T08:04:15.626153Z","shell.execute_reply":"2023-11-29T08:04:30.258009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential, Model\n# полносвязный слой и слой выпрямляющий матрицу в вектор\nfrom tensorflow.keras.layers import Dense, Flatten, Input\n# слой выключения нейронов и слой нормализации выходных данных (нормализует данные в пределах текущей выборки)\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, SpatialDropout2D, GaussianDropout\n# слои свертки и подвыборки\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D\n# работа с обратной связью от обучающейся нейронной сети\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n# вспомогательные инструменты\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.regularizers import *\nimport numpy as np","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 2: Distribution Strategy #\n\nA TPU has eight different *cores* and each of these cores acts as its own accelerator. (A TPU is sort of like having eight GPUs in one machine.) We tell TensorFlow how to make use of all these cores at once through a **distribution strategy**. Run the following cell to create the distribution strategy that we'll later apply to our model.","metadata":{}},{"cell_type":"code","source":"# Detect TPU, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-11-29T08:04:34.300246Z","iopub.execute_input":"2023-11-29T08:04:34.300786Z","iopub.status.idle":"2023-11-29T08:04:41.900968Z","shell.execute_reply.started":"2023-11-29T08:04:34.300754Z","shell.execute_reply":"2023-11-29T08:04:41.900040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path() #получаем путь к наборам данных\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-11-29T08:05:35.319450Z","iopub.execute_input":"2023-11-29T08:05:35.319779Z","iopub.status.idle":"2023-11-29T08:05:35.328375Z","shell.execute_reply.started":"2023-11-29T08:05:35.319752Z","shell.execute_reply":"2023-11-29T08:05:35.327397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [192, 192] # при таком размере графическому процессору не хватит памяти. Используйте TPU\nEPOCHS = 80\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE # находим количество шагов за эпоху","metadata":{"execution":{"iopub.status.busy":"2023-11-29T08:05:46.872247Z","iopub.execute_input":"2023-11-29T08:05:46.872569Z","iopub.status.idle":"2023-11-29T08:05:46.877512Z","shell.execute_reply.started":"2023-11-29T08:05:46.872541Z","shell.execute_reply":"2023-11-29T08:05:46.876562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Декодирует изображение в vyjujvthye. vfnhbwe (тензор)\n    Нормализует данные и преобразовывает изображения к указанному размеру\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3) # Декодирование изображения в формате JPEG в тензор uint8.\n    image = tf.cast(image, tf.float32) / 255.0  # преобразовать изображение в плавающее в диапазоне [0, 1]\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # явный размер, необходимый для TPU\n#     image = tf.keras.applications.inception_resnet_v2.preprocess_input(image)\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # [] означает отдельный элемент\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT) # парсим отдельный пример в указанном формате\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # возвращает набор данных пар (изображение, метка)\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # [] означает отдельный элемент\n        # класс отсутствует, задача этого конкурса - предсказать классы цветов для тестового набора данных\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"Читает из TFRecords. Для оптимальной производительности одновременное чтение из нескольких\n    файлов без учета порядка данных. Порядок не имеет значения, поскольку мы все равно будем перетасовывать данные\"\"\"\n\n    ignore_order = tf.data.Options() # Представляет параметры для tf.data.Dataset.\n    if not ordered:\n        ignore_order.experimental_deterministic = False # отключить порядок, увеличить скорость\n\n    dataset = tf.data.TFRecordDataset(filenames) # автоматически чередует чтение из нескольких файлов\n    dataset = dataset.with_options(ignore_order) # использует данные сразу после их поступления, а не в исходном порядке\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    # возвращает набор данных пар (изображение, метка), если метка = Истина, или пар (изображение, идентификатор), если метка = Ложь\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/train/*.tfrec'), labeled=True)\n    dataset = dataset.repeat() # набор обучающих данных должен повторяться в течение нескольких эпох\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/val/*.tfrec'), labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache() # кешируем набор\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/test/*.tfrec'), labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","metadata":{"execution":{"iopub.status.busy":"2023-11-29T08:06:10.342073Z","iopub.execute_input":"2023-11-29T08:06:10.342406Z","iopub.status.idle":"2023-11-29T08:06:10.528627Z","shell.execute_reply.started":"2023-11-29T08:06:10.342377Z","shell.execute_reply":"2023-11-29T08:06:10.527645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\ndata_augmentation = tf.keras.Sequential([\n  tf.keras.layers.RandomFlip(\"horizontal\"),\n  tf.keras.layers.RandomRotation(0.2),\n  tf.keras.layers.RandomZoom(0.2),\n])\n\n\ndef prepare(ds,  augment=False):\n  # Batch all datasets.\n  #ds = ds.batch(BATCH_SIZE)\n\n  # Use data augmentation only on the training set.\n  if augment:\n    ds = ds.map(lambda x, y: (data_augmentation(x, training=True), y), \n                num_parallel_calls=AUTOTUNE)\n\n  # Use buffered prefetching on all datasets.\n  return ds.prefetch(buffer_size=AUTOTUNE)\n\ntrain_ds = prepare(training_dataset, augment=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:36:26.528651Z","iopub.execute_input":"2023-11-29T09:36:26.529585Z","iopub.status.idle":"2023-11-29T09:36:26.996796Z","shell.execute_reply.started":"2023-11-29T09:36:26.529545Z","shell.execute_reply":"2023-11-29T09:36:26.995608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = next(iter(training_dataset))\nimage = tf.expand_dims(image[0], 0)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:55:11.098506Z","iopub.execute_input":"2023-11-29T09:55:11.098874Z","iopub.status.idle":"2023-11-29T09:55:11.212872Z","shell.execute_reply.started":"2023-11-29T09:55:11.098830Z","shell.execute_reply":"2023-11-29T09:55:11.211809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimage = next(iter(training_dataset))\nimage = tf.expand_dims(image[0], 0)\nplt.figure(figsize=(10, 10))\nfor i in range(9):\n  augmented_image = data_augmentation(image[0])\n  ax = plt.subplot(3, 3, i + 1)\n  plt.imshow(augmented_image[0])\n  plt.axis(\"off\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:55:41.662062Z","iopub.execute_input":"2023-11-29T09:55:41.662453Z","iopub.status.idle":"2023-11-29T09:55:43.506956Z","shell.execute_reply.started":"2023-11-29T09:55:41.662420Z","shell.execute_reply":"2023-11-29T09:55:43.505921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow.keras as tfk\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense, Conv2D, Conv3D, DepthwiseConv2D, SeparableConv2D, Conv3DTranspose\nfrom tensorflow.keras.layers import Flatten, MaxPool2D, AvgPool2D, GlobalAvgPool2D, UpSampling2D, BatchNormalization\nfrom tensorflow.keras.layers import Concatenate, Add, Dropout, ReLU, Lambda, Activation, LeakyReLU, PReLU\nfrom time import time\nimport numpy as np\n\ndef densenet(img_shape, n_classes, f=32):\n  repetitions = 6, 12, 24, 16\n\n  def bn_rl_conv(x, f, k=1, s=1, p='same'):\n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    x = Conv2D(f, k, strides=s, padding=p)(x)\n    return x\n\n\n  def dense_block(tensor, r):\n    for _ in range(r):\n      x = bn_rl_conv(tensor, 4*f)\n      x = bn_rl_conv(x, f, 3)\n      tensor = Concatenate()([tensor, x])\n    return tensor\n\n\n  def transition_block(x):\n    x = bn_rl_conv(x, K.int_shape(x)[-1] // 2)\n    x = AvgPool2D(2, strides=2, padding='same')(x)\n    return x\n\n\n  input = Input(img_shape)\n\n  x = Conv2D(64, 7, strides=2, padding='same')(input)\n  x = MaxPool2D(3, strides=2, padding='same')(x)\n\n  for r in repetitions:\n    d = dense_block(x, r)\n    x = transition_block(d)\n\n  x = GlobalAvgPool2D()(d)\n\n  output = Dense(n_classes, activation='softmax')(x)\n\n  model = Model(input, output)\n  return model","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:35:20.420041Z","iopub.execute_input":"2023-11-29T09:35:20.420435Z","iopub.status.idle":"2023-11-29T09:35:20.431706Z","shell.execute_reply.started":"2023-11-29T09:35:20.420399Z","shell.execute_reply":"2023-11-29T09:35:20.430591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = 192, 192, 3\nn_classes = 104\n\nwith strategy.scope():\n    model = densenet(input_shape, n_classes)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:35:30.068670Z","iopub.execute_input":"2023-11-29T09:35:30.069821Z","iopub.status.idle":"2023-11-29T09:35:42.752314Z","shell.execute_reply.started":"2023-11-29T09:35:30.069776Z","shell.execute_reply":"2023-11-29T09:35:42.751251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks_list = [EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n                  ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3),\n                  ]\n\nmodel.compile(\n    optimizer='nadam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nhistorical = model.fit(train_ds, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          callbacks=callbacks_list,\n          validation_data=validation_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:36:45.173266Z","iopub.execute_input":"2023-11-29T09:36:45.174134Z","iopub.status.idle":"2023-11-29T09:51:25.457985Z","shell.execute_reply.started":"2023-11-29T09:36:45.174093Z","shell.execute_reply":"2023-11-29T09:51:25.456611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Поскольку мы разделяем набор данных и выполняем итерацию отдельно для изображений и идентификаторов, порядок имеет значение.\ntest_ds = get_test_dataset(ordered=True) \n\nprint('Вычисляем предсказания...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Создание файла submission.csv...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # все в одной партии\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"execution":{"iopub.status.busy":"2023-11-29T09:52:27.029829Z","iopub.execute_input":"2023-11-29T09:52:27.031064Z","iopub.status.idle":"2023-11-29T09:52:57.144791Z","shell.execute_reply.started":"2023-11-29T09:52:27.031019Z","shell.execute_reply":"2023-11-29T09:52:57.143725Z"},"trusted":true},"execution_count":null,"outputs":[]}]}