{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras as tfk\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Dense, Flatten, Input, Concatenate\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, SpatialDropout2D, GaussianDropout\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, MaxPool2D, GlobalAvgPool2D, AvgPool2D\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.layers import ReLU, Add\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.regularizers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport numpy as np\n\nimport os\nimport random\nfrom tensorflow.random import set_seed\ndef seed_everything(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\n#     session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n#     sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)\n#     tf.compat.v1.keras.backend.set_session(sess)\n\n# seed = 42\n# seed_everything(seed)\n\nfrom tensorflow.keras.preprocessing import image\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\n%matplotlib inline \nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set environment","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU:', tpu.master())\nexcept ValueError:\n    tpu = None\n    \nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint('REPLICAS:', strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing data","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [192, 192]\nEPOCHS = 55\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Декодирует изображение в многомерную матрицу (тензор)\n    Нормализует данные и преобразовывает изображения к указанному размеру\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3) # Декодирование изображения в формате JPEG в тензор uint8.\n    image = tf.cast(image, tf.float32) / 255.0  # преобразовать изображение в плавающее в диапазоне [0, 1]\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # явный размер, необходимый для TPU\n\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # [] означает отдельный элемент\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT) # парсим отдельный пример в указанном формате\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    label = tf.cast(example['class'], tf.int32)\n    \n    return image, label # возвращает набор данных пар (изображение, метка)\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string означает байтовую строку\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # [] означает отдельный элемент\n        # класс отсутствует, задача этого конкурса - предсказать классы цветов для тестового набора данных\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) # преобразуем изображение к нужному нам формату\n    idnum = example['id']\n    \n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"Читает из TFRecords. Для оптимальной производительности одновременное чтение из нескольких\n    файлов без учета порядка данных. Порядок не имеет значения, поскольку мы все равно будем перетасовывать данные\"\"\"\n\n    ignore_order = tf.data.Options() # Представляет параметры для tf.data.Dataset.\n    if not ordered:\n        ignore_order.experimental_deterministic = False # отключить порядок, увеличить скорость\n\n    dataset = tf.data.TFRecordDataset(filenames) # автоматически чередует чтение из нескольких файлов\n    dataset = dataset.with_options(ignore_order) # использует данные сразу после их поступления, а не в исходном порядке\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    # возвращает набор данных пар (изображение, метка), если метка = Истина, или пар (изображение, идентификатор), если метка = Ложь\n    \n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/train/*.tfrec'), labeled=True)\n    dataset = dataset.repeat() # набор обучающих данных должен повторяться в течение нескольких эпох\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    \n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/val/*.tfrec'), labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache() # кешируем набор\n    \n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/test/*.tfrec'), labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    \n    return dataset\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation","metadata":{}},{"cell_type":"code","source":"def augment_data(ds):\n    transforms = Sequential([\n        tfk.layers.RandomFlip(\"horizontal_and_vertical\"),\n        tfk.layers.RandomRotation(0.2),\n        tfk.layers.RandomBrightness(0.2, value_range=(0.0, 1.0)),\n        tfk.layers.RandomContrast(0.2),\n#         tfk.layers.RandomCrop(30, 30),\n    ])\n    \n    ds = ds.map(lambda x, y: (transforms(x, training=True), y))\n    return ds\n    \ntraining_dataset = augment_data(training_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    filters = 32\n    repetitions = 6, 12, 24, 16\n    \n    def bn_relu_conv(x, fil, ker=1, stride=1, pad='same'):\n        x = BatchNormalization()(x)\n        x = ReLU()(x)\n        x = Conv2D(fil, ker, strides=stride, padding=pad)(x)\n        return x\n    \n    def dense_block(tensor, r):\n        for _ in range(r):\n            x = bn_relu_conv(tensor, 4*filters)\n            x = bn_relu_conv(x, filters, 3)\n            tensor = Concatenate()([tensor, x])\n        return tensor\n    \n    def transition_block(x):\n        x = bn_relu_conv(x, K.int_shape(x)[-1] // 2)\n        x = AvgPool2D(2, strides=2, padding='same')(x)\n        return x\n    \n    \n    inp = Input(shape=(*IMAGE_SIZE, 3))\n    \n    x = Conv2D(64, 7, strides=2, padding='same')(inp)\n    x = MaxPool2D(3, strides=2, padding='same')(x)\n    \n    for r in repetitions:\n        d = dense_block(x, r)\n        x = transition_block(d)\n    \n    x = GlobalAvgPool2D()(d)\n    \n    output = Dense(104, activation='softmax')(x)\n    \n    return Model(inp, output)\n\nwith strategy.scope():\n    model = get_model()\n    \nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = 1e-5\nLR_MAX = 3e-4 \nLR_MIN = 1e-5\nLR_RAMPUP_EPOCHS = 20 #4\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .9 # 0.7\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [\n#     ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=5),\n#     EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n    lr_callback,\n    ModelCheckpoint(filepath='best_weights.h5', monitor='loss', save_best_only=True)\n]\n\nmodel.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\n\nhistory = model.fit(training_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS, \n                    callbacks=callbacks,\n                    validation_data=validation_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['sparse_categorical_accuracy'], \n         label='Доля правильных ответов на обучающем наборе')\nplt.plot(history.history['val_sparse_categorical_accuracy'], \n         label='Доля правильных ответов на проверочном наборе')\nplt.xlabel('Эпоха обучения')\nplt.ylabel('Доля правильных ответов')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'], \n         label='Оценка потерь на обучающем наборе')\nplt.plot(history.history['val_loss'], \n         label='Оценка потерь на проверочном наборе')\nplt.xlabel('Эпоха обучения')\nplt.ylabel('Оценка потерь')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import ReLU, Add\n\ndef get_model():\n    \n    def conv_bn_relu(x, filters, ker=1, stride=1, pad='same'):\n        x = Conv2D(filters, ker, strides=stride, padding=pad)(x)\n        x = BatchNormalization()(x)\n        x = ReLU()(x)\n        return x\n    \n    def indentity_block(tensor, filters):\n        x = conv_bn_relu(tensor, filters)\n        x = conv_bn_relu(x, filters, 3)\n        x = Conv2D(4*filters, 1)(x)\n        x = BatchNormalization()(x)\n        \n        x = Add()([x, tensor])\n        out = ReLU()(x)\n        return out\n    \n    def conv_block(tensor, filters, stride):\n        x = conv_bn_relu(tensor, filters)\n        x = conv_bn_relu(x, filters, 3, stride)\n        x = Conv2D(4*filters, 1)(x)\n        x = BatchNormalization()(x)\n        \n        shortcut = Conv2D(4*filters, 1, strides=stride)(tensor)\n        shortcut = BatchNormalization()(shortcut)\n        \n        x = Add()([x, shortcut])\n        out = ReLU()(x)\n        return out\n    \n    def resnet_block(x, filters, repeats, stride=2):\n        x = conv_block(x, filters, stride)\n        for _ in range(repeats):\n            x = indentity_block(x, filters)\n        return x\n    \n    inp = Input(shape=(*IMAGE_SIZE, 3))\n    \n    x = conv_bn_relu(inp, 64, 7, 2)\n    x = MaxPool2D(3, strides=2, padding='same')(x)\n    \n    x = resnet_block(x, 64, 3, 1)\n#     x = Dropout(0.3)(x)\n    x = resnet_block(x, 128, 4)\n#     x = Dropout(0.3)(x)\n    x = resnet_block(x, 256, 6)\n#     x = Dropout(0.3)(x)\n    x = resnet_block(x, 512, 3)\n    \n    x = GlobalAvgPool2D()(x)\n    \n#     x = Dense(1024, activation='relu')(x)\n#     x = Dropout(0.3)(x)\n#     x = Dense(1024, activation='relu')(x)\n        \n    output = Dense(104, activation='softmax')(x)\n    \n    return Model(inp, output)\n\n# with strategy.scope():\n#     model = get_model()\n    \n# model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# callbacks = [\n#     ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=5),\n# #     EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n#     ModelCheckpoint(filepath='best_weights.h5', monitor='loss', save_best_only=True)\n# ]\n\n# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\n\n# tfk.utils.plot_model(\n#     model, to_file='model.png', show_shapes=True, show_layer_names=True,\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit(training_dataset, \n#                     steps_per_epoch=STEPS_PER_EPOCH, \n#                     epochs=EPOCHS, \n#                     callbacks=callbacks,\n#                     validation_data=validation_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) \n\nmodel.load_weights('best_weights.h5')\nprint('Вычисляем предсказания...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Создание файла submission.csv...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # все в одной партии\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}