{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TF 2.2 блокнот\n[По сути, это перевод стартового блокнота от команды TensorFlow](https://www.kaggle.com/philculliton/a-simple-petals-tf-2-2-notebook)","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Dense, Flatten, Input\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, SpatialDropout2D, GaussianDropout\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.regularizers import *\nimport numpy as np\nimport os\nfrom tensorflow.random import set_seed\ndef seed_everything(seed):\n    np.random.seed(seed)\n    set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\n\nfrom tensorflow.keras.preprocessing import image\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\n%matplotlib inline \nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-05-31T02:07:31.108077Z","iopub.execute_input":"2022-05-31T02:07:31.108691Z","iopub.status.idle":"2022-05-31T02:07:37.332034Z","shell.execute_reply.started":"2022-05-31T02:07:31.108602Z","shell.execute_reply":"2022-05-31T02:07:37.331125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Определяем, какой ускоритель можем использовать","metadata":{}},{"cell_type":"code","source":"\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:07:37.333761Z","iopub.execute_input":"2022-05-31T02:07:37.334034Z","iopub.status.idle":"2022-05-31T02:07:43.296195Z","shell.execute_reply.started":"2022-05-31T02:07:37.334004Z","shell.execute_reply":"2022-05-31T02:07:43.295603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get my data path","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path() ","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:07:43.297500Z","iopub.execute_input":"2022-05-31T02:07:43.297697Z","iopub.status.idle":"2022-05-31T02:07:43.724843Z","shell.execute_reply.started":"2022-05-31T02:07:43.297674Z","shell.execute_reply":"2022-05-31T02:07:43.724140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set some parameters","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [192, 192] \nEPOCHS = 100\nBATCH_SIZE = 24 * strategy.num_replicas_in_sync\n\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE ","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:07:43.726324Z","iopub.execute_input":"2022-05-31T02:07:43.727012Z","iopub.status.idle":"2022-05-31T02:07:43.731484Z","shell.execute_reply.started":"2022-05-31T02:07:43.726983Z","shell.execute_reply":"2022-05-31T02:07:43.730692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Загружаем данные\n\nЭти данные загружаются из Kaggle и автоматически сегментируются для максимального распараллеливания.","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Декодирует изображение в vyjujvthye. vfnhbwe (тензор)\n    Нормализует данные и преобразовывает изображения к указанному размеру\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3) \n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) \n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"class\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT) \n    image = decode_image(example['image']) \n    label = tf.cast(example['class'], tf.int32)\n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"id\": tf.io.FixedLenFeature([], tf.string), \n        \n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) \n    idnum = example['id']\n    return image, idnum \n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"Читает из TFRecords. Для оптимальной производительности одновременное чтение из нескольких\n    файлов без учета порядка данных. Порядок не имеет значения, поскольку мы все равно будем перетасовывать данные\"\"\"\n\n    ignore_order = tf.data.Options() \n    if not ordered:\n        ignore_order.experimental_deterministic = False \n\n    dataset = tf.data.TFRecordDataset(filenames) \n    dataset = dataset.with_options(ignore_order) \n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    \n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/train/*.tfrec'), labeled=True)\n    dataset = dataset.repeat() # набор обучающих данных должен повторяться в течение нескольких эпох\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/val/*.tfrec'), labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache() # кешируем набор\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/test/*.tfrec'), labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:07:43.732780Z","iopub.execute_input":"2022-05-31T02:07:43.733068Z","iopub.status.idle":"2022-05-31T02:07:44.143411Z","shell.execute_reply.started":"2022-05-31T02:07:43.733030Z","shell.execute_reply":"2022-05-31T02:07:44.142349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Построить модель на TPU (или GPU, или CPU...) с Tensorflow 2.1!","metadata":{}},{"cell_type":"code","source":"def get_model():\n\n    model = Sequential()\n\n    model.add(Conv2D(64, (3, 3), input_shape=(*IMAGE_SIZE, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(64, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(64, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(GaussianDropout(0.2))\n\n    model.add(Conv2D(128, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(128, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(128, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(GaussianDropout(0.3))\n\n    model.add(Conv2D(256, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(256, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(256, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(GaussianDropout(0.45))\n\n    \n    model.add(Conv2D(512, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(Conv2D(512, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n   \n    model.add(Conv2D(512, (3, 3), activation='relu'))\n    model.add(BatchNormalization())\n    \n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(GaussianDropout(0.3))\n    \n    \n    \n    model.add(Flatten())\n    \n    model.add(Dense(1024, activation='relu'))\n    \n    model.add(BatchNormalization())\n    \n    model.add(GaussianDropout(0.3))\n    \n    model.add(Dense(512, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(GaussianDropout(0.2))\n    \n    model.add(Dense(104, activation='softmax'))\n    return model\n\n\nwith strategy.scope():    \n    model = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:07:44.146238Z","iopub.execute_input":"2022-05-31T02:07:44.146450Z","iopub.status.idle":"2022-05-31T02:07:47.013553Z","shell.execute_reply.started":"2022-05-31T02:07:44.146426Z","shell.execute_reply":"2022-05-31T02:07:47.012658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks_list = [EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n                  ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3),\n                  ]\n\nmodel.compile(\n    optimizer='nadam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nhistorical = model.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          callbacks=callbacks_list,\n          validation_data=validation_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:07:47.014792Z","iopub.execute_input":"2022-05-31T02:07:47.015053Z","iopub.status.idle":"2022-05-31T02:13:49.369300Z","shell.execute_reply.started":"2022-05-31T02:07:47.015023Z","shell.execute_reply":"2022-05-31T02:13:49.368430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    plt.plot(historical.history['sparse_categorical_accuracy'], \n         label='accuracy')\n    plt.xlabel('Эпоха обучения')\n    plt.ylabel('Доля правильных ответов')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:13:49.370630Z","iopub.execute_input":"2022-05-31T02:13:49.370876Z","iopub.status.idle":"2022-05-31T02:13:49.621952Z","shell.execute_reply.started":"2022-05-31T02:13:49.370848Z","shell.execute_reply":"2022-05-31T02:13:49.621171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Вычислите свои прогнозы на тестовом наборе!\n\nCоздадим файл, который можно будет отправить на конкурс.","metadata":{}},{"cell_type":"code","source":"# Поскольку мы разделяем набор данных и выполняем итерацию отдельно для изображений и идентификаторов, порядок имеет значение.\ntest_ds = get_test_dataset(ordered=True) \n\nprint('Вычисляем предсказания...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Создание файла submission.csv...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # все в одной партии\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"execution":{"iopub.status.busy":"2022-05-31T02:13:49.623369Z","iopub.execute_input":"2022-05-31T02:13:49.623610Z","iopub.status.idle":"2022-05-31T02:14:16.775353Z","shell.execute_reply.started":"2022-05-31T02:13:49.623582Z","shell.execute_reply":"2022-05-31T02:14:16.774501Z"},"trusted":true},"execution_count":null,"outputs":[]}]}