{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Блокнот для отправки на проверку\nОбучение модели: https://www.kaggle.com/nurkasimov/cassava-leaf-disease-tpu-efficientnetb4"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install --quiet /kaggle/input/kerasapplications\n!pip install --quiet /kaggle/input/efficientnet-git","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom efficientnet.tfkeras import EfficientNetB3, EfficientNetB4, EfficientNetB5\nimport tensorflow.keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Обнаружение TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Определение основных переменных и методом"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nWORK_PATH = '../input/cassava-leaf-disease-classification'\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\nCHANNELS = 3\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=CHANNELS)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Загрузка тестовых данных"},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_FILENAMES = tf.io.gfile.glob(WORK_PATH + '/test_tfrecords/ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Построение модели"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n  \n    base_model = EfficientNetB5(weights=None, include_top=False, input_shape = (None, None, 3), pooling='avg')\n    # base_model.trainable = False\n    \n    model = tf.keras.Sequential([\n        base_model,\n        tf.keras.layers.Dropout(.3),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')  \n    ])\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(lr = 0.001),\n        loss='sparse_categorical_crossentropy',  \n        metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Загрузка весов ранее обученной модели"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('../input/cassava-leaf-disease-tpu-efficientnetb4/EffNetB5_best_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_float32(image, label):\n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Вычисление предсказаний и формирование таблицы с результатами "},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True).map(to_float32)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\n\ntest_preds = np.argmax(probabilities, axis=-1)\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\nimage_names = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\nsubmission = pd.DataFrame({'image_id': image_names, 'label': test_preds})\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}