{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Set below variable to whatever accelerator you use\n\nGet a TPU if you can, otherwise you'll be waiting a while","metadata":{}},{"cell_type":"code","source":"ACCELERATOR = 'tpu' # 'tpu-vm', 'tpu', 'gpu'","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:03.184528Z","iopub.execute_input":"2023-03-30T19:37:03.185264Z","iopub.status.idle":"2023-03-30T19:37:03.191993Z","shell.execute_reply.started":"2023-03-30T19:37:03.185222Z","shell.execute_reply":"2023-03-30T19:37:03.190870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# These are your independent variables","metadata":{}},{"cell_type":"code","source":"USE_AUGMENTATION = False\nUSE_EXTERNAL_DATA = False\nUSE_CLASS_WEIGHTS = False\n# USE_10_CROP_TESTING = False\n\nRESNET_DEPTH = 50 # 50, 101, 152","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:03.484316Z","iopub.execute_input":"2023-03-30T19:37:03.484695Z","iopub.status.idle":"2023-03-30T19:37:03.489888Z","shell.execute_reply.started":"2023-03-30T19:37:03.484658Z","shell.execute_reply":"2023-03-30T19:37:03.488677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Sizes","metadata":{}},{"cell_type":"markdown","source":"### Higher resolution likely gives higher accuracy with the expense of longer training time","metadata":{}},{"cell_type":"markdown","source":"### Also high resolution images require more memory","metadata":{}},{"cell_type":"markdown","source":"#### Crop Size is the size used by random cropping","metadata":{}},{"cell_type":"code","source":"INCLUDED_SCALES = [192, 224, 331, 512]\nTEST_SCALE = 512\nMIN_CROP = 0.75","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:04.187856Z","iopub.execute_input":"2023-03-30T19:37:04.188223Z","iopub.status.idle":"2023-03-30T19:37:04.194588Z","shell.execute_reply.started":"2023-03-30T19:37:04.188190Z","shell.execute_reply":"2023-03-30T19:37:04.192340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# These relate to the training process.\nThere's probably some golden values that make training faster","metadata":{}},{"cell_type":"markdown","source":"### Depending on other variables, 16 may be rather low for epochs.\n### But those take time, P100 takes ~3 min per epoch with 331x331 images","metadata":{}},{"cell_type":"code","source":"if ACCELERATOR[0:3] == 'tpu':\n    BATCH_SIZE = 128\nelse:\n    BATCH_SIZE = 16\n\nSTEPS_PER_EXECUTION = 16\nEPOCHS = 16\nLR_PATIENCE = 3\nINITIAL_LR = 0.0001","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:04.839011Z","iopub.execute_input":"2023-03-30T19:37:04.839382Z","iopub.status.idle":"2023-03-30T19:37:04.845387Z","shell.execute_reply.started":"2023-03-30T19:37:04.839348Z","shell.execute_reply":"2023-03-30T19:37:04.844020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### If using a TPU VM, you need to install certain versions of packages","metadata":{}},{"cell_type":"code","source":"# For TPU VM\nif ACCELERATOR == 'tpu-vm':\n    !pip install /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n    !pip install scikit-learn","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:05.246917Z","iopub.execute_input":"2023-03-30T19:37:05.247548Z","iopub.status.idle":"2023-03-30T19:37:05.254083Z","shell.execute_reply.started":"2023-03-30T19:37:05.247511Z","shell.execute_reply":"2023-03-30T19:37:05.252700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normal imports","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport numpy as np\nfrom sklearn.utils import class_weight\n\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:05.931386Z","iopub.execute_input":"2023-03-30T19:37:05.932165Z","iopub.status.idle":"2023-03-30T19:37:14.104895Z","shell.execute_reply.started":"2023-03-30T19:37:05.932119Z","shell.execute_reply":"2023-03-30T19:37:14.102850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get computation strategy for acceleration","metadata":{}},{"cell_type":"code","source":"# For TPU VM\nif ACCELERATOR == 'tpu-vm':\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    strategy = tf.distribute.TPUStrategy(tpu)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:16.018996Z","iopub.execute_input":"2023-03-30T19:37:16.019784Z","iopub.status.idle":"2023-03-30T19:37:16.029181Z","shell.execute_reply.started":"2023-03-30T19:37:16.019743Z","shell.execute_reply":"2023-03-30T19:37:16.024802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For TPU\nif ACCELERATOR == 'tpu':\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:16.509524Z","iopub.execute_input":"2023-03-30T19:37:16.510217Z","iopub.status.idle":"2023-03-30T19:37:16.515652Z","shell.execute_reply.started":"2023-03-30T19:37:16.510180Z","shell.execute_reply":"2023-03-30T19:37:16.514435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For GPU\nif ACCELERATOR == 'gpu':\n    strategy = tf.distribute.get_strategy()\n    tpu = False","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:16.945587Z","iopub.execute_input":"2023-03-30T19:37:16.946267Z","iopub.status.idle":"2023-03-30T19:37:16.957831Z","shell.execute_reply.started":"2023-03-30T19:37:16.946230Z","shell.execute_reply":"2023-03-30T19:37:16.956680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Data","metadata":{}},{"cell_type":"code","source":"COMPETITION_DATA_PATH = KaggleDatasets().get_gcs_path(\"tpu-getting-started\")\nEXTERNAL_DATA_PATH = KaggleDatasets().get_gcs_path(\"tf-flower-photo-tfrec\")","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:17.841518Z","iopub.execute_input":"2023-03-30T19:37:17.842214Z","iopub.status.idle":"2023-03-30T19:37:18.580476Z","shell.execute_reply.started":"2023-03-30T19:37:17.842178Z","shell.execute_reply":"2023-03-30T19:37:18.579457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMPETITION_DATA_PATH","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:21.060433Z","iopub.execute_input":"2023-03-30T19:37:21.061154Z","iopub.status.idle":"2023-03-30T19:37:21.068632Z","shell.execute_reply.started":"2023-03-30T19:37:21.061112Z","shell.execute_reply":"2023-03-30T19:37:21.067406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE_PATHS = [f'/tfrecords-jpeg-{size}x{size}' for size in INCLUDED_SCALES]\nTEST_SIZE_PATH = f'/tfrecords-jpeg-{TEST_SCALE}x{TEST_SCALE}'","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:21.464317Z","iopub.execute_input":"2023-03-30T19:37:21.465036Z","iopub.status.idle":"2023-03-30T19:37:21.469855Z","shell.execute_reply.started":"2023-03-30T19:37:21.464996Z","shell.execute_reply":"2023-03-30T19:37:21.468696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FILENAMES = [tf.io.gfile.glob(COMPETITION_DATA_PATH + path + '/train/*.tfrec') for path in IMAGE_SIZE_PATHS]\nVAL_FILENAMES = tf.io.gfile.glob(COMPETITION_DATA_PATH + TEST_SIZE_PATH + '/val/*.tfrec')\n\nTEST_FILENAMES = tf.io.gfile.glob(COMPETITION_DATA_PATH + TEST_SIZE_PATH + '/test/*.tfrec')\n\n# if USE_EXTERNAL_DATA:\n#     IMAGENET_FILES = tf.io.gfile.glob([EXTERNAL_DATA_PATH + '/imagenet_no_test' + path + '/*.tfrec' for path in IMAGE_SIZE_PATHS])\n#     INATURELIST_FILES = tf.io.gfile.glob([EXTERNAL_DATA_PATH + '/inaturalist_no_test' + path + '/*.tfrec' for path in IMAGE_SIZE_PATHS])\n#     OPENIMAGE_FILES = tf.io.gfile.glob([EXTERNAL_DATA_PATH + '/openimage_no_test' + path + '/*.tfrec' for path in IMAGE_SIZE_PATHS])\n#     OXFORD_FILES = tf.io.gfile.glob([EXTERNAL_DATA_PATH + '/oxford_102_no_test' + path + '/*.tfrec' for path in IMAGE_SIZE_PATHS])\n#     TENSORFLOW_FILES = tf.io.gfile.glob([EXTERNAL_DATA_PATH + '/tf_flowers_no_test' + path + '/*.tfrec' for path in IMAGE_SIZE_PATHS])\n\n#     TRAIN_FILENAMES = TRAIN_FILENAMES + IMAGENET_FILES + INATURELIST_FILES + OXFORD_FILES + TENSORFLOW_FILES","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:21.882419Z","iopub.execute_input":"2023-03-30T19:37:21.883583Z","iopub.status.idle":"2023-03-30T19:37:23.121369Z","shell.execute_reply.started":"2023-03-30T19:37:21.883526Z","shell.execute_reply":"2023-03-30T19:37:23.120318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\ndef count_files(filenames):\n    return sum([int(re.compile('(\\d+)\\.tfrec').search(i).groups()[0]) for i in filenames])","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:23.124039Z","iopub.execute_input":"2023-03-30T19:37:23.124403Z","iopub.status.idle":"2023-03-30T19:37:23.131041Z","shell.execute_reply.started":"2023-03-30T19:37:23.124364Z","shell.execute_reply":"2023-03-30T19:37:23.129757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum([count_files(filenames) for filenames in TRAIN_FILENAMES])","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:44.144543Z","iopub.execute_input":"2023-03-30T19:37:44.145462Z","iopub.status.idle":"2023-03-30T19:37:44.154256Z","shell.execute_reply.started":"2023-03-30T19:37:44.145409Z","shell.execute_reply":"2023-03-30T19:37:44.153218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_files(VAL_FILENAMES)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:49.386720Z","iopub.execute_input":"2023-03-30T19:37:49.387693Z","iopub.status.idle":"2023-03-30T19:37:49.394534Z","shell.execute_reply.started":"2023-03-30T19:37:49.387647Z","shell.execute_reply":"2023-03-30T19:37:49.393377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_files(TEST_FILENAMES)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:37:51.912913Z","iopub.execute_input":"2023-03-30T19:37:51.913273Z","iopub.status.idle":"2023-03-30T19:37:51.920442Z","shell.execute_reply.started":"2023-03-30T19:37:51.913240Z","shell.execute_reply":"2023-03-30T19:37:51.919423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_BATCHES = sum([count_files(filenames) // BATCH_SIZE for filenames in TRAIN_FILENAMES]) \nVAL_BATCHES = count_files(VAL_FILENAMES) // BATCH_SIZE\nTEST_BATCHES = count_files(TEST_FILENAMES) // BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:45:06.563558Z","iopub.execute_input":"2023-03-30T19:45:06.564538Z","iopub.status.idle":"2023-03-30T19:45:06.570333Z","shell.execute_reply.started":"2023-03-30T19:45:06.564500Z","shell.execute_reply":"2023-03-30T19:45:06.569291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parse and Preprocess Data","metadata":{}},{"cell_type":"code","source":"def parse(labeled=True):\n    features = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'class': tf.io.FixedLenFeature([], tf.int64),\n    } if labeled else {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'id': tf.io.FixedLenFeature([], tf.string)\n    }\n    def parse_func(raw):\n        data = tf.io.parse_example(raw, features)\n        image = tf.image.decode_jpeg(data['image'], channels=3)\n        #image = tf.image.resize(image, (IMAGE_SIZE, IMAGE_SIZE))\n        label = data['class' if labeled else 'id']\n        return image, label\n    return parse_func","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:38:12.967789Z","iopub.execute_input":"2023-03-30T19:38:12.968155Z","iopub.status.idle":"2023-03-30T19:38:12.977894Z","shell.execute_reply.started":"2023-03-30T19:38:12.968123Z","shell.execute_reply":"2023-03-30T19:38:12.976868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeled_parse = parse()\nunlabeled_parse = parse(labeled=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:38:18.908847Z","iopub.execute_input":"2023-03-30T19:38:18.909218Z","iopub.status.idle":"2023-03-30T19:38:18.914468Z","shell.execute_reply.started":"2023-03-30T19:38:18.909183Z","shell.execute_reply":"2023-03-30T19:38:18.913110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_list = [tf.data.TFRecordDataset(filenames) for filenames in TRAIN_FILENAMES]\nds_list = [ds.shuffle(128) for ds in ds_list]\nds_list = [ds.map(labeled_parse) for ds in ds_list]\nds_list = [ds.batch(BATCH_SIZE, drop_remainder=True) for ds in ds_list]\nds = tf.data.Dataset.sample_from_datasets(ds_list)\n\n\n\n#ds = ds.shuffle(count_files(TRAIN_FILENAMES))\n#ds = ds.map(labeled_parse)\n\n#ds = ds.map(lambda image, label: (tf.image.resize(image, (IMAGE_SIZE, IMAGE_SIZE)), label))\n\n# def augment(image, label):\n#     rank = image.get_shape().ndims\n    \n#     if (rank == 4):\n#         size = tf.shape(image)[1]\n#         bs = image.shape[0]\n#     else: # rank == 3\n#         size = tf.shape(image)[0]\n        \n#     min_crop = tf.cast(tf.cast(size, tf.dtypes.float64) * MIN_CROP, tf.dtypes.int32)\n    \n#     image = tf.image.random_flip_left_right(image)\n#     c = tf.random.uniform([], minval=min_crop, maxval=size, dtype=tf.dtypes.int32)\n    \n#     if rank == 4:\n#         image = tf.image.random_crop(image, (bs, c, c, 3))\n#     else:\n#         image = tf.image.random_crop(image, (c, c, 3))\n    \n#     if rank == 4:\n#         image.set_shape([bs, None, None, 3])\n#     else:\n#         image.set_shape([None, None, 3])\n        \n#     return image, label\n\n#ds = ds.batch(BATCH_SIZE, drop_remainder=True)\n\n# if USE_AUGMENTATION:\n#     ds = ds.map(augment)\n\n#ds = ds.batch(BATCH_SIZE, drop_remainder=True)\n#ds = ds.prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:43:30.741455Z","iopub.execute_input":"2023-03-30T19:43:30.742134Z","iopub.status.idle":"2023-03-30T19:43:31.104768Z","shell.execute_reply.started":"2023-03-30T19:43:30.742095Z","shell.execute_reply":"2023-03-30T19:43:31.103761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if USE_CLASS_WEIGHTS:\n#     ds_freq = dict(zip(range(104), class_weight.compute_class_weight('balanced', classes=range(104), y=[l.numpy() for l in tf.data.TFRecordDataset(TRAIN_FILENAMES).map(labeled_parse).map(lambda image, label: label)])))","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:43:34.261270Z","iopub.execute_input":"2023-03-30T19:43:34.261661Z","iopub.status.idle":"2023-03-30T19:43:34.266062Z","shell.execute_reply.started":"2023-03-30T19:43:34.261604Z","shell.execute_reply":"2023-03-30T19:43:34.265023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vds = tf.data.TFRecordDataset(VAL_FILENAMES)\nvds = vds.map(labeled_parse)\n# if USE_AUGMENTATION:\n#     vds = vds.map(lambda image, label: (tf.image.resize(image, (CROP_SIZE, CROP_SIZE)), label))\nvds = vds.batch(BATCH_SIZE, drop_remainder=True)\nvds = vds.prefetch(tf.data.AUTOTUNE)\n# if tpu:\n#     vds = vds.cache()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:43:34.847882Z","iopub.execute_input":"2023-03-30T19:43:34.848252Z","iopub.status.idle":"2023-03-30T19:43:34.876735Z","shell.execute_reply.started":"2023-03-30T19:43:34.848219Z","shell.execute_reply":"2023-03-30T19:43:34.875795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tds = tf.data.TFRecordDataset(TEST_FILENAMES)\ntds = tds.map(unlabeled_parse)\ntds = tds.map(lambda image, label: (tf.image.resize(image, (TEST_SCALE, TEST_SCALE)), label))","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:43:36.806329Z","iopub.execute_input":"2023-03-30T19:43:36.807051Z","iopub.status.idle":"2023-03-30T19:43:36.859321Z","shell.execute_reply.started":"2023-03-30T19:43:36.807011Z","shell.execute_reply":"2023-03-30T19:43:36.858384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Flatten, Dense, Conv2D, MaxPool2D, GlobalMaxPool2D, GlobalAveragePooling2D, Dropout","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:43:40.123824Z","iopub.execute_input":"2023-03-30T19:43:40.124206Z","iopub.status.idle":"2023-03-30T19:43:40.132212Z","shell.execute_reply.started":"2023-03-30T19:43:40.124170Z","shell.execute_reply":"2023-03-30T19:43:40.131057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    x = Input(shape=(None, None, 3))\n    \n    if RESNET_DEPTH == 50:\n        resnet = tf.keras.applications.ResNet50(include_top=False, input_shape=[None, None, 3], weights=None)\n    elif RESNET_DEPTH == 101:\n        resnet = tf.keras.applications.ResNet101(include_top=False, input_shape=[None, None, 3], weights=None)\n    elif RESNET_DEPTH == 152:\n        resnet = tf.keras.applications.ResNet152(include_top=False, input_shape=[None, None, 3], weights=None)\n\n    y = tf.keras.applications.resnet50.preprocess_input(x)\n    features = resnet(y)\n    y = GlobalAveragePooling2D()(features)\n    hidden = Dense(2048, activation='relu')\n    classifier = Dense(104, activation='softmax')\n    y = classifier(Dropout(0.5)(hidden(y)))\n    #extract = classifier(hidden(features))\n\n    model = Model(inputs=x, outputs=y)\n    #extraction = Model(inputs=x, outputs=extract)\n    \n    test_x = Input(shape=(TEST_SCALE, TEST_SCALE, 3))\n    test_y = model(test_x)\n    test_model = Model(inputs=test_x, outputs=test_y)\n    \n#     if USE_10_CROP_TESTING:\n\n#         x = Input(shape=(IMAGE_SIZE, IMAGE_SIZE, 3))\n#         c = tf.image.central_crop(x, CROP_SIZE / IMAGE_SIZE)\n#         c = tf.image.resize(c, (CROP_SIZE, CROP_SIZE))\n#         tl = tf.image.crop_to_bounding_box(x, 0, 0, CROP_SIZE, CROP_SIZE)\n#         tr = tf.image.crop_to_bounding_box(x, 0, IMAGE_SIZE - CROP_SIZE, CROP_SIZE, CROP_SIZE)\n#         bl = tf.image.crop_to_bounding_box(x, IMAGE_SIZE - CROP_SIZE, 0, CROP_SIZE, CROP_SIZE)\n#         br = tf.image.crop_to_bounding_box(x, IMAGE_SIZE - CROP_SIZE, IMAGE_SIZE - CROP_SIZE, CROP_SIZE, CROP_SIZE)\n\n#         cf = tf.image.flip_left_right(c)\n#         tlf = tf.image.flip_left_right(tl)\n#         trf = tf.image.flip_left_right(tr)\n#         blf = tf.image.flip_left_right(bl)\n#         brf = tf.image.flip_left_right(br)\n\n#         y = tf.reduce_mean(tf.stack([model(c), model(tl), model(tr), model(bl), model(br), model(cf), model(tlf), model(trf), model(blf), model(brf)], axis=1), axis=1)\n#         test_model = Model(inputs=x, outputs=y)\n#     else:\n#         test_model = model","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:43:58.438529Z","iopub.execute_input":"2023-03-30T19:43:58.439227Z","iopub.status.idle":"2023-03-30T19:44:01.093230Z","shell.execute_reply.started":"2023-03-30T19:43:58.439190Z","shell.execute_reply":"2023-03-30T19:44:01.091933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:44:01.095360Z","iopub.execute_input":"2023-03-30T19:44:01.095727Z","iopub.status.idle":"2023-03-30T19:44:01.149339Z","shell.execute_reply.started":"2023-03-30T19:44:01.095695Z","shell.execute_reply":"2023-03-30T19:44:01.148267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:44:10.067787Z","iopub.execute_input":"2023-03-30T19:44:10.068708Z","iopub.status.idle":"2023-03-30T19:44:10.101548Z","shell.execute_reply.started":"2023-03-30T19:44:10.068653Z","shell.execute_reply":"2023-03-30T19:44:10.100503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model.compile(steps_per_execution=STEPS_PER_EXECUTION, optimizer=tf.keras.optimizers.Adam(learning_rate=INITIAL_LR), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\n    learning_callback = tf.keras.callbacks.ReduceLROnPlateau(verbose=1, patience=LR_PATIENCE)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:44:12.904202Z","iopub.execute_input":"2023-03-30T19:44:12.904775Z","iopub.status.idle":"2023-03-30T19:44:12.940454Z","shell.execute_reply.started":"2023-03-30T19:44:12.904731Z","shell.execute_reply":"2023-03-30T19:44:12.939475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if USE_CLASS_WEIGHTS:\n#     historical = model.fit(ds.repeat(), class_weight=ds_freq, epochs=EPOCHS, validation_data=vds, callbacks=[learning_callback], steps_per_epoch=TRAIN_BATCHES, validation_steps=VAL_BATCHES)\n# else:\nhistorical = model.fit(ds.repeat(), epochs=EPOCHS, validation_data=vds, callbacks=[learning_callback], steps_per_epoch=TRAIN_BATCHES, validation_steps=VAL_BATCHES)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T19:45:16.635218Z","iopub.execute_input":"2023-03-30T19:45:16.635593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_historical(historical):\n    loss_values = historical.history['loss']\n    val_loss_values = historical.history['val_loss']\n    epochs = range(1, len(loss_values)+1)\n\n    plt.plot(epochs, loss_values, label='Training Loss')\n    plt.plot(epochs, val_loss_values, label='Validation Loss')\n\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-28T06:02:08.737267Z","iopub.execute_input":"2023-03-28T06:02:08.738124Z","iopub.status.idle":"2023-03-28T06:02:09.204525Z","shell.execute_reply.started":"2023-03-28T06:02:08.738079Z","shell.execute_reply":"2023-03-28T06:02:09.202927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_historical(historical)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T06:02:09.763339Z","iopub.execute_input":"2023-03-28T06:02:09.764589Z","iopub.status.idle":"2023-03-28T06:02:10.003689Z","shell.execute_reply.started":"2023-03-28T06:02:09.764541Z","shell.execute_reply":"2023-03-28T06:02:10.002200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('history.npy',historical.history)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T06:02:29.285556Z","iopub.execute_input":"2023-03-28T06:02:29.286466Z","iopub.status.idle":"2023-03-28T06:02:29.292324Z","shell.execute_reply.started":"2023-03-28T06:02:29.286424Z","shell.execute_reply":"2023-03-28T06:02:29.291021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-03-28T06:02:30.149787Z","iopub.execute_input":"2023-03-28T06:02:30.150623Z","iopub.status.idle":"2023-03-28T06:02:33.313405Z","shell.execute_reply.started":"2023-03-28T06:02:30.150571Z","shell.execute_reply":"2023-03-28T06:02:33.311893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Computing predictions...')\ntest_images_ds = tds.map(lambda image, idnum: image).batch(BATCH_SIZE)\nprobabilities = test_model.predict(test_images_ds, steps=TEST_BATCHES+1)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = tds.map(lambda image, idnum: idnum)\ntest_ids = next(iter(test_ids_ds.batch(count_files(TEST_FILENAMES)))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"execution":{"iopub.status.busy":"2023-03-28T06:02:34.921230Z","iopub.execute_input":"2023-03-28T06:02:34.922346Z","iopub.status.idle":"2023-03-28T06:03:18.230335Z","shell.execute_reply.started":"2023-03-28T06:02:34.922307Z","shell.execute_reply":"2023-03-28T06:03:18.229052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}