{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"%matplotlib inline\n%load_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from google.cloud import storage\nimport json\nimport matplotlib.gridspec as gridspec\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import classification_report\nimport subprocess\nimport sys\nimport tensorflow as tf\nimport time\nfrom tqdm.notebook import tqdm\n\nfrom tensorflow.keras.backend import dot","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def print_output(output):\n    \"\"\"Prints output from string.\"\"\"\n    for l in output.split('\\n'):\n        print(l)\n\ndef print_pred_metrics(label_actual, label_pred):\n    \"\"\"Prints prediction evaluation metrics and report.\"\"\"\n    print(classification_report(label_actual, label_pred))\n#     print(pd.crosstab(label_actual, label_pred, margins=True))\n        \ndef run_command(command):\n    \"\"\"Runs command line command as a subprocess returning output as string.\"\"\"\n    STDOUT = subprocess.PIPE\n    process = subprocess.run(command, shell=True, check=False,\n                             stdout=STDOUT, stderr=STDOUT, universal_newlines=True)\n    return process.stdout\n\ndef show_images(imgs, titles=None, hw=(3,3), rc=(4,4)):\n    \"\"\"Show list of images with optiona list of titles.\"\"\"\n    h, w = hw\n    r, c = rc\n    fig=plt.figure(figsize=(w*c, h*r))\n    gs1 = gridspec.GridSpec(r, c, fig, hspace=0.2, wspace=0.05)\n    for i in range(r*c):\n        img = imgs[i].squeeze()\n        ax = fig.add_subplot(gs1[i])\n        if titles != None:\n            ax.set_title(titles[i], {'fontsize': 10})\n        plt.imshow(img)\n        plt.axis('off')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"output = run_command('pip freeze | grep efficientnet')\nif output == '':\n    print_output(run_command('pip install efficientnet'))\nelse:\n    print_output(output)\nfrom efficientnet import tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"KAGGLE = os.getenv('KAGGLE_KERNEL_RUN_TYPE') != None\n\nBUCKET = 'flowers-caleb'\nclient = storage.Client(project='fastai-caleb')\nbucket = client.get_bucket(BUCKET)\n\nif KAGGLE:\n    from kaggle_datasets import KaggleDatasets\n    DATASET_DIR = Path('/kaggle/input/flowers-caleb')\n    GCS_DATASET_DIR = KaggleDatasets().get_gcs_path(DATASET_DIR.parts[-1])\n    MODEL_BUCKET = GCS_DATASET_DIR.split('/')[-1]\n    PATH = Path('/kaggle/input/flower-classification-with-tpus')\n    TFRECORD_DIR = KaggleDatasets().get_gcs_path(PATH.parts[-1])\n    TPU_NAME = None\nelse:\n    DATASET_DIR = Path('./flowers-caleb')\n    MODEL_BUCKET = BUCKET\n    PATH = Path('/home/jupyter/.fastai/data/flowers')\n    TFRECORD_DIR = f'gs://{BUCKET}'\n    TPU_NAME = 'dfdc-1'\n    \nSIZES = {s: f'{s}x{s}' for s in [192, 224, 331, 512]}\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(TPU_NAME)\n    print('Running on TPU ', tpu.master())\nexcept:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n    \n    # CONVERT DEGREES TO RADIANS\n    pi = tf.constant(3.14159265359, tf.float32)\n    rotation = pi * rotation / 180.\n    shear = pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return dot(dot(rotation_matrix, shear_matrix), dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def transform(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = tf.shape(image)[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = tf.cast(idx2,dtype='int32')\n    idx2 = tf.clip_by_value(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def augment(example):\n    new_example = example.copy()\n    image = transform(new_example['image'])\n    image = tf.image.random_brightness(image, 0.3)\n    image = tf.image.random_contrast(image, 0.9, 1.1)\n    image = tf.image.random_hue(image, 0.1)\n    image = tf.image.random_jpeg_quality(image, 70, 100)\n    image = tf.image.random_saturation(image, 0.95, 1.05)\n    new_example['image'] = image\n    del example\n    \n    return new_example\n\ndef get_preprocess_fn(input_size=(224, 224), batch_size=128, norm=None, test=False):\n    \n    def imagenet_norm(image):\n        mean = tf.constant([0.485, 0.456, 0.406])\n        std = tf.constant([0.229, 0.224, 0.225])\n        \n        return (image / tf.constant(255, tf.float32) - mean) / std\n    \n    norm_fn = {'per_image': tf.image.per_image_standardization,\n               'imagenet': imagenet_norm,\n               None: tf.image.per_image_standardization\n              }\n\n    def preprocess(batch):\n        image = tf.image.resize(batch['image'], input_size)\n        image = norm_fn[norm](image)\n\n        if test:\n            return image\n        \n        else:\n            image = tf.reshape(image, (batch_size, *input_size, 3))\n            label = tf.cast(batch['label'], tf.float32)\n            label = tf.reshape(label, (batch_size,))\n                \n            return image, label\n        \n    return preprocess\n    \nCLASSES = tf.constant(pd.read_csv(DATASET_DIR/'classes.csv').values.squeeze(), tf.string)\n\ndef get_parse_fn(split):\n    def parse_fn(example):\n        features = {\"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n                    \"id\": tf.io.FixedLenFeature([], tf.string),\n                    \"class\": tf.io.FixedLenFeature([], tf.int64)}\n        \n        if split == 'test':\n            del features['class']\n        \n        example = tf.io.parse_single_example(example, features)\n        example['image'] = tf.image.decode_jpeg(example['image'], channels=3)\n        \n        if split != 'test':\n            example['label'] = tf.cast(example['class'], tf.int32)\n            example['class'] = CLASSES[example['label']]\n        return example\n\n    return parse_fn\n\ndef get_ds(split, img_size=224, batch_size=128, shuffle=False):\n    file_pat = f'{TFRECORD_DIR}/tfrecords-jpeg-{SIZES[img_size]}/{split}/*.tfrec'\n    \n    options = tf.data.Options()\n    options.experimental_deterministic = not shuffle\n    \n    ds = (tf.data.Dataset.list_files(file_pat, shuffle=shuffle)\n          .with_options(options)\n          .interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n          .map(get_parse_fn(split), num_parallel_calls=AUTO)\n         )\n    \n    if split == 'train':\n        ds = ds.repeat().map(augment, num_parallel_calls=AUTO).shuffle(2048)\n    \n    return ds.batch(batch_size).prefetch(AUTO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ds = get_ds('val')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for b in ds.take(1):\n    b=b\nb_aug = tf.map_fn(augment, b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"show_images(b['image'].numpy(), b['class'].numpy().tolist(), hw=(2,2), rc=(2,8))\nshow_images(b_aug['image'].numpy(), b_aug['class'].numpy().tolist(), hw=(2,2), rc=(2,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"img_size = 512 \ninput_size = (512, 512)\nbatch_size = 128\nweights = 'imagenet'\n\nds_train = get_ds('train', img_size=img_size, batch_size=batch_size, shuffle=True)\nds_valid = get_ds('val', img_size=img_size, batch_size=batch_size)\n\npreprocess = get_preprocess_fn(batch_size=batch_size,\n                               input_size=input_size, norm=weights)\n\nds_train_fit = ds_train.map(preprocess, num_parallel_calls=AUTO)\nds_valid_fit = ds_valid.map(preprocess, num_parallel_calls=AUTO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_prefix = 'model_efnb6_512_01'\nmodel_dir = f'gs://{MODEL_BUCKET}/{model_prefix}'\ncheckpoint_dir = f'{model_dir}/checkpoints'\ncheckpoint_fn = checkpoint_dir + '/' + 'cp-{epoch:04d}.ckpt'\ntensorboard_cb = tf.keras.callbacks.TensorBoard(log_dir=model_dir, write_graph=False, profile_batch=0)\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(checkpoint_fn, save_weights_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if False:\n    for b in bucket.list_blobs(prefix=f'{model_prefix}/checkpoints/cp-0052.ckpt'):\n        print(b.name)\n        b.download_to_filename(DATASET_DIR/b.name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if False:\n    for p in (DATASET_DIR/model_prefix/'checkpoints').glob('cp-0049*'):\n        p.unlink()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"cp_to_load = f'{checkpoint_dir}/cp-0052.ckpt'\n\nwith strategy.scope():\n    \n    opt = tf.keras.optimizers.Adam(1e-5)\n    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n    metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n    \n    cnn = efn.EfficientNetB6(weights=None,include_top=False,pooling='avg', input_shape=(*input_size, 3))\n    \n#     for l in cnn.layers[:-32]:\n#         l.trainable = False\n    cnn.trainable = True\n\n    model = tf.keras.Sequential([\n        cnn,\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(512, activation='selu', kernel_initializer=\"lecun_normal\"),\n        tf.keras.layers.AlphaDropout(0.5),\n        tf.keras.layers.Dense(512, activation='selu', kernel_initializer=\"lecun_normal\"),\n        tf.keras.layers.AlphaDropout(0.5),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n    \n    if cp_to_load is not None:\n        model.load_weights(cp_to_load)\n    \n    model.compile(loss=loss_fn, optimizer=opt, metrics=metrics)\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for split in ['train', 'val', 'test']:\n    items = 0\n    for b in bucket.list_blobs(prefix=f'tfrecords-jpeg-{SIZES[img_size]}/{split}'):\n        items += int(b.name.split('.')[0][-3:])\n#         print(b.name)\n    print(split, items, items // batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.optimizer.learning_rate = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false},"cell_type":"code","source":"if False:\n    history = model.fit(ds_train_fit,\n                        steps_per_epoch=200,\n                        epochs=57,\n                        initial_epoch=52,\n                        validation_data=ds_valid_fit,\n                        validation_steps=29,\n                        callbacks=[checkpoint_cb, tensorboard_cb]\n                       )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":false},"cell_type":"code","source":"split = 'test'\n\nds_pred = get_ds(split, img_size=img_size, batch_size=batch_size)\n\npreprocess = get_preprocess_fn(batch_size=batch_size,\n                               input_size=input_size, norm=weights, test=(split == 'test'))\n\nds_pred_pp = ds_pred.map(preprocess, num_parallel_calls=AUTO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# make sure example order is deterministic so we can line up training data with predictions\nassert np.array_equal(np.concatenate([b['id'] for b in ds_pred.as_numpy_iterator()]),\n               np.concatenate([b['id'] for b in ds_pred.as_numpy_iterator()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"id_list = []\nimg_list = []\nlabel_list = []\nclass_list = []\n\n# TTA = 2\n# if TTA is not None:\n#     predictions = []\n#     for b in tqdm(ds_pred.take(1)):\n#         id_list.extend(b['id'].numpy().squeeze())\n#         if split == 'val':\n#             label_list.extend(b['label'].numpy().squeeze())\n#             class_list.extend(b['class'].numpy().squeeze())\n#         avg_preds = []\n#         for i in range(TTA):\n#             b_aug = (tf.data.Dataset.from_tensors(b).unbatch()\n#                      .map(augment, num_parallel_calls=AUTO).batch(batch_size)\n#                      .map(preprocess, num_parallel_calls=AUTO))\n#             preds = model.predict(b_aug)\n#             avg_preds.append(preds)\n#         predictions.extend(np.mean(np.stack(avg_preds), axis=0))\n#     predictions = np.stack(predictions, axis=0)\n# else:\n\npredictions = model.predict(ds_pred_pp)\nfor b in ds_pred.as_numpy_iterator():\n    id_list.extend(b['id'].squeeze())\n    img_list.extend(b['image'].squeeze())\n    if split == 'val':\n        label_list.extend(b['label'].squeeze())\n        class_list.extend(b['class'].squeeze())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_pred = pd.DataFrame({'id': [n.decode() for n in id_list]})\n\ndf_pred['label'] = np.argmax(predictions, axis=1)\ndf_pred['class'] = [n.decode() for n in np.tile(np.expand_dims(CLASSES.numpy(), axis=0),\n                                (len(df_pred.label),))[:,df_pred.label].squeeze()]\ndf_pred['pred_prob'] = np.take_along_axis(predictions, np.expand_dims(df_pred.label, axis=1), axis=1)\n    \nif split == 'val':\n    df_pred['actual_class'] = [n.decode() for n in class_list]\n    df_pred['actual_label'] = label_list\n    if len(img_list) > 0:\n        df_pred['image'] = img_list\n    \ndf_pred[['id', 'label']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Error Analysis "},{"metadata":{"trusted":false},"cell_type":"code","source":"if split == 'val':\n    class_report = classification_report(df_pred.actual_label, df_pred.label, output_dict=True)\n    df_cl_rep = pd.DataFrame(class_report).T.iloc[:103]\n    df_cl_rep['f1-error'] = (1 - df_cl_rep['f1-score']) * df_cl_rep.support\n    df_cl_rep = df_cl_rep.sort_values('f1-error', ascending=False)\n    df_pred_g = pd.DataFrame(df_pred.groupby(['actual_label', 'label']).count()['id'])\n    print(df_cl_rep.head(10))\n    \n    error_label = int(df_cl_rep.index[0])\n    df_errors = df_pred[(df_pred.actual_label == error_label) & (df_pred.label != error_label)].copy()\n    df_errors['n_class_err'] = df_errors.label.map(df_errors.groupby('label').count()['id'])\n    if len(img_list) > 0:\n        df_errors['image'] = df_errors.id.map(df_pred.set_index('id').image)\n    df_errors = df_errors.sort_values(['n_class_err', 'label'], ascending=[False, False])\n    print('\\n',df_errors[[c for c in df_errors.columns if c != 'image']])\n    \n    show_images(df_errors.image.iloc[:16].to_list(),\n                df_errors['class'].iloc[:16].to_list(),\n                hw=(2,2), rc=(2,8))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Update Dataset"},{"metadata":{"trusted":false},"cell_type":"code","source":"if False:\n    if not KAGGLE:\n        print_output(run_command(f'kaggle d version -r tar -p {DATASET_DIR} -m \"add model checkpoint\"'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save Notebook"},{"metadata":{"trusted":false},"cell_type":"code","source":"%%javascript\nIPython.notebook.save_notebook()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Commit Kernel"},{"metadata":{"trusted":false},"cell_type":"code","source":"if True:\n    if not KAGGLE:\n\n        data = {'id': 'calebeverett/efficientnetb6-with-transformation',\n                      'title': 'EfficientnetB6 with Transformation',\n                      'code_file': 'flowers.ipynb',\n                      'language': 'python',\n                      'kernel_type': 'notebook',\n                      'is_private': 'false',\n                      'enable_gpu': 'true',\n                      'enable_internet': 'true',\n                      'dataset_sources': ['calebeverett/flowers-caleb'],\n                      'competition_sources': ['flower-classification-with-tpus'],\n                     ' kernel_sources': []}\n        \n        with open('kernel-metadata.json', 'w') as f:\n            json.dump(data, f)\n\n        print_output(run_command('kaggle k push'))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"tf21","language":"python","name":"tf21"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":true,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":4}