{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Librairies","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n\nimport os\nfrom glob import glob\n\nimport matplotlib.pyplot as plt\n\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n\n## removes tensorflow warning\nimport tensorflow as tf \n\nimport IPython.display as display\n\n","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:34.881907Z","iopub.execute_input":"2021-10-18T17:01:34.88256Z","iopub.status.idle":"2021-10-18T17:01:34.890273Z","shell.execute_reply.started":"2021-10-18T17:01:34.882515Z","shell.execute_reply":"2021-10-18T17:01:34.889385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's first define some variables, counting images from TFRecords might be fastidious, so i directly computed sizes\n","metadata":{}},{"cell_type":"code","source":"print(tf.__version__)\nprint(\"Num GPUs available : \", len(tf.config.list_physical_devices('GPU')))\n\nBATCH = 32\nNB_CLASS = 104\nEPOCH = 18\nTRAIN_IMAGE_NUMBER = 12753\nVAL_IMAGE_NUMBER = 3712\nTEST_IMAGE_NUMBER = 7382","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:37.169546Z","iopub.execute_input":"2021-10-18T17:01:37.169827Z","iopub.status.idle":"2021-10-18T17:01:37.176327Z","shell.execute_reply.started":"2021-10-18T17:01:37.169778Z","shell.execute_reply":"2021-10-18T17:01:37.175282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading files","metadata":{}},{"cell_type":"code","source":"local_path = os.path.join('/kaggle','input','tpu-getting-started','tfrecords-jpeg-224x224')\n\n\ntrain_filenames = [ fileu for fileu in glob(os.path.join(local_path,'train/*'))]\nval_filenames = [ fileu for fileu in glob(os.path.join(local_path,'val/*'))]\ntest_filenames = [ fileu for fileu in glob(os.path.join(local_path,'test/*'))]\n\n\nprint(f'Total Training TFRECORD FILES : {len(train_filenames)}')\nprint(f'Total Validation TFRECORD FILES : {len(val_filenames)}')\nprint(f'Total Testing TFRECORD FILES : {len(test_filenames)}')","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:38.353699Z","iopub.execute_input":"2021-10-18T17:01:38.354214Z","iopub.status.idle":"2021-10-18T17:01:38.367822Z","shell.execute_reply.started":"2021-10-18T17:01:38.354173Z","shell.execute_reply":"2021-10-18T17:01:38.366883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SETUP for TPU utilisation\n\n\n1. `num_parallel_reads=AUTO` instructs the API to read from multiple files if available. It figures out how many automatically.\n2. `experimental_deterministic = False` disables data order enforcement. We will be shuffling the data anyway so order is not important. With this setting the API can use any TFRecord as soon as it is streamed in.\n3. However for predictions we should keep the order so this variable must be set to `True`","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nignore_order = tf.data.Options()\n\nignore_order.experimental_deterministic = False\n","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:39.846519Z","iopub.execute_input":"2021-10-18T17:01:39.846925Z","iopub.status.idle":"2021-10-18T17:01:39.852649Z","shell.execute_reply.started":"2021-10-18T17:01:39.846872Z","shell.execute_reply":"2021-10-18T17:01:39.85183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    \n    image = tf.image.decode_jpeg(image,channels=3)\n    image = tf.cast(image,tf.float32)\n    image = tf.reshape(image,[224,224,3])\n    \n    return image\n\n\ndef read_training_tfrecord(example_proto):\n    image_feature_description = {\n        'class' : tf.io.FixedLenFeature([] , tf.int64),\n        'image' : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example_proto, image_feature_description)\n    \n    image = decode_image(example['image'])\n    \n    label = tf.cast(example['class'], tf.int32)\n    \n    label = tf.one_hot(label, NB_CLASS)\n        \n        \n    return image, label\n\n\ndef read_testing_tfrecord(example_proto):\n    \n    image_feature_description = {\n        'id' : tf.io.FixedLenFeature([], tf.string),\n        'image' : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example_proto, image_feature_description)\n    \n    image = decode_image(example['image'])\n    \n    ids = example['id']\n    \n    return image, ids\n    \n    \n ## We do not want to the data to be shuffled for testing, so we separate process   \ndef load_dataset(filenames,style):\n    \n    \n    dataset = tf.data.TFRecordDataset(filenames,num_parallel_reads=AUTO)\n    \n    \n    if style =='training' or style=='validation' : \n        \n        dataset = dataset.with_options(ignore_order)\n    \n        dataset = dataset.map(read_training_tfrecord)\n            \n        dataset = dataset.cache().shuffle(1000).prefetch(buffer_size=32)\n\n        dataset = dataset.batch(BATCH)\n        \n    else :\n        \n        ignore_order.experimental_deterministic = True\n        \n        dataset = dataset.with_options(ignore_order)\n        \n        dataset = dataset.map(read_testing_tfrecord)\n        \n        dataset = dataset.prefetch(buffer_size=32).batch(BATCH)\n        \n    \n    return dataset\n\n    ","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:47.844678Z","iopub.execute_input":"2021-10-18T17:01:47.845186Z","iopub.status.idle":"2021-10-18T17:01:47.857847Z","shell.execute_reply.started":"2021-10-18T17:01:47.84514Z","shell.execute_reply":"2021-10-18T17:01:47.857039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = load_dataset(train_filenames,style = 'training')\n\nval_dataset = load_dataset(val_filenames, style = 'validation')\n\n\ntest_dataset = load_dataset(test_filenames, style = 'testing')","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:50.071542Z","iopub.execute_input":"2021-10-18T17:01:50.072451Z","iopub.status.idle":"2021-10-18T17:01:50.31653Z","shell.execute_reply.started":"2021-10-18T17:01:50.072408Z","shell.execute_reply":"2021-10-18T17:01:50.315778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simple Visualisation","metadata":{}},{"cell_type":"code","source":"def display_batch(image_batch,label_batch):\n    plt.figure(figsize=(10,10))\n    \n    for k in range(16):\n        \n        ax = plt.subplot(4,4,k+1)\n        \n        plt.imshow(image_batch[k] /255.)\n        \n        \n        #plt.title('Classes : '+ str(id_batch[k]))\n        #because one_hot_encoding applied\n        plt.axis('off')\n\nimage_batch, label_batch = next(iter(train_dataset))     \ndisplay_batch(image_batch, label_batch)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-10-18T17:01:51.730749Z","iopub.execute_input":"2021-10-18T17:01:51.731253Z","iopub.status.idle":"2021-10-18T17:01:54.08289Z","shell.execute_reply.started":"2021-10-18T17:01:51.731213Z","shell.execute_reply":"2021-10-18T17:01:54.081915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building model\n\n\nFor the model, let's use Xception pre-trained model on imageNet, Inception module is really powerful.  \nConcerning top layers, no real research/comparison were performed. So one way to improve the model would be to try different architecture. ","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\n\nfrom tensorflow.keras.optimizers import schedules\n\nimport tensorflow_addons as tfa\n\n\n\n## Possible imporvements\n\n# - Using sparseCorssEntropy instead of hot encoding\n# - training also the base model\n# - Solid LearningRateScheduler","metadata":{"execution":{"iopub.status.busy":"2021-10-18T14:57:47.472953Z","iopub.execute_input":"2021-10-18T14:57:47.473819Z","iopub.status.idle":"2021-10-18T14:57:47.800227Z","shell.execute_reply.started":"2021-10-18T14:57:47.473761Z","shell.execute_reply":"2021-10-18T14:57:47.799447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef build_model():\n    \n    base_model = tf.keras.applications.Xception(\n        include_top=False,\n        weights=\"imagenet\",\n        input_tensor=None,\n        input_shape=(224,224,3),\n    )\n    \n    base_model.trainable = False\n        \n    data_aug = Sequential([\n        \n        layers.Rescaling(1./127.5, offset=-1),\n        \n        layers.RandomFlip(\"horizontal_and_vertical\"),\n        \n        layers.RandomRotation(0.3)])\n    \n    model = Sequential([\n        \n        data_aug,\n              \n        base_model,\n        \n        layers.GlobalAveragePooling2D(),\n    \n        layers.Dense(1024, activation = 'relu'),\n    \n        layers.Dropout(0.2),\n        \n        layers.Dense(256, activation = 'relu'),\n        \n        layers.Dropout(0.2),\n    \n        layers.Dense(NB_CLASS,activation = 'softmax')\n    ])\n    \n\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-18T14:57:47.816192Z","iopub.execute_input":"2021-10-18T14:57:47.817072Z","iopub.status.idle":"2021-10-18T14:57:47.825992Z","shell.execute_reply.started":"2021-10-18T14:57:47.817028Z","shell.execute_reply":"2021-10-18T14:57:47.825005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PrintLR(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        print('\\nLearning rate for epoch {} is {}'.format(epoch + 1,model.optimizer.lr.numpy()))\n##not really usefull","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Early Stopping isn't really useful here, because i didn't trained on many epoch\n2. The ExponnetialDecay is also not really useful, however one must note that even if Adam optimizer modify itself learning rate, adding a scheduler fixes the upper limit of learning rate modified \n3. About the metric, again it is not useful to work with F1_score directly, however it gives a glimpse of your model capacity ( because this is the exact metric used by kaggle on this competition","metadata":{}},{"cell_type":"code","source":"def training():\n\n    init_lr = 0.001\n\n    \n    lr_scheduler = schedules.ExponentialDecay(\n        init_lr, decay_steps=int(np.ceil(TRAIN_IMAGE_NUMBER/BATCH)), decay_rate=0.94, staircase=True)\n\n    early_stopping_cb = tf.keras.callbacks.EarlyStopping(\n        patience=3, restore_best_weights=True)\n\n    checkpoint_path = './training_1/cp.ckpt'\n\n    checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n        checkpoint_path,save_weights_only=True, save_best_only=True, verbose = 1\n    )\n\n    with tf.device('/device:GPU:0'):\n        model = build_model()\n\n        model.compile(optimizer=keras.optimizers.Adam(learning_rate = lr_scheduler),\n                      loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False),\n                      metrics=tfa.metrics.F1Score(num_classes=NB_CLASS, average = 'macro', threshold=0.5)\n        )\n   \n        \n    history = model.fit(\n        train_dataset,\n        epochs=EPOCH,\n        validation_data=val_dataset,\n        callbacks=[checkpoint_cb, early_stopping_cb],\n)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-18T14:58:07.353253Z","iopub.execute_input":"2021-10-18T14:58:07.353945Z","iopub.status.idle":"2021-10-18T14:58:08.530433Z","shell.execute_reply.started":"2021-10-18T14:58:07.353878Z","shell.execute_reply":"2021-10-18T14:58:08.529602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Predictions","metadata":{}},{"cell_type":"code","source":"def test(model):\n    testing_image = test_dataset.map(lambda image, ids : image)\n\n    testing_ids = test_dataset.map(lambda image, ids: ids).unbatch()\n\n\n    test_ids = next(iter(testing_ids.batch(TEST_IMAGE_NUMBER))).numpy().astype('U')\n\n\n    predictions_raw = model.predict(testing_image, batch_size = BATCH)\n\n    pred = np.argmax(predictions_raw, axis=-1)\n\n\n    predictions_raw = model.predict(testing_image, batch_size = BATCH)\n\n    pred = np.argmax(predictions_raw, axis=-1)\n    \n    \n    submission_df = pd.DataFrame(data ={'id': test_ids, 'label' : pred} ).set_index('id')\n\n\n    submission_df.to_csv('submission.csv')\n\n    return\n","metadata":{"execution":{"iopub.status.busy":"2021-10-18T16:57:25.026671Z","iopub.execute_input":"2021-10-18T16:57:25.027324Z","iopub.status.idle":"2021-10-18T16:57:25.035016Z","shell.execute_reply.started":"2021-10-18T16:57:25.027279Z","shell.execute_reply":"2021-10-18T16:57:25.034029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = training()","metadata":{"execution":{"iopub.status.busy":"2021-10-18T16:56:37.817737Z","iopub.execute_input":"2021-10-18T16:56:37.818525Z","iopub.status.idle":"2021-10-18T16:56:39.55945Z","shell.execute_reply.started":"2021-10-18T16:56:37.818478Z","shell.execute_reply":"2021-10-18T16:56:39.558662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test(model)","metadata":{"execution":{"iopub.status.busy":"2021-10-18T16:57:26.74686Z","iopub.execute_input":"2021-10-18T16:57:26.747295Z","iopub.status.idle":"2021-10-18T16:58:11.351485Z","shell.execute_reply.started":"2021-10-18T16:57:26.747244Z","shell.execute_reply":"2021-10-18T16:58:11.350558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-10-19T08:06:01.533337Z","iopub.execute_input":"2021-10-19T08:06:01.534115Z","iopub.status.idle":"2021-10-19T08:06:01.559526Z","shell.execute_reply.started":"2021-10-19T08:06:01.534024Z","shell.execute_reply":"2021-10-19T08:06:01.558805Z"},"trusted":true},"execution_count":null,"outputs":[]}]}