{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom matplotlib import image\nfrom matplotlib import pyplot\nimport json\nimport os\nimport cv2\nimport random\nimport concurrent.futures\nimport time\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom itertools import cycle\nfrom sklearn import svm, datasets\nfrom sklearn.metrics import roc_curve, auc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.preprocessing import label_binarize\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom scipy import interp\nfrom sklearn.metrics import roc_auc_score\n\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/hotel-id-csv-sorted-appended/x.csv\", dtype=str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes=np.unique(train['hotel_id'].values.astype(int))\ndic = dict()\ni = 0\nfor cs in classes:\n    dic.update({str(cs): i})\n    i+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels=[]\nfor ele in train['hotel_id'].values:\n    labels.append(dic[ele])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(labels),\n                                                 labels)\nclass_weight_dict = dict(enumerate(class_weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Splitting into train and validation sets\nnames = train['name_path'].values\nVAL_SPLIT = 0.25\ntrain_names, val_names, train_labels, val_labels = train_test_split(names[:10000], labels[:10000], \\\n                                                    test_size=VAL_SPLIT, random_state=42,\\\n                                                                       stratify=labels[:10000])\n\ntrain_labels = np.array(train_labels)\nval_labels = np.array(val_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMSIZE = 256\ndef read_img(image):\n    img = tf.keras.preprocessing.image.load_img(image, color_mode='rgb', target_size=(IMSIZE, IMSIZE))\n    img = tf.keras.preprocessing.image.img_to_array(img)\n    img = tf.image.convert_image_dtype(img, dtype=tf.uint8, saturate=True)\n    return img\ndef prepare_dataset(namelist, labels, path):\n    start = time.time()\n    labels = np.array(labels)\n    labels = tf.convert_to_tensor(labels)\n    labels = tf.cast(labels, tf.uint16)\n    imgs = []\n    with concurrent.futures.ThreadPoolExecutor(max_workers = 16) as executor:\n        i = 0\n        for value in executor.map(read_img, namelist):\n            i+=1\n            print(\"\\rFetching: [{}/{}]\".format(i, len(namelist)), end=\"\", flush=True)\n            imgs.append(value)\n        imgs = tf.convert_to_tensor(imgs)\n    print(\"\\nExecution time: \",time.time() - start, \"s\")\n    return imgs, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tf.device('/cpu:0'):\n    path = '../input/hotel-id-2021-fgvc8'\n    print(\"Training count\", len(train_names))\n    print(\"Validation count\", len(val_names))\n    train_images, train_labels = prepare_dataset(train_names, train_labels, path)\n    #val_images, val_labels = prepare_dataset(val_names, val_labels, path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training Image tensor shape\", train_images.shape)\nprint(\"Training Labels tensor shape\", train_labels.shape)\n#print(\"Testing Image tensor shape\", val_images.shape)\n#print(\"Tesing Labels tensor shape\", val_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Only use for TPU session\"\"\"\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\nprint(\"All devices: \", tf.config.list_logical_devices('TPU'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 1000\nrandom_rotation = tf.keras.layers.experimental.preprocessing.RandomRotation(3.142/2, seed=SEED)\nrandom_flip = tf.keras.layers.experimental.preprocessing.RandomFlip(mode=\"horizontal_and_vertical\", seed=SEED)\nrandom_zoom = tf.keras.layers.experimental.preprocessing.RandomZoom((0, 0.35), seed=SEED)\nrandom_translate = tf.keras.layers.experimental.preprocessing.RandomTranslation((-0.25, 0.25), (-0.25, 0.25), seed=SEED)\n\ndef preprocess(imgs, label):\n    imgs = random_rotation.call(imgs)\n    imgs = random_flip.call(imgs)\n    imgs = random_zoom.call(imgs)\n    imgs = random_translate.call(imgs)\n    return imgs, label\n\ndef normalize(imgs, label):\n    return tf.cast(imgs, tf.float16)/255, label\n\nstrategy = tf.distribute.experimental.TPUStrategy(resolver)\nwith tf.device('/cpu:0'):\n    TRAIN_BATCH_SIZE = 128\n    VAL_BATCH_SIZE = 64\n    TRAIN_SIZE = len(train_images)\n    VAL_SIZE = len(val_names)\n    train_dataset = tf.data.Dataset.from_tensor_slices((tf.cast(train_images, tf.uint8), \\\n                                                        tf.cast(train_labels, tf.uint16)))\n    del train_images, train_labels\n    train_dataset = train_dataset.shuffle(TRAIN_SIZE).repeat()\\\n                                 .batch(TRAIN_BATCH_SIZE)\\\n                                 .shuffle(TRAIN_BATCH_SIZE)\n    #                .map(preprocess, num_parallel_calls=tf.data.experimental.AUTOTUNE)\\\n    #                .cache()\n    train_dataset = train_dataset.map(normalize, \\\n                                      num_parallel_calls=tf.data.experimental.AUTOTUNE)\\\n                                 .prefetch(tf.data.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tf.device('/cpu:0'):\n    val_images, val_labels = prepare_dataset(val_names, val_labels, path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Testing Image tensor shape\", val_images.shape)\nprint(\"Tesing Labels tensor shape\", val_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tf.device('/cpu:0'):\n    val_dataset = tf.data.Dataset.from_tensor_slices((tf.cast(val_images, tf.uint8), \\\n                                                      tf.cast(val_labels, tf.uint16)))\n    del val_images, val_labels\n    val_dataset = val_dataset.repeat().batch(VAL_BATCH_SIZE)\n    val_dataset = val_dataset.map(normalize, \\\n                                  num_parallel_calls=tf.data.experimental.AUTOTUNE)\\\n                             .prefetch(tf.data.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def IResNet_brain_module(input, layercount):\n    n = layercount\n    ############################################################################\n    # Parallel Block 1\n    x1_1 = tf.keras.layers.Conv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input)\n    x1_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x1_1)\n    x1 = tf.keras.layers.add([x1_1, x1_2])\n    ############################################################################\n    # Parallel Block 2\n    x2_1 = tf.keras.layers.Conv2D(n, (5, 5), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input)\n    x2_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x2_1)\n    x2 = tf.keras.layers.add([x2_1, x2_2, x1])\n    ############################################################################\n    # Parallel Block 3\n    x3_1 = tf.keras.layers.Conv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input)\n    x3_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input)\n    x3 = tf.keras.layers.add([x3_1, x3_2, x2, x1])\n    ############################################################################\n    # Parallel Block 4\n    x4_1 = tf.keras.layers.Conv2D(n, (7, 7), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input)\n    x4_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x4_1)\n    x4 = tf.keras.layers.add([x4_1, x4_2, x3, x2, x1])\n    ############################################################################\n    mod = tf.keras.layers.concatenate([x1, x2, x3, x4], axis = -1)\n    mod = tf.keras.layers.BatchNormalization()(mod)\n    return mod\ndef IResNet_connection_module(input1, input2, input3, input4, layercount):\n    n = layercount\n    ############################################################################\n    # Parallel Block 1\n    x1 = tf.keras.layers.Conv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input1)\n    ############################################################################\n    # Parallel Block 2\n    x2 = tf.keras.layers.Conv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input2)\n    ############################################################################\n    # Parallel Block 3\n    x3 = tf.keras.layers.Conv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input3)\n    ############################################################################\n    # Parallel Block 4\n    x4 = tf.keras.layers.Conv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(input4)\n    ############################################################################\n    mod = tf.keras.layers.add([x1, x2, x3, x4])\n    mod = tf.keras.layers.BatchNormalization()(mod)\n    return mod\ndef IResNet_reduction_module(input, layercount):\n    n = layercount\n    ############################################################################\n    # Reduction module\n    R1= tf.keras.layers.SeparableConv2D(n, (5, 5), strides = 2, activation = 'relu', kernel_regularizer='l2', bias_regularizer='l1')(input)\n    R1 = tf.keras.layers.BatchNormalization()(R1) \n    R1 = tf.keras.layers.SeparableConv2D(n, (3, 3), strides = 2, activation = 'relu', kernel_regularizer='l2', bias_regularizer='l1')(R1)\n    mod = tf.keras.layers.BatchNormalization()(R1)\n    R1 = tf.keras.layers.SeparableConv2D(2*n, (3, 3), strides = 2, activation = 'relu', kernel_regularizer='l2', bias_regularizer='l1')(R1)\n    mod = tf.keras.layers.BatchNormalization()(R1)\n    return mod\ndef IResNet_classifier_module(input, num_classes, activation):\n    ############################################################################\n    # Classifier module\n    R1= tf.keras.layers.Flatten()(input)\n    mod = tf.keras.layers.Dense(num_classes, activation=activation)(R1)\n    return mod","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMSIZE = 256\nCHANNEL = 3\nwith strategy.scope():\n    inp = tf.keras.layers.Input(shape=(IMSIZE, IMSIZE, CHANNEL))\n    #img_inputs_1 = tf.keras.layers.experimental.preprocessing.RandomRotation(3.142/2, seed=SEED)(inp)\n    #img_inputs_2 = tf.keras.layers.experimental.preprocessing.RandomFlip(mode=\"horizontal_and_vertical\", seed=SEED)(img_inputs_1)\n    #img_inputs_3 = tf.keras.layers.experimental.preprocessing.RandomZoom((0, 0.35), seed=SEED)(img_inputs_2)\n    #connect = IResNet_connection_module(img_inputs_1, img_inputs_2, img_inputs_3, inp, 32)\n    module = IResNet_brain_module(inp, 32) \n    red = IResNet_reduction_module(module, 64)\n    red = IResNet_reduction_module(red, 64)\n    cla = IResNet_classifier_module(red, 7770, 'softmax')\n    model = tf.keras.models.Model(inputs=inp, outputs=cla, name = \"IResNetv1\")\n    model.summary()\n    tf.keras.utils.plot_model(model, show_shapes=True,to_file='./img.png')\n    metric_auc = tf.keras.metrics.AUC(num_thresholds=200, name='auc')\n    model.compile(optimizer=tf.keras.optimizers.Adam(0.001),\\\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(), \\\n              metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMSIZE = 256\nCHANNEL = 3\nimport tensorflow_addons as tfa\n\nwith strategy.scope():\n    base_model = tf.keras.applications.InceptionV3(include_top=False,\\\n                                                   weights='imagenet', pooling = 'max')\n    base_model.trainable = True\n    for layer in base_model.layers[:100]:\n        layer.trainable = False\n    model = tf.keras.Sequential([\n        tf.keras.layers.Input(shape=(IMSIZE, IMSIZE, CHANNEL)),\n        #tf.keras.layers.experimental.preprocessing.RandomRotation(3.142/2, seed=SEED),\n        #tf.keras.layers.experimental.preprocessing.RandomFlip(mode=\"horizontal_and_vertical\", seed=SEED),\n        #tf.keras.layers.experimental.preprocessing.RandomZoom((0, 0.35), seed=SEED),\n        #tf.keras.layers.experimental.preprocessing.RandomTranslation((-0.25, 0.25), (-0.25, 0.25), seed=SEED),\n        base_model,\n        tf.keras.layers.Dense(7770, activation='softmax')\n    ])\n    metric_auc = tf.keras.metrics.AUC(num_thresholds=200, name='auc')\n    model.compile(optimizer=tf.keras.optimizers.RMSprop(),\\\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(), \\\n              metrics=['acc'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_checkpoint_callback_acc = tf.keras.callbacks.ModelCheckpoint(\n    filepath='./best-acc-model.h5',\n    save_weights_only=False,\n    monitor='val_acc',\n    mode='max',\n    save_best_only=True)\nmodel_checkpoint_callback_loss = tf.keras.callbacks.ModelCheckpoint(\n    filepath='./best-loss-model.h5',\n    save_weights_only=False,\n    monitor='val_loss',\n    mode='min',\n    save_best_only=True)\nmodel_checkpoint_callback_auc = tf.keras.callbacks.ModelCheckpoint(\n    filepath='./best-auc-model.h5',\n    save_weights_only=False,\n    monitor='val_auc',\n    mode='max',\n    save_best_only=True)\nlr_decay_plateau = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\",\n    factor=0.85,\n    patience=4,\n    verbose=1,\n    mode=\"min\",\n)\ndist_train_dataset = strategy.experimental_distribute_dataset(train_dataset)\ndist_val_dataset = strategy.experimental_distribute_dataset(val_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_dataset, steps_per_epoch = int(TRAIN_SIZE/TRAIN_BATCH_SIZE), \\\n                    validation_data=val_dataset, validation_steps=int(VAL_SIZE/VAL_BATCH_SIZE),\\\n                    epochs=150, callbacks=[model_checkpoint_callback_acc, \\\n                                           model_checkpoint_callback_auc, \\\n                                           model_checkpoint_callback_loss, \\\n                                           lr_decay_plateau])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}