{"cells":[{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install iterative-stratification","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"import time\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom collections import namedtuple\nfrom sklearn import metrics\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import train_test_split, GroupKFold\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nfrom glob import glob\nimport tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras.layers import Dense, Input, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nimport logging\n# no extensive logging \nlogging.getLogger().setLevel(logging.NOTSET)\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"DROPOUT = 0.5 # use aggressive dropout\n# BATCH_SIZE = 16 # per TPU core\n\nEPOCHS = 15\n### Different learning rate for transformer and head ###\n# LR_EFNET = 1e-2\nLR_HEAD = 1e-3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def append_path(pre):\n    return np.vectorize(lambda file: os.path.join(GCS_DS_PATH, pre, file))\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\ndef int_div_round_up(a, b):\n    return (a + b - 1) // b\n\ndef onehot(size, target):\n    vec = np.zeros(size, dtype=np.float32)\n    vec[target] = 1.\n    return vec\n\nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Connect to TPU","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def connect_to_TPU():\n    \"\"\"Detect hardware, return appropriate distribution strategy\"\"\"\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    global_batch_size = 16 * strategy.num_replicas_in_sync\n\n    return tpu, strategy, global_batch_size\n\n\ntpu, strategy, BATCH_SIZE = connect_to_TPU()\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n# Data access\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create and Load the Data","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\n\ndataset = []\n\nfor label, kind in enumerate(['Cover', 'JMiPOD', 'JUNIWARD', 'UERD']):\n    for path in glob('../input/alaska2-image-steganalysis/Cover/*.jpg'):\n        dataset.append({\n            'kind': kind,\n            'image_name': path.split('/')[-1],\n            'label': label\n        })\n\nrandom.shuffle(dataset)\ndataset = pd.DataFrame(dataset)\n\ngkf = GroupKFold(n_splits=5)\n\ndataset.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(gkf.split(X=dataset.index, y=dataset['label'], groups=dataset['image_name'])):\n    dataset.loc[dataset.iloc[val_index].index, 'fold'] = fold_number\n\n# fold_gkf = pd.read_csv('../input/alaska2-public-baseline/groupkfold_by_shonenkov.csv')\nfold_gkf = dataset.copy()\nfold_gkf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"fold_number = 0\n# train_df = fold_gkf[fold_gkf['fold'] != fold_number]\ntrain_df = fold_gkf[fold_gkf['fold'] == fold_number]\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mskf = MultilabelStratifiedKFold(n_splits=8, random_state=42)\n\ntrain_data = None\nvalid_data = None\n\nfor train_idx, val_idx in mskf.split(train_df['image_name'], train_df[['label', 'kind']]):\n    \n    train_data = train_df.iloc[train_idx]\n    valid_data = train_df.iloc[val_idx]\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/alaska2-image-steganalysis/sample_submission.csv')\n# train_filenames = np.array(os.listdir(\"/kaggle/input/alaska2-image-steganalysis/Cover/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\n\ntrain_paths = []\ntrain_labels = []\n\nfor i in range(len(train_data['kind'])):\n    kind = train_data['kind'].iloc[i]\n    im_id = train_data['image_name'].iloc[i]\n    label = onehot(4, train_data['label'].iloc[i])\n    path = os.path.join(GCS_DS_PATH, kind, im_id)\n    \n    train_paths.append(path)\n    train_labels.append(label)\n    \nlen(train_paths), len(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\n\nvalid_paths = []\nvalid_labels = []\n\nfor i in range(len(valid_data['kind'])):\n    kind = valid_data['kind'].iloc[i]\n    im_id = valid_data['image_name'].iloc[i]\n    label = onehot(4, valid_data['label'].iloc[i])\n    path = os.path.join(GCS_DS_PATH, kind, im_id)\n    \n    valid_paths.append(path)\n    valid_labels.append(label)\n    \n# len(valid_paths), len(valid_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"test_paths = append_path('Test')(sub.Id.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_paths = np.array(train_paths[0:1000])\ntrain_labels = np.array(train_labels[0:1000])\nvalid_paths = np.array(valid_paths[0:1000])\nvalid_labels = np.array(valid_labels[0:1000])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create Distributed Dataset","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"LABEL_MAP = {\"Cover\": 0,\n            \"JMiPOD\": 1,\n            \"JUNIWARD\": 2,\n            \"UERD\": 3}\n\ndef decode_image(filename, label, image_size=(512, 512)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef decode_test_image(filename, image_size=(512, 512)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    return image\n    \ndef data_augment(image, label=None):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    \n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    return (tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE, drop_remainder=True)\n    .cache()\n    .prefetch(AUTO))\n\ndef get_validation_dataset(repeated=False):\n    return (tf.data.Dataset\n    .from_tensor_slices((valid_paths, valid_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE, drop_remainder=repeated)\n    .cache()\n    .prefetch(AUTO))\n\ndef get_test_dataset(ordered=False):\n    return (tf.data.Dataset\n        .from_tensor_slices(test_paths)\n        .map(decode_test_image, num_parallel_calls=AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_dataset  = get_training_dataset()\nvalid_dataset  = get_validation_dataset(repeated=True)\ntest_dataset  = get_test_dataset()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Building ","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\n\ndef build_model():\n    base_model = efn.EfficientNetB0(weights='imagenet',include_top=False, input_shape=(512, 512, 3))\n    base_model.trainable = False\n    \n    inputs = Input(shape=(512, 512, 3))\n    efnet_feat = base_model(inputs)\n    x = GlobalAveragePooling2D()(efnet_feat)\n    outputs = Dense(4, activation='softmax', name='custome_head')(x)\n    model = tf.keras.Model(inputs=[inputs], outputs=[outputs])\n    \n    return model\n\n\nwith strategy.scope():               \n    model = build_model()\n    optimizer_head = Adam(learning_rate=LR_HEAD)\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def define_losses_and_metrics():\n    with strategy.scope():\n        loss_object = tf.keras.losses.CategoricalCrossentropy(\n            reduction=tf.keras.losses.Reduction.NONE, from_logits=False)\n\n        def compute_loss(labels, predictions):\n            per_example_loss = loss_object(labels, predictions)\n            loss = tf.nn.compute_average_loss(\n                per_example_loss, global_batch_size = BATCH_SIZE)\n            return loss\n        train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')\n        valid_accuracy_metric = tf.keras.metrics.AUC(name='val_AUC')\n    return compute_loss, train_accuracy_metric, valid_accuracy_metric\n\ntrain_loss, train_accuracy_metric, valid_accuracy_metric = define_losses_and_metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"STEPS_PER_TPU_CALL = len(train_paths) // 128 \nVALIDATION_STEPS_PER_TPU_CALL = len(valid_paths) // 128\n\n@tf.function\ndef train_step(data_iter):\n    def train_step_fn(inputs):\n        features, labels = inputs\n\n        # calculate the 2 gradients ( note persistent, and del)\n        with tf.GradientTape(persistent=True) as tape:\n            predictions = model(features, training=True)\n            loss = train_loss(labels, predictions)\n        gradients = tape.gradient(loss, model.trainable_variables)\n        del tape # not sure if we should delete it.\n\n        ### make the gradients step\n        optimizer_head.apply_gradients(zip(gradients, \n                                           model.trainable_variables))\n\n        train_accuracy_metric.update_state(labels, predictions)\n        \n    # this loop runs on the TPU\n    for _ in tf.range(STEPS_PER_TPU_CALL):\n        strategy.run(train_step_fn, args=(next(data_iter),))\n\ndef predict(dataset):  \n    predictions = []\n    for tensor in dataset:\n        predictions.append(distributed_prediction_step(tensor))\n    ### stack replicas and batches\n    predictions = np.vstack(list(map(np.vstack,predictions)))\n    return predictions\n\n@tf.function\ndef distributed_prediction_step(data):\n    predictions = strategy.run(prediction_step, args=(data,))\n    return strategy.experimental_local_results(predictions)\n\ndef prediction_step(inputs):\n    features = inputs  # note datasets used in prediction do not have labels\n    predictions = model(features, training=False)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"@tf.function\ndef valid_step(data_iter):\n    def valid_step_fn(images, labels):\n        probabilities = model(images, training=False)\n        \n        # update metrics\n        valid_accuracy_metric.update_state(labels, probabilities)\n    # this loop runs on the TPU\n    for _ in tf.range(VALIDATION_STEPS_PER_TPU_CALL):\n        strategy.run(valid_step_fn, next(data_iter))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Start Training","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"start_time = epoch_start_time = time.time()\nSTEPS_PER_EPOCH = len(train_paths) // BATCH_SIZE # we can use BATCH_SIZE instead this is for exp for now\n\n\nHistory = namedtuple('History', 'history')\nhistory = History(history={'loss': [], 'categorical_auc': [], 'val_categorical_auc': []})\n\nprint(\"Training steps per epoch:\", STEPS_PER_EPOCH, \"in increments of\", STEPS_PER_TPU_CALL)\nprint(\"Validation images:\", len(valid_paths),\n      \"Batch size:\", BATCH_SIZE,\n      \"Validation steps:\", len(valid_paths) // BATCH_SIZE, \"in increments of\", VALIDATION_STEPS_PER_TPU_CALL)\n\nepoch = 0\ntrain_data_iter = iter(train_dataset)\nvalid_data_iter = iter(valid_dataset)\n\nstep = 0\nepoch_steps = 0\nbest_weights = None\n\nwhile True:\n    \n    # run training step\n    train_step(train_data_iter)\n    epoch_steps += STEPS_PER_TPU_CALL\n    step += STEPS_PER_TPU_CALL\n    print('=', end='', flush=True)\n        \n    # validation run at the end of each epoch\n    if (step // STEPS_PER_EPOCH) > epoch:\n        print('|', end='', flush=True)\n    \n        # validation run\n        valid_epoch_steps = 0\n        val_preds = []\n        val_lables = []\n        for _ in range(1):\n            valid_step(valid_data_iter)\n            valid_epoch_steps += VALIDATION_STEPS_PER_TPU_CALL\n            print('=', end='', flush=True)\n    \n        # compute metrics\n        history.history['categorical_auc'].append(train_accuracy_metric.result().numpy())\n        history.history['val_categorical_auc'].append(valid_accuracy_metric.result().numpy())\n\n        ## save weights if it is the best yet\n        if history.history['val_categorical_auc'][-1] == max(history.history['val_categorical_auc']):\n            best_weights = model.get_weights()\n        \n        ### Restore best weighths ###\n        model.set_weights(best_weights)\n        \n        epoch_time = time.time() - epoch_start_time\n        print('\\nEPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))\n        print('time: {:0.1f}s'.format(epoch_time),\n             'auc: {:0.4f}'.format(history.history['categorical_auc'][-1]),\n              'val_auc: {:0.4f}'.format(history.history['val_categorical_auc'][-1]),\n              'steps/val_steps: {:d}/{:d}'.format(epoch_steps, valid_epoch_steps), flush=True)\n        \n        ### Reset (train) metrics ###\n        train_accuracy_metric.reset_states()\n        valid_accuracy_metric.reset_states()\n        \n        # set up next epoch\n        epoch = step // STEPS_PER_EPOCH\n        epoch_steps = 0\n        epoch_start_time = time.time()\n        if epoch >= EPOCHS:\n            break\n\noptimized_ctl_training_time = time.time() - start_time\nprint(\"OPTIMIZED CTL TRAINING TIME: {:0.1f}s\".format(optimized_ctl_training_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"model.save(\"efnetB0_exp_1.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make Predictions ","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\npreds = predict(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = 1 - preds[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s = 0\nfinal_preds = np.zeros((5000))\nfor i in range(8):\n    end = s + 5000\n    final_preds += preds[s:end]\n    s = end","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"sub.Label = final_preds / 8\nsub.to_csv('submission.csv', index=False)\nsub.head(n=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Label'].hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}