{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Base TensorFlow EfficientNetB0 on noisy-student weights + TTA","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!pip install -U -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport efficientnet.tfkeras as efn\nfrom tqdm import tqdm\nfrom pathlib import Path\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n\nprint(\"tf version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# there are some issues with dataset - unable to load it to GCS now\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\nGCS_DS_PATH","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    print(\"Running on TPU\", tpu.master())\nexcept ValueError:\n    strategy = tf.distribute.get_strategy()\n    print(\"Running on GPU\")\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH = Path(\"..\") / \"input\" / \"siim-isic-melanoma-classification\"\nSHAPE = (224, 224, 3)\nBATCH_SIZE = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(BASE_PATH / \"train.csv\")\ntest_df = pd.read_csv(BASE_PATH / \"test.csv\")\n\ntrain_df[\"image_path\"] = train_df.apply(lambda x: str(\n    BASE_PATH / \"jpeg\" / \"train\" / f\"{x.image_name}.jpg\"), axis=1)\ntest_df[\"image_path\"] = test_df.apply(lambda x: str(\n    BASE_PATH / \"jpeg\" / \"test\" / f\"{x.image_name}.jpg\"), axis=1)\n\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_data_gen = dict(\n    rescale=1/255.,\n    horizontal_flip=True,\n    vertical_flip=True,\n)\n\ntrain_datagen = ImageDataGenerator(\n    **base_data_gen,\n    zoom_range=0.1,\n    validation_split=0.1,\n)\n# for TTA\ntest_datagen = ImageDataGenerator(\n    **base_data_gen\n)\n\ngen_args = dict(\n    dataframe=train_df,\n    x_col=\"image_path\",\n    y_col=\"benign_malignant\",\n    batch_size=BATCH_SIZE,\n    seed=42,\n    target_size=SHAPE[:2],\n    color_mode=\"grayscale\" if SHAPE[-1] == 1 else \"rgb\",\n    class_mode=\"binary\",\n    shuffle=True,\n)\n\ntrain_gen = train_datagen.flow_from_dataframe(\n    **gen_args,\n    subset=\"training\"\n)\n\nvalid_gen = train_datagen.flow_from_dataframe(\n    **gen_args,\n    subset=\"validation\"\n)\n\ntest_gen = test_datagen.flow_from_dataframe(\n    dataframe=test_df,\n    x_col=\"image_path\",\n    y_col=None,\n    batch_size=BATCH_SIZE,\n    target_size=SHAPE[:2],\n    color_mode=\"grayscale\" if SHAPE[-1] == 1 else \"rgb\",\n    shuffle=False,\n    class_mode=None\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_examples(images_arr):\n  fig, axes = plt.subplots(1, 5, figsize=(15,15))\n  axes = axes.flatten()\n  for img, ax in zip(images_arr, axes):\n    ax.imshow(img.reshape(SHAPE), cmap=\"gray\")\n    ax.axis(\"off\")\n  plt.tight_layout()\n  plt.show()\n\n\nsample_training_images, _ = next(train_gen)\nplot_examples(sample_training_images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### EfficientNetB0 on nosy-student","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_shape):\n    base_model = efn.EfficientNetB0(\n        weights=\"noisy-student\", include_top=False, input_shape=input_shape\n    )\n    model = tf.keras.Sequential([\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1, activation=\"sigmoid\")\n    ])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = create_model(input_shape=SHAPE)\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss=\"binary_crossentropy\",\n        metrics=[\"accuracy\"]\n    )\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ncallbacks = [\n    ModelCheckpoint(filepath=\"weights.hdf5\", verbose=1, save_best_only=True),\n    EarlyStopping(monitor=\"val_loss\", patience=5),\n]\nsteps_per_epoch = len(train_gen.filenames) // BATCH_SIZE\nvalidation_steps = len(valid_gen.filenames) // BATCH_SIZE\n\nhistory = model.fit(\n    train_gen,\n    steps_per_epoch=steps_per_epoch,\n    epochs=1000,\n    validation_data=valid_gen,\n    validation_steps=validation_steps,\n    verbose=1,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].plot()\nhistory_df[['acc', 'val_acc']].plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TTA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tta = []\ntta_steps = 10\ntest_steps = len(test_gen.filenames) // BATCH_SIZE\n\nfor _ in tqdm(range(tta_steps)):\n    test_gen.reset()\n    preds = model.predict_generator(\n        generator=test_gen,\n        steps=test_steps\n    )\n    tta.append(preds)\n\ntta_mean = np.mean(tta, axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = train_gen.class_indices\n\nsub_df =  pd.read_csv(BASE_PATH / \"sample_submission.csv\")\nsub_df.target = tta_mean\nsub_df.to_csv(\"submission.csv\", index=False)\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}