{"metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"version": "3.6.3", "pygments_lexer": "ipython3", "codemirror_mode": {"version": 3, "name": "ipython"}, "mimetype": "text/x-python", "name": "python", "file_extension": ".py", "nbconvert_exporter": "python"}, "anaconda-cloud": {}}, "nbformat_minor": 1, "nbformat": 4, "cells": [{"metadata": {"_uuid": "b268443f012666e63ce063e35a67401756445580", "_cell_guid": "6ed6a394-2108-42e8-bd45-48744857642f"}, "cell_type": "markdown", "source": ["# Xception model\n", "- for weight see: https://www.kaggle.com/c/cdiscount-image-classification-challenge/discussion/41021"]}, {"metadata": {"_uuid": "de66f0b97a4f0d7ee254c2e868fbf76bbcb80331", "_cell_guid": "ce296850-1da1-4ca0-ad9f-de717bf7548e", "collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["import os\n", "import pickle\n", "from keras.applications.xception import Xception\n", "from keras.layers import Flatten, Dense, AveragePooling2D, Dropout, GlobalAveragePooling2D\n", "from keras.models import Model\n", "from keras.optimizers import Adam, SGD\n", "from keras.callbacks import ModelCheckpoint\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard\n", "import math\n", "\n", "from keras.models import load_model\n", "import keras.backend as K\n", "from keras.metrics import top_k_categorical_accuracy\n", "import tensorflow as tf\n", "\n", "# MultiGPU model build on top of\n", "# https://github.com/sallamander/multi-gpu-keras-tf/\n", "from multiGPU import MultiGPUModel\n", "import numpy as np"]}, {"metadata": {"_uuid": "592e6263f6dbfcbe6794281f099ef2fdc9b554a3", "_cell_guid": "29d79883-f0ac-436c-b965-85d6948e74cf", "collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["model_name = \"xception_v2\"\n", "models_savename = \"./models/\" + model_name\n", "\n", "train_data_dir = '/path/to/train/dir'\n", "val_data_dir = '/path/to/val/dir'\n", "classnames = pickle.load(open(\"/path/to/val/\", \"rb\"))\n", "batch_size = 86 * 3  # 258\n", "img_width = 180\n", "img_height = 180"]}, {"metadata": {"_uuid": "02a9f51810e42fc73e14d7319b360f2adcf8e0ee", "_cell_guid": "1e9f782a-ec7f-4bcb-bfc6-fd44d3ef1387", "collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["model0 = Xception(include_top=False, weights='imagenet',\n", "                    input_tensor=None, input_shape=(img_width, img_height, 3))\n", "\n", "for lay in model0.layers:\n", "    lay.trainable = False\n", "    \n", "x = model0.output\n", "x = GlobalAveragePooling2D(name='avg_pool')(x)\n", "\n", "x = Dropout(0.2)(x)\n", "x = Dense(len(classnames), activation='softmax', name='predictions')(x)\n", "model0 = Model(model0.input, x)\n", "\n", "# Train on 3GPUs\n", "model = MultiGPUModel(model0, [0, 1, 2], int(batch_size/3))"]}, {"metadata": {"_uuid": "96de159fdb8c274a7a0e90dfcd0ecccb31217a23", "_cell_guid": "501eda5f-674c-410a-be61-d5725c8d7d09", "collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["# Data generator\n", "train_datagen = ImageDataGenerator(\n", "        rescale=1./255,\n", "        zoom_range=0.1,\n", "        width_shift_range=0.1,\n", "        height_shift_range=0.1,\n", "        horizontal_flip=True)\n", "\n", "train_generator = train_datagen.flow_from_directory(\n", "        train_data_dir,\n", "        target_size = (img_width, img_height),\n", "        batch_size = batch_size,\n", "        shuffle = True,\n", "        classes = classnames,\n", "        class_mode = 'categorical')\n", "\n", "val_datagen = ImageDataGenerator(rescale=1./255)\n", "\n", "validation_generator = val_datagen.flow_from_directory(\n", "        val_data_dir,\n", "        target_size=(img_width, img_height),\n", "        batch_size=batch_size,\n", "        shuffle = True,\n", "        classes = classnames,\n", "        class_mode = 'categorical')"]}, {"metadata": {"_uuid": "57e42e4ba919cc5ca70b6c5e5f487f265ee7e7b7", "_cell_guid": "9eebbe17-7cce-4d7c-9fb8-7a5c992e2ade", "collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["os.makedirs(\"./models\", exist_ok=True)\n", "callbacks = [ModelCheckpoint(monitor='val_loss',\n", "                             filepath= models_savename + '_{epoch:03d}-{val_loss:.7f}.hdf5',\n", "                             save_best_only=False,\n", "                             save_weights_only=False,\n", "                             mode='max'),\n", "             TensorBoard(log_dir='logs/{}'.format(model_name))]"]}, {"metadata": {"_uuid": "52ab43e2c62317126a2d37103502f0b1344e01da", "_cell_guid": "0d9cd460-13cd-488f-9599-6087117e74a1", "collapsed": true, "scrolled": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["# Train head\n", "model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.001, momentum=0.9), metrics=[top_k_categorical_accuracy, 'accuracy'])\n", "model.fit_generator(generator=train_generator,\n", "                    steps_per_epoch=math.ceil(2000000 / batch_size),\n", "                    verbose=1,\n", "                    callbacks=callbacks,\n", "                    validation_data=validation_generator,\n", "                    initial_epoch=0,\n", "                    epochs=3,\n", "                    use_multiprocessing=True,\n", "                    max_queue_size=10,\n", "                    workers = 8,\n", "                    validation_steps=math.ceil(10000 / batch_size))\n", "\n", "# train xception blocks 11+\n", "for clayer in model.layers[4].layers:\n", "    print(\"trainable:\", clayer.name)\n", "    if clayer.name.split(\"_\")[0] in [\"block{}\".format(i) for i in range(10, 15)]:\n", "        clayer.trainable = True\n", "\n", "model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.00025), \n", "              metrics=[top_k_categorical_accuracy, 'accuracy'])\n", "model.fit_generator(generator=train_generator,\n", "                    steps_per_epoch=math.ceil(2000000 / batch_size),\n", "                    verbose=1,\n", "                    callbacks=callbacks,\n", "                    validation_data=validation_generator,\n", "                    initial_epoch=3,\n", "                    epochs=8,\n", "                    use_multiprocessing=True,\n", "                    max_queue_size=10,\n", "                    workers = 8,\n", "                    validation_steps=math.ceil(10000 / batch_size))\n", "\n", "# Train the whole model\n", "for clayer in model.layers[4].layers:\n", "    clayer.trainable = True\n", "\n", "# Note you need to recompile the whole thing. Otherwise you are not traing first layers    \n", "model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.00025), \n", "              metrics=[top_k_categorical_accuracy, 'accuracy'])\n", "\n", "\n", "init_epochs = 8  # We pretrained the model already\n", "\n", "# Keep training for as long as you like.\n", "for i in range(100):\n", "    # gradually decrease the learning rate\n", "    K.set_value(model.optimizer.lr, 0.95 * K.get_value(model.optimizer.lr))\n", "    start_epoch = (i * 2)\n", "    epochs = ((i + 1) * 2)    \n", "    model.fit_generator(generator=train_generator,\n", "                        steps_per_epoch=math.ceil(2000000 / batch_size),\n", "                        verbose=1,\n", "                        callbacks=callbacks,\n", "                        validation_data=validation_generator,\n", "                        initial_epoch=start_epoch + init_epochs,\n", "                        epochs=epochs + init_epochs,\n", "                        use_multiprocessing=True,\n", "                        max_queue_size=10,\n", "                        workers = 8,\n", "                        validation_steps=math.ceil(10000 / batch_size))\n"]}]}