{"cells":[{"metadata":{"trusted":true,"_uuid":"4fcddcfd1bb4f18ff0c7bb43c3ec29fa0384c37a"},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Model\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Input, Flatten\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, History\nfrom keras.applications.xception import Xception\nfrom keras.applications.resnet50 import ResNet50\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f616c6f3f8b4f7b3aa4e0a533c2dba7bd18f1a8a"},"cell_type":"markdown","source":"# Constants and Helper Functions"},{"metadata":{"trusted":true,"_uuid":"7d0d9ce964e6ad3071c2721a23887e8f20fcddf4"},"cell_type":"code","source":"data_dir = '../input/dog-breed-identification'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e2ae757fbfdcd2b7aa581b858ec99111f7ad936"},"cell_type":"code","source":"def plot_results(h):\n    plt.plot(h['loss'], 'blue', label='Train Loss')\n    plt.plot(h['val_loss'], 'orange', label='Val. Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv(data_dir + '/labels.csv')\ndf['filename'] = df.id + '.jpg'\n\n# top_dogs = df['breed'].value_counts()\n# top_dogs = top_dogs[:16].index # top N\n# df = df[df['breed'].isin(top_dogs)].reset_index(drop=True)\nprint(df.shape)\nprint(df.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"400d20236451b2562f3589624c3f1281b82e177e"},"cell_type":"code","source":"# Constants\ninput_shape = (224,224,3)\nnum_classes = len(set(df.breed.values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e88cefa847cc87a5e0f3728efa55e29351b3f41"},"cell_type":"code","source":"def get_data_generators():\n    train_datagen = ImageDataGenerator(\n        rescale=1./255,\n        width_shift_range=0.2,\n        height_shift_range=0.2,\n        rotation_range=40,\n        shear_range=0.2,\n        zoom_range=0.2,\n        horizontal_flip=True,\n        fill_mode='nearest',\n        validation_split=0.2\n    )\n    \n    train_generator = train_datagen.flow_from_dataframe(\n        df,\n        directory=data_dir + '/train/train',\n        x_col='filename',\n        y_col='breed',\n        target_size=(input_shape[0], input_shape[1]),\n        batch_size=32,\n        class_mode='categorical',\n        shuffle=True,\n        seed=42,\n        subset='training'\n    )\n    valid_generator = train_datagen.flow_from_dataframe(\n        df,\n        directory=data_dir + '/train/train',\n        x_col='filename',\n        y_col='breed',\n        target_size=(input_shape[0], input_shape[1]),\n        batch_size=32,\n        class_mode='categorical',\n        shuffle=True,\n        seed=42,\n        subset='validation'\n    )\n    \n    return train_generator, valid_generator","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"a4bf357b8459be2d478792a329b5bda0b0809d52"},"cell_type":"code","source":"train_generator, valid_generator = get_data_generators()\n\nfor s in sorted(train_generator.class_indices):\n    if train_generator.class_indices[s] != valid_generator.class_indices[s]:\n        raise Exception(f'MisMatch: {train_generator.class_indices[s]} {valid_generator.class_indices[s]}')\n\nunique_train_classes = set(train_generator.classes)\nunique_valid_classes = set(valid_generator.classes)\n\nif len(unique_train_classes) != num_classes or len(unique_valid_classes) != num_classes:\n    raise Exception('Train and Valid do not contain all classes.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2b8d1cea015016150d9c4e26953063dedae758d"},"cell_type":"code","source":"def get_model():\n    pretrained_model = Xception(weights='../input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5',\n                                include_top=False,\n                                input_shape=input_shape)\n    for l in pretrained_model.layers:\n        l.trainable = False\n    \n    X_input = Input(shape=input_shape)\n    X = pretrained_model(X_input)\n    X = Flatten()(X)\n    X = Dense(4005, activation='relu')(X)\n#     X = Dense(2048, activation='relu')(X)\n#     X = Dropout(0.5)(X)\n    out = Dense(num_classes, activation='softmax')(X)\n    return Model(X_input, [out])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa9ec77ff99efe131c23b0b0348d284b1dc51bd2"},"cell_type":"code","source":"model = get_model()\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nearly_stop_callback = EarlyStopping(monitor='val_loss',\n                                    verbose=1,\n                                    patience=7,\n                                    min_delta=1e-3)\ncheckpoint_callback = ModelCheckpoint('./best-model.h5', save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b322d909bac2556bdfafabb376a78fab377e031b"},"cell_type":"code","source":"history = model.fit_generator(\n    train_generator,\n    epochs=100,\n    callbacks=[\n        early_stop_callback,\n        checkpoint_callback\n    ],\n    validation_data=valid_generator,\n    verbose=1,\n    shuffle=True,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa78b92c8092b5b80bd2b8f8ad2de315206f6709"},"cell_type":"code","source":"all_history = {\n    'loss': [],\n    'val_loss': []\n}\nall_history['loss'] = all_history['loss'] + history.history['loss']\nall_history['val_loss'] = all_history['val_loss'] + history.history['val_loss']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf5297b31e0b37d2c3ccd78ca1812ed58eb46e9b"},"cell_type":"code","source":"plot_results(all_history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"527a985ad0827e4e2021120fd0f702ec7dfeed1f"},"cell_type":"markdown","source":"# FastAI Version - this is 10x better"},{"metadata":{"trusted":true,"_uuid":"fed4eb15b8b77876b78d85c20c95f2d089b104e8"},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\n\n# Move trained resnet so pytorch can find it\n# Copy training images to a new folder for fastai\n!cp ../input/resnet-from-fastai/resnet34-333f7ec4.pth /tmp/.torch/models/resnet34-333f7ec4.pth\n!cp -r ../input/dog-breed-identification/train ../train-jpg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b8f44acec8bc9ebb3ff2bdafec3c3a9fbfba245"},"cell_type":"code","source":"bs = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b70942a63fa0be35542fee78a04018d333743b3d"},"cell_type":"code","source":"data = ImageDataBunch.from_df(path='../train-jpg',\n                              df=df, \n                              fn_col='filename',\n                              label_col='breed',\n                              folder='train',\n                              ds_tfms=get_transforms(), \n                              size=224, \n                              bs=bs, \n                              num_workers=0)\ndata = data.normalize(imagenet_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"926d25858188736f26f2247f5a924a1dd6ac39df"},"cell_type":"code","source":"data.show_batch(rows=3, figsize=(7,6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdd36dc522bf9aeca40eacdaf94b82cf4231dc29"},"cell_type":"code","source":"print(data.classes)\nlen(data.classes),data.c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6435291c5c90f66db8828963941f63f6b1c00ea6"},"cell_type":"code","source":"learn = create_cnn(data, models.resnet34, metrics=error_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e79ee4de10baf8772d51ab38ebbf326bd0717087"},"cell_type":"code","source":"learn.fit_one_cycle(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4966275d4a15b2186de59fa4b3cfe5b2e35909a"},"cell_type":"code","source":"learn.save('stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c631ab47c71d602fe0c934f2f13fd7d57fbbd6ce"},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\n\nlosses, idxs = interp.top_losses()\n\nlen(data.valid_ds)==len(losses)==len(idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"336681114551576132ee06d9386c97c7d5b96b7f"},"cell_type":"code","source":"interp.plot_top_losses(9, figsize=(15,11))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5432a7e5292f5710f3d3e3d26223acc94fd4e222"},"cell_type":"code","source":"interp.plot_confusion_matrix(figsize=(12,12), dpi=60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"397f8e38e0e177f355aa051c9eab26058e02a19b"},"cell_type":"code","source":"interp.most_confused(min_val=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d784a94676c8398cb6df9327d800edcbd880c8e"},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b871749f6a9f295a171fc6f8c7611a4fb3ef66b6"},"cell_type":"code","source":"learn.fit_one_cycle(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"fbee4adf55f3c17f5b1d03b8fd09a2d92f590ec5"},"cell_type":"code","source":"learn.load('stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6480383255715730e2bb4bddc477b27fedcc986c"},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f834ef423042ea0aa7f92b58c3af85a4afc68e4a"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b55cfd1a07be9025b71e3522eaf0ef0d95a613a"},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit_one_cycle(2, max_lr=slice(1e-5, 1e-4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9c5770414fcaf4fb75c377b040ae022912118b1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}