{"cells":[{"metadata":{"_uuid":"c45233e219834bfabde0f9c49e7ee5ee5d3aa2e1"},"cell_type":"markdown","source":"**Important: This notebook will only work with fastai-0.7.x. Do not try to run any fastai-1.x **"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n#print(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"!pip install fastai==0.7.0\n!pip install torchtext==0.2.3\n!pip install torch==0.3.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a38297079d1c9c539075d26d68b513f77e661f47"},"cell_type":"code","source":"from fastai.transforms import *\nfrom fastai.conv_learner import * \nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed5e0d5c11e80c288bf0c3fab8069c1a7b501afe"},"cell_type":"code","source":"from fastai.imports import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e765d9268ca392ec59a8f84156a9ac4f2de3c5e4"},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33ea5aaa8bfe3a5f090f8e921ac25af807b7520a"},"cell_type":"code","source":"torch.backends.cudnn.enabled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05d58225b8d7ad52cdfa4d8204011c7304b71928"},"cell_type":"code","source":"os.listdir(\"../input\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3f023b57bbe086136022cfa36e075fcd183b396"},"cell_type":"code","source":"filenames = os.listdir('../input/train/train')[:5]\nfilenames","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ec772b2a0bcc3e8f4698be634faf3a5086ab27a"},"cell_type":"code","source":"path = \"../input/train/train\"\nimg = plt.imread(f'{path}/{filenames[0]}')\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7710250a81c9d6e6423109355b8fc14ec27eb078"},"cell_type":"code","source":"img.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11d5ac1c4186fc7d085d1008c9cc3b7b0163e119"},"cell_type":"code","source":"shutil.rmtree(f'{path}tmp', ignore_errors=True)#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"136521eada99f136740c2c8f5e0ecf0e867faea0"},"cell_type":"code","source":"img[:4,:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7a4d416b7a1556c437f6468d5a32ce798adb953"},"cell_type":"code","source":"PATH = \"../input/\"\nTMP_PATH = \"/tmp/tmp\"\nMODEL_PATH = \"/tmp/model/\"\nsz=224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"927d58a31ae43ddce5d2cad96b79abd96426df99"},"cell_type":"code","source":"fnames = np.array([ f'train/train/{i}'  for i in sorted(os.listdir(f'{PATH}train/train'))])\nlabel = np.array([0 if 'cat' in fname else 1 for fname in fnames]).astype(np.double)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd4e652091b46e237f710164ef2664fae2ad78fb"},"cell_type":"markdown","source":"## **The following error can be resolved by the solution [here](https://forums.fast.ai/t/windows-10-installation-notes-windows-command-and-wsl-bash/6500/55)**"},{"metadata":{"trusted":true,"_uuid":"ffa7b5798464b9973ac9496f9bf11926c91735b8","scrolled":true},"cell_type":"code","source":"arch = resnet34\ndata = ImageClassifierData.from_names_and_array(path=PATH,\n                                                fnames=fnames ,\n                                                y=label,\n                                                classes=['dogs', 'cats'],\n                                                test_name=(f'{PATH}test1/test1'),\n                                      tfms=tfms_from_model(arch , sz))\nlearn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)\nlearn.fit(0.01 , 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1c00cfe5f9ac7258a4fe4bebd6b8e049c5e96b6"},"cell_type":"code","source":"data.val_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d612faad12bcdc00e377758390aebf4588c19cd8"},"cell_type":"code","source":"data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9db4e348af9390601d26e57cf10bd669ac04074e"},"cell_type":"code","source":"log_preds = learn.predict()\nlog_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bfccc293477f19b498d942b0dc82312d4e4089e"},"cell_type":"code","source":"log_preds[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67e001702e5e08fd3639c5ebd033de4d7670ddcd"},"cell_type":"code","source":"preds = np.argmax(log_preds, axis=1)  # from log probabilities to 0 or 1\nprobs = np.exp(log_preds[:,1])        # pr(dog)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfb2fd0cfc8c99a12057b8318b7052da33b32d01"},"cell_type":"code","source":"def rand_by_mask(mask): return np.random.choice(np.where(mask)[0], 4, replace=False)\ndef rand_by_correct(is_correct): return rand_by_mask((preds == data.val_y)==is_correct)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be3215df4be0d388b00d2e4c5c25ea48758edaa2"},"cell_type":"code","source":"def plots(ims, figsize=(12,6), rows=1, titles=None):\n    f = plt.figure(figsize=figsize)\n    for i in range(len(ims)):\n        sp = f.add_subplot(rows, len(ims)//rows, i+1)\n        sp.axis('Off')\n        if titles is not None: sp.set_title(titles[i], fontsize=16)\n        plt.imshow(ims[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e1ccae3e851d36ffddfb165576fc8e8af465a44"},"cell_type":"code","source":"def load_img_id(ds, idx): return np.array(PIL.Image.open(PATH+ds.fnames[idx]))\n\ndef plot_val_with_title(idxs, title):\n    imgs = [load_img_id(data.val_ds,x) for x in idxs]\n    title_probs = [probs[x] for x in idxs]\n    print(title)\n    return plots(imgs, rows=1, titles=title_probs, figsize=(16,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11b8f7d7ba43d1f845781073a6b96720aadda273"},"cell_type":"code","source":"plot_val_with_title(rand_by_correct(True), \"Correctly classified\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0388ea6ecc3f812f098f83eda31e2201bdceb70"},"cell_type":"code","source":"# 2. A few incorrect labels at random\nplot_val_with_title(rand_by_correct(False), \"Incorrectly classified\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f694857fd2741f5225fed887b107f6e05782da5"},"cell_type":"code","source":"def most_by_mask(mask, mult):\n    idxs = np.where(mask)[0]\n    return idxs[np.argsort(mult * probs[idxs])[:4]]\n\ndef most_by_correct(y, is_correct): \n    mult = -1 if (y==1)==is_correct else 1\n    return most_by_mask(((preds == data.val_y)==is_correct) & (data.val_y == y), mult)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0783b218e347632ce8b9901e4189855de6b4552b"},"cell_type":"code","source":"plot_val_with_title(most_by_correct(0, True), \"Most correct cats\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2fd58898bfbedfe5192536b5306a710360c6003"},"cell_type":"code","source":"plot_val_with_title(most_by_correct(1, True), \"Most correct dogs\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0e4e4ec4db990e60a4839207bb6307166418406"},"cell_type":"code","source":"plot_val_with_title(most_by_correct(0, False), \"Most incorrect cats\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f643bd6ef1a24ff9e27e7e38cafd1dfb96335f61"},"cell_type":"code","source":"plot_val_with_title(most_by_correct(1, False), \"Most incorrect dogs\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a966fc7f873ff4caef11543d790af20fd30794eb"},"cell_type":"code","source":"most_uncertain = np.argsort(np.abs(probs -0.5))[:4]\nplot_val_with_title(most_uncertain, \"Most uncertain predictions\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3ff0ed962c0b5b980b21ec1c8b8d4aab142bccf"},"cell_type":"markdown","source":"### **Choosing a learning rate**\nThe learning rate determines how quickly or how slowly you want to update the weights (or parameters). Learning rate is one of the most difficult parameters to set, because it significantly affects model performance.\n\nThe method `learn.lr_find()` helps you find an optimal learning rate. It uses the technique developed in the 2015 paper Cyclical Learning Rates for Training Neural Networks, where we simply keep increasing the learning rate from a very small value, until the loss stops decreasing. We can plot the learning rate across batches to see what this looks like.\n\nWe first create a new learner, since we want to know how to set the learning rate for a new (untrained) model."},{"metadata":{"trusted":true,"_uuid":"14c4a3a873aafaa9fac9121c65cbb695335e89c5"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a5293a965903b93a08e7ac55527be0e07c71ecc"},"cell_type":"code","source":"lrf=learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f750b00d774e3c7c4dc6a22496863e299dc6efe"},"cell_type":"markdown","source":"Our `learn` object contains an attribute `sched` that contains our learning rate scheduler, and has some convenient plotting functionality including this one:"},{"metadata":{"trusted":true,"_uuid":"f96f6e7bd05484e4b0ecd6f55f82712d3976e3c1"},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ebbd78659e16fb66d1af24f52a859c67abba630"},"cell_type":"markdown","source":"\nNote that in the previous plot iteration is one iteration (or minibatch) of SGD. In one epoch there are (num_train_samples/num_iterations) of SGD.\n\nWe can see the plot of loss versus learning rate to see where our loss stops decreasing:"},{"metadata":{"trusted":true,"_uuid":"9aeba0dbc308abd111f45acbe3a473f799c75eac"},"cell_type":"code","source":"learn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ac104e90a4f0ae6e32e83bffe32f491e429737b"},"cell_type":"markdown","source":"## **Improving our model**\n### **Data augmentation**\nIf you try training for more epochs, you'll notice that we start to overfit, which means that our model is learning to recognize the specific images in the training set, rather than generalizing such that we also get good results on the validation set. One way to fix this is to effectively create more data, through data augmentation. This refers to randomly changing the images in ways that shouldn't impact their interpretation, such as horizontal flipping, zooming, and rotating.\n\nWe can do this by passing `aug_tfms` (augmentation transforms) to tfms_from_model, with a list of functions to apply that randomly change the image however we wish. For photos that are largely taken from the side (e.g. most photos of dogs and cats, as opposed to photos taken from the top down, such as satellite imagery) we can use the pre-defined list of functions `transforms_side_on`. We can also specify random zooming of images up to specified scale by adding the `max_zoom` parameter."},{"metadata":{"trusted":true,"_uuid":"ff200c57cf2157b302f4839369613c620f9b271d"},"cell_type":"code","source":"tfms = tfms_from_model(resnet34, sz, aug_tfms=transforms_side_on, max_zoom=1.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f586c6f3bf52021af5c258092074fa25796dc9d0"},"cell_type":"code","source":"def get_augs():\n    data = ImageClassifierData.from_names_and_array(\n        path=PATH, \n        fnames=fnames, \n        y=label, \n        classes=['dogs', 'cats'], \n        test_name=f'{PATH}test1/test1', \n        tfms=tfms,\n        num_workers=1,\n        bs=2\n    )\n    x,_ = next(iter(data.aug_dl))\n    return data.trn_ds.denorm(x)[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b85a386e8360fdd4000c129b91b4df9b5763ef6"},"cell_type":"code","source":"ims = np.stack([get_augs() for i in range(6)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e55944ec895cad972cadfc5f0ca5fcd0280e5875"},"cell_type":"code","source":"plots(ims , rows=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8892cde543991d902d186c272370c99dee2bd896"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}