{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"### FAST.AI IMPLEMENTATION"},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../working\"))\n\n# Any results you write to the current directory are saved as output.\n# install package from kernel, 'switch internet on' and '!pip install package_name'\n## submission code - in new kernel on 'submission' page\n# submission = pd.read_csv(\"../input/submissions/submission_1.csv\")\n# submission.to_csv('submission.csv', index=False)\n# print(os.listdir(\"../input\"))\n# print(os.listdir(\"../working\"))\n## submission kernels\n# https://www.kaggle.com/anuragtr/kernel007cee2e21/edit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(os.listdir(\"../input/working\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a href=\"./submission.csv\"> DOWNLOAD SUBMISSION.CSV </a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\nfrom fastai.vision import *\nfrom fastai.metrics import *\nfrom fastai.callbacks import *\n# doc(ImageDataBunch) # to see documentation of a function","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint(train_df.shape)\nprint(test_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].hist()\ntrain_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1. Implementation - resnet34/resnet50 - Default"},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_seed(seed_value, use_cuda):\n    np.random.seed(seed_value) # cpu vars\n    torch.manual_seed(seed_value) # cpu  vars\n    random.seed(seed_value) # Python\n    if use_cuda: \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) # gpu vars\n        torch.backends.cudnn.deterministic = True  #needed\n        torch.backends.cudnn.benchmark = False\nrandom_seed(120, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from_csv\nbs = 64\n# bs = 16   # uncomment this line if you run out of memory even after clicking Kernel->Restart\nimg_size = 224\npath = '../input/aptos2019-blindness-detection/'\n# tfms = get_transforms(flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.) # customized transformations\ndata = ImageDataBunch.from_csv(path, folder='train_images', csv_labels='train.csv', size=img_size, suffix='.png', ds_tfms=get_transforms()).normalize(imagenet_stats)\n    # ref: https://github.com/fastai/course-v3/blob/master/nbs/dl1/lesson1-pets.ipynb\n        # https://docs.fast.ai/data_block.html\n        # https://docs.fast.ai/vision.data.html","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data.show_batch(rows=3, figsize=(7,6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(data.classes)\nlen(data.classes),data.c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa = KappaScore()\nkappa.weights = \"quadratic\"\n# learn = cnn_learner(data, models.resnet34, metrics=[accuracy, kappa]) # or 'error_rate'\n# learn = cnn_learner(data, models.resnet34, metrics=error_rate) # use this for only 'error_rate'\nlearn = cnn_learner(data, models.resnet152, metrics=[accuracy, kappa]) # or 'error_rate'\nlearn.model\n# no layers are unfrozen, just few added at the end","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# callback to save best model\n# learn.fit_one_cycle(4)\nlearn.model_dir = '../working'\nlearn.fit_one_cycle(4, callbacks=[SaveModelCallback(learn, every='improvement', monitor='kappa_score', name='resnet-def-best')]) # or 'error_rate' 'accuracy'..","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" resnet34;img=224;bs=64;ep=4;seed=120                                                                       0.786885 acc; 0.835128 kappa \"\"\"\n\"\"\" resnet152;img=224;bs=64;ep=4;seed=120                                                                       0.788251 acc; 0.865084 kappa \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## not required when SaveModelCallback implemented\n# learn.model_dir = '../working'\n# learn.save('resnet34-1')\n# # learn = learn.load(\"resnet34-1\")\n# print(os.listdir(\"../input/working\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\nlosses,idxs = interp.top_losses()\nlen(data.valid_ds)==len(losses)==len(idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# interp.plot_top_losses(9, figsize=(15,11))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# interp.plot_confusion_matrix(figsize=(12,12), dpi=60)\ninterp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.most_confused(min_val=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Implementation - resnet34/resnet50 - Unfreezing, fine-tuning, and learning rates"},{"metadata":{"trusted":true},"cell_type":"code","source":"# continuing from default model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# find optimum lr\nkappa = KappaScore()\nkappa.weights = \"quadratic\"\nlearn = cnn_learner(data, models.resnet152, metrics=[accuracy, kappa])\n# learn = cnn_learner(data, models.resnet34, metrics=error_rate)\nlearn.unfreeze() # must before lr_find\nlearn.model_dir = '../working'\nlearn.load('resnet-def-best') # load default model; new training starts with weights of this model\n    # Poonam's reply - https://forums.fast.ai/t/why-do-we-need-to-unfreeze-the-learner-everytime-before-retarining-even-if-learn-fit-one-cycle-works-fine-without-learn-unfreeze/41614/2\nlearn.lr_find()\nlearn.recorder.plot()\n# note: if optimum lr found without loading default, it's incorrect","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# architecture & training\nlearn.unfreeze()\n# learn.fit_one_cycle(8, max_lr=slice(1e-06, 1e-04))\nlearn.fit_one_cycle(4, max_lr=slice(1e-06, 1e-05), callbacks=[SaveModelCallback(learn, every='improvement', monitor='kappa_score', name='resnet-def-best2')])\n\"\"\" Author - \"I tried to pick something well before it started getting worse. So I decided to pick 1e-6. But there's no point training \nall the layers at that rate, because we know that the later layers worked just fine before when we were training much more quickly.\" \n\"And the first part of the slice should be a value from your learning rate finder which is well before things started getting worse. \nSo you can see things are starting to get worse maybe about here(1e-4): So I picked something that's at least 10 times smaller(1e-6) than that.\" \nSo he picked (1e-6,1e-4)    \"\"\"\n    # ref: https://github.com/hiromis/notes/blob/master/Lesson1.md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" resnet34;img=224;bs=64;ep=8;lr(1e-06, 1e-04)                                                                    0.799180 acc; kappa 0.862905\"\"\"\n\"\"\" resnet152;img=224;bs=64;ep=8;lr(1e-06, 1e-05)                                                                     acc; kappa \"\"\"\n\n# even after increasing epochs to 40, accuracy stays similar\n\n\n\n\n# things to try:\n    # unfreeze limited layers\n    # https://www.kdnuggets.com/2019/05/boost-your-image-classification-model.html","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## not required when SaveModelCallback implemented\n# learn.model_dir = '../working'\n# learn.save('resnet34-2')\n# # learn = learn.load(\"stage-1\")\n# print(os.listdir(\"../input/working\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\nlosses,idxs = interp.top_losses()\nlen(data.valid_ds)==len(losses)==len(idxs)\ninterp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict & Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nlearn.data.add_test(ImageList.from_df(sample_df,'../input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))\npreds,y = learn.get_preds(ds_type=DatasetType.Test)\n# then implement your own logic here(highest prob is final class)/or from below notebooks\n    # https://www.kaggle.com/axel81/fastai-ensembler\n    # https://www.kaggle.com/demonplus/fast-ai-starter-with-resnet-50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sam_sub_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsam_sub_df[\"id_code\"]=sam_sub_df[\"id_code\"]\nprint(sam_sub_df.shape)\nsam_sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results=pd.DataFrame({\"id_code\":sam_sub_df[\"id_code\"],\n                      \"diagnosis\":np.argmax(preds,axis=1)})\nresults['id_code'] = results['id_code']\nresults['id_code'] = results['id_code'].astype('str')\nresults['diagnosis'] = results['diagnosis'].astype('int')\nresults.to_csv(\"submission.csv\",index=False)\nresults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" submission \"\"\"\n\"\"\"  error;  Kappa   -     LB \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Things to try:\n# https://www.kdnuggets.com/2019/05/boost-your-image-classification-model.html","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ROUGH"},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.random.seed(4)\nmsk = np.random.rand(len(dfx)) < 0.8\ntrain = dfx[msk]\ntest = dfx[~msk]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}