{"cells":[{"metadata":{"trusted":true,"_uuid":"a503853adef770ddb1513e85ad2d377f8965c1db","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport os\nfrom sklearn.metrics import f1_score\n\nfrom fastai import *\nfrom fastai.vision import *\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport cv2\n\nfrom tqdm import tqdm\nfrom skmultilearn.model_selection import iterative_train_test_split\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n%load_ext autoreload\n%autoreload","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b220f04907e1c156e1e1235dc0b53f56b63661e"},"cell_type":"markdown","source":"Defining a metric so after epoch I get the validation ROC-AUC score"},{"metadata":{"trusted":true,"_uuid":"5caf6cf236cf9d098331763b434806cd73d13823"},"cell_type":"code","source":"model_path='.'\npath='../input/'\ntrain_folder=f'{path}train'\ntest_folder=f'{path}test'\ntrain_lbl=f'{path}train_labels.csv'\nORG_SIZE=96\n\nbs=64\nnum_workers=None # Apprently 2 cpus per kaggle node, so 4 threads I think\nsz=96","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1c089e47c2325cf783092cc235b9b5e9831022a"},"cell_type":"markdown","source":"In Case I want to run quick tests use a subsample:"},{"metadata":{"trusted":true,"_uuid":"ecb840384eb1e0aa90fe881f8c212beb135ad6c4"},"cell_type":"code","source":"df_trn=pd.read_csv(train_lbl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7982793320648eb1ecf956b46da3f2da0d81aaef"},"cell_type":"code","source":"tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,\n                      max_lighting=0.05, max_warp=0.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"546c687ba7770602c688c01a58427c47f48c80ac"},"cell_type":"code","source":"data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);\nstats=data.batch_stats()        \ndata.normalize(stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e8fec9df84b343e53c9e4379881967f713f17873"},"cell_type":"code","source":"data.show_batch(rows=5, figsize=(12,9))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b83ed47a0109f7d51f1c475958a5191f87c63b01"},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"881873d04d04ba407205e66c9f535dce0938adb2"},"cell_type":"code","source":"def auc_score(y_pred,y_true,tens=True):\n    score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])\n    if tens:\n        score=tensor(score)\n    else:\n        score=score\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95ac8eb66b25e87a0f4828640a7d6ef405e313f1"},"cell_type":"code","source":"from torchvision.models import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e87abf1fbf9b061cb26fb05afc6932e253d4135c"},"cell_type":"code","source":"learn = create_cnn(\n    data,\n    densenet201,\n    path='.',    \n    metrics=[auc_score], \n    ps=0.5\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8f689bfd03e79e5f8f130032468a85788b7acce"},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c701d08ba0f3657fa651476f83f2a53a4ffb34d1"},"cell_type":"code","source":"lr = 2e-03","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcfa743233605eafacfcbc383da90c7bc5fbd473"},"cell_type":"code","source":"learn.fit_one_cycle(1,lr)\nlearn.recorder.plot()\nlearn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c87540b8006dffcda2f33bab39add4a704b1740d"},"cell_type":"code","source":"learn.unfreeze()\nlearn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c87540b8006dffcda2f33bab39add4a704b1740d"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"237be2254fe3cb29c8e04430963e6883995b277c"},"cell_type":"markdown","source":"### Warm up with frozen weight is done on a subset so we dont have to waste an entire epoch"},{"metadata":{"trusted":true,"_uuid":"b63495c9c8dd4e12d91cda8a2702f6c1a5bdc168"},"cell_type":"code","source":"learn.fit_one_cycle(1,slice(1e-4,1e-3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b63495c9c8dd4e12d91cda8a2702f6c1a5bdc168"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b63495c9c8dd4e12d91cda8a2702f6c1a5bdc168"},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c4e009e09fb77b08997ac8e9ee69660c28676be"},"cell_type":"markdown","source":"### Predit the validation data using TTA\nHere for every image we want to predict on, n_augs images are augmented form the original image.\nWe can then compare the predictions on for example the image and the image flipped / roated / slightly different crop/ lighting/stretched etc. \nFor now only the diherdral and rotations are used. THis gives a nice extra percent or two when compared to the auc above after training where not TTA is used. \nI also test if mean or max is better to use on the image and its augments but it can't conclude anything yet."},{"metadata":{"trusted":true,"_uuid":"b6917d2f83141d472e04073b59a9b5e1e6b26c57"},"cell_type":"code","source":"preds,y=learn.get_preds()\npred_score=auc_score(preds,y)\npred_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6917d2f83141d472e04073b59a9b5e1e6b26c57"},"cell_type":"code","source":"preds,y=learn.TTA()\npred_score_tta=auc_score(preds,y)\npred_score_tta","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bdfd3b80d33057d155e7da32391817a7e88ff80f"},"cell_type":"markdown","source":"### Now predict on test set"},{"metadata":{"trusted":true,"_uuid":"6c1890c9509ca13a01b6bba3bba296b5a9b34fa7"},"cell_type":"code","source":"preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a85d720a61b996e3ee308c4a012345ef69fc461","scrolled":true},"cell_type":"code","source":"preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88a566de210643baa4a44519e67b6e377c9d26d7"},"cell_type":"markdown","source":"### prepare submission\nI now load in the sample submission and put my predictions in the label column and save to a new file."},{"metadata":{"_uuid":"c7f2e327a7943ed6e8d7f8bfc7bd3f9682186b7c"},"cell_type":"markdown","source":"Sometimes its important in which order the ids in the submissions are so to make sure I don't mess up I put them in the same order. My first submission had a 50% score so I somewhere messed up the order oder the matching of id to label.\nsince fname_clean is the id we can just use that as index when adding the correct label in our dataframe. "},{"metadata":{"trusted":true,"_uuid":"7c64426066b5d158024e4bbf05a6a05c10777ce7"},"cell_type":"code","source":"sub=pd.read_csv(f'{path}/sample_submission.csv').set_index('id')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20c89b9c62d4d98fe4cae6195e80338abfb3a553"},"cell_type":"code","source":"clean_fname=np.vectorize(lambda fname: str(fname).split('/')[-1].split('.')[0])\nfname_cleaned=clean_fname(data.test_ds.items)\nfname_cleaned=fname_cleaned.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"25f186dca18d7855cf8b3910a45acd3de9f1f34e"},"cell_type":"markdown","source":"## I add the score to the name of the file so I can later plot the leaderboard score versus my validation score\nIn the fastai course Jeremy mentions that if you have a monotonic relation between validation and LB score the way you set up your validation set matches what the test set consists of."},{"metadata":{"trusted":true,"_uuid":"1e9f92573996df4ff7a5f047e7ffae6ef6e53b73"},"cell_type":"code","source":"sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])\nsub.to_csv(f'submission_{pred_score}.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a934db6adbf52935aab26a9c6400bf0530d3bb9"},"cell_type":"code","source":"sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])\nsub.to_csv(f'submission_{pred_score_tta}.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15af9ff7c16ef0e5bda33d4595b482c91d21f305"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3584d685b32adf5ca42d8d2df7451e26750bc82"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}