{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-14T07:10:36.571607Z","iopub.execute_input":"2022-07-14T07:10:36.572006Z","iopub.status.idle":"2022-07-14T07:10:36.603119Z","shell.execute_reply.started":"2022-07-14T07:10:36.571922Z","shell.execute_reply":"2022-07-14T07:10:36.601857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nimport pandas as pd\n%matplotlib inline\nset_seed(3865)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:10:40.839714Z","iopub.execute_input":"2022-07-14T07:10:40.840095Z","iopub.status.idle":"2022-07-14T07:10:43.683573Z","shell.execute_reply.started":"2022-07-14T07:10:40.840065Z","shell.execute_reply":"2022-07-14T07:10:43.682335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nwith zipfile.ZipFile('../input/dogs-vs-cats-redux-kernels-edition/train.zip', 'r') as zip_ref:\n    zip_ref.extractall('.')\nwith zipfile.ZipFile('../input/dogs-vs-cats-redux-kernels-edition/test.zip', 'r') as zip_ref:\n    zip_ref.extractall('.')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:10:45.776274Z","iopub.execute_input":"2022-07-14T07:10:45.776858Z","iopub.status.idle":"2022-07-14T07:11:04.800243Z","shell.execute_reply.started":"2022-07-14T07:10:45.776823Z","shell.execute_reply":"2022-07-14T07:11:04.799076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# some bad images should be deleted from the train set\nos.remove('./train/dog.10797.jpg')\nos.remove('./train/dog.10747.jpg')\nos.remove('./train/dog.10237.jpg')\nos.remove('./train/dog.9517.jpg')\nos.remove('./train/dog.8736.jpg')\nos.remove('./train/dog.5604.jpg')\nos.remove('./train/dog.1043.jpg')\nos.remove('./train/cat.4338.jpg')\nos.remove('./train/dog.10161.jpg')\nos.remove('./train/dog.10190.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:06.742923Z","iopub.execute_input":"2022-07-14T07:11:06.743655Z","iopub.status.idle":"2022-07-14T07:11:06.751716Z","shell.execute_reply.started":"2022-07-14T07:11:06.743621Z","shell.execute_reply":"2022-07-14T07:11:06.750425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tpath = \"./train\"\nftrain = get_image_files(tpath)\nprint('Train set size:', len(ftrain))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:10.434402Z","iopub.execute_input":"2022-07-14T07:11:10.435241Z","iopub.status.idle":"2022-07-14T07:11:10.772742Z","shell.execute_reply.started":"2022-07-14T07:11:10.435188Z","shell.execute_reply":"2022-07-14T07:11:10.771567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as Alb\nclass AlbTransform(Transform):\n    def __init__(self, aug): self.aug = aug\n    def encodes(self, img: PILImage):\n        aug_img = self.aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)\n    \ndef get_augs(): return Alb.Compose([\n    Alb.ShiftScaleRotate(rotate_limit=20, border_mode=0, value=(0,0,0) ),\n    Alb.Transpose(),\n    Alb.Flip(),\n    Alb.RandomRotate90(),\n    Alb.RandomBrightnessContrast(),\n    Alb.HueSaturationValue(\n      hue_shift_limit=5, \n      sat_shift_limit=5, \n      val_shift_limit=5 ),    \n])\n\nitem_tfms = [Resize(224), AlbTransform(get_augs())] \nbatch_tfms = Normalize.from_stats(*imagenet_stats) ","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:13.511646Z","iopub.execute_input":"2022-07-14T07:11:13.518861Z","iopub.status.idle":"2022-07-14T07:11:15.781836Z","shell.execute_reply.started":"2022-07-14T07:11:13.518726Z","shell.execute_reply":"2022-07-14T07:11:15.780217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = ImageDataLoaders.from_name_re( path=tpath, \n    fnames=ftrain, pat=r'(.+)\\.\\d+.jpg$', valid_pct=0.1, \n    item_tfms=item_tfms, batch_tfms=batch_tfms, bs=64, shuffle=True )","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:20.567399Z","iopub.execute_input":"2022-07-14T07:11:20.567779Z","iopub.status.idle":"2022-07-14T07:11:22.662184Z","shell.execute_reply.started":"2022-07-14T07:11:20.567748Z","shell.execute_reply":"2022-07-14T07:11:22.661218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uncomment to test data loaders\n# dls.train.show_batch(max_n=12)\nprint('train items:', len(dls.train.items), 'validation items:', len(dls.valid.items))\n# dls.valid.show_batch(max_n=12)\n# dls.vocab","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:25.584518Z","iopub.execute_input":"2022-07-14T07:11:25.584943Z","iopub.status.idle":"2022-07-14T07:11:25.591067Z","shell.execute_reply.started":"2022-07-14T07:11:25.584908Z","shell.execute_reply":"2022-07-14T07:11:25.590006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = cnn_learner(dls, resnet34, metrics=error_rate)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:28.935917Z","iopub.execute_input":"2022-07-14T07:11:28.936328Z","iopub.status.idle":"2022-07-14T07:11:32.201170Z","shell.execute_reply.started":"2022-07-14T07:11:28.936297Z","shell.execute_reply":"2022-07-14T07:11:32.199965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare test data\nftest = get_image_files('test')\n# ftest = ftest[:50] # scale down for debug\nprint('Testing', len(ftest), 'items')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:39.484659Z","iopub.execute_input":"2022-07-14T07:11:39.485714Z","iopub.status.idle":"2022-07-14T07:11:39.614143Z","shell.execute_reply.started":"2022-07-14T07:11:39.485662Z","shell.execute_reply":"2022-07-14T07:11:39.612979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make dataloader for test data\ntst_dl = dls.test_dl(ftest, with_labels=False, shuffle=False)\n# uncomment to see if dataloader is working\ntst_dl.show_batch(max_n=12)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:11:42.780705Z","iopub.execute_input":"2022-07-14T07:11:42.781631Z","iopub.status.idle":"2022-07-14T07:11:44.429277Z","shell.execute_reply.started":"2022-07-14T07:11:42.781597Z","shell.execute_reply":"2022-07-14T07:11:44.427896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"startTime = time.time()\npreds = learn.tta(dl=tst_dl, n=1, use_max=False)\nprint('TTA in:', time.time()-startTime, 'secs')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T07:16:05.005747Z","iopub.execute_input":"2022-07-14T07:16:05.006196Z","iopub.status.idle":"2022-07-14T08:03:18.453619Z","shell.execute_reply.started":"2022-07-14T07:16:05.006160Z","shell.execute_reply":"2022-07-14T08:03:18.451069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for this competition should not submit 0/1, should submit probability of \"dog\"\nsubm_df = pd.DataFrame()\nsubm_df['id'] = [item.stem for item in tst_dl.items]\nsubm_df['label'] = preds[0][:,1].clip(0.005, 0.995)\nsubm_df.to_csv('submission.csv', header=True, index=False)\nsubm_df","metadata":{"execution":{"iopub.status.busy":"2022-07-14T08:05:48.074450Z","iopub.execute_input":"2022-07-14T08:05:48.075115Z","iopub.status.idle":"2022-07-14T08:05:48.199074Z","shell.execute_reply.started":"2022-07-14T08:05:48.075064Z","shell.execute_reply":"2022-07-14T08:05:48.197842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df['label'] = torch.softmax(preds[0], dim=1)[:, 1]\nsubm_df.to_csv('submission-softmax.csv', header=True, index=False)\nsubm_df","metadata":{"execution":{"iopub.status.busy":"2022-07-14T08:05:52.433826Z","iopub.execute_input":"2022-07-14T08:05:52.434207Z","iopub.status.idle":"2022-07-14T08:05:52.485131Z","shell.execute_reply.started":"2022-07-14T08:05:52.434176Z","shell.execute_reply":"2022-07-14T08:05:52.483878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cleanup data\nfrom shutil import rmtree\nrmtree('./train', ignore_errors=True)\nrmtree('./test', ignore_errors=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T08:05:56.923338Z","iopub.execute_input":"2022-07-14T08:05:56.923760Z","iopub.status.idle":"2022-07-14T08:05:58.234450Z","shell.execute_reply.started":"2022-07-14T08:05:56.923728Z","shell.execute_reply":"2022-07-14T08:05:58.233205Z"},"trusted":true},"execution_count":null,"outputs":[]}]}