{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## fastai import statements for vision not including fastbook\n\nfrom fastai.vision.all import *\n#from fastbook import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import data"},{"metadata":{"trusted":true},"cell_type":"code","source":"## ../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json\ntrain_path = \"../input/herbarium-2020-fgvc7/nybg2020/train/images/\"\ntest_path = \"../input/herbarium-2020-fgvc7/nybg2020/test/images/\"\nimage_path = \"../input/herbarium-2020-fgvc7/nybg2020/train/images/000/00/437000.jpg\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## From EDA:"},{"metadata":{},"cell_type":"markdown","source":"Number of Categories, Number of images per category  \n[[3, 1],  \n [3726, 2],  \n [2660, 3],  \n [3769, 4],  \n [1434, 5],  \n [1243, 6],  \n [1000, 7],  \n [1833, 8],  \n [757, 9],  \n [683, 10]]  "},{"metadata":{},"cell_type":"markdown","source":"|  % categories      | n.images /category | Total images       |\n|--------------------|--------------------|--------------------|\n| 25% (8k)           | 1-4                | 21k                |\n| 50% (16k)          | 1-9                | 70k                |\n| 75% (24k)          | 1-27               | 198k               |\n| 80% (25k)          | 1-36               | 248k               |\n| 90% (29k)          | 1-82               | 423k               |\n| 100% (32k)         | 1-1795             | 1000k              |"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Import dfs\ndf_all = pd.read_csv(\"../input/processed-2/df_all.csv\")\ndf_all.drop(\"Unnamed: 0\",axis=1, inplace=True)\ndf_all_cat = pd.read_csv(\"../input/processed-2/df_one_cat.csv\")\ndf_all.shape,df_all_cat.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Determin size of dataset\n## Reduce size of dater (takes a long time 60s)\ndf_red = df_all.groupby(\"category_id\").head(n=8)\n#df_red = df_all.groupby(\"category_id\").apply(lambda x: x.sample(min(20,len(x)))).reset_index(drop=True)\nprint(\"Number of categories:\",len(df_red[\"category_id\"].unique()), \"\\nLength of df:\", len(df_red))\ndf_red\ndf_red[df_red[\"category_id\"]==23718]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Managing distribution of data for training"},{"metadata":{"trusted":true},"cell_type":"code","source":"## declarations\nimport random\nrandom.seed(42)\n\nuse_all_cat = True\nmin_specimens = 1\nmax_specimens = None\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Keep max 8 images per category and keep all cats\ndf = df_all.groupby(\"category_id\").apply(lambda x: x.sample(min(8,len(x)))).reset_index(drop=True)\ndf.sort_values(\"len_rows\", inplace=True)\ndf.reset_index(drop=True, inplace=True)\n#df = df[(df[\"len_rows\"] >= min_specimens) & (df[\"len_rows\"] < max_specimens)]\ndf.reset_index(drop=True, inplace=True)\nctg_unq = list(df[\"category_id\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Determine valid and test dataset\n\ntrain_ind_1 = [random.sample(df[df[\"category_id\"]==ctg].index.tolist(),1)[0] for ctg in ctg_unq]\n\n## get rest of train and valid dset\navl_ind = list(set(range(len(df)))-set(train_ind_1))\nvalid_ind = random.sample(avl_ind,int(0.2*len(df)))\ntrain_ind = list(set(range(len(df)))-set(valid_ind))\nprint(\"Valid ind is not in Train indices?\", not set(valid_ind).issubset(set(train_ind)))\nprint(\"Total images:\", len(df_all), \"\\nTotal filtered images:\", len(df), \n      \"\\nTotal selected cat:\", len(train_ind_1), \"\\nSingle images in df:\", len(df[df[\"len_rows\"]==1]))\nprint(\"Validation count:\", len(valid_ind), \"\\nTraining count:\", len(train_ind))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## check: All of valid_cat should be in train_cat\nvalid_set = set(df.loc[valid_ind,\"category_id\"])\ntrain_set = set(df.loc[train_ind,\"category_id\"])\nprint(\"All Valid categ in Train categories?\", valid_set.issubset(train_set))\nprint(\"\\nTotal selected cat:\", len(train_ind_1), \n      \"\\nTotal select valid categories: \", len(valid_set),\"\\nTotal select Train categories:\",len(train_set))\nprint(len(valid_set.intersection(train_set)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Make a new column \"Is_valid\"\ndf.loc[valid_ind,[\"is_valid\"]] = True\ndf.loc[train_ind,[\"is_valid\"]] = False\ndf.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_cpu_gpu_usage():\n    !gpustat -cp\n    !free -m\n    #!top -bn1 | grep \"Cpu(s)\" | sed \"s/.*, *\\([0-9.]*\\)%* id.*/\\1/\" | awk '{print 100 - $1\"%\"}'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Block"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Writting the splitter so that valid data has categories as in train\n\ndef splitter(df):\n    train_ind = df.index[df['is_valid']==False].tolist()\n    valid_ind = df.index[df['is_valid']==True].tolist()\n    \n    valid_cats = set([df[\"category_id\"].iloc[i] for i in valid_ind])\n    train_cats = set([df[\"category_id\"].iloc[i] for i in train_ind])\n    if not valid_cats.issubset(train_cats):\n        raise Exception(\"something is wrong\")\n    return train_ind,valid_ind\n\ntrain,valid = splitter(df)\nlen(train), len(valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_x(r): return \"../input/herbarium-2020-fgvc7/nybg2020/train/\"+r[\"filepath\"]\ndef get_y(r): return r[\"category_id\"]\ndblock = DataBlock(blocks=(ImageBlock, CategoryBlock),#documentation???\n    get_x = get_x,\n    get_y = get_y,\n    splitter=splitter,\n    item_tfms=Resize(256))\n    #item_tfms=RandomResizedCrop(256, min_scale=0.08),\n    #batch_tfms=aug_transforms(size=224, min_scale=0.5, mult=2, pad_mode='zeros')) # next iter mult=2\n\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Create dsets and dls\ndsets = dblock.datasets(df)\ndls = dblock.dataloaders(df,bs=128)\n#x,y = dsets.train[0]\n#x,y,x.shape,y.shape, len(dsets.train)\n#dblock.summary(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Showing one batch prep\nx1,y1 = dls.train.one_batch()\ndls.show_batch(nrows=2,ncols=3)\n#x.shape,y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## NN Learner\nfrom fastai.callback.fp16 import *\n\nf1_score_multi = F1Score(average=\"macro\") ## convert class to functie\nlearn = cnn_learner(dls,resnet50,metrics=f1_score_multi).to_fp16() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## check memory usage\nprint_cpu_gpu_usage()\nlearn.fine_tune(9, base_lr=3e-3, freeze_epochs=1)\nprint_cpu_gpu_usage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Export\nlearn.export()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Size  of files and folders\n!ls -l export.pkl\n!ls -l df.csv\n!du -sh ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}