{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Make sample with less categories\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## fastai import statements for vision not including fastbook\n\nfrom fastai.vision.all import *\n#from fastbook import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Understanding length of paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"## ../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json\ntrain_path = \"../input/herbarium-2020-fgvc7/nybg2020/train/images/\"\ntest_path = \"../input/herbarium-2020-fgvc7/nybg2020/test/images/\"\nimage_path = \"../input/herbarium-2020-fgvc7/nybg2020/train/images/000/00/437000.jpg\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Extracting Json contents"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_json_path = train_path+\"../metadata.json\"\ntest_json_path = test_path+\"../metadata.json\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(train_json_path, encoding=\"utf8\", errors='ignore') as f:\n     tr_metadata = json.load(f)\n        \n# with open(test_json_path, encoding=\"utf8\", errors='ignore') as f:\n#      te_metadata = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Length of each of the keys!\nprint([(name,len(tr_metadata[name])) for name in tr_metadata.keys()])\n#[(name,len(te_metadata[name])) for name in te_metadata.keys()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num = 1030746\nprint(\"annotations\",tr_metadata[\"annotations\"][num], \"type\", type(tr_metadata[\"annotations\"]))\nprint(\"images\",tr_metadata[\"images\"][num])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## \"Licenses\" is a list but \"info\" is not a list. Both have not more than 1 indice at max.\nprint(\"categories\",tr_metadata[\"categories\"][0:2])# index till 32000 interesting.\nprint(\"licenses\",tr_metadata[\"licenses\"],\"\\n\")\nprint(\"regions\",tr_metadata[\"regions\"][0], \"\\n\")\nprint(tr_metadata[\"info\"], type(tr_metadata[\"info\"]))\n#print(te_metadata[\"info\"], type(te_metadata[\"info\"]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Length characteristics of the meta data\nn_tr_img = len(tr_metadata[\"annotations\"])\nlen(tr_metadata[\"annotations\"])==len(tr_metadata[\"images\"]), n_tr_img/10**6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im = Image.open(\"../input/herbarium-2020-fgvc7/nybg2020/train/images/000/00/437000.jpg\")\nim.to_thumb(250,250)\n                ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**From EDR:**\nNumber of Categories, Number of images per category  \n[[3, 1],  \n [3726, 2],  \n [2660, 3],  \n [3769, 4],  \n [1434, 5],  \n [1243, 6],  \n [1000, 7],  \n [1833, 8],  \n [757, 9],  \n [683, 10]]  \n"},{"metadata":{},"cell_type":"markdown","source":"## dict --> Df (tot_spec, min_specs, maxspec)"},{"metadata":{"trusted":true},"cell_type":"code","source":"## generate random numbers from 0 to len(pics) needed\nimport random\n\nreq_spec = 10000\ntot_spec = 5*req_spec#len(tr_metadata[\"annotations\"]) \nmin_specimens = 2; \nmax_specimens = 3;\n\nrandom.seed(42)\nrng_img = random.sample(range(0,n_tr_img),tot_spec) #finalnum not included\nrng_img[0:5], max(rng_img), min(rng_img), n_tr_img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## never grow a DF, make a list (of lists) and then convert it to pandas (https://stackoverflow.com/a/56746204/5986651)\nlst_df = [[tr_metadata[\"annotations\"][num][\"category_id\"], tr_metadata[\"annotations\"][num][\"image_id\"], tr_metadata[\"images\"][num][\"file_name\"]] for i,num in enumerate(rng_img)]\nlst_df[0:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Convert list to DF (\"category_id\" and \"image_id\" are from \"annotations\". The \"filepath\" is from )\ndf = pd.DataFrame.from_records(lst_df)\ndf.columns  = [\"category_id\", \"image_id\", \"filepath\"]\ndf[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make basic dataframe with train and valid"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Sort by length of rows\ndf[\"len_rows\"] = df.groupby(\"category_id\")[[\"category_id\"]].transform(len)\ndf = df.sort_values(\"len_rows\")\ndf.reset_index(drop=True, inplace=True) # otherwise you have issues with df.index\n#df = df.drop(\"c\",axis=1)\nprint(\"No. of Single images:\", len(df[df[\"len_rows\"]==1]))\nprint(\"min specimens:\",df.len_rows.min(), \"max_specimens:\",df.len_rows.max())\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modify DataFrame based on number of specimens per category"},{"metadata":{"trusted":true},"cell_type":"code","source":"## make copy of df before filtering\ndf_unfiltered = df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Get unique categories matching the min_spec and max_spec\nif max_specimens == None: max_specimens = df.len_rows.max()\nif min_specimens == None: min_specimens = df.len_rows.min()\n    \ndf = df[(df[\"len_rows\"] >= min_specimens) & (df[\"len_rows\"] < max_specimens)]\ndf.reset_index(drop=True, inplace=True)\nctg_unq = list(df[\"category_id\"].unique())\nctg_unq_unfiltered = df_unfiltered[\"category_id\"].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Determine valid and test dataset\n\ntrain_ind_1 = [random.sample(df[df[\"category_id\"]==ctg].index.tolist(),1) for ctg in random.sample(ctg_unq,len(ctg_unq))]\ntrain_ind_1 = [item for sublist in train_ind_1 for item in sublist]\n\n## get rest of train and valid dset\navl_ind = list(set(range(len(df)))-set(train_ind_1))\nvalid_ind = random.sample(avl_ind,int(0.2*len(df)))\ntrain_ind = list(set(range(len(df)))-set(valid_ind))\nprint(\"Valid ind is not in Train indices?\", not set(valid_ind).issubset(set(train_ind)))\nprint(\"Total images:\", len(df_unfiltered), \"\\nTotal filtered images:\", len(df), \"\\nTotal Categories:\", len(ctg_unq_unfiltered),\n      \"\\nTotal selected cat:\", len(train_ind_1), \"\\nSingle images in df:\", len(df[df[\"len_rows\"]==1]))\nprint(\"Validation count:\", len(valid_ind), \"Training count:\", len(train_ind))\nprint(df[\"len_rows\"].unique()[0:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## check: All of valid_cat should be in train_cat\nvalid_set = set(df.loc[valid_ind,\"category_id\"])\ntrain_set = set(df.loc[train_ind,\"category_id\"])\nprint(\"All Valid categ in Train categories?\", valid_set.issubset(train_set))\nprint(\"\\nTotal Categories:\", len(ctg_unq_unfiltered), \"\\nTotal selected cat:\", len(train_ind_1), \n      \"\\nTotal select valid categories: \", len(valid_set),\"\\nTotal select Train categories:\",len(train_set))\nprint(len(valid_set.intersection(train_set)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Make a new column \"Is_valid\"\ndf.loc[valid_ind,[\"is_valid\"]] = True\ndf.loc[train_ind,[\"is_valid\"]] = False\ndf.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv(\"df.csv\")\ndf_unfiltered.to_csv(\"df_unfiltered.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Block"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Writting the splitter so that it valid data has categories as in train\n\ndef splitter(df):\n    train_ind = df.index[df['is_valid']==False].tolist()\n    valid_ind = df.index[df['is_valid']==True].tolist()\n    \n    valid_cats = set([df[\"category_id\"].iloc[i] for i in valid_ind])\n    train_cats = set([df[\"category_id\"].iloc[i] for i in train_ind])\n    if not valid_cats.issubset(train_cats):\n        raise Exception(\"something is wrong\")\n    return train_ind,valid_ind\n\ntrain,valid = splitter(df)\nlen(train), len(valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_x(r): return \"../input/herbarium-2020-fgvc7/nybg2020/train/\"+r[\"filepath\"]\ndef get_y(r): return r[\"category_id\"]\ndblock = DataBlock(blocks=(ImageBlock, CategoryBlock),#documentation???\n    get_x = get_x,\n    get_y = get_y,\n    splitter=splitter,\n    item_tfms=Resize(256))\n    #item_tfms=RandomResizedCrop(256, min_scale=0.08),\n    #batch_tfms=aug_transforms(size=224, min_scale=0.5, mult=2, pad_mode='zeros')) # next iter mult=2\n\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#aug_transforms??","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Create dsets and dls\ndsets = dblock.datasets(df)\ndls = dblock.dataloaders(df,bs=128)\nx,y = dsets.train[0]\n#x,y,x.shape,y.shape, len(dsets.train)\n#dblock.summary(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x,y = dsets.valid[0]\nx,y,x.shape,y.shape, len(dsets.valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nx1,y1 = dls.train.one_batch()\nx.shape,y.shape\n# x1,y1 = first(dls.train)\n# x1.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_,axs = plt.subplots(1,3,figsize=(12,4))\nfor ax,method in zip(axs.flatten(), [ResizeMethod.Squish, ResizeMethod.Pad, ResizeMethod.Crop]):\n    rsz = Resize(256, method=method)\n    show_image(rsz(img, split_idx=0), ctx=ax, title=method);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"crop = RandomResizedCrop(256, min_scale=0.5)\n_,axs = plt.subplots(3,3,figsize=(9,9))\nfor ax in axs.flatten():\n    cropped = crop(img)\n    show_image(cropped, ctx=ax);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"timg = TensorImage(array(img)).permute(2,0,1).float()/255.\ndef _batch_ex(bs): return TensorImage(timg[None].expand(bs, *timg.shape).clone())\n\n#tfms = aug_transforms(pad_mode='zeros', mult=2, min_scale=0.5)\ntfms = aug_transforms(size=224, min_scale=0.5, mult=2, pad_mode='zeros')\n\ny = _batch_ex(9)\nfor t in tfms: y = t(y, split_idx=0)\n_,axs = plt.subplots(1,3, figsize=(12,3))\nfor i,ax in enumerate(axs.flatten()): show_image(y[i], ctx=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls.show_batch(nrows=2,ncols=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## NN Learner \n# f1_score_multi = F1Score(average=\"macro\") ## convert class to functie\n# learn = cnn_learner(dls,resnet18,metrics=f1_score_multi)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lr1,_ = learn.lr_find()\n#print(lr1)\n# lr1 = 0.005","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## fit one cycle with freeze\n#learn.fit_one_cycle(1,lr1) # no need to call freeze","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## find new learning rate after unfreezing\n#learn.unfreeze()\n#lr2, lr_st2 = learn.lr_find()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(lr2,lr_st2)\n# lr2 = max(lr2,lr_st2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## learn the rest\n# learn.fit_one_cycle(10,lr_max=slice(lr2/10,lr2))# trying lr_min/100 is also an issue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## NN Learner\nf1_score_multi = F1Score(average=\"macro\") ## convert class to functie\nlearn = cnn_learner(dls,resnet50,metrics=f1_score_multi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#resnet18","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nlearn.lr_find()\nlearn.fine_tune(10, base_lr=3e-3, freeze_epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Export\nlearn.export()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Size  of files and folders\n!ls -l export.pkl\n!ls -l df.csv\n!du -sh ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}