{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\nimport sys\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.getcwd()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## fastai import statements for vision not including fastbook\nfrom fastai.vision.all import *\n#from fastbook import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Understanding the path structure and lenghts"},{"metadata":{"trusted":true},"cell_type":"code","source":"## ../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json\ntrain_path = \"../input/herbarium-2020-fgvc7/nybg2020/train/images/\"\ntest_path = \"../input/herbarium-2020-fgvc7/nybg2020/test/images/\"\nimage_path = \"../input/herbarium-2020-fgvc7/nybg2020/train/images/000/00/437000.jpg\"\n\nexport_path = \"../input/200kp5maxspec89epochresnet50/export.pkl\"\n#export_path = \"../input/min-5-df-and-export/export-min-5.pkl\"\n#df_path = \"../input/100k-10-epochs/df-100k-10-epochs.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## See one image\nimport os\n#print(os.getcwd())\nprint(\"Number of folders in train/images/ is\", len(next(os.walk(train_path))[1]))\nprint(\"Number of folders in train/images/000 is\", len(next(os.walk(train_path+\"000/\"))[1]))\nprint(\"Number of files in train/images/000/00/ is\", len(next(os.walk(train_path+\"000/00/\"))[2]))\nprint(\"\")\nprint(\"Number of folders in test/images/ is\", len(next(os.walk(test_path))[1]))\nprint(\"Number of files in test/images/000/ is\", len(next(os.walk(test_path+\"001/\"))[2]))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Understanding Json file contents"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_json_path = train_path+\"../metadata.json\"\ntest_json_path = test_path+\"../metadata.json\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz_tr = os.path.getsize(train_json_path)\nsz_te = os.path.getsize(test_json_path)\nprint(\"Size of train metadata is, \",sz_tr/10**6,\"MB\", \"Size of test metadata is\", sz_te/10**6, \"MB\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(train_json_path, encoding=\"utf8\", errors='ignore') as f:\n     tr_metadata = json.load(f)\n        \nwith open(test_json_path, encoding=\"utf8\", errors='ignore') as f:\n     te_metadata = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_metadata.keys(),te_metadata.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Length of each of the keys!\nprint([(name,len(tr_metadata[name])) for name in tr_metadata.keys()])\n[(name,len(te_metadata[name])) for name in te_metadata.keys()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num = 1030746\nprint(\"annotations\",tr_metadata[\"annotations\"][num], \"type\", type(tr_metadata[\"annotations\"]))\nprint(\"images\",tr_metadata[\"images\"][num])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_metadata[\"annotations\"][num][\"category_id\"], tr_metadata[\"images\"][num][\"file_name\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## \"Licenses\" is a list but \"info\" is not a list. Both have not more than 1 indice at max.\nprint(\"categories\",tr_metadata[\"categories\"][0:2])# index till 32000 interesting.\nprint(\"licenses\",tr_metadata[\"licenses\"],\"\\n\")\nprint(\"regions\",tr_metadata[\"regions\"][0], \"\\n\")\nprint(tr_metadata[\"info\"], type(tr_metadata[\"info\"]))\nprint(te_metadata[\"info\"], type(te_metadata[\"info\"]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Length characteristics of the meta data\nn_tr_img = len(tr_metadata[\"annotations\"])\nlen(tr_metadata[\"annotations\"])==len(tr_metadata[\"images\"]), n_tr_img/10**6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im = Image.open(\"../input/herbarium-2020-fgvc7/nybg2020/train/images/000/00/437000.jpg\")\nim.to_thumb(250,250)\n                ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make Test df (image id, filename, category_od)"},{"metadata":{"trusted":true},"cell_type":"code","source":"## never grow a DF, make a list (of lists) and then convert it to pandas (https://stackoverflow.com/a/56746204/5986651)\nlst_df_test = [[te_metadata[\"images\"][num][\"id\"], te_metadata[\"images\"][num][\"file_name\"]] for num in range(len(te_metadata[\"images\"]))]\nlst_df_test[0:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Convert list to DF (\"category_id\" and \"image_id\" are from \"annotations\". The \"filepath\" is from )\ndft= pd.DataFrame.from_records(lst_df_test)\ndft.columns  = [\"image_id\", \"filepath\"]\nprint(len(dft))\ndft[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## size of dft\nfrom pympler import asizeof\nasizeof.asizeof(dft)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Split dataframes into several to avoid memory issues\nno_splits = 100\ndfs = np.array_split(dft,100,axis=0)\nlen(dfs),len(dfs[1]), len(dft),type(dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## check CPU GPU USAGE\ndef print_cpu_gpu_usage():\n    !gpustat -cp\n    !free -m\n    #!top -bn1 | grep \"Cpu(s)\" | sed \"s/.*, *\\([0-9.]*\\)%* id.*/\\1/\" | awk '{print 100 - $1\"%\"}'\n    \nprint_cpu_gpu_usage()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Load learner\ndef get_x(r): return \"../input/herbarium-2020-fgvc7/nybg2020/test/\"+r[\"filepath\"]\ndef get_y(r): return r[\"category_id\"]\n\nlearn_inf = load_learner(export_path, cpu=False)\n#learn_inf = load_learner(\"export.pkl\", cpu=False)\n## check if you are running with CPU or GPU?\nlearn_inf.dls.device","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Size of inference object\nasizeof.asizeof(learn_inf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_cpu_gpu_usage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## predict function\ndef predict_func(dfv):\n    valid_dl = learn_inf.dls.test_dl(dfv, bs=256) ## Make DL\n    pred_tens = learn_inf.get_preds(dl=valid_dl) ## Get preds (proltys)\n    pred_argmax = pred_tens[0].argmax(dim=1) ## argmax\n    pred_categ = learn_inf.dls.vocab[pred_argmax] ## Get Category ID\n    dfv.loc[:,\"pred_cat\"] = pred_categ ## Add to dfv\n    print(pred_argmax.shape)\n    \n    #print(\"Accuracy is\",sum(dfv[\"pred_cat\"] == dfv[\"category_id\"])/len(dfv)) ## Print Accuracy for Valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Predict for each small dataframe\nfor i in range(len(dfs)):\n    predict_func(dfs[i])\n    print_cpu_gpu_usage()\n    print(dfs[i][0:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Joining the dataframe\ndft = pd.concat(dfs)\n#dft = dfs[0].append(dfs[1:len(dfs)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Save output\n\ndft.to_csv(\"my_results.csv\")\ndft.drop(\"filepath\",1, inplace=True)\ndft.columns = [\"Id\",\"Predicted\"]\ndft.to_csv(\"my_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_cpu_gpu_usage","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}