{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n#loading json file for train data\ndata_path_train = \"../input/herbarium-2021-fgvc8/train/metadata.json\"\nwith open(data_path_train) as json_file:\n    meta_train = json.load(json_file)\n#creating seperate df from json\nannotations_train =  pd.json_normalize(meta_train ['annotations'])\n\ncategories_train =  pd.json_normalize(meta_train ['categories'])\ncategories_train.rename(columns={'id':'category_id'}, inplace=True)\n\nimages_train =  pd.json_normalize(meta_train ['images'])\nimages_train.rename(columns={'id':'image_id'}, inplace=True)\n\ndf_merge = pd.merge(images_train[['image_id','file_name']],annotations_train[['category_id', 'image_id']] , on='image_id')\ndf_merge = pd.merge(df_merge[['image_id','file_name','category_id']],categories_train[['category_id','name','family','order']] , on='category_id')\n#df_merge = df_merge.drop(df_merge[df_merge.order =='Unknown'].index)\n\ndf_merge['file_path']=\"../input/herbarium-2021-fgvc8/train/\"+df_merge['file_name']\n\n\n\nspecies= df_merge['category_id'].value_counts()>51\ns_list = species.index[species==True].tolist()\ndf_more= df_merge.loc[df_merge['category_id'].isin(s_list)]\n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:35:01.64181Z","iopub.execute_input":"2021-05-26T09:35:01.642181Z","iopub.status.idle":"2021-05-26T09:35:51.817229Z","shell.execute_reply.started":"2021-05-26T09:35:01.642144Z","shell.execute_reply":"2021-05-26T09:35:51.816221Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_more","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:35:51.818901Z","iopub.execute_input":"2021-05-26T09:35:51.819173Z","iopub.status.idle":"2021-05-26T09:35:51.837582Z","shell.execute_reply.started":"2021-05-26T09:35:51.819146Z","shell.execute_reply":"2021-05-26T09:35:51.836205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_more['category_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:35:51.839388Z","iopub.execute_input":"2021-05-26T09:35:51.839822Z","iopub.status.idle":"2021-05-26T09:35:51.867916Z","shell.execute_reply.started":"2021-05-26T09:35:51.839782Z","shell.execute_reply":"2021-05-26T09:35:51.867165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stratified_sample_df(df, col, n_samples):\n    n = min(n_samples, df[col].value_counts().min())\n    df_ = df.groupby(col).apply(lambda x: x.sample(n))\n    df_.index = df_.index.droplevel(0)\n    return df_","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:35:51.868953Z","iopub.execute_input":"2021-05-26T09:35:51.86927Z","iopub.status.idle":"2021-05-26T09:35:51.875205Z","shell.execute_reply.started":"2021-05-26T09:35:51.86924Z","shell.execute_reply":"2021-05-26T09:35:51.873895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = stratified_sample_df(df_more, 'category_id', 2000)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:35:51.878877Z","iopub.execute_input":"2021-05-26T09:35:51.879396Z","iopub.status.idle":"2021-05-26T09:36:03.240919Z","shell.execute_reply.started":"2021-05-26T09:35:51.879349Z","shell.execute_reply":"2021-05-26T09:36:03.23999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:36:03.242407Z","iopub.execute_input":"2021-05-26T09:36:03.242658Z","iopub.status.idle":"2021-05-26T09:36:03.26099Z","shell.execute_reply.started":"2021-05-26T09:36:03.242635Z","shell.execute_reply":"2021-05-26T09:36:03.260149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(data['category_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-05-26T09:36:03.26232Z","iopub.execute_input":"2021-05-26T09:36:03.262604Z","iopub.status.idle":"2021-05-26T09:36:03.278969Z","shell.execute_reply.started":"2021-05-26T09:36:03.262578Z","shell.execute_reply":"2021-05-26T09:36:03.278007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}