{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Machine Learning and Data Science Imports\nimport tensorflow_probability as tfp\nimport tensorflow_datasets as tfds\nimport tensorflow_addons as tfa\nimport tensorflow_hub as hub\nfrom skimage import exposure\nimport pandas as pd; pd.options.mode.chained_assignment = None\nimport numpy as np\nimport scipy\n\n# Built In Imports\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport IPython\nimport urllib\nimport json\nimport zipfile\nimport pickle\nimport shutil\nimport string\nimport math\nimport tqdm\nimport time\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib\nimport plotly\nimport PIL\nimport cv2\n\n# PRESETS\nFIG_FONT = dict(family=\"Helvetica, Arial\", size=14, color=\"#7f7f7f\")\nLABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette(\"Spectral\", 15)]\nLABEL_COLORS_WOUT_NO_FINDING = LABEL_COLORS[:8]+LABEL_COLORS[9:]\n\n# Other Imports\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm.notebook import tqdm\nimport pydicom","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_folder_path = \"/kaggle/input/iwildcam2021-fgvc8/metadata\"\nlist_of_files = list(os.listdir(json_folder_path))\n\nfor file_name in list_of_files:\n    json_path = os.path.join(json_folder_path, file_name)\n    print(f\"Current json processed: {file_name}\")\n    with open(json_path) as json_file:\n        # read each json\n        json_data = json.load(json_file)\n        # for each item in the json\n        for item in json_data.items():\n            # prepare the dataframe name\n            file_name_split = file_name.split(\".\")[0]\n            file_name_split = file_name_split.split(\"_\")\n            file_name_str = file_name_split[1] + \"_\" + file_name_split[2]\n            print(f\"\\tCurrent json item processed: {item[0]} length: {len(item[1])}\")\n            data_frame_name = f\"{file_name_str}_{item[0]}_df\"\n            print(f\"\\tDynamic dataframe created: {data_frame_name}\")\n            # dynamic creation of a dataframe, using vars()[data_frame_name]\n            vars()[data_frame_name] = pd.json_normalize(json_data.get(item[0]))\n            # output the dataframe\n            vars()[data_frame_name].to_csv(f\"{data_frame_name}\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(megadetector_results_images_df.shape)\npd.set_option('display.max_colwidth', None)\nmegadetector_results_images_df.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox_train = megadetector_results_images_df[megadetector_results_images_df.max_detection_conf > .7]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox_train.head(117)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Just testing we have access to the images\nimport matplotlib.pyplot as plt\nimport matplotlib.image as img\n\nimage = img.imread('/kaggle/input/iwildcam2021-fgvc8/train/905cd794-21bc-11ea-a13a-137349068a90.jpg')\nplt.imshow(image)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Function to break the multiple Json detections to individual rows\n\ndef explode(df, lst_cols, fill_value='', preserve_index=False):\n    # make sure `lst_cols` is list-alike\n    if (lst_cols is not None\n        and len(lst_cols) > 0\n        and not isinstance(lst_cols, (list, tuple, np.ndarray, pd.Series))):\n        lst_cols = [lst_cols]\n    # all columns except `lst_cols`\n    idx_cols = df.columns.difference(lst_cols)\n    # calculate lengths of lists\n    lens = df[lst_cols[0]].str.len()\n    # preserve original index values    \n    idx = np.repeat(df.index.values, lens)\n    # create \"exploded\" DF\n    res = (pd.DataFrame({\n                col:np.repeat(df[col].values, lens)\n                for col in idx_cols},\n                index=idx)\n             .assign(**{col:np.concatenate(df.loc[lens>0, col].values)\n                            for col in lst_cols}))\n    # append those rows that have empty lists\n    if (lens == 0).any():\n        # at least one list in cells is empty\n        res = (res.append(df.loc[lens==0, idx_cols], sort=False)\n                  .fillna(fill_value))\n    # revert the original index order\n    res = res.sort_index()\n    # reset index if requested\n    if not preserve_index:        \n        res = res.reset_index(drop=True)\n    return res","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Execute function\nbbox_train = explode(bbox_train, ['detections'], fill_value='')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#break out detections to components\nbbox_3 = bbox_train['detections'].values.tolist()\nxyz = pd.DataFrame(bbox_3,columns =['category', 'bbox','conf'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Rejoin results from previous cell\nTrain_DF=pd.concat([bbox_train, xyz], axis=1)\nTrain_DF.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Remove unneeded fields\nTrain_DF.drop(['max_detection_conf','detections'], axis=1, inplace=True)\nTrain_DF.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Filter out poor detection\nTrain_DF = Train_DF[Train_DF.conf > .69]\nTrain_DF.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Add image path to DF\nTrain_DF['image_path'] = f'/kaggle/input/iwildcam2021-fgvc8/train/'+Train_DF.id+('.jpg')\nTrain_DF.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Split the list for BBox into mins/maxes\nTrain_DF[['X_min','Y_max', 'W', 'H']] = pd.DataFrame(Train_DF.bbox.tolist(), index= Train_DF.index)\nTrain_DF.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Randomize dataframe rows (because I'm going to split the DF to Train Test)\n\nTrain_DF=Train_DF.sample(frac=1)\nTrain_DF.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_DF = Train_DF.rename(columns={'id': 'image_id'})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annotations_annotations_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_Merge = pd.merge(Train_DF, \n                       train_annotations_annotations_df, \n                       on='image_id', how='left')\nTrain_Merge.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annotations_categories_df['category_id'] = train_annotations_categories_df['id']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Merge the Animal types associated to the image detection\nTrain_Merge = pd.merge(Train_Merge, \n                       train_annotations_categories_df, \n                       on='category_id', how='left')\nTrain_Merge.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop NA values\nprint(len(Train_Merge))\nTrain_No_Null = Train_Merge[Train_Merge['category_id'].notna()]\nprint(len(Train_No_Null))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.merge(train_annotations_categories_df, Train_No_Null, how=\"inner\", on=[\"category_id\"])\ndictionary = pd.DataFrame(result, columns=['name_x', 'category_id'])\ndictionary=dictionary.drop_duplicates(subset=['category_id'])\ndictionary['new_category_id'] = range(len(dictionary))\ndictionary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_No_Null=pd.merge(dictionary,Train_No_Null, how=\"inner\", on=[\"category_id\"])\nTrain_No_Null.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_No_Null['category_id'] = Train_No_Null['category_id'].astype(str)\nUnique_Category_List=Train_No_Null['category_id'].unique()\nUnique_Category_List=Unique_Category_List.tolist()\nprint(Unique_Category_List)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_DF_Final = Train_No_Null.copy()\nTrain_DF_Final = Train_DF_Final[0:0]\nTest_DF_Final = Train_DF_Final[0:0]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for value in Unique_Category_List:\n    temp_df=Train_No_Null[Train_No_Null.category_id == value].head(5)\n    Train_DF_Final=Train_DF_Final.append(temp_df)\n    del temp_df\nTrain_DF_Final","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for value in Unique_Category_List:\n    temp_df=Train_No_Null[Train_No_Null.category_id == value].tail(2)\n    Test_DF_Final=Test_DF_Final.append(temp_df)\n    del temp_df\nTest_DF_Final","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#used to create the yolo label values\n'''\nTrain_No_Null['x_mid'] = Train_No_Null.apply(lambda row: (row.X_min+((row.W)/2)), axis =1)\nTrain_No_Null['y_mid'] = Train_No_Null.apply(lambda row: ((1-(row.Y_max))-((row.H)/2)), axis =1)\n\n\nTrain_No_Null.head()'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.mkdir(\"/kaggle/working/text_folder/\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#used to create the yolo label values\n#Train_No_Null=pd.merge(dictionary,Train_No_Null, how=\"inner\", on=[\"category_id\"])\n#Yolo_Label_df=Train_No_Null[[\"image_id\", \"new_category_id\", \"x_mid\", \"y_mid\", \"W\", \"H\"]]\n#Train_No_Null\n#Yolo_Label_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#used to create the yolo label values\n'''for index, row in tqdm(Yolo_Label_df.iterrows()):\n    file_name = \"/kaggle/working/text_folder/\" + row['image_id']+\".txt\"\n    pd.DataFrame(row).T.to_csv(file_name, columns=[\"new_category_id\", \"x_mid\", \"y_mid\", \"W\", \"H\"], header=None, index=None, sep=' ', mode='a')'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(Test_DF_Final.image_path.unique())\ntrain_files += list(Train_DF_Final.image_path.unique())\nlen(train_files), len(val_files)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/animals/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/animals/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/animals/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/animals/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/animal-detect-yolo-labels/text_folder'\nfor file in tqdm(train_files):\n    try:\n        shutil.copy(file, '/kaggle/working/animals/images/train')\n        filename = file.split('/')[-1].split('.')[0]\n        shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/animals/labels/train')\n    except:\n        pass\n    \nfor file in tqdm(val_files):\n    try:\n        shutil.copy(file, '/kaggle/working/animals/images/val')\n        filename = file.split('/')[-1].split('.')[0]\n        shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/animals/labels/val')\n    except:\n        pass","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install zip_files\n#!zip-folder --auto-root --outfile /kaggle/working/text_folder.zip /kaggle/working/text_folder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from IPython.display import FileLink\nFileLink(r'./text_folder.zip')'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" '''import os\n arr = os.listdir('./animals/images/')\n print(arr)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''os.chdir('/kaggle/working/')\nimport os\nos.remove(\"./animals.yaml\")'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from shutil import rmtree\n\nrmtree(\"./animals\")'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\npath, dirs, files = next(os.walk(\"/kaggle/working/animals/images/train\"))\nfile_count = len(files)\nprint(file_count)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(Train_DF_Final.new_category_id, Train_DF_Final.name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))\nclasses","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/animals/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/animals/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 198,\n    names = classes\n    )\n\nwith open(join( cwd , 'animals.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'animals.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python train.py --img 400 --batch 40 --epochs 100 --data /kaggle/working/animals.yaml --weights yolov5x.pt --cache","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Detect Me On It\n\n!python detect.py --weights 'runs/train/exp/weights/best.pt'\\\n--img 500\\\n--conf 0.15\\\n--iou 0.5\\\n--source /kaggle/working/vinbigdata/images/valy\\\n--save-conf\\\n--save-txt","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#WORKING QUERIES BELOW /// IGNORE","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr = os.listdir('/kaggle/working/animals/labels/train')\nprint(arr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Just testing we have access to the images\nimport matplotlib.pyplot as plt\nimport matplotlib.image as img\n\nimage = img.imread('/kaggle/working/animals/images/train/90006c84-21bc-11ea-a13a-137349068a90.jpg')\nplt.imshow(image)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path_list = Train_DF_Final['image_path'].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counter=0\nfor file in image_path_list:\n    if os.path.exists(file)==True:\n        print('true')\n    else:\n        print('fail')'''\n\nos.path.exists('../input/iwildcam2021-fgvc8/train/86760c00-21bc-11ea-a13a-137349068a90.jpg')\n\n../input/iwildcam2021-fgvc8/train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Just testing we have access to the images\nimport matplotlib.pyplot as plt\nimport matplotlib.image as img\n\nimage = img.imread('/kaggle/input/iwildcam2021-fgvc8/train/86760c00-21bc-11ea-a13a-137349068a90.jpg')\nplt.imshow(image)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}