{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !rm -rf *\n# !mkdir /kaggle/working/datasets\n# !mkdir /kaggle/working/datasets/train\n# !mkdir /kaggle/working/datasets/validation\n\n# !mkdir /kaggle/working/datasets/train/images\n# !mkdir /kaggle/working/datasets/train/labels\n\n# !mkdir /kaggle/working/datasets/validation/images\n# !mkdir /kaggle/working/datasets/validation/labels\n\n# !ln -s /kaggle/input/dlsprint2/badlad/images /kaggle/working/datasets/badlad/images\n# !ln -s /kaggle/input/dlsprint2/badlad/labels/yolov8_format/train /kaggle/working/datasets/badlad/labels\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-29T07:23:47.988899Z","iopub.execute_input":"2023-07-29T07:23:47.989846Z","iopub.status.idle":"2023-07-29T07:23:56.988091Z","shell.execute_reply.started":"2023-07-29T07:23:47.989799Z","shell.execute_reply":"2023-07-29T07:23:56.986628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# from tqdm import tqdm\n\n# def all_files_in_folder_symlink(source_dir, target_dir):\n#     files = os.listdir(source_dir)\n\n#     for file in tqdm(files):\n#         source_file = os.path.join(source_dir, file)\n#         target_file = os.path.join(target_dir, file)\n#         os.symlink(source_file, target_file)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:56.990585Z","iopub.execute_input":"2023-07-29T07:23:56.991028Z","iopub.status.idle":"2023-07-29T07:23:56.998190Z","shell.execute_reply.started":"2023-07-29T07:23:56.990987Z","shell.execute_reply":"2023-07-29T07:23:56.996551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_files_in_folder_symlink(\"/kaggle/input/dlsprint2/badlad/labels/yolov8_format/train\",\"/kaggle/working/datasets/badlad/labels/train\")\n# all_files_in_folder_symlink(\"/kaggle/input/dlsprint2/badlad/images/train\",\"/kaggle/working/datasets/badlad/images/train\")\n# all_files_in_folder_symlink(\"/kaggle/input/dlsprint2/badlad/images/test\",\"/kaggle/working/datasets/badlad/images/test\")","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.000543Z","iopub.execute_input":"2023-07-29T07:23:57.001140Z","iopub.status.idle":"2023-07-29T07:23:57.014819Z","shell.execute_reply.started":"2023-07-29T07:23:57.001091Z","shell.execute_reply":"2023-07-29T07:23:57.013245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# file_content = \"\"\"\n# path: /kaggle/working/datasets/badlad\n# train: images/train\n# val: images/train\n\n# names:\n#   0: paragraph\n#   1: text_box\n#   2: image\n#   3: table\n\n\n# \"\"\"\n\n# with open(\"yolov8.yaml\", mode=\"w\") as f:\n#     f.write(file_content)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.018623Z","iopub.execute_input":"2023-07-29T07:23:57.019129Z","iopub.status.idle":"2023-07-29T07:23:57.028258Z","shell.execute_reply.started":"2023-07-29T07:23:57.019085Z","shell.execute_reply":"2023-07-29T07:23:57.026818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import wandb\n# wandb.init(mode=\"disabled\")","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.030191Z","iopub.execute_input":"2023-07-29T07:23:57.031021Z","iopub.status.idle":"2023-07-29T07:23:57.043099Z","shell.execute_reply.started":"2023-07-29T07:23:57.030973Z","shell.execute_reply":"2023-07-29T07:23:57.042119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import yaml\n# from yaml.loader import SafeLoader\n\n# with open('yolov8.yaml') as f:\n#     data = yaml.load(f, Loader=SafeLoader)\n#     print(data)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.045341Z","iopub.execute_input":"2023-07-29T07:23:57.045834Z","iopub.status.idle":"2023-07-29T07:23:57.059714Z","shell.execute_reply.started":"2023-07-29T07:23:57.045792Z","shell.execute_reply":"2023-07-29T07:23:57.058344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"# import yaml\n# from yaml.loader import SafeLoader\n\n# with open('yolov8.yaml') as f:\n#     data = yaml.load(f, Loader=SafeLoader)\n#     print(data)\n\n# import torch\n# device = 'cuda' if torch.cuda.is_available() else 'cpu'\n# training_data = \"/kaggle/working/yolov8.yaml\"\n# print('training starts')","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.060938Z","iopub.execute_input":"2023-07-29T07:23:57.061283Z","iopub.status.idle":"2023-07-29T07:23:57.074514Z","shell.execute_reply.started":"2023-07-29T07:23:57.061254Z","shell.execute_reply":"2023-07-29T07:23:57.073473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from ultralytics import YOLO\n\n\n# model = YOLO(\"yolov8m-seg.pt\")\n\n# # model.train(data=\"/kaggle/working/yolov8.yaml\", epochs=3)\n\n# results = model.train( \n#         batch=8,\n#         device= 0,\n#         data=training_data,\n#         save=True,\n#         save_period=4,\n#         epochs=8,\n#         imgsz=640\n     \n#     )","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.075949Z","iopub.execute_input":"2023-07-29T07:23:57.076294Z","iopub.status.idle":"2023-07-29T07:23:57.092414Z","shell.execute_reply.started":"2023-07-29T07:23:57.076253Z","shell.execute_reply":"2023-07-29T07:23:57.091083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame({'Id':Id, 'Predicted':Prediction}, index=None)\n# submission.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.094292Z","iopub.execute_input":"2023-07-29T07:23:57.094729Z","iopub.status.idle":"2023-07-29T07:23:57.106138Z","shell.execute_reply.started":"2023-07-29T07:23:57.094697Z","shell.execute_reply":"2023-07-29T07:23:57.104547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:23:57.111680Z","iopub.execute_input":"2023-07-29T07:23:57.112629Z","iopub.status.idle":"2023-07-29T07:23:57.120092Z","shell.execute_reply.started":"2023-07-29T07:23:57.112590Z","shell.execute_reply":"2023-07-29T07:23:57.118529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pycocotools\n!pip install ultralytics -q","metadata":{"execution":{"iopub.status.busy":"2023-07-31T18:46:12.482710Z","iopub.execute_input":"2023-07-31T18:46:12.483650Z","iopub.status.idle":"2023-07-31T18:47:06.906880Z","shell.execute_reply.started":"2023-07-31T18:46:12.483620Z","shell.execute_reply":"2023-07-31T18:47:06.905542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2>Create Data </h2>","metadata":{}},{"cell_type":"code","source":"# def visualize_image(index, coco, path):\n#     img_ids = coco.getImgIds(imgIds=[index])\n#     images = coco.loadImgs(img_ids)\n#     plt.figure(figsize=(15, 12))\n\n#     plt.subplot(2,3,1)\n#     normal_image = os.path.join(path, images[0]['file_name'])\n#     normal_image = cv2.imread(normal_image)\n#     height, width = normal_image.shape[0], normal_image.shape[1]\n#     plt.imshow(normal_image)\n#     plt.title('original image')\n    \n#     masks = {0:np.zeros((height, width)), \n#       1:np.zeros((height, width)), \n#       2:np.zeros((height, width)),\n#       3:np.zeros((height, width))}\n    \n#     annot_id = coco.getAnnIds(imgIds=img_ids)\n#     annotations = coco.loadAnns(annot_id)\n#     for ann in annotations:\n#         label = ann['category_id']\n#         m = coco.annToMask(ann)\n#         masks[label] += m\n    \n#     plt.subplot(2,3,2)\n#     plt.imshow(masks[0])\n#     plt.title('Paragraph')\n\n#     plt.subplot(2,3,3)\n#     plt.imshow(masks[1])\n#     plt.title('text_box')\n\n#     plt.subplot(2,3,4)\n#     plt.imshow(masks[2])\n#     plt.title('image')\n    \n#     plt.subplot(2,3,5)\n#     plt.imshow(masks[3])\n#     plt.title('table')\n    \n#     plt.subplot(2,3,6)\n#     plt.imshow(normal_image); plt.axis('off')\n#     coco.showAnns(annotations)\n#     plt.title('Annotations')\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:24:58.056848Z","iopub.execute_input":"2023-07-29T07:24:58.057418Z","iopub.status.idle":"2023-07-29T07:24:58.081937Z","shell.execute_reply.started":"2023-07-29T07:24:58.057341Z","shell.execute_reply":"2023-07-29T07:24:58.079939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def sliding_window(image, kernel_height, y_stride):\n#     y_start, y_end = 0, kernel_height\n#     isheight = 1\n#     while isheight:\n#         yield image[y_start:y_end], y_start, y_end\n#         if y_end == image.shape[0]:\n#             isheight = 0\n#         y_start += y_stride\n#         y_end = min(y_start+kernel_height, image.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:24:58.084791Z","iopub.execute_input":"2023-07-29T07:24:58.085397Z","iopub.status.idle":"2023-07-29T07:24:58.106549Z","shell.execute_reply.started":"2023-07-29T07:24:58.085321Z","shell.execute_reply":"2023-07-29T07:24:58.104967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def mask_to_yolo(Image, s, e, annotation):\n#     ht, wt = Image.shape[:2]\n#     yolo_label = ''\n#     for ann in annotation:\n#         x1, y1, w, h = ann['bbox']\n#         x2, y2 = x1+w, y1+h\n#         if y2 <= s:\n#             pass\n#         elif y1 >= e:\n#             pass\n#         else:\n#             y_cor = []\n#             x_cor = [x/wt for x in ann['segmentation'][0][0::2]]\n#             for y in ann['segmentation'][0][1::2]:\n#                 if y < s:\n#                     y_cor.append(0)\n#                 elif y > e:\n#                     y_cor.append(ht/ht)\n#                 else:\n#                     y_cor.append((y-s)/ht)\n#             points = list(zip(x_cor, y_cor))\n#             yolo_label += str(ann['category_id']) + ' ' +''.join([f\"{x:.6f} {y:.6f} \" for x, y in points]) + '\\n'\n#     return yolo_label","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:24:58.108104Z","iopub.execute_input":"2023-07-29T07:24:58.108962Z","iopub.status.idle":"2023-07-29T07:24:58.122439Z","shell.execute_reply.started":"2023-07-29T07:24:58.108924Z","shell.execute_reply":"2023-07-29T07:24:58.121349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_mask_from_yolo(label_file, ht, wt):\n#     with open(label_file, 'r') as file:\n#         content = file.readlines()\n    \n#     final_mask = {0:np.zeros((ht, wt), dtype=np.uint8), \n#                   1:np.zeros((ht, wt), dtype=np.uint8), \n#                   2:np.zeros((ht, wt), dtype=np.uint8),\n#                   3:np.zeros((ht, wt), dtype=np.uint8)}\n    \n#     for line in content:\n#         data = line.split()\n#         cat = int(data[0])\n#         x_cor = [float(x)*wt for x in data[1::2]]\n#         y_cor = [float(y)*ht for y in data[2::2]]\n#         points = np.array(list(zip(x_cor, y_cor)), dtype=np.int32)\n#         final_mask[cat] = cv2.fillPoly(final_mask[cat], pts=[points], color=1)\n        \n#     return final_mask","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:24:58.124205Z","iopub.execute_input":"2023-07-29T07:24:58.125634Z","iopub.status.idle":"2023-07-29T07:24:58.143576Z","shell.execute_reply.started":"2023-07-29T07:24:58.125575Z","shell.execute_reply":"2023-07-29T07:24:58.142323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import cv2\n# import numpy as np\n# import matplotlib.pyplot as plt\n# from pycocotools.coco import COCO\n# from tqdm.notebook import tqdm\n# from sklearn.model_selection import train_test_split\n# from sklearn.model_selection import StratifiedGroupKFold\n# sgkf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pycocotools.coco import COCO\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-07-31T18:47:51.664298Z","iopub.execute_input":"2023-07-31T18:47:51.664684Z","iopub.status.idle":"2023-07-31T18:47:52.470285Z","shell.execute_reply.started":"2023-07-31T18:47:51.664641Z","shell.execute_reply":"2023-07-31T18:47:52.469187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coco = COCO(\"../input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ann_ids = coco.getAnnIds()\nanns = coco.loadAnns(ann_ids)\n\ncat_ids = [ann[\"category_id\"] for ann in anns]\nimg_ids = [ann[\"image_id\"] for ann in anns]\n\nann_ids = pd.Series(ann_ids)\ncat_ids = pd.Series(cat_ids)\nimg_ids = pd.Series(img_ids)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nFOLDS = 5\nSEED = 3000\n\nsgkf = StratifiedGroupKFold(n_splits=FOLDS, shuffle=True, random_state=3000)\n\ncounts = cat_ids.value_counts()\n\nprint(f\"Number of images: {len(img_ids.unique())}\")\n\nfor cls, count in zip(counts.index, counts):\n    print(f\"Number of instances of class {cls}: {count}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = []\nnumber_of_images = []\nparagraph = []\ntext_box = []\nimage = []\ntable = []\n\nfor fold, (_, val_idx) in enumerate(sgkf.split(ann_ids, cat_ids, img_ids)):\n    folds.append(fold)\n    val_ann_ids = ann_ids[val_idx]\n    val_cat_ids = cat_ids[val_idx]\n    val_img_ids = set(img_ids[val_idx])\n    \n    os.makedirs(f\"val_{fold}/images\")\n    os.makedirs(f\"val_{fold}/labels\")\n    for img in tqdm(coco.loadImgs(val_img_ids)):\n        img_src = \"/kaggle/input/dlsprint2/badlad/images/train/\" + img[\"file_name\"]\n        img_dst = f\"val_{fold}/images/\" + img[\"file_name\"]\n        os.symlink(img_src, img_dst)\n\n        label_src = \"/kaggle/input/dlsprint2/badlad/labels/yolov8_format/train/\" + img[\"file_name\"][:-4] + \".txt\"\n        label_dst = f\"val_{fold}/labels/\" + img[\"file_name\"][:-4] + \".txt\"\n        os.symlink(label_src, label_dst)\n    \n    number_of_images.append(len(val_img_ids))\n  \n    paragraph.append(sum(val_cat_ids == 0))\n    text_box.append(sum(val_cat_ids == 1))\n    image.append(sum(val_cat_ids == 2))\n    table.append(sum(val_cat_ids == 3))\n\ndf = pd.DataFrame({\n    \"Fold\": folds,\n    \"Number of Images\": number_of_images,\n    \"Paragraph\": paragraph,\n    \"text_box\": text_box,\n    \"image\": image,\n    \"table\": table,\n})\n\ndf.set_index(\"Fold\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\n\nwandb.init(mode=\"disabled\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_image_path = '/kaggle/input/dlsprint2/badlad/images/train'\n# train_label_path = '/kaggle/input/dlsprint2/badlad/labels/yolov8_format/train'\n# json_file = '/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json'\n# img_path = '/kaggle/working/datasets/train/images'\n# lbl_path = '/kaggle/working/datasets/train/labels'\n# train_coco = COCO(json_file)","metadata":{"execution":{"iopub.status.busy":"2023-07-31T18:47:55.000927Z","iopub.execute_input":"2023-07-31T18:47:55.001642Z","iopub.status.idle":"2023-07-31T18:48:02.083603Z","shell.execute_reply.started":"2023-07-31T18:47:55.001607Z","shell.execute_reply":"2023-07-31T18:48:02.082345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile badlad.yaml\npath: /kaggle/working/\ntrain:\n  - val_0/\n  - val_1/\n  - val_2/\n  - val_3/\nval: val_4/\n\nnames:\n  0: paragraph\n  1: text_box\n  2: image\n  3: table","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n# training_data = \"/kaggle/working/yolov8.yaml\"\nprint('training starts')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ultralytics import YOLO\n\n\nmodel = YOLO(\"/kaggle/input/yolo-runs4/best.pt\")\n\n# model.train(data=\"/kaggle/working/yolov8.yaml\", epochs=3)\n\nresults = model.train( \n        batch=8,\n        device= device,\n        data=\"/kaggle/working/badlad.yaml\",\n        save=True,\n        save_period=2,\n        epochs=4,\n        imgsz=640,\n    \n#         augmentation\n        translate=0.4,\n        scale=0.3,\n        fliplr=0.3,\n        mosaic=0.5,\n    \n#         optimizer\n        lr0=0.018,\n        patience=3,\n        optimizer=\"SGD\",\n        momentum=0.947,\n        weight_decay=0.0005\n    )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ann_id = train_coco.getAnnIds(catIds=[1])\n\n# annot = train_coco.loadAnns(ann_id)\n# cat_id = [ann['category_id'] for ann in annot]\n# img_id = [ann['image_id'] for ann in annot]\n\n# ann_id = pd.Series(ann_id)\n# cat_id = pd.Series(cat_id)\n# img_id = pd.Series(img_id)\n\n# data_fold = []\n# for fold, (_, val_index) in enumerate(sgkf.split(ann_id, cat_id, img_id)):\n#     val_cat_id = cat_id[val_index]\n#     val_ann_id = ann_id[val_index]\n#     val_img_id = img_id[val_index]\n#     Fold = [fold]*len(val_img_id)\n#     data_fold.append(pd.DataFrame({'fold':Fold,'ann_id':val_ann_id, 'cat_id':val_cat_id, 'img_id':val_img_id}, index=None))\n    \n# final_data = pd.concat([df for df in data_fold], axis=0, ignore_index=True)\n# final_data","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:08.394755Z","iopub.execute_input":"2023-07-29T07:25:08.395116Z","iopub.status.idle":"2023-07-29T07:25:08.401204Z","shell.execute_reply.started":"2023-07-29T07:25:08.395083Z","shell.execute_reply":"2023-07-29T07:25:08.399761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-31T18:48:13.467543Z","iopub.execute_input":"2023-07-31T18:48:13.467912Z","iopub.status.idle":"2023-07-31T18:48:13.497585Z","shell.execute_reply.started":"2023-07-31T18:48:13.467880Z","shell.execute_reply":"2023-07-31T18:48:13.496583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:08.447792Z","iopub.execute_input":"2023-07-29T07:25:08.448899Z","iopub.status.idle":"2023-07-29T07:25:14.096373Z","shell.execute_reply.started":"2023-07-29T07:25:08.448855Z","shell.execute_reply":"2023-07-29T07:25:14.094841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for Id in tqdm(val_img):\n# # ids = train_coco.getImgIds(imgIds=[20])\n#     image_dict = train_coco.loadImgs(ids=Id)\n#     annot = train_coco.loadAnns(train_coco.getAnnIds(imgIds=Id))\n#     image = cv2.imread(os.path.join(train_image_path, image_dict[0]['file_name']))\n#     height, width = image.shape[0], image.shape[1]\n\n#     k_ht = height//3\n#     stride = k_ht - int(k_ht*0.3)\n#     count = 1\n\n#     for slc, start, end in sliding_window(image, k_ht, stride):\n#         f_name, ext = image_dict[0]['file_name'].split('.')\n#         label_file = os.path.join(lbl_path, f_name+str(count)+'.txt')\n#         image_file = os.path.join(img_path, f_name+str(count)+'.'+ext)\n#         count += 1\n#         line = mask_to_yolo(slc, start, end, annot)\n#         if len(line) > 0:  \n#             with open(label_file, \"w\") as f:\n#                 f.write(line)\n#             cv2.imwrite(image_file, slc)\n\n# for Id in tqdm(train_img[0:2000]):\n#     image_dict = train_coco.loadImgs(ids=Id)\n    \n#     src_img_file = os.path.join(train_image_path, image_dict[0]['file_name'])\n#     tar_img_file = os.path.join('/kaggle/working/datasets/validation/images', image_dict[0]['file_name'])\n    \n#     label_file = image_dict[0]['file_name'].split('.')[0]+'.txt'\n    \n#     src_lbl_file = os.path.join(train_label_path, label_file)\n#     tar_lbl_file = os.path.join('/kaggle/working/datasets/validation/labels', label_file)\n    \n#     os.symlink(src_img_file, tar_img_file)\n#     os.symlink(src_lbl_file, tar_lbl_file)\n\n# for Id in tqdm(val_img):\n#     image_dict = train_coco.loadImgs(ids=Id)\n    \n#     src_img_file = os.path.join(train_image_path, image_dict[0]['file_name'])\n#     tar_img_file = os.path.join('/kaggle/working/datasets/train/images', image_dict[0]['file_name'])\n    \n#     label_file = image_dict[0]['file_name'].split('.')[0]+'.txt'\n    \n#     src_lbl_file = os.path.join(train_label_path, label_file)\n#     tar_lbl_file = os.path.join('/kaggle/working/datasets/train/labels', label_file)\n    \n#     os.symlink(src_img_file, tar_img_file)\n#     os.symlink(src_lbl_file, tar_lbl_file)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:14.098134Z","iopub.execute_input":"2023-07-29T07:25:14.099416Z","iopub.status.idle":"2023-07-29T07:25:14.128467Z","shell.execute_reply.started":"2023-07-29T07:25:14.099340Z","shell.execute_reply":"2023-07-29T07:25:14.127097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:14.130695Z","iopub.execute_input":"2023-07-29T07:25:14.131238Z","iopub.status.idle":"2023-07-29T07:25:14.166145Z","shell.execute_reply.started":"2023-07-29T07:25:14.131190Z","shell.execute_reply":"2023-07-29T07:25:14.164589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# file_content = \"\"\"\n# path: /kaggle/working/datasets/\n# train: train/\n# val: validation/\n\n# names:\n#   0: paragraph\n#   1: text_box\n#   2: image\n#   3: table\n\n\n# \"\"\"\n\n# with open(\"yolov8.yaml\", mode=\"w\") as f:\n#     f.write(file_content)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:14.168388Z","iopub.execute_input":"2023-07-29T07:25:14.168785Z","iopub.status.idle":"2023-07-29T07:25:14.175432Z","shell.execute_reply.started":"2023-07-29T07:25:14.168751Z","shell.execute_reply":"2023-07-29T07:25:14.174124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import wandb\n# wandb.init(mode=\"disabled\")\n\n# import yaml\n# from yaml.loader import SafeLoader\n\n# with open('yolov8.yaml') as f:\n#     data = yaml.load(f, Loader=SafeLoader)\n#     print(data)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:14.176903Z","iopub.execute_input":"2023-07-29T07:25:14.177225Z","iopub.status.idle":"2023-07-29T07:25:16.015414Z","shell.execute_reply.started":"2023-07-29T07:25:14.177196Z","shell.execute_reply":"2023-07-29T07:25:16.013772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:16.018122Z","iopub.execute_input":"2023-07-29T07:25:16.019186Z","iopub.status.idle":"2023-07-29T07:25:19.237214Z","shell.execute_reply.started":"2023-07-29T07:25:16.019131Z","shell.execute_reply":"2023-07-29T07:25:19.234193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-29T07:25:19.240091Z","iopub.execute_input":"2023-07-29T07:25:19.241165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_image = cv2.imread('/kaggle/working/datasets/train/images/5059a52e-6c02-4c74-9cac-91022acebb182.png')\n# h, w = n_image.shape[:2]\n# masks = create_mask_from_yolo('/kaggle/working/datasets/train/labels/5059a52e-6c02-4c74-9cac-91022acebb182.txt', h, w)\n\n# plt.figure(figsize=(8,5))\n# plt.subplot(2,3,1)\n# plt.imshow(n_image)\n# plt.title('original_image')\n\n# plt.subplot(2,3,2)\n# plt.imshow(masks[0])\n# plt.title('Paragraph')\n\n# plt.subplot(2,3,3)\n# plt.imshow(masks[1])\n# plt.title('text_box')\n\n# plt.subplot(2,3,4)\n# plt.imshow(masks[2])\n# plt.title('image')\n\n# plt.subplot(2,3,5)\n# plt.imshow(masks[3])\n# plt.title('table')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize_image(20, train_coco, path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# f_name = image_dict[0]['file_name']\n# f_name = f_name.replace('png', 'txt')\n# f_name = os.path.join('/kaggle/input/dlsprint2/badlad/labels/yolov8_format/train', f_name)\n# with open(f_name, 'r') as f:\n#     data = f.readlines()\n\n# for d in data:\n#     num = d.split()\n#     cat = int(num[0])\n#     pts = [float(x)*image_dict[0]['width'] if idx%2==0 else float(x)*image_dict[0]['height'] for idx, x in enumerate(num[1:]) ]\n# #     print(pts)\n    \n# m = create_mask_from_yolo(f_name, image_dict[0]['height'], image_dict[0]['width'])\n# plt.imshow(m[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}