{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31703,"databundleVersionId":2871752,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"There are three video folders, video_0 and video_1 is for training purpose and video_2 is for test.csv. We have to output a file similar to example_sample_submission.csv","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nprint(\"ok\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-16T18:38:18.861061Z","iopub.execute_input":"2024-11-16T18:38:18.861754Z","iopub.status.idle":"2024-11-16T18:38:19.217358Z","shell.execute_reply.started":"2024-11-16T18:38:18.861701Z","shell.execute_reply":"2024-11-16T18:38:19.216428Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T22:39:37.491755Z","iopub.execute_input":"2024-11-14T22:39:37.492689Z","iopub.status.idle":"2024-11-14T22:39:45.708289Z","shell.execute_reply.started":"2024-11-14T22:39:37.492644Z","shell.execute_reply":"2024-11-14T22:39:45.70694Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = [{'x': 559, 'y': 213, 'width': 50, 'height': 32}]\nprint(len(x))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Just to see what the file contains","metadata":{}},{"cell_type":"code","source":"train_file = open(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\").read()\ntrain_file","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T02:37:56.882797Z","iopub.execute_input":"2024-11-18T02:37:56.883707Z","iopub.status.idle":"2024-11-18T02:37:56.912659Z","shell.execute_reply.started":"2024-11-18T02:37:56.883666Z","shell.execute_reply":"2024-11-18T02:37:56.911793Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Reading the training data file","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-11-18T02:38:06.769789Z","iopub.execute_input":"2024-11-18T02:38:06.770511Z","iopub.status.idle":"2024-11-18T02:38:06.819547Z","shell.execute_reply.started":"2024-11-18T02:38:06.770475Z","shell.execute_reply":"2024-11-18T02:38:06.818634Z"},"trusted":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"It may seems like that annotations is empty but there are fields below which have values as shown below. Empty annotation means that there is no coral reefs in it.","metadata":{}},{"cell_type":"markdown","source":"Example of image where annotation is not empty, this means that the image has a coral reefs in it.","metadata":{}},{"cell_type":"code","source":"train.iloc[17]","metadata":{"execution":{"iopub.status.busy":"2024-11-18T02:38:27.287243Z","iopub.execute_input":"2024-11-18T02:38:27.287633Z","iopub.status.idle":"2024-11-18T02:38:27.296129Z","shell.execute_reply.started":"2024-11-18T02:38:27.287598Z","shell.execute_reply":"2024-11-18T02:38:27.294801Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/test.csv\")\ntest","metadata":{"execution":{"iopub.status.busy":"2024-11-18T02:38:30.534698Z","iopub.execute_input":"2024-11-18T02:38:30.535443Z","iopub.status.idle":"2024-11-18T02:38:30.553048Z","shell.execute_reply.started":"2024-11-18T02:38:30.5354Z","shell.execute_reply":"2024-11-18T02:38:30.552227Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Loading test examples, that we have to execute on our system.","metadata":{}},{"cell_type":"code","source":"example_test = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/example_sample_submission.csv\")\nexample_test","metadata":{"execution":{"iopub.status.busy":"2024-11-10T23:29:52.816381Z","iopub.execute_input":"2024-11-10T23:29:52.817284Z","iopub.status.idle":"2024-11-10T23:29:52.829987Z","shell.execute_reply.started":"2024-11-10T23:29:52.817223Z","shell.execute_reply":"2024-11-10T23:29:52.82916Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T18:32:40.145891Z","iopub.execute_input":"2024-11-16T18:32:40.146702Z","iopub.status.idle":"2024-11-16T18:32:41.224116Z","shell.execute_reply.started":"2024-11-16T18:32:40.146647Z","shell.execute_reply":"2024-11-16T18:32:41.223175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install ultralytics\n","metadata":{"trusted":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install opencv-python","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:32:16.254181Z","iopub.execute_input":"2024-11-16T20:32:16.254607Z","iopub.status.idle":"2024-11-16T20:32:27.766948Z","shell.execute_reply.started":"2024-11-16T20:32:16.254553Z","shell.execute_reply":"2024-11-16T20:32:27.765741Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install --upgrade ultralytics","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:32:27.768645Z","iopub.execute_input":"2024-11-16T20:32:27.768997Z","iopub.status.idle":"2024-11-16T20:32:39.789457Z","shell.execute_reply.started":"2024-11-16T20:32:27.768955Z","shell.execute_reply":"2024-11-16T20:32:39.788368Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport PIL \nfrom PIL import Image\nfrom IPython.display import display\nimport matplotlib.pyplot as plt\nimport os \nimport pathlib ","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:32:39.791429Z","iopub.execute_input":"2024-11-16T20:32:39.791755Z","iopub.status.idle":"2024-11-16T20:32:44.87605Z","shell.execute_reply.started":"2024-11-16T20:32:39.79172Z","shell.execute_reply":"2024-11-16T20:32:44.875275Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Sample run before traning","metadata":{}},{"cell_type":"code","source":"model = YOLO(\"yolov8m.pt\") # medium size I have chosen","metadata":{"execution":{"iopub.status.busy":"2024-11-11T01:59:46.857771Z","iopub.execute_input":"2024-11-11T01:59:46.858078Z","iopub.status.idle":"2024-11-11T01:59:47.00256Z","shell.execute_reply.started":"2024-11-11T01:59:46.858046Z","shell.execute_reply":"2024-11-11T01:59:47.001775Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# results = model.predict()\nresults=model.predict(source=\"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_1/10925.jpg\",\n              save=True, conf=.25)\n\n# conf: object confidence threshold for detection\n#Iou: intersection over union (IoU) threshold for NMS\nresult = results[0]\n# Plotting results\nres_plotted = results[0].plot()\nres_plotted = cv2.cvtColor(res_plotted, cv2.COLOR_BGR2RGB)\ndisplay(Image.fromarray(res_plotted))","metadata":{"execution":{"iopub.status.busy":"2024-11-18T02:43:06.908231Z","iopub.execute_input":"2024-11-18T02:43:06.908921Z","iopub.status.idle":"2024-11-18T02:43:07.153904Z","shell.execute_reply.started":"2024-11-18T02:43:06.908881Z","shell.execute_reply":"2024-11-18T02:43:07.152747Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**As expected nothing was detected, as we need to train the model on our data before model could detect correct data.**","metadata":{}},{"cell_type":"code","source":"len(result.boxes)","metadata":{"execution":{"iopub.status.busy":"2024-11-18T02:39:13.331769Z","iopub.execute_input":"2024-11-18T02:39:13.332376Z","iopub.status.idle":"2024-11-18T02:39:13.338126Z","shell.execute_reply.started":"2024-11-18T02:39:13.332337Z","shell.execute_reply":"2024-11-18T02:39:13.337241Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**This should give error as no object was detected**, we will run this example again after the traning of model to visualize the difference.","metadata":{}},{"cell_type":"code","source":"box = result.boxes[0]","metadata":{"execution":{"iopub.status.busy":"2024-10-30T02:06:46.053811Z","iopub.execute_input":"2024-10-30T02:06:46.054167Z","iopub.status.idle":"2024-10-30T02:06:46.36021Z","shell.execute_reply.started":"2024-10-30T02:06:46.054134Z","shell.execute_reply":"2024-10-30T02:06:46.358817Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for result in results:\n    boxes = result.boxes  # Boxes object for bbox outputs\n    masks = result.masks  # Masks object for segmentation masks outputs\n    probs = result.probs  # Class probabilities for classification outputs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T22:25:28.617938Z","iopub.execute_input":"2024-11-10T22:25:28.618669Z","iopub.status.idle":"2024-11-10T22:25:28.623523Z","shell.execute_reply.started":"2024-11-10T22:25:28.618626Z","shell.execute_reply":"2024-11-10T22:25:28.62235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxes","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# initialization","metadata":{}},{"cell_type":"code","source":"!pip install opencv-python\n!pip install ultralytics\n!pip install --upgrade ultralytics\n!pip install --upgrade ultralytics ray","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:27:05.825704Z","iopub.execute_input":"2024-11-17T20:27:05.826068Z","iopub.status.idle":"2024-11-17T20:28:04.409508Z","shell.execute_reply.started":"2024-11-17T20:27:05.826032Z","shell.execute_reply":"2024-11-17T20:28:04.408559Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport PIL \nfrom PIL import Image\nfrom IPython.display import display\nimport matplotlib.pyplot as plt\nimport os \nimport pathlib ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:28:06.629011Z","iopub.execute_input":"2024-11-17T20:28:06.629792Z","iopub.status.idle":"2024-11-17T20:28:10.918755Z","shell.execute_reply.started":"2024-11-17T20:28:06.629749Z","shell.execute_reply":"2024-11-17T20:28:10.91792Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Preparing Data**","metadata":{}},{"cell_type":"markdown","source":"dataset/\n├── images/\n│   ├── train/\n│   │   ├── image1.jpg\n│   │   ├── image2.jpg\n│   │   └── ...\n│   └── val/\n│       ├── image101.jpg\n│       ├── image102.jpg\n│       └── ...\n└── labels/\n    ├── train/\n    │   ├── image1.txt\n    │   ├── image2.txt\n    │   └── ...\n    └── val/\n        ├── image101.txt\n        ├── image102.txt\n        └── ...\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# Load the CSV\ntrain = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\n\n# Specify directories\nimages_dir = \"/kaggle/working/images/train/\"\nlabels_dir = \"/kaggle/working/labels/train/\"\nos.makedirs(images_dir, exist_ok=True)\nos.makedirs(labels_dir, exist_ok=True)\nprint(\"ok\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:28:16.133842Z","iopub.execute_input":"2024-11-17T20:28:16.134432Z","iopub.status.idle":"2024-11-17T20:28:16.198575Z","shell.execute_reply.started":"2024-11-17T20:28:16.134384Z","shell.execute_reply":"2024-11-17T20:28:16.197685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define function to convert bounding box format\ndef convert_to_yolo_format(row, image_width, image_height):\n    annotations = eval(row['annotations'])  # Convert string representation to list\n    yolo_annotations = []\n    \n    for annotation in annotations:\n        x = annotation['x']\n        y = annotation['y']\n        width = annotation['width']\n        height = annotation['height']\n        \n        # Calculate YOLO format\n        x_center = (x + width / 2) / image_width\n        y_center = (y + height / 2) / image_height\n        width /= image_width\n        height /= image_height\n        \n        yolo_annotations.append(f\"0 {x_center} {y_center} {width} {height}\")\n    \n    return yolo_annotations\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:28:18.218276Z","iopub.execute_input":"2024-11-17T20:28:18.218658Z","iopub.status.idle":"2024-11-17T20:28:18.225711Z","shell.execute_reply.started":"2024-11-17T20:28:18.218624Z","shell.execute_reply":"2024-11-17T20:28:18.224631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Process each row to create label files\nfor _, row in train.iterrows():\n    image_id = row['image_id']\n    annotations = row['annotations']\n    # Check if there are annotations\n    if annotations == \"[]\":\n        continue  # Skip images with no annotations\n\n    # Set image dimensions (adjust this if known)\n    image_width = 1280\n    image_height = 720\n    \n    # Convert annotations to YOLO format\n    yolo_annotations = convert_to_yolo_format(row, image_width, image_height)\n    # Write YOLO annotations to label file\n    label_path = os.path.join(labels_dir, f\"{image_id}.txt\")\n    with open(label_path, 'w') as f:\n        f.write(\"\\n\".join(yolo_annotations))\n\nprint(\"ok\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:28:20.092333Z","iopub.execute_input":"2024-11-17T20:28:20.093094Z","iopub.status.idle":"2024-11-17T20:28:22.361505Z","shell.execute_reply.started":"2024-11-17T20:28:20.093055Z","shell.execute_reply":"2024-11-17T20:28:22.360571Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Using Symlinks","metadata":{}},{"cell_type":"code","source":"import os\nfrom glob import glob\n\n# Define source directories and target directory\nsource_dir = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/\"\ntrain_images_dir = \"/kaggle/working/images/train/\"\nos.makedirs(train_images_dir, exist_ok=True)\n\n# Create symlinks for each image from video folders to the train directory\nfor video_folder in [\"video_0\", \"video_1\"]:\n    # Extract the folder ID (e.g., \"0\" from \"video_0\")\n    folder_id = video_folder.split('_')[1]\n    video_path = os.path.join(source_dir, video_folder)\n    images = glob(os.path.join(video_path, \"*.jpg\"))\n    \n    for img_path in images:\n        img_name = os.path.basename(img_path)\n        # Create the new symlink name as folder_id + \"-\" + img_name\n        new_img_name = f\"{folder_id}-{img_name}\"\n        symlink_path = os.path.join(train_images_dir, new_img_name)\n        \n        # Create a symlink instead of copying\n        if not os.path.exists(symlink_path):  # Avoid overwriting if it already exists\n            os.symlink(img_path, symlink_path)\nprint(\"ok\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:28:24.427853Z","iopub.execute_input":"2024-11-17T20:28:24.428259Z","iopub.status.idle":"2024-11-17T20:28:26.2033Z","shell.execute_reply.started":"2024-11-17T20:28:24.428213Z","shell.execute_reply":"2024-11-17T20:28:26.202125Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import yaml\n\n# Define the configuration as a dictionary\ndata_config = {\n    'path': '/kaggle/working/',\n    'train': 'images/train',\n    'val': 'images/train',  # Ensure this is correct if you have validation images\n    'names': ['coral']  # List of class names, replace 'coral' with your actual class names if needed\n}\n\n# Write the dictionary to a YAML file\nwith open('/kaggle/working/data.yaml', 'w') as file:\n    yaml.dump(data_config, file)\n\nprint(\"data.yaml created successfully at /kaggle/working/data.yaml\")\n\n# Verify the contents of data.yaml\nwith open('/kaggle/working/data.yaml', 'r') as file:\n    data_yaml_content = file.read()\n    print(data_yaml_content)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:28:30.718374Z","iopub.execute_input":"2024-11-17T20:28:30.718792Z","iopub.status.idle":"2024-11-17T20:28:30.727514Z","shell.execute_reply.started":"2024-11-17T20:28:30.718755Z","shell.execute_reply":"2024-11-17T20:28:30.726501Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Checking Total Number Of Label Files**","metadata":{}},{"cell_type":"code","source":"import os\n\n# Path to the labels directory\nlabels_dir = \"/kaggle/working/labels/train\"\n\n# Check if any label files exist and print a few examples\nlabel_files = os.listdir(labels_dir)\nprint(f\"Number of label files: {len(label_files)}\")\n\n# Print the contents of a sample label file, if it exists\nif label_files:\n    sample_label_path = os.path.join(labels_dir, label_files[0])\n    with open(sample_label_path, 'r') as f:\n        print(f\"Contents of {sample_label_path}:\\n\", f.read())\nelse:\n    print(\"No label files found.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T07:31:30.756825Z","iopub.execute_input":"2024-11-17T07:31:30.757687Z","iopub.status.idle":"2024-11-17T07:31:30.768547Z","shell.execute_reply.started":"2024-11-17T07:31:30.757646Z","shell.execute_reply":"2024-11-17T07:31:30.76737Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Training begins**","metadata":{}},{"cell_type":"markdown","source":"# YOLO MEDIUM","metadata":{}},{"cell_type":"markdown","source":"Printing the models","metadata":{}},{"cell_type":"code","source":"print(\"*************************small***********************\")\ns_model = YOLO('yolov8s.pt')\nprint(s_model.model)\n\nprint(\"*************************medium***********************\")\nm_model = YOLO('yolov8m.pt')\nprint(m_model.model)\n\nprint(\"*************************large***********************\")\nl_model = YOLO('yolov8l.pt')\nprint(l_model.model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T07:36:58.087658Z","iopub.execute_input":"2024-11-17T07:36:58.088537Z","iopub.status.idle":"2024-11-17T07:37:01.97455Z","shell.execute_reply.started":"2024-11-17T07:36:58.088496Z","shell.execute_reply":"2024-11-17T07:37:01.9736Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8s.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(data=\"/kaggle/working/data.yaml\", seed=0, epochs = 5, batch = -1, optimizer = 'auto')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T18:40:34.142355Z","iopub.execute_input":"2024-11-16T18:40:34.143315Z","iopub.status.idle":"2024-11-16T19:17:24.124143Z","shell.execute_reply.started":"2024-11-16T18:40:34.14327Z","shell.execute_reply":"2024-11-16T19:17:24.123218Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(data=\"/kaggle/working/data.yaml\", seed=0, epochs = 5, batch = -1, optimizer = 'auto')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T22:23:55.803611Z","iopub.status.idle":"2024-11-16T22:23:55.80395Z","shell.execute_reply.started":"2024-11-16T22:23:55.803779Z","shell.execute_reply":"2024-11-16T22:23:55.803796Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# YOLO Large","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8l.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(data=\"/kaggle/working/data.yaml\", seed=0, epochs = 5, batch = -1, optimizer = 'auto')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**YOLO Medium is giving better performance out of the box for some reason.**","metadata":{}},{"cell_type":"markdown","source":"S,M,L WITH SPECIFIC PARAMETERS","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nS_model = YOLO('yolov8s.pt')\n\n# Training The Final Model\nS_model = S_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # Use AdamW for improved convergence\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Proceeding with yolo medium","metadata":{}},{"cell_type":"markdown","source":"# Base Case Medium Model going forward","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # Use AdamW for improved convergence\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-12T23:01:43.64727Z","iopub.execute_input":"2024-11-12T23:01:43.647821Z","iopub.status.idle":"2024-11-12T23:54:16.709225Z","shell.execute_reply.started":"2024-11-12T23:01:43.64778Z","shell.execute_reply":"2024-11-12T23:54:16.708056Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# SGD","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # CHANGED\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T22:50:16.996565Z","iopub.execute_input":"2024-11-14T22:50:16.996966Z","iopub.status.idle":"2024-11-14T23:43:09.772517Z","shell.execute_reply.started":"2024-11-14T22:50:16.996926Z","shell.execute_reply":"2024-11-14T23:43:09.771379Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Base case SGD going forward","metadata":{}},{"cell_type":"markdown","source":"removed warmup ephocs","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    # removed warmup epochs\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T00:06:06.083768Z","iopub.execute_input":"2024-11-15T00:06:06.08423Z","iopub.status.idle":"2024-11-15T00:59:05.447237Z","shell.execute_reply.started":"2024-11-15T00:06:06.084186Z","shell.execute_reply":"2024-11-15T00:59:05.446266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Base case Augmentation","metadata":{}},{"cell_type":"code","source":"import yaml\n\n# Define the configuration as a dictionary\ndata_config = {\n    'path': '/kaggle/working/',\n    'train': 'images/train',\n    'val': 'images/train',  # Ensure this is correct if you have validation images\n    'names': ['coral'],  # List of class names, replace 'coral' with your actual class names if needed\n    'augment': {\n        'flipud': 0.5,           # Flip images vertically with a probability of 50%\n        'fliplr': 0.5,           # Flip images horizontally with a probability of 50%\n        'mosaic': 0.8,           # Use mosaic augmentation 80% of the time\n        'mixup': 0.2,            # Use mixup augmentation 20% of the time\n        'degrees': 10.0,         # Range of random rotation in degrees\n        'translate': 0.1,        # Range of random translation, as a fraction of the image size\n        'scale': 0.5,            # Range of random scaling, as a fraction of the image size\n        'shear': 2.0,            # Range of random shear in degrees\n        'perspective': 0.0001,   # Perspective change range for transforming the image\n        'hsv_h': 0.015,          # Hue adjustment range in the HSV color space\n        'hsv_s': 0.7,            # Saturation adjustment range in the HSV color space\n        'hsv_v': 0.4,            # Brightness adjustment range in the HSV color space\n        'brightness': 0.2,       # Adjust brightness by ±20%\n        'contrast': 0.2,         # Adjust contrast by ±20%\n        'blur': 0.2,             # Probability of applying a blur effect\n        'noise': 0.1,            # Probability of adding Gaussian noise\n        'sharpness': 0.3,        # Range of sharpness adjustment\n        'motion_blur': 0.1,      # Probability of applying motion blur\n        'grayscale': 0.1,        # Probability of converting the image to grayscale\n        'cutout': 0.5,           # Probability of applying cutout augmentation\n        'cutmix': 0.3,           # Probability of applying cutmix augmentation\n        'mosaic_ratio': 0.4      # Specifies how much the mosaic affects the combined images\n    }\n}\n\n# Write the dictionary to a YAML file\nwith open('/kaggle/working/data.yaml', 'w') as file:\n    yaml.dump(data_config, file)\n\nprint(\"data.yaml created successfully at /kaggle/working/data.yaml\")\n\n# Verify the contents of data.yaml\nwith open('/kaggle/working/data.yaml', 'r') as file:\n    data_yaml_content = file.read()\n    print(data_yaml_content)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T01:33:12.907884Z","iopub.execute_input":"2024-11-15T01:33:12.908614Z","iopub.status.idle":"2024-11-15T01:33:12.922044Z","shell.execute_reply.started":"2024-11-15T01:33:12.908571Z","shell.execute_reply":"2024-11-15T01:33:12.921051Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # CHANGED\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T01:33:16.705731Z","iopub.execute_input":"2024-11-15T01:33:16.706145Z","iopub.status.idle":"2024-11-15T02:26:22.636026Z","shell.execute_reply.started":"2024-11-15T01:33:16.706105Z","shell.execute_reply":"2024-11-15T02:26:22.63504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"augment=True,                       # Enable basic augmentations\n    mosaic=0.8,                         # Mosaic probability\n    mixup=0.2                           # Mixup probability","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # CHANGED\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n\n    augment=True,                       # Enable basic augmentations\n    mosaic=0.8,                         # Mosaic probability\n    mixup=0.2                           # Mixup probability\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T02:27:00.787259Z","iopub.execute_input":"2024-11-15T02:27:00.787657Z","iopub.status.idle":"2024-11-15T03:21:38.513023Z","shell.execute_reply.started":"2024-11-15T02:27:00.787619Z","shell.execute_reply":"2024-11-15T03:21:38.511893Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"More augmentations","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # CHANGED\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n\n    augment=True,                       # Enable basic augmentations\n    \n    # Augmentation parameters\n    hsv_h=0.015,                        # Hue shift (0.0 - 0.5), where 0.015 is a small shift\n    hsv_s=0.7,                          # Saturation shift (0.0 - 0.9)\n    hsv_v=0.4,                          # Brightness shift (0.0 - 0.9)\n    degrees=10.0,                       # Range of rotation in degrees (-180 to 180)\n    translate=0.1,                      # Range of translation (0.0 - 1.0), fraction of image size\n    scale=0.5,                          # Scale factor range (0.0 - 1.0)\n    shear=2.0,                          # Shear angle in degrees\n    perspective=0.0001,                 # Perspective distortion (0.0 - 0.001)\n    flipud=0.5,                         # Probability of vertical flip (0.0 - 1.0)\n    fliplr=0.5,                         # Probability of horizontal flip (0.0 - 1.0)\n    mosaic=1.0,                         # Probability of using mosaic augmentation (0.0 - 1.0)\n    mixup=0.2,                          # Probability of using mixup augmentation (0.0 - 1.0)\n    copy_paste=0.0,                     # Probability of copy-paste augmentation (0.0 - 1.0)\n    auto_augment='randaugment',         # Auto augment policy ('randaugment', 'trivialaugment', etc.)\n    erasing=0.4,                        # Probability of random erasing (cutout) augmentation\n    crop_fraction=1.0 \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T03:37:27.143366Z","iopub.execute_input":"2024-11-15T03:37:27.144174Z","iopub.status.idle":"2024-11-15T04:32:20.310794Z","shell.execute_reply.started":"2024-11-15T03:37:27.144131Z","shell.execute_reply":"2024-11-15T04:32:20.309792Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Normalization","metadata":{}},{"cell_type":"markdown","source":"It is already part of yolo v8 model","metadata":{}},{"cell_type":"markdown","source":"# Regularization","metadata":{}},{"cell_type":"markdown","source":"**DROPOUT**\nThis would require me to alter the underlying model of the yolo, and dropout works best when data is less compared to the model size\nBut still test out the droput=0.1","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    dropout=0.1,#added\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    dropout=0.01,#added\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Testing various parameters other than Augmentation, Optimizer, ephocs, batch size","metadata":{}},{"cell_type":"markdown","source":"change the Confidence value, lower or make it hight maybe\nconf=0.3","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    conf=0.3 #higher, default is .25\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T20:33:22.820519Z","iopub.execute_input":"2024-11-16T20:33:22.820938Z","iopub.status.idle":"2024-11-16T21:29:37.894808Z","shell.execute_reply.started":"2024-11-16T20:33:22.8209Z","shell.execute_reply":"2024-11-16T21:29:37.893789Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"conf=0.2","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    conf=0.2 #lower, default is .2\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T21:29:37.896707Z","iopub.execute_input":"2024-11-16T21:29:37.897057Z","iopub.status.idle":"2024-11-16T22:23:55.427779Z","shell.execute_reply.started":"2024-11-16T21:29:37.897023Z","shell.execute_reply":"2024-11-16T22:23:55.426767Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Learning Rates","metadata":{}},{"cell_type":"markdown","source":"lr0=0.005","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    conf=0.25               #default is .2\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T23:25:42.231482Z","iopub.execute_input":"2024-11-16T23:25:42.231819Z","iopub.status.idle":"2024-11-17T00:20:27.243269Z","shell.execute_reply.started":"2024-11-16T23:25:42.231784Z","shell.execute_reply":"2024-11-17T00:20:27.242115Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"lr0=0.01","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.01, # Higher, earlier 0.002 then it was .005\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.0005,     # Regularization\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    conf=0.25               #default is .2\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T01:41:07.280613Z","iopub.execute_input":"2024-11-17T01:41:07.280957Z","iopub.status.idle":"2024-11-17T02:35:49.237652Z","shell.execute_reply.started":"2024-11-17T01:41:07.280922Z","shell.execute_reply":"2024-11-17T02:35:49.23672Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Weight Decay","metadata":{}},{"cell_type":"markdown","source":"weight_decay=0.001","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.001, # Increased, initial 0.0005\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T22:30:42.357249Z","iopub.execute_input":"2024-11-16T22:30:42.358129Z","iopub.status.idle":"2024-11-16T23:25:42.229483Z","shell.execute_reply.started":"2024-11-16T22:30:42.358088Z","shell.execute_reply":"2024-11-16T23:25:42.228382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"weight_decay=0.01,","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.01, # Increased further, initial 0.0005 then 0.001\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T00:46:06.781188Z","iopub.execute_input":"2024-11-17T00:46:06.781588Z","iopub.status.idle":"2024-11-17T01:41:07.278486Z","shell.execute_reply.started":"2024-11-17T00:46:06.781549Z","shell.execute_reply":"2024-11-17T01:41:07.277363Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"weight_decay=0.05","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # SGD\n    lr0=0.002,               # Slightly higher initial learning rate than the large model\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.05, # Increased further, initial 0.0005 then 0.001 then .01\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True              # Cosine learning rate decay for smoother optimization\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T02:35:49.239483Z","iopub.execute_input":"2024-11-17T02:35:49.239815Z","iopub.status.idle":"2024-11-17T03:30:48.4793Z","shell.execute_reply.started":"2024-11-17T02:35:49.239782Z","shell.execute_reply":"2024-11-17T03:30:48.478374Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Learning rate schedules\n\nWhile Cosine Decay is the default and works well in YOLOv8, other schedules may be better suited for specific datasets, tasks, or training requirements. For example:\n\nUse OneCycleLR or Warmup for large models or when initializing from scratch.\nUse Step Decay for tasks with well-defined training phases.\nUse Plateau Decay for models that train on highly variable datasets.","metadata":{}},{"cell_type":"markdown","source":"# RANDOMIZE THE IMAGES USED WITHIN A BATCH, BECAUSE IMAGES ARE RELATED TO ONE OTHER AND MAY IMPACT THE GRADIENT CALCULATIONS.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nfrom PIL import Image\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torch.optim import SGD\nfrom torchvision import transforms\nfrom tqdm import tqdm\nfrom ultralytics import YOLO\n\n# Define the custom Dataset\nclass VideoFrameDataset(Dataset):\n    def __init__(self, image_dir, label_dir, transform=None):\n        self.image_dir = image_dir\n        self.label_dir = label_dir\n        self.transform = transform\n        \n        # Organize images by video_id\n        self.data = {}\n        self.all_images = []\n        for image_file in os.listdir(image_dir):\n            video_id = image_file.split('-')[0]  # Extract video_id from filename\n            if video_id not in self.data:\n                self.data[video_id] = []\n            self.data[video_id].append(image_file)\n        \n        # Flatten the dataset for indexing\n        for video_id, images in self.data.items():\n            for image in images:\n                self.all_images.append((video_id, image))\n\n    def __len__(self):\n        return len(self.all_images)\n\n    def __getitem__(self, idx):\n        video_id, image_file = self.all_images[idx]\n        image_path = os.path.join(self.image_dir, image_file)\n        label_path = os.path.join(self.label_dir, image_file.replace('.jpg', '.txt'))  # Assuming labels are in .txt format\n        \n        # Load image\n        image = Image.open(image_path).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n        \n        # Load label\n        with open(label_path, 'r') as f:\n            label = f.read()  # Modify based on your label format\n        \n        return image, label, video_id\n\n# Define the BalancedBatchSampler\nclass BalancedBatchSampler(Sampler):\n    def __init__(self, data, batch_size):\n        self.data = data\n        self.batch_size = batch_size\n        \n        # Group indices by video_id\n        self.grouped_data = {}\n        for idx, (video_id, _) in enumerate(data.all_images):\n            if video_id not in self.grouped_data:\n                self.grouped_data[video_id] = []\n            self.grouped_data[video_id].append(idx)\n        \n        # Shuffle indices within each video\n        for video_id in self.grouped_data:\n            random.shuffle(self.grouped_data[video_id])\n\n    def __iter__(self):\n        # Generate balanced batches with equal frames from each video\n        video_ids = list(self.grouped_data.keys())\n        video_pointers = {video_id: 0 for video_id in video_ids}\n        batch = []\n        \n        while True:\n            for video_id in video_ids:\n                if video_pointers[video_id] < len(self.grouped_data[video_id]):\n                    batch.append(self.grouped_data[video_id][video_pointers[video_id]])\n                    video_pointers[video_id] += 1\n                    \n                # When batch is full, yield it\n                if len(batch) == self.batch_size:\n                    yield batch\n                    batch = []\n            \n            # Stop when all videos are exhausted\n            if all(video_pointers[video_id] >= len(self.grouped_data[video_id]) for video_id in video_ids):\n                break\n\n    def __len__(self):\n        return sum(len(indices) for indices in self.grouped_data.values()) // self.batch_size\n\n# Paths for images and labels\nimage_dir = \"/kaggle/working/images/train\"\nlabel_dir = \"/kaggle/working/labels/train\"\n\n# Define data transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Adjust normalization values if needed\n])\n\n# Create the dataset\ndataset = VideoFrameDataset(image_dir=image_dir, label_dir=label_dir, transform=transform)\n\n# Create the sampler and DataLoader\nbatch_size = 32\nsampler = BalancedBatchSampler(dataset, batch_size=batch_size)\ndataloader = DataLoader(dataset, batch_size=batch_size, sampler=sampler, num_workers=4)\n\n# Load the YOLOv8 model\nmodel = YOLO('yolov8m.pt')\n\n# Move model to device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel.to(device)\n\n# Define optimizer and learning rate scheduler\noptimizer = SGD(model.parameters(), lr=0.002, momentum=0.9, weight_decay=0.0005)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=5)  # 5 epochs\n\n# Training loop\nepochs = 5\nfor epoch in range(epochs):\n    print(f\"Epoch {epoch + 1}/{epochs}\")\n    model.train()\n    running_loss = 0.0\n\n    for batch in tqdm(dataloader, desc=\"Training\"):\n        # Unpack batch\n        images, labels, video_ids = batch\n        images = images.to(device)\n\n        # Forward pass\n        preds = model(images)\n\n        # Compute YOLO loss\n        loss = model.loss(preds, labels)\n\n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        # Track loss\n        running_loss += loss.item()\n\n    # Step the scheduler\n    scheduler.step()\n\n    # Print loss for the epoch\n    print(f\"Epoch [{epoch + 1}/{epochs}] Loss: {running_loss / len(dataloader):.4f}\")\n\n# Save the trained model\nmodel.save('custom_yolov8m.pt')\n\nprint(\"Training complete. Model saved as 'custom_yolov8m.pt'.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T00:38:47.984522Z","iopub.execute_input":"2024-11-17T00:38:47.98489Z","iopub.status.idle":"2024-11-17T00:41:32.929764Z","shell.execute_reply.started":"2024-11-17T00:38:47.984856Z","shell.execute_reply":"2024-11-17T00:41:32.928007Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# FINAL VERSION\n","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=50             # Epochs to wait for no observable improvement for early stopping of training\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T04:08:28.039425Z","iopub.execute_input":"2024-11-17T04:08:28.040156Z","iopub.status.idle":"2024-11-17T05:03:22.08545Z","shell.execute_reply.started":"2024-11-17T04:08:28.040113Z","shell.execute_reply":"2024-11-17T05:03:22.084314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"sgd","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=50             # Epochs to wait for no observable improvement for early stopping of training\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T06:22:13.9096Z","iopub.execute_input":"2024-11-17T06:22:13.910037Z","iopub.status.idle":"2024-11-17T07:17:16.604582Z","shell.execute_reply.started":"2024-11-17T06:22:13.909998Z","shell.execute_reply":"2024-11-17T07:17:16.603606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"lrf=0.001","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=5,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.001,                # Learning rate factor remains the same\n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=50             # Epochs to wait for no observable improvement for early stopping of training\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T05:07:16.281798Z","iopub.execute_input":"2024-11-17T05:07:16.2822Z","iopub.status.idle":"2024-11-17T06:02:26.73695Z","shell.execute_reply.started":"2024-11-17T05:07:16.28216Z","shell.execute_reply":"2024-11-17T06:02:26.736016Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"AUGMENTATION + ADAMW","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=100,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    \n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    \n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=50,             # Epochs to wait for no observable improvement for early stopping of training\n\n    conf=0.3,\n\n    augment=True,                       # Enable basic augmentations\n    \n    # Augmentation parameters\n    hsv_h=0.015,                        # Hue shift (0.0 - 0.5), where 0.015 is a small shift\n    hsv_s=0.7,                          # Saturation shift (0.0 - 0.9)\n    hsv_v=0.4,                          # Brightness shift (0.0 - 0.9)\n    degrees=10.0,                       # Range of rotation in degrees (-180 to 180)\n    translate=0.1,                      # Range of translation (0.0 - 1.0), fraction of image size\n    scale=0.5,                          # Scale factor range (0.0 - 1.0)\n    shear=2.0,                          # Shear angle in degrees\n    perspective=0.0001,                 # Perspective distortion (0.0 - 0.001)\n    flipud=0.5,                         # Probability of vertical flip (0.0 - 1.0)\n    fliplr=0.5,                         # Probability of horizontal flip (0.0 - 1.0)\n    mosaic=1.0,                         # Probability of using mosaic augmentation (0.0 - 1.0)\n    mixup=0.2,                          # Probability of using mixup augmentation (0.0 - 1.0)\n    copy_paste=0.0,                     # Probability of copy-paste augmentation (0.0 - 1.0)\n    auto_augment='randaugment',         # Auto augment policy ('randaugment', 'trivialaugment', etc.)\n    erasing=0.4,                        # Probability of random erasing (cutout) augmentation\n    crop_fraction=1.0 \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T07:37:24.011498Z","iopub.execute_input":"2024-11-17T07:37:24.011881Z","iopub.status.idle":"2024-11-17T14:58:03.164197Z","shell.execute_reply.started":"2024-11-17T07:37:24.011845Z","shell.execute_reply":"2024-11-17T14:58:03.162604Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"NO AUGMENTATION + SGD","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=100,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='SGD',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    \n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    \n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=50,             # Epochs to wait for no observable improvement for early stopping of training\n\n    conf=0.25\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T14:59:34.837127Z","iopub.execute_input":"2024-11-17T14:59:34.837906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"No Augmentation, ADAMW","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=50,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    \n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    \n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=10,             # Epochs to wait for no observable improvement for early stopping of training\n\n    conf=0.25\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"TESTING SMALL WITH THE PARAMETERS","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8s.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=30,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # CHANGED\n    lr0=0.007, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    \n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    \n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=10,             # Epochs to wait for no observable improvement for early stopping of training\n\n    conf=0.3,\n\n    augment=True,                       # Enable basic augmentations\n    \n    # Augmentation parameters\n    hsv_h=0.015,                        # Hue shift (0.0 - 0.5), where 0.015 is a small shift\n    hsv_s=0.7,                          # Saturation shift (0.0 - 0.9)\n    hsv_v=0.4,                          # Brightness shift (0.0 - 0.9)\n    degrees=10.0,                       # Range of rotation in degrees (-180 to 180)\n    translate=0.1,                      # Range of translation (0.0 - 1.0), fraction of image size\n    scale=0.5,                          # Scale factor range (0.0 - 1.0)\n    shear=2.0,                          # Shear angle in degrees\n    perspective=0.0001,                 # Perspective distortion (0.0 - 0.001)\n    flipud=0.5,                         # Probability of vertical flip (0.0 - 1.0)\n    fliplr=0.5,                         # Probability of horizontal flip (0.0 - 1.0)\n    mosaic=1.0,                         # Probability of using mosaic augmentation (0.0 - 1.0)\n    mixup=0.2,                          # Probability of using mixup augmentation (0.0 - 1.0)\n    copy_paste=0.0,                     # Probability of copy-paste augmentation (0.0 - 1.0)\n    auto_augment='randaugment',         # Auto augment policy ('randaugment', 'trivialaugment', etc.)\n    erasing=0.4,                        # Probability of random erasing (cutout) augmentation\n    crop_fraction=1.0 \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T20:34:04.782967Z","iopub.execute_input":"2024-11-17T20:34:04.783387Z","iopub.status.idle":"2024-11-17T21:22:11.867121Z","shell.execute_reply.started":"2024-11-17T20:34:04.783348Z","shell.execute_reply":"2024-11-17T21:22:11.865458Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Augmentation + SGD","metadata":{}},{"cell_type":"code","source":"# Build from YAML and transfer weights\nFinal_model = YOLO('yolov8m.pt')\n\n# Training The Final Model\nResult_Final_model = Final_model.train(\n    data=\"/kaggle/working/data.yaml\",\n    seed=0,                   # Random seed for reproducibility\n    epochs=30,               # Medium model requires fewer epochs; start with 30 but we have patience = 50 thats why keeping it 100 as we don't need to worry\n    batch=32,                # Use a larger batch size if memory allows (medium model uses less memory)\n    optimizer='AdamW',       # CHANGED\n    lr0=0.005, # Higher, earlier 0.002\n    lrf=0.01,                # Learning rate factor remains the same\n    \n    weight_decay=0.001,     # Regularization, increased from 0.0005\n    \n    warmup_epochs=3,         # Warmup epochs to stabilize initial training\n    cos_lr=True,              # Cosine learning rate decay for smoother optimization\n    patience=50,             # Epochs to wait for no observable improvement for early stopping of training\n\n    conf=0.25,\n\n    augment=True,                       # Enable basic augmentations\n    \n    # Augmentation parameters\n    hsv_h=0.015,                        # Hue shift (0.0 - 0.5), where 0.015 is a small shift\n    hsv_s=0.7,                          # Saturation shift (0.0 - 0.9)\n    hsv_v=0.4,                          # Brightness shift (0.0 - 0.9)\n    degrees=10.0,                       # Range of rotation in degrees (-180 to 180)\n    translate=0.1,                      # Range of translation (0.0 - 1.0), fraction of image size\n    scale=0.5,                          # Scale factor range (0.0 - 1.0)\n    shear=2.0,                          # Shear angle in degrees\n    perspective=0.0001,                 # Perspective distortion (0.0 - 0.001)\n    flipud=0.5,                         # Probability of vertical flip (0.0 - 1.0)\n    fliplr=0.5,                         # Probability of horizontal flip (0.0 - 1.0)\n    mosaic=1.0,                         # Probability of using mosaic augmentation (0.0 - 1.0)\n    mixup=0.2,                          # Probability of using mixup augmentation (0.0 - 1.0)\n    copy_paste=0.0,                     # Probability of copy-paste augmentation (0.0 - 1.0)\n    auto_augment='randaugment',         # Auto augment policy ('randaugment', 'trivialaugment', etc.)\n    erasing=0.4,                        # Probability of random erasing (cutout) augmentation\n    crop_fraction=1.0 \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T21:22:58.738102Z","iopub.execute_input":"2024-11-17T21:22:58.738511Z","iopub.status.idle":"2024-11-18T02:31:00.006808Z","shell.execute_reply.started":"2024-11-17T21:22:58.738473Z","shell.execute_reply":"2024-11-18T02:31:00.005692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!nvidia-smi --gpu-reset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T22:49:53.853784Z","iopub.execute_input":"2024-11-14T22:49:53.854219Z","iopub.status.idle":"2024-11-14T22:49:54.945076Z","shell.execute_reply.started":"2024-11-14T22:49:53.854175Z","shell.execute_reply":"2024-11-14T22:49:54.944071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python val.py --data {yaml_file} --weights runs/detect/train4/weights/best.pt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T02:35:10.651019Z","iopub.execute_input":"2024-11-18T02:35:10.651999Z","iopub.status.idle":"2024-11-18T02:35:11.834613Z","shell.execute_reply.started":"2024-11-18T02:35:10.651959Z","shell.execute_reply":"2024-11-18T02:35:11.833456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the best model weights after training\nmodel = YOLO('runs/detect/train4/weights/best.pt')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T02:35:30.651815Z","iopub.execute_input":"2024-11-18T02:35:30.65229Z","iopub.status.idle":"2024-11-18T02:35:30.802359Z","shell.execute_reply.started":"2024-11-18T02:35:30.652248Z","shell.execute_reply":"2024-11-18T02:35:30.801519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ndef read_and_print_files(directory, n=None):\n    \"\"\"\n    Reads files from a directory and prints their content.\n\n    Args:\n        directory (str): The path to the directory.\n        n (int, optional): Number of files to process. If None, processes all files.\n    \"\"\"\n    # List all files in the directory\n    file_list = [os.path.join(directory, file) for file in os.listdir(directory) if os.path.isfile(os.path.join(directory, file))]\n    \n    # Sort files alphabetically (optional, for consistent processing order)\n    file_list.sort()\n    \n    # Process only the first n files if n is specified\n    if n is not None:\n        file_list = file_list[:n]\n    \n    for file in file_list:\n        print(f\"\\nReading file: {file}\")\n        try:\n            with open(file, 'r') as f:\n                # Read and print each line\n                for line in f:\n                    # Remove extra spaces and split integers (optional processing)\n                    line_content = line.strip()\n                    print(line_content)\n        except Exception as e:\n            print(f\"Error reading file {file}: {e}\")\n\n# Directory path\nworkingdir = \"/kaggle/working/labels/train\"\n\n# Specify the number of files to read (or None for all files)\nn_files_to_read = 10\n\n# Call the function\nread_and_print_files(workingdir, n_files_to_read if n_files_to_read > 0 else None)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T02:56:34.482642Z","iopub.execute_input":"2024-11-18T02:56:34.483021Z","iopub.status.idle":"2024-11-18T02:56:34.539761Z","shell.execute_reply.started":"2024-11-18T02:56:34.482984Z","shell.execute_reply":"2024-11-18T02:56:34.538854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Sample test with image of choice**","metadata":{}},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom PIL import Image","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"id = \"1-1008\"\n\n# Prediction step\nresults = model.predict(\n    source=\"/kaggle/working/images/train/\"+id+\".jpg\",\n    save=True,\n    conf=0.25\n)\n\n# Accessing prediction results\nresult = results[0]\n\n# Plotting the predictions\nres_plotted = result.plot()\nres_plotted = cv2.cvtColor(res_plotted, cv2.COLOR_BGR2RGB)\n\n# Load the original label file\nlabel_path = \"/kaggle/working/labels/train/\"+id+\".txt\"\nimage_path = \"/kaggle/working/images/train/\"+id+\".jpg\"\n\ntry:\n    print(f\"Reading original bounding box data from: {label_path}\")\n    with open(label_path, 'r') as f:\n        original_boxes = f.readlines()\n    \n    # Load the original image\n    original_image = cv2.imread(image_path)\n    original_image = cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB)\n    \n    height, width, _ = original_image.shape\n    \n    # Draw original bounding boxes on the original image\n    for box in original_boxes:\n        data = list(map(float, box.strip().split()))\n        cls, x_center, y_center, w, h = data\n        x1 = int((x_center - w / 2) * width)\n        y1 = int((y_center - h / 2) * height)\n        x2 = int((x_center + w / 2) * width)\n        y2 = int((y_center + h / 2) * height)\n        \n        # Draw rectangle for the original label\n        cv2.rectangle(res_plotted, (x1, y1), (x2, y2), color=(0, 0, 255), thickness=2)  # Blue color for original labels\n        \nexcept FileNotFoundError:\n    print(f\"Error: Label file {label_path} not found.\")\nexcept Exception as e:\n    print(f\"An error occurred while processing the label file: {e}\")\n\n# Overlay predictions on top of the original bounding box image\nheight, width, _ = res_plotted.shape\n\n# Draw predictions again with red color\nfor box in result.boxes:\n    x1, y1, x2, y2 = map(int, box.xyxy[0].tolist())\n    conf = box.conf[0] if box.conf is not None else 0.0\n    cls = int(box.cls[0]) if box.cls is not None else -1\n    cv2.rectangle(res_plotted, (x1, y1), (x2, y2), color=(255, 0, 0), thickness=2)  # Red color for predictions\n    cv2.putText(\n        res_plotted,\n        f\"{cls} {conf:.2f}\",\n        (x1, y1 - 10),\n        cv2.FONT_HERSHEY_SIMPLEX,\n        0.5,\n        (255, 0, 0),\n        1,\n        cv2.LINE_AA,\n    )\n\n# Display the image with predictions (red) and original labels (blue)\nplt.figure(figsize=(10, 10))\nplt.imshow(res_plotted)\nplt.axis(\"off\")\nplt.title(\"Image with Predictions (Red) and Original Labels (Blue)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T02:56:52.039444Z","iopub.execute_input":"2024-11-18T02:56:52.039822Z","iopub.status.idle":"2024-11-18T02:56:52.50693Z","shell.execute_reply.started":"2024-11-18T02:56:52.039786Z","shell.execute_reply":"2024-11-18T02:56:52.506098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Creating The submission file","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport cv2\nfrom PIL import Image\nfrom ultralytics import YOLO\n\n# Load the test CSV file\ntest_df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/test.csv\")\n\n# Load the trained model\nmodel = YOLO('runs/detect/train4/weights/best.pt')\n\n# Define the base directory for images\nbase_image_dir = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/\"\n\n# List to hold results for submission\nsubmission_data = []\n\n# Loop through each row in the test DataFrame\nfor index, row in test_df.iterrows():\n    # Construct the image path based on video_id and video_frame\n    video_id = row['video_id']\n    video_frame = row['video_frame']\n    image_path = f\"{base_image_dir}video_{video_id-1}/{video_frame}.jpg\"\n    \n    # Run the prediction on the current image\n    results = model.predict(source=image_path, conf=0.2, iou=0.5)\n    \n    # Prepare annotations for the current image\n    annotations = []\n    for box in results[0].boxes.data:  # Iterate over each detected box\n        confidence = float(box[4].item())  # Confidence score\n        x_min = float(box[0].item())\n        y_min = float(box[1].item())\n        x_max = float(box[2].item())\n        y_max = float(box[3].item())\n        \n        # Convert to width and height\n        width = x_max - x_min\n        height = y_max - y_min\n        \n        # Format as \"confidence x_min y_min width height\"\n        annotations.append(f\"{confidence:.2f} {x_min} {y_min} {width} {height}\")\n    \n    # Join all annotations with a space separator, or leave blank if no detections\n    annotation_str = \" \".join(annotations) if annotations else \"\"\n    \n    # Append result to submission data\n    submission_data.append({\"row_num\": index, \"annotations\": annotation_str})\n\n# Create DataFrame for submission and save to CSV\nsubmission_df = pd.DataFrame(submission_data)\nsubmission_df.to_csv(\"submission.csv\", index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T02:59:16.669637Z","iopub.execute_input":"2024-11-18T02:59:16.670008Z","iopub.status.idle":"2024-11-18T02:59:17.383766Z","shell.execute_reply.started":"2024-11-18T02:59:16.669975Z","shell.execute_reply":"2024-11-18T02:59:17.382851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cat submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T04:28:53.663254Z","iopub.execute_input":"2024-11-11T04:28:53.663626Z","iopub.status.idle":"2024-11-11T04:28:54.732263Z","shell.execute_reply.started":"2024-11-11T04:28:53.663591Z","shell.execute_reply":"2024-11-11T04:28:54.731229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T04:14:34.560213Z","iopub.execute_input":"2024-11-11T04:14:34.56124Z","iopub.status.idle":"2024-11-11T04:14:34.566401Z","shell.execute_reply.started":"2024-11-11T04:14:34.56118Z","shell.execute_reply":"2024-11-11T04:14:34.565304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T04:14:37.371881Z","iopub.execute_input":"2024-11-11T04:14:37.372292Z","iopub.status.idle":"2024-11-11T04:14:37.386979Z","shell.execute_reply.started":"2024-11-11T04:14:37.372254Z","shell.execute_reply":"2024-11-11T04:14:37.38616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\nfor (pixel_array, sample_prediction_df) in iter_test:\n    sample_prediction_df['annotations'] = '0.5 0 0 100 100'  # make your predictions here\n    env.predict(sample_prediction_df)   # register your predictions\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}