{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":9988,"databundleVersionId":868324,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":203.461915,"end_time":"2025-09-03T07:42:09.130815","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-09-03T07:38:45.668900","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"H = 768\nWORK_DIR = \"/kaggle/working/\"","metadata":{"papermill":{"duration":0.012146,"end_time":"2025-09-03T07:38:49.712732","exception":false,"start_time":"2025-09-03T07:38:49.700586","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T07:56:13.515376Z","iopub.execute_input":"2025-09-06T07:56:13.516010Z","iopub.status.idle":"2025-09-06T07:56:13.520265Z","shell.execute_reply.started":"2025-09-06T07:56:13.515956Z","shell.execute_reply":"2025-09-06T07:56:13.519521Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Main Training**","metadata":{"papermill":{"duration":0.002712,"end_time":"2025-09-03T07:38:49.718704","exception":false,"start_time":"2025-09-03T07:38:49.715992","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ndef encode_to_bounding_box(encode_value, check_test = False):\n    \"\"\"Function help convert Encode Pixel value to Bounding Box for YoLo Training for a picture\"\"\"\n    encode_value = list(map(int, encode_value.split()))\n    x_min, y_min, x_max, y_max = 1000, 1000, -1, -1\n    for i in range(0, len(encode_value), 2):\n        start = encode_value[i]\n        for j in range(start, start + encode_value[i+1]):\n            r = (j-1) % H\n            c = (j-1) // H\n            if x_min > c: x_min = c\n            if y_min > r: y_min = r\n            if x_max < c: x_max = c\n            if y_max < r: y_max = r\n    if x_min > 0 and y_min > 0: \n        x_min, y_min = x_min - 1, y_min - 1\n    if x_max > 0 and y_max > 0:\n        x_max, y_max = x_max + 1, y_max + 1\n    if check_test == True:\n        return [x_min, y_min, x_max, y_max]\n    x_center = (x_min + x_max) / 2 / H\n    y_center = (y_min + y_max) / 2 / H\n    w = (x_max - x_min) / H\n    h = (y_max - y_min) / H\n    bb = [0, x_center, y_center, w, h]\n    return \" \".join(map(str, bb))\n","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:14.075424Z","iopub.execute_input":"2025-09-06T07:56:14.075681Z","iopub.status.idle":"2025-09-06T07:56:14.342932Z","shell.execute_reply.started":"2025-09-06T07:56:14.075659Z","shell.execute_reply":"2025-09-06T07:56:14.342147Z"},"papermill":{"duration":1.372439,"end_time":"2025-09-03T07:38:51.094067","exception":false,"start_time":"2025-09-03T07:38:49.721628","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\n\nimages_path = os.path.join(WORK_DIR, \"images\")\nimg_train = os.path.join(images_path, \"train\")\nimg_val = os.path.join(images_path, \"val\")\nimg_test = os.path.join(images_path, \"test\")\n\nlabels_path = os.path.join(WORK_DIR, \"labels\")\nlb_train = os.path.join(labels_path, \"train\")\nlb_val = os.path.join(labels_path, \"val\")\nlb_test = os.path.join(labels_path, \"test\")\n\nif os.path.exists(images_path) and os.path.exists(labels_path):\n    shutil.rmtree(images_path)\n    shutil.rmtree(labels_path)\nfor path in [images_path, img_train, img_val, img_test,\n         labels_path, lb_train, lb_val, lb_test]:\n    os.makedirs(path, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:14.580861Z","iopub.execute_input":"2025-09-06T07:56:14.581182Z","iopub.status.idle":"2025-09-06T07:56:14.587126Z","shell.execute_reply.started":"2025-09-06T07:56:14.581162Z","shell.execute_reply":"2025-09-06T07:56:14.586420Z"},"papermill":{"duration":0.011368,"end_time":"2025-09-03T07:38:51.108887","exception":false,"start_time":"2025-09-03T07:38:51.097519","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_list = pd.read_csv(\"/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv\")\nhas_ship_train = 50000\nhas_ship_val = 10000\nhas_ship_test = 1000","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:16.719245Z","iopub.execute_input":"2025-09-06T07:56:16.719497Z","iopub.status.idle":"2025-09-06T07:56:17.807772Z","shell.execute_reply.started":"2025-09-06T07:56:16.719477Z","shell.execute_reply":"2025-09-06T07:56:17.807018Z"},"papermill":{"duration":0.989598,"end_time":"2025-09-03T07:38:52.101521","exception":false,"start_time":"2025-09-03T07:38:51.111923","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import albumentations as A\nimport cv2\nimport matplotlib.pyplot as plt\nsatellite_transforms = A.Compose([\n            # 1. Sương mù và bụi khí quyển (giảm tương phản, làm mờ)\n            A.OneOf([\n                # Sương mù nhẹ\n                A.RandomFog(\n                    fog_coef_lower=0.1, \n                    fog_coef_upper=0.4, \n                    alpha_coef=0.1,\n                    p=1.0\n                ),\n                # Bụi khí quyển - giảm tương phản\n                A.Compose([\n                    A.RandomBrightnessContrast(\n                        brightness_limit=0.1, \n                        contrast_limit=(-0.3, -0.1), # Giảm contrast\n                        p=1.0\n                    ),\n                    A.GaussianBlur(blur_limit=(1, 3), p=0.8),  # Làm mờ nhẹ\n                ]),\n            ], p=0.4),\n            \n            # 2. Glint từ mặt nước (lóa sáng cục bộ)\n            A.OneOf([\n                # Sun flare mô phỏng phản xạ mặt nước\n                A.RandomSunFlare(\n                    flare_roi=(0, 0, 1, 1),  # Toàn ảnh\n                    angle_lower=0,\n                    angle_upper=1,\n                    num_flare_circles_lower=1,\n                    num_flare_circles_upper=3,\n                    src_radius=50,\n                    p=1.0\n                ),\n            ], p=0.4),\n            \n            # 3. Motion blur do vệ tinh di chuyển\n            A.OneOf([\n                A.MotionBlur(blur_limit=(3, 7), p=1.0),  # Motion blur\n                A.Compose([\n                    A.MotionBlur(blur_limit=(2, 4), p=1.0),\n                    A.GaussianBlur(blur_limit=(1, 2), p=0.5),  # Thêm blur nhẹ\n                ]),\n            ], p=0.2),\n            \n            # # 4. Nhiễu sensor và nhiễu điện tử\n            A.OneOf([\n                # Nhiễu Gaussian\n                A.GaussNoise(\n                    var_limit=(10.0, 50.0),\n                    mean=0,\n                    per_channel=True,\n                    p=1.0\n                ),\n                # Nhiễu speckle (đặc trưng của ảnh vệ tinh)\n                A.MultiplicativeNoise(\n                    multiplier=(0.95, 1.05),\n                    per_channel=False,\n                    p=1.0\n                ),\n                # Nhiễu ISO cao\n                A.ISONoise(\n                    color_shift=(0.01, 0.05),\n                    intensity=(0.1, 0.5),\n                    p=1.0\n                ),\n            ], p=0.015),\n            \n            # 5. Vấn đề về độ phơi sáng và bão hòa\n            A.OneOf([\n                # Over-exposure\n                A.RandomBrightnessContrast(\n                    brightness_limit=(0.2, 0.4),\n                    contrast_limit=(0.1, 0.3),\n                    p=1.0\n                ),\n                # Under-exposure  \n                A.RandomBrightnessContrast(\n                    brightness_limit=(-0.3, -0.1),\n                    contrast_limit=(0.1, 0.3),\n                    p=1.0\n                ),\n                # Saturation issues\n                A.HueSaturationValue(\n                    hue_shift_limit=5,\n                    sat_shift_limit=30,\n                    val_shift_limit=20,\n                    p=1.0\n                ),\n            ], p=0.05),\n            \n            # 6. Atmospheric effects (hiệu ứng khí quyển)\n            A.OneOf([\n                # Haze effect\n                A.Compose([\n                    A.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=-0.2, p=1.0),\n                    A.HueSaturationValue(sat_shift_limit=-20, p=1.0),  # Giảm saturation\n                ]),\n                # Atmospheric scattering\n                A.Compose([\n                    A.ToSepia(p=0.3),  # Tạo hiệu ứng scattering nhẹ\n                    A.RandomGamma(gamma_limit=(80, 120), p=1.0),\n                ]),\n            ], p=0.05),  \n        ])","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:17.808951Z","iopub.execute_input":"2025-09-06T07:56:17.809211Z","iopub.status.idle":"2025-09-06T07:56:23.081751Z","shell.execute_reply.started":"2025-09-06T07:56:17.809193Z","shell.execute_reply":"2025-09-06T07:56:23.081028Z"},"papermill":{"duration":5.893257,"end_time":"2025-09-03T07:38:57.997925","exception":false,"start_time":"2025-09-03T07:38:52.104668","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport os\n\na_path = \"/kaggle/input/airbus-ship-detection/train_v2/0005d01c8.jpg\"\n\ndef apply_and_visualize(a_path, satellite_transforms):\n    # Đọc ảnh BGR bằng OpenCV\n    img = cv2.imread(a_path)\n    if img is None:\n        raise ValueError(f\"Không đọc được ảnh từ {a_path}\")\n    \n    # Chuyển sang RGB\n    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    # Áp dụng transform\n    transformed = satellite_transforms(image=img_rgb)\n    img_transformed = transformed[\"image\"]\n\n    # Đường dẫn lưu\n    orig_path = \"/kaggle/working/original.jpg\"\n    trans_path = \"/kaggle/working/transformed.jpg\"\n\n    # Lưu ảnh (cv2 dùng BGR nên cần convert lại)\n    cv2.imwrite(orig_path, cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR))\n    cv2.imwrite(trans_path, cv2.cvtColor(img_transformed, cv2.COLOR_RGB2BGR))\n\n    # In dung lượng file\n    orig_size = os.path.getsize(orig_path) / 1024  # KB\n    trans_size = os.path.getsize(trans_path) / 1024  # KB\n    print(f\"Dung lượng ảnh gốc: {orig_size:.2f} KB\")\n    print(f\"Dung lượng ảnh transformed: {trans_size:.2f} KB\")\n\n    # Hiển thị\n    plt.figure(figsize=(10,5))\n    plt.subplot(1,2,1)\n    plt.imshow(img_rgb)\n    plt.title(\"Original\")\n    plt.axis(\"off\")\n\n    plt.subplot(1,2,2)\n    plt.imshow(img_transformed)\n    plt.title(\"Transformed\")\n    plt.axis(\"off\")\n\n    plt.show()\n\napply_and_visualize(a_path, satellite_transforms)\n","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:23.083380Z","iopub.execute_input":"2025-09-06T07:56:23.083658Z","iopub.status.idle":"2025-09-06T07:56:23.546176Z","shell.execute_reply.started":"2025-09-06T07:56:23.083640Z","shell.execute_reply":"2025-09-06T07:56:23.545386Z"},"papermill":{"duration":0.549955,"end_time":"2025-09-03T07:38:58.564192","exception":false,"start_time":"2025-09-03T07:38:58.014237","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\nfrom tqdm import tqdm\ndef offline_aug(train_path, satellite_transforms):\n    \"\"\"\n    Mở ảnh từ đường dẫn, áp dụng satellite_transforms (albumentations),\n    rồi ghi đè kết quả lên chính file đó.\n    \"\"\"\n    files = os.listdir(train_path)\n    print(len(files))\n    processed = 0\n    for file_name in tqdm(files, desc=\"Augmenting\", unit=\"img\"):\n        if random.random() < 0.7: continue\n        else:\n            full_img_path = os.path.join(train_path, file_name)\n            image = cv2.imread(full_img_path)\n            os.remove(full_img_path)\n            if image is None:\n                raise FileNotFoundError(f\"Không tìm thấy ảnh: {full_img_path}\")\n            # Áp dụng transform (albumentations cần đầu vào là dict)\n            transformed = satellite_transforms(image=image)\n            transformed_image = transformed[\"image\"]\n            # Ghi đè ảnh (giữ nguyên BGR cho OpenCV)\n            cv2.imwrite(full_img_path, transformed_image)\n        processed += 1\n        if processed % 10000 == 0: print(\"Processed: \", processed)\n    print(\"Offline Data Augmentation is succesful !\")","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:23.546817Z","iopub.execute_input":"2025-09-06T07:56:23.547016Z","iopub.status.idle":"2025-09-06T07:56:23.552554Z","shell.execute_reply.started":"2025-09-06T07:56:23.547000Z","shell.execute_reply":"2025-09-06T07:56:23.551822Z"},"papermill":{"duration":0.0116,"end_time":"2025-09-03T07:38:58.580820","exception":false,"start_time":"2025-09-03T07:38:58.569220","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"index_seg = 0\ndef setup_dataset(has_ship_len, index_seg, img_path, lb_path, tag = 'train'):\n    has = 0\n    multiple = False\n    while has < has_ship_len:\n        if pd.isna(train_list.iloc[index_seg,1]) == False:\n            img_name = train_list.iloc[index_seg,0]\n            \n            src_img_path = \"/kaggle/input/airbus-ship-detection/train_v2/\"+img_name\n            full_img_path = os.path.join(img_path, img_name)\n            shutil.copy(src_img_path, full_img_path) #copy ảnh vào thư mục cần \n            \n            full_lb_path = os.path.join(lb_path, img_name.replace(\".jpg\", \".txt\"))\n            bbs = []\n            while train_list.iloc[index_seg,0] == img_name:\n                multiple = True\n                bbs.append(encode_to_bounding_box(train_list.iloc[index_seg,1]))\n                index_seg += 1\n            with open(full_lb_path, \"w\") as f:\n                for item in bbs:\n                    f.write(item + \"\\n\")\n        has += 1\n        if has % 2000 == 0: print(\"Processed \",tag,\": \",has)\n        if multiple == False: index_seg += 1\n        else: multiple = False\n    print(\"Check index_seg:\", index_seg)\n    return index_seg\n\nindex_seg = setup_dataset(has_ship_train, index_seg, img_train, lb_train, 'train') #SETUP TRAIN\nindex_seg = setup_dataset(has_ship_val, index_seg, img_val, lb_val, 'validation') #SETUP VALIDATION\nsetup_dataset(has_ship_test, index_seg, img_test, lb_test, 'test') #SETUP TEST","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:56:23.553833Z","iopub.execute_input":"2025-09-06T07:56:23.554084Z","iopub.status.idle":"2025-09-06T07:59:04.667172Z","shell.execute_reply.started":"2025-09-06T07:56:23.554067Z","shell.execute_reply":"2025-09-06T07:59:04.666444Z"},"papermill":{"duration":115.147061,"end_time":"2025-09-03T07:40:53.732136","exception":false,"start_time":"2025-09-03T07:38:58.585075","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# offline_aug(img_train, satellite_transforms) #DATA AUGMENTATION FOR TRAIN DATA","metadata":{"execution":{"iopub.status.busy":"2025-09-06T07:59:04.667999Z","iopub.execute_input":"2025-09-06T07:59:04.668219Z","iopub.status.idle":"2025-09-06T08:04:58.978549Z","shell.execute_reply.started":"2025-09-06T07:59:04.668203Z","shell.execute_reply":"2025-09-06T08:04:58.977700Z"},"papermill":{"duration":0.010832,"end_time":"2025-09-03T07:40:53.750169","exception":false,"start_time":"2025-09-03T07:40:53.739337","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}