{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qU wandb\n!pip install -qU bbox-utility # https://github.com/awsaf49/bbox\n!pip install -q imagesize","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:46:48.431213Z","iopub.execute_input":"2022-03-18T11:46:48.431556Z","iopub.status.idle":"2022-03-18T11:47:17.919885Z","shell.execute_reply.started":"2022-03-18T11:46:48.431452Z","shell.execute_reply":"2022-03-18T11:47:17.918841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\n\nimport os\nimport cv2\nimport random\nimport glob\nimport wandb\n\nimport imagesize\nimport shutil\nimport yaml\n\nfrom matplotlib.colors import Normalize\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nfrom joblib import Parallel, delayed\nfrom kaggle_secrets import UserSecretsClient\nfrom bbox.utils import coco2yolo, coco2voc, voc2yolo, yolo2voc\nfrom bbox.utils import draw_bboxes, load_image\nfrom bbox.utils import clip_bbox, str2annot, annot2str\nfrom sklearn.model_selection import KFold\nfrom scipy.stats import gaussian_kde\n\n%matplotlib inline\ntrain_jpg_path = \"../input/happy-whale-and-dolphin/train_images\"\ntest_jpg_peth = \"../input/happy-whale-and-dolphin/test_images\"\n\nROOT_DIR = \"../input/whale-categorization-playground\"\nIMAGE_DIR = \"/kaggle/data1/images\"\nLABEL_DIR = \"/kaggle/data1/labels\"\n\ncwd = \"/kaggle/working\"\ntrain_output = \"/kaggle/working/output/train\"\ntest_output = \"/kaggle/working/output/test\"\n\n#sample_submission = pd.read_csv(\"../input/happy-whale-and-dolphin/sample_submission.csv\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-18T11:47:17.921838Z","iopub.execute_input":"2022-03-18T11:47:17.922120Z","iopub.status.idle":"2022-03-18T11:47:20.338875Z","shell.execute_reply.started":"2022-03-18T11:47:17.922082Z","shell.execute_reply":"2022-03-18T11:47:20.338107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Loading and EDA","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\ndf_train.species.replace({\"globis\": \"short_finned_pilot_whale\",\n                          \"beluga\": \"beluga_whale\",\n                          \"pilot_whale\": \"short_finned_pilot_whale\",\n                          \"kiler_whale\": \"killer_whale\",\n                          \"bottlenose_dolpin\": \"bottlenose_dolphin\"}, inplace=True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:20.340392Z","iopub.execute_input":"2022-03-18T11:47:20.340659Z","iopub.status.idle":"2022-03-18T11:47:20.450768Z","shell.execute_reply.started":"2022-03-18T11:47:20.340626Z","shell.execute_reply":"2022-03-18T11:47:20.449961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:20.452956Z","iopub.execute_input":"2022-03-18T11:47:20.453215Z","iopub.status.idle":"2022-03-18T11:47:20.486665Z","shell.execute_reply.started":"2022-03-18T11:47:20.453180Z","shell.execute_reply":"2022-03-18T11:47:20.485868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of images in the train folder: {len(os.listdir(train_jpg_path))}\")\nprint(f\"Number of images in the test folder: {len(os.listdir(test_jpg_peth))}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:20.488194Z","iopub.execute_input":"2022-03-18T11:47:20.488506Z","iopub.status.idle":"2022-03-18T11:47:21.645001Z","shell.execute_reply.started":"2022-03-18T11:47:20.488461Z","shell.execute_reply":"2022-03-18T11:47:21.644156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_examples = 5\n#random.seed(161)\n\ntrain_pic_list = os.listdir(train_jpg_path)\nfig = plt.figure(figsize=(25,25))\n\nfor i in range(cnt_examples):\n    ax = fig.add_subplot(cnt_examples,1,i+1)\n    example_pic = random.choice(train_pic_list)\n    img_plt = plt.imshow(plt.imread(f'{train_jpg_path}/{example_pic}'))\n    plt.axis('off')\n    ax.set_title(df_train[df_train.image == example_pic].species.values[0])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:21.646221Z","iopub.execute_input":"2022-03-18T11:47:21.646533Z","iopub.status.idle":"2022-03-18T11:47:25.115843Z","shell.execute_reply.started":"2022-03-18T11:47:21.646491Z","shell.execute_reply":"2022-03-18T11:47:25.113541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animal_cnt = df_train.species.value_counts()\nprint(\"Occurences of different species:\")\nprint(animal_cnt)\nprint(f\"Total number of species: {len(animal_cnt)}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:25.116824Z","iopub.execute_input":"2022-03-18T11:47:25.117054Z","iopub.status.idle":"2022-03-18T11:47:25.133024Z","shell.execute_reply.started":"2022-03-18T11:47:25.117023Z","shell.execute_reply":"2022-03-18T11:47:25.132095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"specs = list(animal_cnt.keys())\nvalues = list(animal_cnt.values)\n\ncmap = cm.get_cmap('jet')\nnorm = Normalize(vmin=0,vmax=len(specs))\ncols = np.arange(0,len(specs))\n\nfig = plt.figure(figsize=(12,8))\nax = fig.add_subplot(1,1,1)\nax.set_axisbelow(True)\nplt.grid(visible=True)\nplt.bar(specs, values, color=cmap(norm(cols)))\nplt.xticks(rotation='vertical')\nplt.title('Occurences Of Different Species In The Dataset', fontsize=16, fontname=\"Times New Roman Bold\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:25.134934Z","iopub.execute_input":"2022-03-18T11:47:25.135613Z","iopub.status.idle":"2022-03-18T11:47:25.591609Z","shell.execute_reply.started":"2022-03-18T11:47:25.135578Z","shell.execute_reply":"2022-03-18T11:47:25.590904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_dolphins = 0\ncnt_whales = 0\nfor spec in animal_cnt.keys():\n    cnt = animal_cnt[spec]\n    if spec.split('_')[-1] == 'dolphin':\n        cnt_dolphins += cnt\n    else:\n        cnt_whales += cnt\n        \nprint(f\"Number of dolphins in the set: {cnt_dolphins}\")\nprint(f\"Number of whales in the set: {cnt_whales}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:25.595754Z","iopub.execute_input":"2022-03-18T11:47:25.596374Z","iopub.status.idle":"2022-03-18T11:47:25.605418Z","shell.execute_reply.started":"2022-03-18T11:47:25.596330Z","shell.execute_reply":"2022-03-18T11:47:25.604728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Creating Bounding Boxes","metadata":{}},{"cell_type":"markdown","source":"Based on this [notebook](https://www.kaggle.com/awsaf49/happywhale-boundingbox-yolov5) about how to create bounding boxes with YOLOv5 based on the [Whale Flute dataset](https://www.kaggle.com/martinpiotte/humpback-whale-identification-fluke-location) and the [Humpback Whale Identification Challenge](https://www.kaggle.com/c/whale-categorization-playground). Big recommendation!","metadata":{}},{"cell_type":"markdown","source":"## WandB","metadata":{}},{"cell_type":"code","source":"try:\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"WANDB\")\n    wandb.login(key=api_key)\n    anonymous = None\nexcept:\n    wandb.login(anonymous=\"must\")\n    print(\"To use ur W&B account,\\nGo to Add-ons -> Secrets and provide your W&B access token. Use the Label name as WANDB. \\nGet your W&B access token from here: https://wandb.ai/authorize\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:25.609246Z","iopub.execute_input":"2022-03-18T11:47:25.612446Z","iopub.status.idle":"2022-03-18T11:47:27.604323Z","shell.execute_reply.started":"2022-03-18T11:47:25.612402Z","shell.execute_reply":"2022-03-18T11:47:27.603634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Meta Data","metadata":{}},{"cell_type":"code","source":"FOLD = 0\nDIM = 640\nMODEL = \"yolov5x\"\nBATCH = 16\nEPOCHS = 18\nOPTIMIZER = \"Adam\"\n\nPROJECT = \"happywhale-det-public\"\nNAME = f\"{MODEL}-dim{DIM}-fold{FOLD}\"","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:27.607181Z","iopub.execute_input":"2022-03-18T11:47:27.607379Z","iopub.status.idle":"2022-03-18T11:47:27.612080Z","shell.execute_reply.started":"2022-03-18T11:47:27.607354Z","shell.execute_reply":"2022-03-18T11:47:27.611283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Directories","metadata":{}},{"cell_type":"code","source":"!mkdir -p {IMAGE_DIR}\n!mkdir -p {LABEL_DIR}","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:27.613311Z","iopub.execute_input":"2022-03-18T11:47:27.614037Z","iopub.status.idle":"2022-03-18T11:47:28.937376Z","shell.execute_reply.started":"2022-03-18T11:47:27.614002Z","shell.execute_reply":"2022-03-18T11:47:28.936481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Paths","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ndf[\"image_id\"] = df[\"Image\"]\ndf[\"old_image_path\"] = f\"{ROOT_DIR}/train/\" + df.image_id.astype(str)\ndf[\"image_path\"] = f\"{IMAGE_DIR}/\" + df.image_id\ndf[\"label_path\"] = f\"{LABEL_DIR}/\" + df.image_id.str.replace(\"jpg\", \"txt\")\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:28.939196Z","iopub.execute_input":"2022-03-18T11:47:28.939519Z","iopub.status.idle":"2022-03-18T11:47:28.992157Z","shell.execute_reply.started":"2022-03-18T11:47:28.939463Z","shell.execute_reply":"2022-03-18T11:47:28.991512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Write Copies","metadata":{}},{"cell_type":"code","source":"def make_copy(row):\n    shutil.copyfile(row.old_image_path, row.image_path)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:28.993328Z","iopub.execute_input":"2022-03-18T11:47:28.993654Z","iopub.status.idle":"2022-03-18T11:47:28.998505Z","shell.execute_reply.started":"2022-03-18T11:47:28.993616Z","shell.execute_reply":"2022-03-18T11:47:28.997299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_paths = df.old_image_path.tolist()\n_ = Parallel(n_jobs=-1, backend='threading')(delayed(make_copy)(row) for _, row in tqdm(df.iterrows(), total=len(df)))","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:47:28.999962Z","iopub.execute_input":"2022-03-18T11:47:29.000255Z","iopub.status.idle":"2022-03-18T11:48:17.327709Z","shell.execute_reply.started":"2022-03-18T11:47:29.000219Z","shell.execute_reply":"2022-03-18T11:48:17.327045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create BBox","metadata":{}},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row[\"width\"], row[\"height\"] = imagesize.get(row[\"image_path\"])\n    return row\n\nnp.random.seed(32)\ncolors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255)) for idx in range(1)]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:17.328698Z","iopub.execute_input":"2022-03-18T11:48:17.328942Z","iopub.status.idle":"2022-03-18T11:48:17.337431Z","shell.execute_reply.started":"2022-03-18T11:48:17.328904Z","shell.execute_reply":"2022-03-18T11:48:17.334749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def point2bbox(points):\n    points = np.array(points)\n    points = points.astype('int')\n    points = points.reshape(-1, 2)\n    xmin, ymin, xmax, ymax = points[:, 0].min(), points[:, 1].min(), points[:, 0].max(), points[:, 1].max()\n    return [[xmin, ymin, xmax, ymax]]\n\nf = open('/kaggle/input/humpback-whale-identification-fluke-location/cropping.txt', 'rt').read()\nid2point = {x.split(',')[0]:x.split(',')[1:] for x in f.split('\\n')}\ndf['point'] = df['image_id'].map(id2point)\ndf = df[~df.point.isna()]\ndf['bbox'] = df.point.map(point2bbox)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:17.340113Z","iopub.execute_input":"2022-03-18T11:48:17.340324Z","iopub.status.idle":"2022-03-18T11:48:17.543367Z","shell.execute_reply.started":"2022-03-18T11:48:17.340297Z","shell.execute_reply":"2022-03-18T11:48:17.542647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Image-Size","metadata":{}},{"cell_type":"code","source":"df = df.progress_apply(get_imgsize, axis=1)\ndisplay(df.head(2))","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:17.544605Z","iopub.execute_input":"2022-03-18T11:48:17.545259Z","iopub.status.idle":"2022-03-18T11:48:18.927299Z","shell.execute_reply.started":"2022-03-18T11:48:17.545223Z","shell.execute_reply":"2022-03-18T11:48:18.926543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:18.928564Z","iopub.execute_input":"2022-03-18T11:48:18.929057Z","iopub.status.idle":"2022-03-18T11:48:18.954057Z","shell.execute_reply.started":"2022-03-18T11:48:18.929014Z","shell.execute_reply":"2022-03-18T11:48:18.953201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Labels","metadata":{}},{"cell_type":"code","source":"cnt = 0\nall_bboxes = []\nbboxes_info = []\nfor row_idx in tqdm(range(df.shape[0])):\n    row = df.iloc[row_idx]\n    image_height = row.height\n    image_width = row.width\n    bboxes_voc = np.array(row.bbox).astype(np.float32).copy()\n    num_bbox = len(bboxes_voc)\n    names = [\"whale\"] * num_bbox\n    labels = np.array([0] * num_bbox)[..., None].astype(str)\n    \n    with open(row.label_path, \"w\") as f:\n        if num_bbox < 1:\n            annot = \"\"\n            f.write(annot)\n            cnt += 1\n            continue\n            \n        bboxes_voc = clip_bbox(bboxes_voc, image_height, image_width)\n        bboxes_yolo = voc2yolo(bboxes_voc, image_height, image_width)\n        all_bboxes.extend(bboxes_yolo.astype(float))\n        bboxes_info.extend([[row.image_id]]*len(bboxes_yolo))\n        annots = np.concatenate([labels, bboxes_yolo], axis=1)\n        f.write(annot2str(annots))\n        \nprint(f\"Missing: {cnt}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:18.955690Z","iopub.execute_input":"2022-03-18T11:48:18.956201Z","iopub.status.idle":"2022-03-18T11:48:21.440548Z","shell.execute_reply.started":"2022-03-18T11:48:18.956160Z","shell.execute_reply":"2022-03-18T11:48:21.439652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Folds","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=6, random_state=161, shuffle=True)\ndf = df.reset_index(drop=True)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df)):\n    df.loc[val_idx, 'fold'] = fold\ndf.fold.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:21.442155Z","iopub.execute_input":"2022-03-18T11:48:21.442453Z","iopub.status.idle":"2022-03-18T11:48:21.463061Z","shell.execute_reply.started":"2022-03-18T11:48:21.442415Z","shell.execute_reply":"2022-03-18T11:48:21.462264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:21.464360Z","iopub.execute_input":"2022-03-18T11:48:21.464862Z","iopub.status.idle":"2022-03-18T11:48:21.480191Z","shell.execute_reply.started":"2022-03-18T11:48:21.464825Z","shell.execute_reply":"2022-03-18T11:48:21.479349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BBox Distribution","metadata":{}},{"cell_type":"code","source":"bbox_df = pd.DataFrame(np.concatenate([bboxes_info, all_bboxes], axis=1),\n                      columns=['image_id','xmid','ymid','w','h'])\nbbox_df[['xmid','ymid','w','h']] = bbox_df[['xmid','ymid','w','h']].astype(float)\nbbox_df['area'] = bbox_df.w * bbox_df.h\nbbox_df = bbox_df.merge(df[['image_id', 'fold']], on='image_id', how='left')\nbbox_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:21.481423Z","iopub.execute_input":"2022-03-18T11:48:21.481948Z","iopub.status.idle":"2022-03-18T11:48:21.520001Z","shell.execute_reply.started":"2022-03-18T11:48:21.481900Z","shell.execute_reply":"2022-03-18T11:48:21.519363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_bboxes = np.array(all_bboxes)\n\nx_val = all_bboxes[...,0]\ny_val = all_bboxes[...,1]\n\nxy = np.vstack([x_val, y_val])\nz = gaussian_kde(xy)(xy)\n\nfig, ax = plt.subplots(figsize = (10, 10))\nax.scatter(x_val, y_val, c=z, s=50, cmap='viridis')\nax.set_xlabel('x_mid')\nax.set_ylabel('y_mid')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:21.521229Z","iopub.execute_input":"2022-03-18T11:48:21.521677Z","iopub.status.idle":"2022-03-18T11:48:21.762233Z","shell.execute_reply.started":"2022-03-18T11:48:21.521642Z","shell.execute_reply":"2022-03-18T11:48:21.761394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization","metadata":{}},{"cell_type":"code","source":"df2 = df.sample(100)\ny = 3\nx = 5\nplt.figure(figsize=(4 * x, 4 * y))\nfor idx in range(x*y):\n    row = df2.iloc[idx]\n    img = load_image(row.image_path)\n    img = cv2.resize(img, (512, 512))\n    image_height = row.height\n    image_width = row.width\n    with open(row.label_path) as f:\n        annot = str2annot(f.read())\n    bboxes_yolo = annot[...,1:]\n    labels = annot[..., 0].astype(int).tolist()\n    names = ['whale']*len(bboxes_yolo)\n    plt.subplot(y, x, idx+1)\n    plt.imshow(draw_bboxes(img = img,\n                          bboxes = bboxes_yolo,\n                          classes = names,\n                          class_ids = labels,\n                          class_name = True,\n                          colors = colors,\n                          bbox_format='yolo',\n                          line_thickness=2))","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:21.763385Z","iopub.execute_input":"2022-03-18T11:48:21.763742Z","iopub.status.idle":"2022-03-18T11:48:24.779840Z","shell.execute_reply.started":"2022-03-18T11:48:21.763702Z","shell.execute_reply":"2022-03-18T11:48:24.776538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fluke Dataset","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:24.781175Z","iopub.execute_input":"2022-03-18T11:48:24.781656Z","iopub.status.idle":"2022-03-18T11:48:24.811666Z","shell.execute_reply.started":"2022-03-18T11:48:24.781612Z","shell.execute_reply":"2022-03-18T11:48:24.810070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files = []\ntrain_df_fluke = df.query(\"fold!=@FOLD\")\nval_df_fluke = df.query(\"fold==@FOLD\")\ntrain_files += list(train_df_fluke.image_path.unique())\nval_files += list(val_df_fluke.image_path.unique())\nlen(train_files), len(val_files)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:24.813163Z","iopub.execute_input":"2022-03-18T11:48:24.813630Z","iopub.status.idle":"2022-03-18T11:48:24.937396Z","shell.execute_reply.started":"2022-03-18T11:48:24.813591Z","shell.execute_reply":"2022-03-18T11:48:24.936772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"with open(os.path.join(cwd, 'train.txt'), 'w') as f:\n    for path in train_df_fluke.image_path.tolist():\n        f.write(path+\"\\n\")\n        \nwith open(os.path.join(cwd, 'val.txt'), 'w') as f:\n    for path in val_df_fluke.image_path.tolist():\n        f.write(path+\"\\n\")\n\ndata = dict(\n    path=cwd,\n    train = os.path.join(cwd, 'train.txt'),\n    val = os.path.join(cwd, 'val.txt'),\n    nc = 1,\n    names = ['whale']\n)\n\nwith open(os.path.join(cwd, 'happywhale.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n    \nf = open(os.path.join(cwd, 'happywhale.yaml'), 'r')\nprint(f\"yaml file:\\n{f.read()}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:24.942083Z","iopub.execute_input":"2022-03-18T11:48:24.942736Z","iopub.status.idle":"2022-03-18T11:48:24.954074Z","shell.execute_reply.started":"2022-03-18T11:48:24.942698Z","shell.execute_reply":"2022-03-18T11:48:24.953309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /kaggle/working/hyp.yaml\nlr0: 0.001\nlrf: 0.01\nmomentum: 0.937\nweight_decay: 0.0005\nwarmup_epochs: 3.0\nwarmup_momentum: 0.8\nwarmup_bias_lr: 0.1\nbox: 0.05\ncls: 0.5\ncls_pw: 1.0\nobj: 1.0\nobj_pw: 1.0\niuo_t: 0.25\nanchor_t: 4.0\nfl_gamma: 0.0\nhsv_h: 0.015\nhsv_s: 0.7\nhsv_v: 0.4\ndegrees: 30.0\ntranslate: 0.10\nscale: 0.80\nshear: 10.0\nperspective: 0.0\nflipud: 0.5\nfliplr: 0.5\nmosaic: 0.75\nmixup: 0.0\ncopy_paste: 0.0","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:24.955330Z","iopub.execute_input":"2022-03-18T11:48:24.955533Z","iopub.status.idle":"2022-03-18T11:48:24.963503Z","shell.execute_reply.started":"2022-03-18T11:48:24.955507Z","shell.execute_reply":"2022-03-18T11:48:24.962647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## YOLOv5","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n!rm -r /kaggle/working/yolov5\n!git clone https://github.com/ultralytics/yolov5\n!cp -r /kaggle/input/yolov5-lib-ds /kaggle/working/yolov5\n%cd yolov5\n%pip install -qr requirements.txt","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:24.966056Z","iopub.execute_input":"2022-03-18T11:48:24.966552Z","iopub.status.idle":"2022-03-18T11:48:38.284341Z","shell.execute_reply.started":"2022-03-18T11:48:24.966516Z","shell.execute_reply":"2022-03-18T11:48:38.283534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolov5 import utils\n_ = utils.notebook_init()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:38.286031Z","iopub.execute_input":"2022-03-18T11:48:38.286307Z","iopub.status.idle":"2022-03-18T11:48:39.959614Z","shell.execute_reply.started":"2022-03-18T11:48:38.286268Z","shell.execute_reply":"2022-03-18T11:48:39.958785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"!python train.py --img {DIM}\\\n--batch {BATCH}\\\n--epochs {EPOCHS}\\\n--optimizer {OPTIMIZER}\\\n--data /kaggle/working/happywhale.yaml\\\n--hyp /kaggle/working/hyp.yaml\\\n--weights {MODEL}.pt\\\n--project {PROJECT} --name {NAME}\\\n--exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-03-18T11:48:39.961455Z","iopub.execute_input":"2022-03-18T11:48:39.962653Z","iopub.status.idle":"2022-03-18T12:43:00.455346Z","shell.execute_reply.started":"2022-03-18T11:48:39.962608Z","shell.execute_reply":"2022-03-18T12:43:00.454505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Output Files","metadata":{}},{"cell_type":"code","source":"OUTPUT_DIR = f\"{PROJECT}/{NAME}\"\n!ls {OUTPUT_DIR}","metadata":{"execution":{"iopub.status.busy":"2022-03-18T12:43:00.458918Z","iopub.execute_input":"2022-03-18T12:43:00.459163Z","iopub.status.idle":"2022-03-18T12:43:01.186959Z","shell.execute_reply.started":"2022-03-18T12:43:00.459131Z","shell.execute_reply":"2022-03-18T12:43:01.186132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {OUTPUT_DIR}/weights/best.pt","metadata":{"execution":{"iopub.status.busy":"2022-03-18T12:43:01.188307Z","iopub.execute_input":"2022-03-18T12:43:01.188550Z","iopub.status.idle":"2022-03-18T12:43:01.899535Z","shell.execute_reply.started":"2022-03-18T12:43:01.188520Z","shell.execute_reply":"2022-03-18T12:43:01.898736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Whale and Dolphin Data","metadata":{}},{"cell_type":"code","source":"df2 = pd.read_csv(f\"/kaggle/input/happywhale-data-distribution/train.csv\")\ndf2[\"image_id\"] = df2[\"image\"]\ndf2[\"label_path\"] = train_output + \"/labels/\" + df2[\"image_id\"].str.replace('jpg','txt')\n\ntest_df2 = pd.read_csv(f\"/kaggle/input/happywhale-data-distribution/test.csv\")\ntest_df2[\"image_id\"] = test_df2[\"image\"]\ntest_df2[\"label_path\"] = test_output + \"/labels/\" + test_df2[\"image_id\"].str.replace('jpg','txt')\n\nprint(\"Train Images: {:,} | Test Images: {:,}\".format(len(df2), len(test_df2)))","metadata":{"execution":{"iopub.status.busy":"2022-03-18T12:43:01.901534Z","iopub.execute_input":"2022-03-18T12:43:01.901823Z","iopub.status.idle":"2022-03-18T12:43:02.434921Z","shell.execute_reply.started":"2022-03-18T12:43:01.901782Z","shell.execute_reply":"2022-03-18T12:43:02.434157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction on Train","metadata":{}},{"cell_type":"code","source":"!rm -rf {train_output}\n!mkdir -p {train_output}","metadata":{"execution":{"iopub.status.busy":"2022-03-18T12:43:02.436056Z","iopub.execute_input":"2022-03-18T12:43:02.436294Z","iopub.status.idle":"2022-03-18T12:43:03.767925Z","shell.execute_reply.started":"2022-03-18T12:43:02.436257Z","shell.execute_reply":"2022-03-18T12:43:03.766939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --img {DIM}\\\n--source /kaggle/input/happy-whale-and-dolphin/train_images\\\n--weights {OUTPUT_DIR}/weights/best.pt\\\n--project /kaggle/working/output --name train\\\n--conf 0.01 --iou 0.4 --max-det 1\\\n--safe-txt --safe-conf\\\n--nosafe\\\n--half\\\n--exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-03-18T12:43:03.769742Z","iopub.execute_input":"2022-03-18T12:43:03.770054Z","iopub.status.idle":"2022-03-18T12:43:10.044654Z","shell.execute_reply.started":"2022-03-18T12:43:03.770012Z","shell.execute_reply":"2022-03-18T12:43:10.043543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **To be continued...**","metadata":{}}]}