{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom itertools import cycle\nimport matplotlib.pylab as plt\nfrom matplotlib.patches import Rectangle\n\nplt.style.use(\"ggplot\")\ncolor_pal = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]\ncolor_cycle = cycle(plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"])","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:46:37.872813Z","iopub.execute_input":"2021-12-10T04:46:37.873284Z","iopub.status.idle":"2021-12-10T04:46:37.879306Z","shell.execute_reply.started":"2021-12-10T04:46:37.87324Z","shell.execute_reply":"2021-12-10T04:46:37.878348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ntest = pd.read_csv(\"../input/tensorflow-great-barrier-reef/test.csv\")\nss = pd.read_csv(\"../input/tensorflow-great-barrier-reef/example_sample_submission.csv\")\n\ntrain.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:46:52.194024Z","iopub.execute_input":"2021-12-10T04:46:52.194565Z","iopub.status.idle":"2021-12-10T04:46:52.262292Z","shell.execute_reply.started":"2021-12-10T04:46:52.19453Z","shell.execute_reply":"2021-12-10T04:46:52.261449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"sum_cots\"] = train[\"annotations\"].apply(lambda x: len(eval(x)))\ntrain[\"video_sequence\"] = (\n    train[\"video_id\"].astype(\"str\") + \"_\" + train[\"sequence\"].astype(\"str\")\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:47:08.034824Z","iopub.execute_input":"2021-12-10T04:47:08.035224Z","iopub.status.idle":"2021-12-10T04:47:08.290345Z","shell.execute_reply.started":"2021-12-10T04:47:08.03517Z","shell.execute_reply":"2021-12-10T04:47:08.289742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_folds(df):\n    df = df.copy()\n    plt.style.use('ggplot')\n    df = df.groupby('fold_id').agg(\n        sum_cots=('sum_cots', 'sum'), duration=('fold_id', 'count'))\n    df['mean_cots'] = df.sum_cots / df.duration\n    fig, axs = plt.subplots(1, 3, figsize=(15, 4))\n    df.sum_cots.plot(kind='bar', ax=axs[0])\n    df.duration.plot(kind='bar', ax=axs[1])\n    df.mean_cots.plot(kind='bar', ax=axs[2])\n    axs[0].set_title('#COTS')\n    axs[1].set_title('#Frames')\n    axs[2].set_title('#COTS/frame')\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:47:24.380737Z","iopub.execute_input":"2021-12-10T04:47:24.381583Z","iopub.status.idle":"2021-12-10T04:47:24.38881Z","shell.execute_reply.started":"2021-12-10T04:47:24.381536Z","shell.execute_reply":"2021-12-10T04:47:24.387653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\n\n\ndef allocate_group_k_fold(df, n_split):\n    df = df.copy()\n    kf = GroupKFold(n_splits=n_split)\n    df['fold_id'] = -1\n    for fold, (train_idx, val_idx) in enumerate(kf.split(df, groups=df.video_sequence)):\n        df.loc[val_idx, 'fold_id'] = fold\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:47:42.992953Z","iopub.execute_input":"2021-12-10T04:47:42.993267Z","iopub.status.idle":"2021-12-10T04:47:43.934611Z","shell.execute_reply.started":"2021-12-10T04:47:42.993232Z","shell.execute_reply":"2021-12-10T04:47:43.932866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_split = 5\ndf = train.copy()\ndf = df.query('sum_cots > 0') # select only annotated frames\ndf.reset_index(inplace=True)\ngroup_k_alloc_df = allocate_group_k_fold(df, n_split)\ndf = plot_folds(group_k_alloc_df)\nplt.suptitle('Visualization of Statistics of each Folds - GroupKFold', fontsize=16)\ndf, df.std()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:47:56.658835Z","iopub.execute_input":"2021-12-10T04:47:56.659147Z","iopub.status.idle":"2021-12-10T04:47:57.237491Z","shell.execute_reply.started":"2021-12-10T04:47:56.659116Z","shell.execute_reply":"2021-12-10T04:47:57.236579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train.copy()\n\n# make annotated flag\ndf[\"annotated\"] = df[\"sum_cots\"].apply(lambda x: min(x, 1))\n\ndfs = []\n\n# calculate non-annotated frame sub_sequence\nfor i, d in df.groupby(\"video_id\"):\n    ad = d.groupby((d[\"annotated\"] != d[\"annotated\"].shift()).cumsum(), as_index=False)[\n        [\"video_frame\", \"annotated\", \"sum_cots\"]\n    ].agg(\n        annotated=(\"annotated\", \"first\"),\n        start_frame=(\"video_frame\", 'first'),\n        end_frame=(\"video_frame\", \"last\"),\n        sum_cots=(\"sum_cots\", \"sum\"),\n        mean_cots=(\"sum_cots\", \"mean\"),\n         )\n    ad[\"video_id\"] = i\n    dfs.append(ad)\n\ndf_annot = pd.concat(dfs)\ndf_annot[\"duration\"] = df_annot[\"end_frame\"] - df_annot[\"start_frame\"] + 1\nsub_sequence = df_annot.query(\"annotated == 1\")\n\nsub_sequence.reset_index(drop=True)\n\nlast_sub_sequence_end = -1\nsub_sequence_id = 0\nsub_sequence_ids = []\ncontinuous = False\nprev_video_id = 0\nfor idx, (\n    annotated,\n    start_frame,\n    end_frame,\n    sum_cots,\n    mean_cots,\n    video_id,\n    duration,\n) in sub_sequence.iterrows():\n    sub_sequence_ids.append(sub_sequence_id)\n    last_sub_sequence_end = end_frame\n    prev_video_id = video_id\n    if not (prev_video_id == video_id and last_sub_sequence_end + 1 == start_frame):\n        sub_sequence_id += 1\n\nsub_sequence.loc[:, \"sub_sequence_id\"] = sub_sequence_ids\nsub_sequence.drop('annotated', axis=1, inplace=True)\nsub_sequence.reset_index(drop=True, inplace=True)\nsub_sequence","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:48:47.721903Z","iopub.execute_input":"2021-12-10T04:48:47.722339Z","iopub.status.idle":"2021-12-10T04:48:47.970933Z","shell.execute_reply.started":"2021-12-10T04:48:47.722306Z","shell.execute_reply":"2021-12-10T04:48:47.970122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.patches as mpatches\n\n\nfig, axes = plt.subplots(3, 1, figsize=(15, 8), sharex=True, sharey=True)\naxes = axes.ravel()\nmax_annotation = df[\"sum_cots\"].max()\nfor i, d in df.groupby([\"video_id\", \"sequence\"]):\n    video_id = d[\"video_id\"].values[0]\n    ax = axes[video_id]\n    d.set_index(\"video_frame\")[\"sum_cots\"].apply(\n        lambda x: x / max_annotation\n    ).plot(ax=ax, c=\"black\", linewidth=0.5)\n\n    ax.set_title(f\"Video ID: {video_id}\")\n\n\n# visualize clippable interval\nfor (\n    annotated,\n    start_frame,\n    end_frame,\n    sum_cots,\n    mean_cots,\n    video_id,\n    duration,\n    sub_sequence_id,\n) in sub_sequence.itertuples():\n    ax = axes[int(video_id)]\n    rect = mpatches.Rectangle(\n        (start_frame, 0), duration, 1, alpha=0.3, facecolor='red'\n    )\n    ax.add_patch(rect)\n    fig.suptitle(\"Sub-Sequences Visualized\", fontsize=15)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:49:41.887027Z","iopub.execute_input":"2021-12-10T04:49:41.887746Z","iopub.status.idle":"2021-12-10T04:49:42.83224Z","shell.execute_reply.started":"2021-12-10T04:49:41.887695Z","shell.execute_reply":"2021-12-10T04:49:42.831347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def allocate_fold(df, n_split, key=\"sum_cots\"):\n    df = df.copy()\n    assert key in df.columns\n    df.sort_values(key, ascending=False, inplace=True)\n    df[\"fold_id\"] = -1\n    for fold_id in range(n_split):\n        index = df.iloc[fold_id::n_split].index\n        df.loc[index, \"fold_id\"] = fold_id\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:49:59.016332Z","iopub.execute_input":"2021-12-10T04:49:59.016628Z","iopub.status.idle":"2021-12-10T04:49:59.023005Z","shell.execute_reply.started":"2021-12-10T04:49:59.016597Z","shell.execute_reply":"2021-12-10T04:49:59.022013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_folds_sub_sequence(df):\n    df = df.copy()\n    plt.style.use('ggplot')\n    df = df.groupby('fold_id').agg(\n        sum_cots=('sum_cots', 'sum'), duration=('duration', 'sum'))\n    df['mean_cots'] = df.sum_cots / df.duration\n    fig, axs = plt.subplots(1, 3, figsize=(15, 5))\n    df.sum_cots.plot(kind='bar', ax=axs[0])\n    df.duration.plot(kind='bar', ax=axs[1])\n    df.mean_cots.plot(kind='bar', ax=axs[2])\n    axs[0].set_title('#COTS')\n    axs[1].set_title('#Frames')\n    axs[2].set_title('#COTS/frame')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:50:12.801636Z","iopub.execute_input":"2021-12-10T04:50:12.802629Z","iopub.status.idle":"2021-12-10T04:50:12.809801Z","shell.execute_reply.started":"2021-12-10T04:50:12.802577Z","shell.execute_reply":"2021-12-10T04:50:12.808855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = allocate_fold(sub_sequence, n_split=5)\ndf = plot_folds_sub_sequence(df)\nplt.suptitle('Statistics of Folds by Round-Robin Algorithm', fontsize=16)\nplt.tight_layout()\ndf, df.agg('std')","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:50:26.139016Z","iopub.execute_input":"2021-12-10T04:50:26.13988Z","iopub.status.idle":"2021-12-10T04:50:26.690683Z","shell.execute_reply.started":"2021-12-10T04:50:26.139833Z","shell.execute_reply":"2021-12-10T04:50:26.689807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_split_statistics(sub_sequence, n_split):\n    df = allocate_fold(sub_sequence, n_split)\n    df = df.groupby('fold_id').agg(\n    sum_cots=('sum_cots', 'sum'), duration=('duration', 'sum'))\n    df['mean_cots'] = df.sum_cots / df.duration\n    return df\n\n\ndeviations = {\"sum_cots\": [], \"duration\": [], \"mean_cots\": []}\nn_splits = np.arange(3, 11)\nfor i in n_splits:\n    data = calc_split_statistics(sub_sequence, i).std()\n    for key in data.keys():\n        deviations[key].append(data[key])\n\nfig, ax = plt.subplots(1, 3, figsize=(15, 4))\nfor i, key in enumerate(deviations.keys()):\n    ax[i].plot(n_splits, deviations[key], label=key)\n    ax[i].set_ylim(bottom=0)\n    ax[i].set_title(key)\n    ax[i].set_xlabel(\"n_splits\")\nplt.suptitle(\"Standard Deviation vs. #Splits\", fontsize=15)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:50:58.943793Z","iopub.execute_input":"2021-12-10T04:50:58.944127Z","iopub.status.idle":"2021-12-10T04:50:59.515892Z","shell.execute_reply.started":"2021-12-10T04:50:58.944093Z","shell.execute_reply":"2021-12-10T04:50:59.515043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.patches as mpatches\n\n\nfig, axes = plt.subplots(3, 1, figsize=(15, 8), sharex=True, sharey=True)\naxes = axes.ravel()\n\ndf = train.copy()\nmax_annotation = df[\"sum_cots\"].max()\nfor i, d in df.groupby([\"video_id\", \"sequence\"]):\n    video_id = d[\"video_id\"].values[0]\n    ax = axes[video_id]\n    d.set_index(\"video_frame\")[\"sum_cots\"].apply(\n        lambda x: x / max_annotation\n    ).plot(ax=ax, c=\"black\", linewidth=0.5)\n    ax.set_title(f\"Video ID: {video_id}\")\n\n\nn_split = 4\ndf = allocate_fold(sub_sequence, n_split)\noof_colors = [\"red\", \"blue\", \"green\", \"yellow\"]\n# visualize clippable interval\nfor (\n    annotated,\n    start_frame,\n    end_frame,\n    sum_cots,\n    mean_cots,\n    video_id,\n    duration,\n    sub_sequence_id,\n    fold_id,\n    ) in df.itertuples():\n    ax = axes[int(video_id)]\n    rect = mpatches.Rectangle(\n        (start_frame, 0), duration, 1, alpha=0.3, facecolor=oof_colors[int(fold_id)]\n    )\n    ax.add_patch(rect)\n\nfig.suptitle(\"Fold-Splitted Sub-Sequences Visualized\", fontsize=15)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:53:22.70594Z","iopub.execute_input":"2021-12-10T04:53:22.706219Z","iopub.status.idle":"2021-12-10T04:53:23.638356Z","shell.execute_reply.started":"2021-12-10T04:53:22.70619Z","shell.execute_reply":"2021-12-10T04:53:23.637398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concatenate sub_sequence table and train table\n\nn_split = 4\ndf = train.copy()\ndfs = []\nalloc_df = allocate_fold(sub_sequence, n_split)\nbb = alloc_df.copy()\nfor video_id, d in df.groupby(\"video_id\"):\n    a = d[\"video_frame\"].values\n    b = bb.query(\"video_id == @video_id\").drop(\"video_id\", axis=1)\n    sub_sequence_low = b[\"start_frame\"].values\n    sub_sequence_high = b[\"end_frame\"].values\n\n    i, j = np.where((a[:, None] >= sub_sequence_low) & (a[:, None] <= sub_sequence_high))\n    dfs.append(\n        pd.DataFrame(\n            np.column_stack([d.values[i], b.values[j]]),\n            columns=d.columns.append(b.columns),\n        )\n    )\n\ndf = pd.concat(dfs)\ndf = df.loc[:, ~df.columns.duplicated()] # remove duplicated columns\n\nfor column in alloc_df.columns:\n    if column != \"mean_cots\":\n        df[column] = df[column].astype(int)\n        \ndf.to_csv(\"train_metadata_ext.csv\", index=False)\ndf[:3]","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:54:22.359782Z","iopub.execute_input":"2021-12-10T04:54:22.36047Z","iopub.status.idle":"2021-12-10T04:54:22.465268Z","shell.execute_reply.started":"2021-12-10T04:54:22.360429Z","shell.execute_reply":"2021-12-10T04:54:22.464361Z"},"trusted":true},"execution_count":null,"outputs":[]}]}