{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# はじめに\n\n\nこのKernelは、**Happywhale - Whale and Dolphin Identification competition** のデータセットを調査します。\n\n<img src=\"https://images.unsplash.com/photo-1570913179118-f3d24be1d1f7?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2143&q=80\" width=500></img>","metadata":{}},{"cell_type":"markdown","source":"# 分析の準備\n\nデータをロードして、事前調査をしてみましょう。\n\n<img src=\"https://images.unsplash.com/photo-1568430328012-21ed450453ea?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=1174&q=80\" width=500></img>","metadata":{}},{"cell_type":"code","source":"!pip install imagesize","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:54:55.11848Z","iopub.execute_input":"2022-02-05T17:54:55.118819Z","iopub.status.idle":"2022-02-05T17:55:03.947482Z","shell.execute_reply.started":"2022-02-05T17:54:55.118781Z","shell.execute_reply":"2022-02-05T17:55:03.946355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nimport imagesize","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:55:03.949733Z","iopub.execute_input":"2022-02-05T17:55:03.950078Z","iopub.status.idle":"2022-02-05T17:55:03.955587Z","shell.execute_reply.started":"2022-02-05T17:55:03.950034Z","shell.execute_reply":"2022-02-05T17:55:03.954899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Files and folders: {os.listdir('/kaggle/input/happy-whale-and-dolphin')}\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:51:54.728302Z","iopub.execute_input":"2022-02-05T17:51:54.729892Z","iopub.status.idle":"2022-02-05T17:51:54.738396Z","shell.execute_reply.started":"2022-02-05T17:51:54.729841Z","shell.execute_reply":"2022-02-05T17:51:54.737586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"まず、`train.csv` と `sample_submission.csv` をロードしてみましょう。","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/train.csv')\nsubmission_df = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/sample_submission.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:51:54.73997Z","iopub.execute_input":"2022-02-05T17:51:54.740204Z","iopub.status.idle":"2022-02-05T17:51:54.917374Z","shell.execute_reply.started":"2022-02-05T17:51:54.740173Z","shell.execute_reply":"2022-02-05T17:51:54.916594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:51:54.919217Z","iopub.execute_input":"2022-02-05T17:51:54.919661Z","iopub.status.idle":"2022-02-05T17:51:54.939216Z","shell.execute_reply.started":"2022-02-05T17:51:54.919624Z","shell.execute_reply":"2022-02-05T17:51:54.938614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:51:54.940234Z","iopub.execute_input":"2022-02-05T17:51:54.940618Z","iopub.status.idle":"2022-02-05T17:51:54.949441Z","shell.execute_reply.started":"2022-02-05T17:51:54.940587Z","shell.execute_reply":"2022-02-05T17:51:54.948666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データ探索\n\n<img src=\"https://images.unsplash.com/photo-1611890129309-31e797820019?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=1170&q=80\" width=500></img>","metadata":{}},{"cell_type":"markdown","source":"訓練データと訓練画像、テスト画像について、もう少し詳しく見てみましょう。","metadata":{}},{"cell_type":"code","source":"print(f\"Images in train index file: {train_df.image.nunique()}\")\nprint(f\"Species in train index file: {train_df.species.nunique()}\")\nprint(f\"Individual IDs in train index file: {train_df.individual_id.nunique()}\")\n\nprint(f\"Images in train images folder: {len(os.listdir('/kaggle/input/happy-whale-and-dolphin/train_images'))}\")\nprint(f\"Images in test images folder: {len(os.listdir('/kaggle/input/happy-whale-and-dolphin/test_images'))}\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:51:54.950716Z","iopub.execute_input":"2022-02-05T17:51:54.951248Z","iopub.status.idle":"2022-02-05T17:51:56.285665Z","shell.execute_reply.started":"2022-02-05T17:51:54.951203Z","shell.execute_reply":"2022-02-05T17:51:56.284731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"全種類に目を向けてみよう。","metadata":{}},{"cell_type":"code","source":"print(f\"Species: {train_df.species.unique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-05T18:04:09.330967Z","iopub.execute_input":"2022-02-05T18:04:09.331772Z","iopub.status.idle":"2022-02-05T18:04:09.344486Z","shell.execute_reply.started":"2022-02-05T18:04:09.331712Z","shell.execute_reply":"2022-02-05T18:04:09.343473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"このコンペの掲示板や他のノートブック（ex: [Happywhale: Data Distribution](https://www.kaggle.com/awsaf49/happywhale-data-distribution/notebook)）での議論から、次のようなことがわかります。\n* シロイルカとグロビスはクジラである。 \n* イルカとクジラの区別がつくので、ベルーガとグロバスの名前を変更する。 \nまた、次のことも観察されます。\n* バンドウイルカはタイプミス（dolpin）です。\n* 殺人鯨は`kiler`と間違って入力されています。\n","metadata":{}},{"cell_type":"code","source":"train_df.loc[train_df.species.str.contains('beluga'), 'species'] = 'beluga_whale'\ntrain_df.loc[train_df.species.str.contains('globis'), 'species'] = 'globis_whale'","metadata":{"execution":{"iopub.status.busy":"2022-02-05T18:11:01.226426Z","iopub.execute_input":"2022-02-05T18:11:01.226765Z","iopub.status.idle":"2022-02-05T18:11:01.299569Z","shell.execute_reply.started":"2022-02-05T18:11:01.22673Z","shell.execute_reply":"2022-02-05T18:11:01.298795Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['class'] = train_df.species.map(lambda x: 'whale' if 'whale' in x else 'dolphin')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:11:02.755382Z","iopub.execute_input":"2022-02-05T18:11:02.755708Z","iopub.status.idle":"2022-02-05T18:11:02.77813Z","shell.execute_reply.started":"2022-02-05T18:11:02.755665Z","shell.execute_reply":"2022-02-05T18:11:02.777092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['species'] = train_df['species'].str.replace('bottlenose_dolpin','bottlenose_dolphin')\ntrain_df['species'] = train_df['species'].str.replace('kiler_whale','killer_whale')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:11:04.659669Z","iopub.execute_input":"2022-02-05T18:11:04.659962Z","iopub.status.idle":"2022-02-05T18:11:04.752916Z","shell.execute_reply.started":"2022-02-05T18:11:04.659933Z","shell.execute_reply":"2022-02-05T18:11:04.752103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"イルカVSクジラって何種類あるのか確認してみよう。","metadata":{}},{"cell_type":"code","source":"temp = train_df.groupby([\"class\"])[\"species\"].nunique()\ndf = pd.DataFrame({'Classes': temp.index,\n                   'Species': temp.values\n                  })\ndf = df.sort_values(['Species'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title('Species distribution - grouped on Dolphins and Whales - train dataset')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Classes', y=\"Species\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T18:31:22.669923Z","iopub.execute_input":"2022-02-05T18:31:22.670223Z","iopub.status.idle":"2022-02-05T18:31:22.867923Z","shell.execute_reply.started":"2022-02-05T18:31:22.67019Z","shell.execute_reply":"2022-02-05T18:31:22.866867Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`train_df` の値分布から `individual_id` というカラムの詳細を確認してみましょう。","metadata":{}},{"cell_type":"code","source":"print(\"Top 10 individual_id\")\ntrain_df.individual_id.value_counts().head(10)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:11:07.952927Z","iopub.execute_input":"2022-02-05T18:11:07.953787Z","iopub.status.idle":"2022-02-05T18:11:07.979819Z","shell.execute_reply.started":"2022-02-05T18:11:07.953744Z","shell.execute_reply":"2022-02-05T18:11:07.978922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(7, 7))\nsns.kdeplot(np.log(train_df.individual_id.value_counts()))\nplt.title(\"Logaritmic distribution of individual_id frequency in images\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:11:15.866371Z","iopub.execute_input":"2022-02-05T18:11:15.867376Z","iopub.status.idle":"2022-02-05T18:11:16.310517Z","shell.execute_reply.started":"2022-02-05T18:11:15.867321Z","shell.execute_reply":"2022-02-05T18:11:16.309489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"また、イルカとクジラは分けて考えましょう。","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(7, 7))\nsns.kdeplot(np.log(train_df.loc[train_df[\"class\"]=='whale'].individual_id.value_counts()))\nsns.kdeplot(np.log(train_df.loc[train_df[\"class\"]=='dolphin'].individual_id.value_counts()))\nax.legend(labels=['whale', 'dolphin'])\nplt.title(\"Logaritmic distribution of individual_id frequency in images\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T18:20:10.636472Z","iopub.execute_input":"2022-02-05T18:20:10.637257Z","iopub.status.idle":"2022-02-05T18:20:11.027603Z","shell.execute_reply.started":"2022-02-05T18:20:10.637214Z","shell.execute_reply":"2022-02-05T18:20:11.026706Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"訓練データセットに含まれる種の頻度も確認してみましょう。","metadata":{}},{"cell_type":"code","source":"df = train_df.groupby([\"class\", \"species\"])[\"image\"].count().reset_index()\ndf.columns = [\"Class\", \"Species\", \"Images\"]\ndf = df.sort_values(['Images'], ascending=False)\nplt.figure(figsize = (12,6))\nplt.title('Species distribution - images per each species - train dataset')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Species', y=\"Images\", hue='Class', data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:33:09.599577Z","iopub.execute_input":"2022-02-05T18:33:09.599884Z","iopub.status.idle":"2022-02-05T18:33:10.37895Z","shell.execute_reply.started":"2022-02-05T18:33:09.59985Z","shell.execute_reply":"2022-02-05T18:33:10.378104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"では、それぞれの種にいくつの個体IDがあるのか見てみましょう。","metadata":{}},{"cell_type":"code","source":"df = train_df.groupby([\"class\", \"species\"])[\"individual_id\"].nunique().reset_index()\ndf.columns = [\"Class\", \"Species\", \"Unique ID Count\"]\ndf = df.sort_values([\"Unique ID Count\"], ascending=False)\n\ndf = df.sort_values(['Unique ID Count'], ascending=False)\nplt.figure(figsize = (12,6))\nplt.title('Species distribution - Individual IDs per each species - train dataset')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Species', y=\"Unique ID Count\", hue='Class', data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:36:27.842838Z","iopub.execute_input":"2022-02-05T18:36:27.843145Z","iopub.status.idle":"2022-02-05T18:36:28.639Z","shell.execute_reply.started":"2022-02-05T18:36:27.843116Z","shell.execute_reply":"2022-02-05T18:36:28.638081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"それでは、トレーニング画像とテスト画像のデータセットで、画像サイズを確認してみましょう。","metadata":{}},{"cell_type":"markdown","source":"`train_df` にリストされたイメージのセットが、 `train_images` フォルダにあるイメージのセットと同じかどうか確認してみましょう。","metadata":{}},{"cell_type":"code","source":"train_df_list = list(train_df.image.unique())\ntrain_images_list = list(os.listdir('/kaggle/input/happy-whale-and-dolphin/train_images'))\ndelta = set(train_df_list) & set(train_images_list)\nminus = set(train_df_list) - set(train_images_list)\nprint(f\"Images in train dataset: {len(train_df_list)}\\nImages in train folder: {len(train_images_list)}\\nIntersection: {len(delta)}\\nDifference: {len(minus)}\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:36:41.387393Z","iopub.execute_input":"2022-02-05T18:36:41.387729Z","iopub.status.idle":"2022-02-05T18:36:41.466728Z","shell.execute_reply.started":"2022-02-05T18:36:41.387687Z","shell.execute_reply":"2022-02-05T18:36:41.465548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`train_df` にインデックスされたすべての画像は images フォルダに存在し、その逆も同様です。","metadata":{}},{"cell_type":"markdown","source":"# 画像データ探索","metadata":{}},{"cell_type":"markdown","source":"まず、どちらの関数（cv2ベースとimagesizeベース）が速く実行されるかをテストします。","metadata":{}},{"cell_type":"code","source":"# image size using cv2 imread shape\ndef read_image_sizes_cv2(file_name):\n    image = cv2.imread('/kaggle/input/happy-whale-and-dolphin/train_images/' + file_name)\n    return list(image.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T17:57:08.709591Z","iopub.execute_input":"2022-02-05T17:57:08.70989Z","iopub.status.idle":"2022-02-05T17:57:08.715433Z","shell.execute_reply.started":"2022-02-05T17:57:08.709861Z","shell.execute_reply":"2022-02-05T17:57:08.714728Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image size using imagesize\ndef get_image_sizes_imagesize(file_name):\n    width, height = imagesize.get('/kaggle/input/happy-whale-and-dolphin/train_images/' + file_name)\n    return [width, height]","metadata":{"execution":{"iopub.status.busy":"2022-02-05T17:57:10.480848Z","iopub.execute_input":"2022-02-05T17:57:10.481417Z","iopub.status.idle":"2022-02-05T17:57:10.485825Z","shell.execute_reply.started":"2022-02-05T17:57:10.481382Z","shell.execute_reply":"2022-02-05T17:57:10.485037Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nsample_size = 100\nstart_time = time.time()\ntrain_sample_df = train_df.sample(sample_size)\nm = np.stack(train_sample_df['image'].apply(read_image_sizes_cv2))\ndf = pd.DataFrame(m,columns=['w','h','c'])\nprint(f\"Total processing time for {sample_size} images (using cv2): {round(time.time()-start_time, 2)} sec.\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T17:58:39.476121Z","iopub.execute_input":"2022-02-05T17:58:39.476713Z","iopub.status.idle":"2022-02-05T17:58:46.488188Z","shell.execute_reply.started":"2022-02-05T17:58:39.476665Z","shell.execute_reply":"2022-02-05T17:58:46.48735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nsample_size = 100\nstart_time = time.time()\ntrain_sample_df = train_df.sample(sample_size)\nm = np.stack(train_sample_df['image'].apply(get_image_sizes_imagesize))\ndf = pd.DataFrame(m,columns=['w','h'])\nprint(f\"Total processing time for {sample_size} images (using imagesize): {round(time.time()-start_time, 2)} sec.\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:00:45.114108Z","iopub.execute_input":"2022-02-05T18:00:45.114421Z","iopub.status.idle":"2022-02-05T18:00:47.990429Z","shell.execute_reply.started":"2022-02-05T18:00:45.11439Z","shell.execute_reply":"2022-02-05T18:00:47.989532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"この関数は、より効果的であるため、イメージサイズベースの関数を使用することにしました。\n2500サンプルで実行する。","metadata":{}},{"cell_type":"code","source":"import time\nsample_size = 2500\nstart_time = time.time()\ntrain_sample_df = train_df.sample(sample_size)\nm = np.stack(train_sample_df['image'].apply(get_image_sizes_imagesize))\ndf = pd.DataFrame(m,columns=['w','h'])\nprint(f\"Total processing time for {sample_size} images (using imagesize): {round(time.time()-start_time, 2)} sec.\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:37:23.005927Z","iopub.execute_input":"2022-02-05T18:37:23.006213Z","iopub.status.idle":"2022-02-05T18:37:51.790369Z","shell.execute_reply.started":"2022-02-05T18:37:23.006184Z","shell.execute_reply":"2022-02-05T18:37:51.78935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img_df = pd.concat([train_sample_df, df], axis=1, sort=False)\nprint(f\"Number of different image size ( images samples): {train_img_df.groupby(['w','h']).count().shape[0]}\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:38:09.29815Z","iopub.execute_input":"2022-02-05T18:38:09.298454Z","iopub.status.idle":"2022-02-05T18:38:09.312817Z","shell.execute_reply.started":"2022-02-05T18:38:09.298422Z","shell.execute_reply":"2022-02-05T18:38:09.312127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"画像サイズが多いようです（全画像数の5％弱しかサンプリングしていません）。\n\n種ごとの幅・高さ、色の分布を可視化してみましょう。","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (12,6))\nplt.title('Species distribution - width per each species - train dataset (5% random data sample)')\nsns.set_color_codes(\"pastel\")\ns = sns.boxplot(x = 'species', y=\"w\", data=train_img_df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:38:12.133731Z","iopub.execute_input":"2022-02-05T18:38:12.134541Z","iopub.status.idle":"2022-02-05T18:38:12.715434Z","shell.execute_reply.started":"2022-02-05T18:38:12.134488Z","shell.execute_reply":"2022-02-05T18:38:12.714576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12,6))\nplt.title('Species distribution - height per each species - train dataset (5% random data sample)')\nsns.set_color_codes(\"pastel\")\ns = sns.boxplot(x = 'species', y=\"h\", data=train_img_df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:38:19.009158Z","iopub.execute_input":"2022-02-05T18:38:19.009531Z","iopub.status.idle":"2022-02-05T18:38:19.601575Z","shell.execute_reply.started":"2022-02-05T18:38:19.009476Z","shell.execute_reply":"2022-02-05T18:38:19.600536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"種ごとの幅と高さの分布を散布図を使って示してみよう。","metadata":{}},{"cell_type":"code","source":"def plot_species_scatter(train_img_df):\n    i = 0\n    sns.set_style('whitegrid')\n    plt.figure()\n    species = list(train_img_df.species.unique())\n    fig, ax = plt.subplots(5, 5,figsize=(15, 12))\n\n    for spec in species:\n        i += 1\n        plt.subplot(5, 5,i)\n        df = train_img_df.loc[train_img_df.species==spec]\n        plt.scatter(df['w'], df['h'], marker='+')\n        plt.xlabel(spec, fontsize=9)\n    plt.show();\nplot_species_scatter(train_img_df.dropna())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:38:26.852327Z","iopub.execute_input":"2022-02-05T18:38:26.852981Z","iopub.status.idle":"2022-02-05T18:38:30.274626Z","shell.execute_reply.started":"2022-02-05T18:38:26.85293Z","shell.execute_reply":"2022-02-05T18:38:30.273736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5%のランダムサンプルでは、色数は常に3色であるようです。","metadata":{}},{"cell_type":"markdown","source":"種族ごとにグループ化された訓練画像のいくつかをサンプルにしてみましょう。 \n\nまず、プロット関数を作成します。","metadata":{}},{"cell_type":"code","source":"def plot_image_samples(species):\n    root_path = \"/kaggle/input/happy-whale-and-dolphin/\"\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    images_folder=\"train_images/\"\n    df = train_df[train_df['species']==species].copy()\n    df.index = range(len(df.index))\n\n    f, ax = plt.subplots(4, 4, figsize=(16,16))\n\n    for i in range(16):\n        file = df.loc[i, 'image']\n        species = df.loc[i, 'species']\n        identifier = df.loc[i, 'individual_id']\n        img = cv2.imread(root_path+images_folder+file)\n        ax[i//4, i%4].imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n        ax[i//4, i%4].set_title(identifier+\" (\"+species+\")\")\n        ax[i//4, i%4].axis('off')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:38:34.108987Z","iopub.execute_input":"2022-02-05T18:38:34.109303Z","iopub.status.idle":"2022-02-05T18:38:34.118824Z","shell.execute_reply.started":"2022-02-05T18:38:34.109266Z","shell.execute_reply":"2022-02-05T18:38:34.117818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"bottlenose_dolphin\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:38:43.176089Z","iopub.execute_input":"2022-02-05T18:38:43.176387Z","iopub.status.idle":"2022-02-05T18:39:00.958551Z","shell.execute_reply.started":"2022-02-05T18:38:43.176356Z","shell.execute_reply":"2022-02-05T18:39:00.957843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"beluga_whale\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:39:08.864527Z","iopub.execute_input":"2022-02-05T18:39:08.864831Z","iopub.status.idle":"2022-02-05T18:39:13.762249Z","shell.execute_reply.started":"2022-02-05T18:39:08.864799Z","shell.execute_reply":"2022-02-05T18:39:13.758635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"humpback_whale\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:39:54.045303Z","iopub.execute_input":"2022-02-05T18:39:54.04556Z","iopub.status.idle":"2022-02-05T18:40:08.005194Z","shell.execute_reply.started":"2022-02-05T18:39:54.045525Z","shell.execute_reply":"2022-02-05T18:40:08.004546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"blue_whale\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:39:47.846138Z","iopub.execute_input":"2022-02-05T18:39:47.846384Z","iopub.status.idle":"2022-02-05T18:39:54.043331Z","shell.execute_reply.started":"2022-02-05T18:39:47.846351Z","shell.execute_reply":"2022-02-05T18:39:54.042385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"killer_whale\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:39:29.814669Z","iopub.execute_input":"2022-02-05T18:39:29.815043Z","iopub.status.idle":"2022-02-05T18:39:47.844963Z","shell.execute_reply.started":"2022-02-05T18:39:29.815002Z","shell.execute_reply":"2022-02-05T18:39:47.844048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"spotted_dolphin\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:39:21.596812Z","iopub.execute_input":"2022-02-05T18:39:21.597089Z","iopub.status.idle":"2022-02-05T18:39:29.812803Z","shell.execute_reply.started":"2022-02-05T18:39:21.59706Z","shell.execute_reply":"2022-02-05T18:39:29.811872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"テスト画像のサンプルも見てみましょう。","metadata":{}},{"cell_type":"code","source":"def plot_image_samples_test():\n    root_path = \"/kaggle/input/happy-whale-and-dolphin/\"\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    images_folder=\"test_images/\"\n\n    f, ax = plt.subplots(4, 4, figsize=(16,16))\n    file_list = list(os.listdir(root_path+images_folder))\n    for i in range(16):\n        file = file_list[i]\n        img = cv2.imread(root_path+images_folder+file)\n        ax[i//4, i%4].imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n        ax[i//4, i%4].set_title(\"Test image: \"+file)\n        ax[i//4, i%4].axis('off')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:40:08.00633Z","iopub.execute_input":"2022-02-05T18:40:08.007014Z","iopub.status.idle":"2022-02-05T18:40:08.015287Z","shell.execute_reply.started":"2022-02-05T18:40:08.006977Z","shell.execute_reply":"2022-02-05T18:40:08.014315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples_test()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:40:08.016951Z","iopub.execute_input":"2022-02-05T18:40:08.017262Z","iopub.status.idle":"2022-02-05T18:40:20.883298Z","shell.execute_reply.started":"2022-02-05T18:40:08.01723Z","shell.execute_reply":"2022-02-05T18:40:20.87943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 提出\n\n<img src=\"https://images.unsplash.com/photo-1602264985195-52b338cb937b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=1170&q=80\" width=500></img>\n\n識別子を回転させて、`new_individual`が最初の選択肢になるようにしましょう。","metadata":{}},{"cell_type":"code","source":"def rotate_values(x):\n    xcopy = x.split()\n    temp = xcopy[4]\n    xcopy[4] = xcopy[0]\n    xcopy[0] = temp\n    xcopy = \" \".join(xcopy)\n    return xcopy","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:41:22.489768Z","iopub.execute_input":"2022-02-05T18:41:22.490093Z","iopub.status.idle":"2022-02-05T18:41:22.495768Z","shell.execute_reply.started":"2022-02-05T18:41:22.490054Z","shell.execute_reply":"2022-02-05T18:41:22.494762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[\"predictions\"] = submission_df[\"predictions\"].apply(lambda x: rotate_values(x))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:41:24.1685Z","iopub.execute_input":"2022-02-05T18:41:24.168876Z","iopub.status.idle":"2022-02-05T18:41:24.209425Z","shell.execute_reply.started":"2022-02-05T18:41:24.168842Z","shell.execute_reply":"2022-02-05T18:41:24.208524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:41:25.820827Z","iopub.execute_input":"2022-02-05T18:41:25.821124Z","iopub.status.idle":"2022-02-05T18:41:25.831806Z","shell.execute_reply.started":"2022-02-05T18:41:25.821092Z","shell.execute_reply":"2022-02-05T18:41:25.831076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"用意した投稿ファイルを出力します。","metadata":{}},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-05T18:41:30.33679Z","iopub.execute_input":"2022-02-05T18:41:30.337812Z","iopub.status.idle":"2022-02-05T18:41:30.5028Z","shell.execute_reply.started":"2022-02-05T18:41:30.337769Z","shell.execute_reply":"2022-02-05T18:41:30.501929Z"},"trusted":true},"execution_count":null,"outputs":[]}]}