{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:07:38.516203Z","iopub.execute_input":"2022-02-02T01:07:38.516931Z","iopub.status.idle":"2022-02-02T01:07:38.523267Z","shell.execute_reply.started":"2022-02-02T01:07:38.516884Z","shell.execute_reply":"2022-02-02T01:07:38.522134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/train.csv')\nsubmission_df = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:49:54.371909Z","iopub.execute_input":"2022-02-02T00:49:54.372234Z","iopub.status.idle":"2022-02-02T00:49:54.515110Z","shell.execute_reply.started":"2022-02-02T00:49:54.372199Z","shell.execute_reply":"2022-02-02T00:49:54.514422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:50:35.240861Z","iopub.execute_input":"2022-02-02T00:50:35.241230Z","iopub.status.idle":"2022-02-02T00:50:35.254901Z","shell.execute_reply.started":"2022-02-02T00:50:35.241186Z","shell.execute_reply":"2022-02-02T00:50:35.254191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:50:50.356029Z","iopub.execute_input":"2022-02-02T00:50:50.356368Z","iopub.status.idle":"2022-02-02T00:50:50.365791Z","shell.execute_reply.started":"2022-02-02T00:50:50.356333Z","shell.execute_reply":"2022-02-02T00:50:50.365213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Images in train index file: {train_df.image.nunique()}\")\nprint(f\"Species in train index file: {train_df.species.nunique()}\")\nprint(f\"Individual IDs in train index file: {train_df.individual_id.nunique()}\")\n\nprint(f\"Images in train images folder: {len(os.listdir('/kaggle/input/happy-whale-and-dolphin/train_images'))}\")\nprint(f\"Images in test images folder: {len(os.listdir('/kaggle/input/happy-whale-and-dolphin/test_images'))}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:51:16.755905Z","iopub.execute_input":"2022-02-02T00:51:16.756598Z","iopub.status.idle":"2022-02-02T00:51:16.838991Z","shell.execute_reply.started":"2022-02-02T00:51:16.756558Z","shell.execute_reply":"2022-02-02T00:51:16.838115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Top 10 individual_id\")\ntrain_df.individual_id.value_counts().head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:53:33.360524Z","iopub.execute_input":"2022-02-02T00:53:33.360855Z","iopub.status.idle":"2022-02-02T00:53:33.384848Z","shell.execute_reply.started":"2022-02-02T00:53:33.360823Z","shell.execute_reply":"2022-02-02T00:53:33.384233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfig, ax = plt.subplots(1, 1, figsize=(7, 7))\nsns.kdeplot(np.log(train_df.individual_id.value_counts()))\nplt.title(\"Logaritmic distribution of individual_id frequency in images\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:03:36.209205Z","iopub.execute_input":"2022-02-02T01:03:36.209613Z","iopub.status.idle":"2022-02-02T01:03:36.625899Z","shell.execute_reply.started":"2022-02-02T01:03:36.209565Z","shell.execute_reply":"2022-02-02T01:03:36.625029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train_df[\"species\"].value_counts()\ndf = pd.DataFrame({'Species': temp.index,\n                   'Images': temp.values\n                  })\nplt.figure(figsize = (12,6))\nplt.title('Species distribution - images per each species - train dataset')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Species', y=\"Images\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:54:44.363269Z","iopub.execute_input":"2022-02-02T00:54:44.363645Z","iopub.status.idle":"2022-02-02T00:54:44.861274Z","shell.execute_reply.started":"2022-02-02T00:54:44.363609Z","shell.execute_reply":"2022-02-02T00:54:44.860553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ここで、各種ごとにいくつの個別IDがあるかを確認\n\ntemp = train_df.groupby([\"species\"])[\"individual_id\"].nunique()\ndf = pd.DataFrame({'Species': temp.index,\n                   'Unique ID Count': temp.values\n                  })\ndf = df.sort_values(['Unique ID Count'], ascending=False)\nplt.figure(figsize = (12,6))\nplt.title('Species distribution - Individual IDs per each species - train dataset')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Species', y=\"Unique ID Count\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:56:40.474798Z","iopub.execute_input":"2022-02-02T00:56:40.475390Z","iopub.status.idle":"2022-02-02T00:56:41.166333Z","shell.execute_reply.started":"2022-02-02T00:56:40.475342Z","shell.execute_reply":"2022-02-02T00:56:41.165602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_df_list = list(train_df.image.unique())\ntrain_images_list = list(os.listdir('/kaggle/input/happy-whale-and-dolphin/train_images'))\ndelta = set(train_df_list) & set(train_images_list)\nminus = set(train_df_list) - set(train_images_list)\nprint(f\"Images in train dataset: {len(train_df_list)}\\nImages in train folder: {len(train_images_list)}\\nIntersection: {len(delta)}\\nDifference: {len(minus)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:00:33.690563Z","iopub.execute_input":"2022-02-02T01:00:33.690923Z","iopub.status.idle":"2022-02-02T01:00:33.770484Z","shell.execute_reply.started":"2022-02-02T01:00:33.690887Z","shell.execute_reply":"2022-02-02T01:00:33.769536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_image_sizes(file_name):\n    image = cv2.imread('/kaggle/input/happy-whale-and-dolphin/train_images/' + file_name)\n    return list(image.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:00:50.962172Z","iopub.execute_input":"2022-02-02T01:00:50.962696Z","iopub.status.idle":"2022-02-02T01:00:50.966413Z","shell.execute_reply.started":"2022-02-02T01:00:50.962644Z","shell.execute_reply":"2022-02-02T01:00:50.965834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = np.stack(train_df['image'].sample(1000).apply(read_image_sizes))\ndf = pd.DataFrame(m,columns=['w','h','c'])","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:00:59.222145Z","iopub.execute_input":"2022-02-02T01:00:59.222468Z","iopub.status.idle":"2022-02-02T01:02:16.598704Z","shell.execute_reply.started":"2022-02-02T01:00:59.222432Z","shell.execute_reply":"2022-02-02T01:02:16.597731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_samples(species):\n    root_path = \"/kaggle/input/happy-whale-and-dolphin/\"\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    df = train_df[train_df['species']==species].copy()\n    df.index = range(len(df.index))\n    \n    f, ax = plt.subplots(4, 4, figsize=(16,16))\n\n    for i in range(16):\n        file = df.loc[i, 'image']\n        species = df.loc[i, 'species']\n        identifier = df.loc[i, 'individual_id']\n        img = cv2.imread(root_path+'train_images/'+file)\n        ax[i//4, i%4].imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n        ax[i//4, i%4].set_title(identifier+\" (\"+species+\")\")\n        ax[i//4, i%4].axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:02:23.526900Z","iopub.execute_input":"2022-02-02T01:02:23.527230Z","iopub.status.idle":"2022-02-02T01:02:23.536596Z","shell.execute_reply.started":"2022-02-02T01:02:23.527183Z","shell.execute_reply":"2022-02-02T01:02:23.535946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_samples(\"bottlenose_dolphin\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:03:44.323018Z","iopub.execute_input":"2022-02-02T01:03:44.323346Z","iopub.status.idle":"2022-02-02T01:04:02.307217Z","shell.execute_reply.started":"2022-02-02T01:03:44.323309Z","shell.execute_reply":"2022-02-02T01:04:02.306130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rotate_values(x):\n    xcopy = x.split()\n    temp = xcopy[4]\n    xcopy[4] = xcopy[0]\n    xcopy[0] = temp\n    xcopy = \" \".join(xcopy)\n    return xcopy","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:06:11.940361Z","iopub.execute_input":"2022-02-02T01:06:11.940725Z","iopub.status.idle":"2022-02-02T01:06:11.946725Z","shell.execute_reply.started":"2022-02-02T01:06:11.940687Z","shell.execute_reply":"2022-02-02T01:06:11.945842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def new_indivisual_out(x):\n    xcopy = x.split()\n    return xcopy[4]","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:10:44.139096Z","iopub.execute_input":"2022-02-02T01:10:44.139968Z","iopub.status.idle":"2022-02-02T01:10:44.145704Z","shell.execute_reply.started":"2022-02-02T01:10:44.139914Z","shell.execute_reply":"2022-02-02T01:10:44.144706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[\"predictions\"] = submission_df[\"predictions\"].apply(lambda x: new_indivisual_out(x))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:10:44.960417Z","iopub.execute_input":"2022-02-02T01:10:44.960872Z","iopub.status.idle":"2022-02-02T01:10:44.990216Z","shell.execute_reply.started":"2022-02-02T01:10:44.960819Z","shell.execute_reply":"2022-02-02T01:10:44.989268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:10:47.128287Z","iopub.execute_input":"2022-02-02T01:10:47.128607Z","iopub.status.idle":"2022-02-02T01:10:47.137894Z","shell.execute_reply.started":"2022-02-02T01:10:47.128571Z","shell.execute_reply":"2022-02-02T01:10:47.137314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:11:23.587747Z","iopub.execute_input":"2022-02-02T01:11:23.588584Z","iopub.status.idle":"2022-02-02T01:11:23.669743Z","shell.execute_reply.started":"2022-02-02T01:11:23.588542Z","shell.execute_reply":"2022-02-02T01:11:23.668901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}