{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What this notebook does:\n\n* Comparing species in the BirdCLEF 2022 tarain dataset and the Pyle's checklist[1]","metadata":{}},{"cell_type":"code","source":"!pip install nb-black > /dev/null","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-24T12:55:17.823256Z","iopub.execute_input":"2022-03-24T12:55:17.823925Z","iopub.status.idle":"2022-03-24T12:55:30.856295Z","shell.execute_reply.started":"2022-03-24T12:55:17.823825Z","shell.execute_reply":"2022-03-24T12:55:30.855587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib_venn import venn2\n\nplt.style.use(\"ggplot\")\n%load_ext lab_black","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-24T12:55:30.859379Z","iopub.execute_input":"2022-03-24T12:55:30.859726Z","iopub.status.idle":"2022-03-24T12:55:32.180307Z","shell.execute_reply.started":"2022-03-24T12:55:30.859680Z","shell.execute_reply":"2022-03-24T12:55:32.179696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"taxonomy = pd.read_csv(\"../input/birdclef-2022/eBird_Taxonomy_v2021.csv\")\nprimary_list = pd.read_csv(\n    \"../input/the-birds-of-the-hawaiian-islands/primary_checklist.csv\"\n)\nprimary_tax = pd.read_csv(\n    \"../input/the-birds-of-the-hawaiian-islands/primary_checklist_taxonomy.csv\"\n)\ntrain = pd.read_csv(\n    \"../input/birdclef-2022-train-metadata-with-audio-metadata/train_ext.csv\"\n)\n\n# exclue extinct species\nextinct = [\"X\", \"x\"]\nprimary_tax = primary_tax.query(\"`HAWAIIAN ISLANDS` not in @extinct\").reset_index(\n    drop=True\n)\n\n# unique species\ntrain_species = train[\"primary_label\"].unique()\nprimary_species = primary_tax[\"SPECIES_CODE\"].unique()\nscored_species = train.query(\"is_scored == True\")[\"primary_label\"].unique()\n\n# taxonomy of species in train dataset\ntrain_tax = pd.merge(\n    pd.DataFrame({\"primary_label\": train_species}),\n    taxonomy,\n    left_on=\"primary_label\",\n    right_on=\"SPECIES_CODE\",\n).drop(\"primary_label\", axis=1)\ntrain_tax.to_csv(\"train_tax.csv\", index=False)\n\n# specify dataset type\nprimary_tax[\"dataset\"] = \"Pyle's checklist\"\ntrain_tax[\"dataset\"] = \"BirdCLEF 2022 train dataset\"\n\n# add eBird URL\nprimary_tax[\"EBIRD_URL\"] = \"https://ebird.org/species/\" + primary_tax[\"SPECIES_CODE\"]\ntrain_tax[\"EBIRD_URL\"] = \"https://ebird.org/species/\" + train_tax[\"SPECIES_CODE\"]\n\n# species are endemic or not\nendemic = primary_tax.query(\"'R' in `HAWAIIAN ISLANDS`\")[\"SPECIES_CODE\"].to_numpy()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-24T13:03:57.483379Z","iopub.execute_input":"2022-03-24T13:03:57.484166Z","iopub.status.idle":"2022-03-24T13:03:57.699782Z","shell.execute_reply.started":"2022-03-24T13:03:57.484116Z","shell.execute_reply":"2022-03-24T13:03:57.699042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"endemic.npy\", endemic)\nnp.load(\"endemic.npy\", allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T13:05:59.108454Z","iopub.execute_input":"2022-03-24T13:05:59.108785Z","iopub.status.idle":"2022-03-24T13:05:59.119479Z","shell.execute_reply.started":"2022-03-24T13:05:59.108720Z","shell.execute_reply":"2022-03-24T13:05:59.118821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Duplication of Species","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(figsize=(6, 6))\nsns.set_color_codes(\"pastel\")\n\ns1, s2 = set(train_species), set(primary_species)\nvenn2(\n    subsets=(s1, s2),\n    set_labels=(\"BirdCLEF '22\", \"Pyle's checklist\"),\n    ax=ax,\n)\nax.set_title(\"Duplication of Species\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-24T12:55:32.530859Z","iopub.execute_input":"2022-03-24T12:55:32.531215Z","iopub.status.idle":"2022-03-24T12:55:32.718560Z","shell.execute_reply.started":"2022-03-24T12:55:32.531187Z","shell.execute_reply":"2022-03-24T12:55:32.716981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* About 95% the species in BirdCLEF 2022 train dataset are encompassed to the Pyle's checklist.","metadata":{}},{"cell_type":"code","source":"s1, s2 = set(train_species), set(primary_species)\nnot_in_list = s1 - s2\n\nprint(f\"Species not in the Pyle's checklist:\")\ntrain_tax.query(\"SPECIES_CODE in @not_in_list\").drop(\n    [\"TAXON_ORDER\", \"CATEGORY\", \"SPECIES_GROUP\", \"REPORT_AS\"], axis=1\n)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-24T12:55:32.725834Z","iopub.execute_input":"2022-03-24T12:55:32.729971Z","iopub.status.idle":"2022-03-24T12:55:32.775785Z","shell.execute_reply.started":"2022-03-24T12:55:32.729887Z","shell.execute_reply":"2022-03-24T12:55:32.774938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* According to [eBird website](https://ebird.org), `Burrowing Parakeet`, `Golden Pheasant`, `Inca Tern`, `Northern Harrier`, and `Rosy-faced Lovebird` are observed in Hawaii. Since Pyle's checklist was edited in 2017, species that is recently observed in Hawaii islands are not in the list.","metadata":{}},{"cell_type":"code","source":"s1, s2 = set(train_species), set(primary_species)\nnot_in_list = s2 - s1\n\nprint(f\"Species not in the BirdCLEF 2022 train data:\")\next_species = primary_tax.query(\"SPECIES_CODE in @not_in_list\").drop(\n    [\"TAXON_ORDER\", \"CATEGORY\", \"SPECIES_GROUP\", \"REPORT_AS\"], axis=1\n)\next_species.to_csv(\"ext_species.csv\", index=False)\next_species","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-24T12:55:32.777699Z","iopub.execute_input":"2022-03-24T12:55:32.778556Z","iopub.status.idle":"2022-03-24T12:55:32.826271Z","shell.execute_reply.started":"2022-03-24T12:55:32.778511Z","shell.execute_reply":"2022-03-24T12:55:32.825593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* According to eBird website, at least some of these birds are obserbed in Hawaii. -> Why they are ommited in the BirdCLEFF 2022 dataset?","metadata":{}},{"cell_type":"markdown","source":"# Taxonomy","metadata":{}},{"cell_type":"code","source":"def plot_taxonomy(primary_tax, train_tax, key, ax):\n    united_tax = pd.concat([primary_tax, train_tax])\n    df = united_tax.groupby([key, \"dataset\"]).count()[\"SPECIES_CODE\"].reset_index()\n    sns.barplot(\n        y=key,\n        x=\"SPECIES_CODE\",\n        data=df,\n        ax=ax,\n        order=primary_tax[key].value_counts().index,\n        hue_order=[\"Pyle's checklist\", \"BirdCLEF 2022 train dataset\"],\n        hue=\"dataset\",\n        dodge=False,\n        alpha=0.8,\n    )\n    ax.legend(ncol=2, loc=\"lower right\", frameon=True)\n\n\n_, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 12))\nsns.set_color_codes(\"pastel\")\n\nplt.suptitle(\"Order & Family\", fontsize=18)\nax1.set_title(\"Bird Order\")\nax2.set_title(\"Bird Family\")\n\nplot_taxonomy(primary_tax, train_tax, \"ORDER1\", ax1)\nplot_taxonomy(primary_tax, train_tax, \"FAMILY\", ax2)\n\nplt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-24T12:55:32.827500Z","iopub.execute_input":"2022-03-24T12:55:32.827925Z","iopub.status.idle":"2022-03-24T12:55:35.367157Z","shell.execute_reply.started":"2022-03-24T12:55:32.827893Z","shell.execute_reply":"2022-03-24T12:55:35.366525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* There is no bias toward any particular orders or families.","metadata":{}},{"cell_type":"markdown","source":"# Discussion\n\n* I assume species in BirdCLEF 2022 dataset are subset of that in Pyle's checklist (except for recently observed species).\n* At least at this time, no rules of thumb could be found for the species selected and excluded from the BirdCLEF 2022 dataset.","metadata":{}},{"cell_type":"markdown","source":"# Reference\n\n[1] Robert L. Pyle and Peter Pyle, \"The Birds of the Hawaiian Islands: Occurrence, History, Distribution, and Status Version 2 - 1 January 2017\", http://hbs.bishopmuseum.org/birds/rlp-monograph/PrimaryChecklist.htm","metadata":{}}]}