{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nimport numpy as np\nimport random,os\nimport torch\nimport pandas as pd\nfrom pathlib import Path\nimport librosa\nfrom matplotlib import pyplot as plt\nimport IPython\n\nfrom PIL import Image\nimport ipywidgets as widgets\nfrom ipywidgets import Layout\nfrom matplotlib import pyplot as plt\n\nfrom glob import glob\nfrom IPython.display import display\n\n#Deep learning from pytorch\nimport torch\nimport torchvision\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.optim as optim\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport requests","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-20T11:12:26.23797Z","iopub.execute_input":"2022-05-20T11:12:26.238404Z","iopub.status.idle":"2022-05-20T11:12:30.931974Z","shell.execute_reply.started":"2022-05-20T11:12:26.23829Z","shell.execute_reply":"2022-05-20T11:12:30.930592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\njson_open = open('/kaggle/input/birdclef-2022/scored_birds.json', 'r')\nbird_target = json.load(json_open)\nbird_target\n\ntrain = pd.read_csv(\"../input/fulllabeldataset/train.csv\",index_col=0)\n#train = train[train.num <= 3].reset_index(drop=True)\ntrain[\"filepath\"] = \"../input/fulllabeldataset/image_128/\" + train[\"path\"]\ntrain[\"sort_index\"] = train.filename_id.apply(lambda x: int(x.split(\"_\")[-1])//5)\ntrain[\"file_id\"] = train.filename_id.apply(lambda x: x.split(\"_\")[0])\ntrain['secondary_labels_array'] =  train.secondary_labels.apply(eval)\ntrain[\"sleng\"] = train.secondary_labels_array.apply(lambda x: len(x))\nunique_key = sorted(bird_target)\nLABEL_IDS = {label: label_id for label_id, label in enumerate(unique_key)}\nINV_LABEL_IDS = {val: key for key,val in LABEL_IDS.items()}\nprint(LABEL_IDS)\ntrain[\"label_id\"] = -1\ntrain.loc[train.primary_label.isin(bird_target),\"label_id\"] = train.loc[train.primary_label.isin(bird_target),\"primary_label\"].map(LABEL_IDS)\ndf_nolabel = train[train.sleng == 0]\ndf_nolabel['label_array'] =  df_nolabel.label_id.apply(lambda x: np.array([x]))\ndf_nolabel['num'] = 0\ndf_nolabel[\"sec1\"] = -1\ndf_nolabel[\"sec2\"] = -1\ndf_nolabel[\"sec3\"] = -1\n\ndf_label = train[train.sleng > 0]\ndf_label['labels_id'] = df_label.secondary_labels_array.apply(lambda x: np.vectorize(lambda s: LABEL_IDS[s] if s in unique_key else -1)(x))\ndf_label.labels_id = df_label.labels_id.apply(lambda x: x[x != -1])\ndf_label['num'] = df_label.labels_id.apply(lambda x: len(x))\ndf_label[\"sec1\"] = df_label.labels_id.apply(lambda x: x[0] if len(x) > 0 else -1)\ndf_label[\"sec2\"] = df_label.labels_id.apply(lambda x: x[1] if len(x) > 1 else -1)\ndf_label[\"sec3\"] = df_label.labels_id.apply(lambda x: x[2] if len(x) > 2 else -1)\n\ntrain = pd.concat([df_label,df_nolabel]).sort_values([\"file_id\",\"sort_index\"]).reset_index(drop=True)\n\ntrain.loc[train.label_id == train.sec1,\"sec1\"] = -1\ntrain.loc[train.label_id == train.sec2,\"sec2\"] = -1\ntrain.loc[train.label_id == train.sec3,\"sec3\"] = -1\n\n#train[\"label\"] = train[[\"sec1\",\"sec2\",\"sec3\"]].apply(lambda x: np.array([x.sec1,x.sec2,x.sec3],dtype=np.int32),axis=1)\n#train[\"label\"] = train.label.apply(lambda x: x[x != -1])\n#train[\"num\"] = train.labels_id.apply(lambda x: len(x))\n\n\ntrain = train[~((train.label_id == -1)&(train.num==0))].reset_index(drop=True)\ntrain.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-05-20T11:12:30.934348Z","iopub.execute_input":"2022-05-20T11:12:30.934634Z","iopub.status.idle":"2022-05-20T11:12:31.786637Z","shell.execute_reply.started":"2022-05-20T11:12:30.934595Z","shell.execute_reply":"2022-05-20T11:12:31.785692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"furudf = pd.read_csv(\"../input/annotationdataset-furu/results.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_hand\"})\nfurudfnum1 = pd.read_csv(\"../input/annotationdataset-furu/results_num1_primary.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_hand\"})\nishiidf = pd.read_csv(\"../input/annotationdataset-ishii/results.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_hand\"})\ntadokorodf = pd.read_csv(\"../input/annotationdataset-tadokoro/results.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_hand\"})\ntakemidf = pd.read_csv(\"../input/annotationdataset-takemi/results.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_hand\"})\nhandlabeldf = pd.concat([furudf,furudfnum1,ishiidf,tadokorodf,takemidf]).drop_duplicates(\"filename_id\")\ntrain = pd.merge(train,handlabeldf,on=[\"filename_id\"],how=\"left\").fillna(0)\n\nfurudf_sec = pd.read_csv(\"../input/annotationdataset-furusec/results_sec1.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec1\"})\nishiidf1_sec = pd.read_csv(\"../input/annotationdatasetishiisec/results_sec1.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec1\"})\nishiidf2_sec = pd.read_csv(\"../input/annotationdatasetishiisec/results_sec2.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec1\"})\nishiidf3_sec = pd.read_csv(\"../input/annotationdatasetishiisec/results_sec3_1.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec1\"})\ntakemidf_sec = pd.read_csv(\"../input/annotationdataset-takemisec/results_sec1.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec1\"})\nhandlabeldf_sec1 = pd.concat([furudf_sec,ishiidf1_sec,ishiidf2_sec,ishiidf3_sec,takemidf_sec]).drop_duplicates(\"filename_id\")\ntrain = pd.merge(train,handlabeldf_sec1,on=[\"filename_id\"],how=\"left\").fillna(0)\n\nishiidf1_sec2 = pd.read_csv(\"../input/annotationdatasetishiisec/results_sec2.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec2\"})\nishiidf2_sec2 = pd.read_csv(\"../input/annotationdatasetishiisec/results_sec3_2.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec2\"})\nhandlabeldf_sec2 = pd.concat([ishiidf1_sec2,ishiidf2_sec2]).drop_duplicates(\"filename_id\")\ntrain = pd.merge(train,handlabeldf_sec2,on=[\"filename_id\"],how=\"left\").fillna(0)\n\nishiidf_sec3 = pd.read_csv(\"../input/annotationdatasetishiisec/results_sec3_3.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec3\"})\ntakemidf_sec3 = pd.read_csv(\"../input/annotationdataset-takemisec2/results_sec3.txt\",header=None).rename(columns={0:\"filename_id\",1:\"primary_label_sec3\"})\nhandlabeldf_sec3 = pd.concat([ishiidf_sec3,takemidf_sec3]).drop_duplicates(\"filename_id\")\ntrain = pd.merge(train,handlabeldf_sec3,on=[\"filename_id\"],how=\"left\").fillna(0)\n\n#skylar houfinに関しては、確率低いもののみハンドラベリング済みなので、高いものは全てTrueにする\ntrain.loc[(train.primary_label.isin([\"skylar\"]))&(train.num==0),\"primary_label_hand\"] = \"skylar\"\ntrain.loc[(train.primary_label.isin([\"houfin\"]))&(train.num==0),\"primary_label_hand\"] = \"houfin\"\ntrain.loc[train.primary_label_hand==0,\"primary_label_hand\"] = \"nolabel\"\ntrain.loc[(train.primary_label_sec1==0)&(train.sec1 == -1),\"primary_label_sec1\"] = \"nocall\"\ntrain.loc[train.primary_label_sec1==0,\"primary_label_sec1\"] = \"nolabel\"\ntrain.loc[(train.primary_label_sec2==0)&(train.sec2 == -1),\"primary_label_sec2\"] = \"nocall\"\ntrain.loc[train.primary_label_sec2==0,\"primary_label_sec2\"] = \"nolabel\"\ntrain.loc[(train.primary_label_sec3==0)&(train.sec3 == -1),\"primary_label_sec3\"] = \"nocall\"\ntrain.loc[train.primary_label_sec3==0,\"primary_label_sec3\"] = \"nolabel\"\n\ntrain.to_csv(\"handlabeltrain.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T12:24:11.046063Z","iopub.execute_input":"2022-05-15T12:24:11.046329Z","iopub.status.idle":"2022-05-15T12:24:11.509276Z","shell.execute_reply.started":"2022-05-15T12:24:11.0463Z","shell.execute_reply":"2022-05-15T12:24:11.508264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-05-15T12:24:11.511041Z","iopub.execute_input":"2022-05-15T12:24:11.511311Z","iopub.status.idle":"2022-05-15T12:24:11.565076Z","shell.execute_reply.started":"2022-05-15T12:24:11.511279Z","shell.execute_reply":"2022-05-15T12:24:11.564286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-15T12:24:11.789762Z","iopub.execute_input":"2022-05-15T12:24:11.790499Z","iopub.status.idle":"2022-05-15T12:24:11.834468Z","shell.execute_reply.started":"2022-05-15T12:24:11.790456Z","shell.execute_reply":"2022-05-15T12:24:11.833436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-12T11:37:36.008378Z","iopub.execute_input":"2022-05-12T11:37:36.009184Z","iopub.status.idle":"2022-05-12T11:37:37.465078Z","shell.execute_reply.started":"2022-05-12T11:37:36.009119Z","shell.execute_reply":"2022-05-12T11:37:37.464068Z"},"trusted":true},"execution_count":null,"outputs":[]}]}