{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"-請在執行一次 \"Run All\" 後進行下一步。\n\n-在最後的單元格輸出中會生成 \"最佳閾值\"。\n\n-將生成的閾值寫入下一個單元格的 BEST_THRESHOLD 中，然後按下 \"Save Version\"。\n\n-將生成的 submission.csv 提交。","metadata":{}},{"cell_type":"code","source":"#📌注意事項：請執行一次 \"Run All\" 後，進行以下值的更改\n#用於表格競賽F1優化的閾值\nBEST_THRESHOLD = 0.100428\n#用於插入 \"nocall\" 的閾值\nBEST_NOCALL_THRESHOLD = 0.169353","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:41:06.374293Z","iopub.execute_input":"2021-05-30T10:41:06.374978Z","iopub.status.idle":"2021-05-30T10:41:06.389724Z","shell.execute_reply.started":"2021-05-30T10:41:06.374833Z","shell.execute_reply":"2021-05-30T10:41:06.388776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install -q --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple scikit-learn==1.0.dev0\n# !pip install -q resnest    ","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:41:06.397826Z","iopub.execute_input":"2021-05-30T10:41:06.398635Z","iopub.status.idle":"2021-05-30T10:41:06.405023Z","shell.execute_reply.started":"2021-05-30T10:41:06.398594Z","shell.execute_reply":"2021-05-30T10:41:06.40356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/scikit-learn-10dev0/scikit_learn-1.0.dev0-cp37-cp37m-manylinux2010_x86_64.whl\n# !pip install -q ../input/resnest-v0-0-5/resnest-0.0.5-py3-none-any.whl\n!pip install -q \"../input/resnest50-fast-package/resnest-0.0.6b20200701/resnest\"","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:41:06.407005Z","iopub.execute_input":"2021-05-30T10:41:06.415112Z","iopub.status.idle":"2021-05-30T10:41:59.328438Z","shell.execute_reply.started":"2021-05-30T10:41:06.415072Z","shell.execute_reply":"2021-05-30T10:41:59.327299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:41:59.330111Z","iopub.execute_input":"2021-05-30T10:41:59.330482Z","iopub.status.idle":"2021-05-30T10:41:59.337614Z","shell.execute_reply.started":"2021-05-30T10:41:59.330441Z","shell.execute_reply":"2021-05-30T10:41:59.33668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nsys.path.append(\"../input/birdclef-toolkit-v0530-1930/lib\")\nimport bird_recognition","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-30T10:41:59.33899Z","iopub.execute_input":"2021-05-30T10:41:59.339366Z","iopub.status.idle":"2021-05-30T10:42:03.220951Z","shell.execute_reply.started":"2021-05-30T10:41:59.33933Z","shell.execute_reply":"2021-05-30T10:42:03.219987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(bird_recognition.evaluation.TARGET_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:42:03.222619Z","iopub.execute_input":"2021-05-30T10:42:03.222997Z","iopub.status.idle":"2021-05-30T10:42:03.227882Z","shell.execute_reply.started":"2021-05-30T10:42:03.222954Z","shell.execute_reply":"2021-05-30T10:42:03.227049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#10幀的版本資料庫\nfilepath_list = [\n    # \"../input/metadata-probability-v0525-2100/birdclef_resnest50_fold0_epoch_33_f1_val_03859_20210524151554.csv\",\n    \n    # 👑fold: 1\n    \"../input/metadata-probability-v0525-2100/birdclef_resnest50_fold1_epoch_34_f1_val_04757_20210524185455.csv\",\n    \n    # \"../input/metadata-probability-v0525-2100/birdclef_resnest50_fold2_epoch_34_f1_val_05027_20210524223209.csv\",\n    # \"../input/metadata-probability-v0525-2100/birdclef_resnest50_fold3_epoch_20_f1_val_04299_20210525010703.csv\",\n    # \"../input/metadata-probability-v0525-2100/birdclef_resnest50_fold4_epoch_34_f1_val_05140_20210525074929.csv\"  \n]\nprob_df = pd.concat([pd.read_csv(_) for _ in filepath_list])","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:42:03.229348Z","iopub.execute_input":"2021-05-30T10:42:03.229843Z","iopub.status.idle":"2021-05-30T10:42:08.337806Z","shell.execute_reply.started":"2021-05-30T10:42:03.229801Z","shell.execute_reply":"2021-05-30T10:42:08.336891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"../input/bird-probabilities/train_bird_call_prob_{0,1,2}.csv\" # CV: 0.7764\nclass TrainingConfig:\n    def __init__(self, debug:bool):\n        #注意：這與Config中插入 'nocall' 時的閾值不同！\n        self.nocall_threshold:float=0.5\n        \n        self.debug = debug\n        self.num_kfolds:int = 5\n        self.num_spieces:int = 397\n        self.num_candidates:int = 5\n        self.max_distance:int = 15 # 20\n        self.weight_rate:float = 1.0\n        self.sampling_strategy:float = None # 1.0\n        self.random_state:int=777\n        self.num_prob:int = 6\n        self.min_rating = None # choose from  [1,2,3,4,5, None]\n        self.use_to_birds=True\n        self.use_add_secondlabel=False # True\n        self.xgb_params={\n            \"objective\": \"binary:logistic\",\n            \"tree_method\": 'gpu_hist',\n            \"n_estimators\": 1000,\n        }\n        self.lgb_params = {\n            'objective': 'binary',\n            'metric': 'binary_logloss',\n            'device':'gpu',\n        }\n        self.weights_filepath_dict = {\n            # 'xgb':[f\"./xgb_{kfold_index}.pkl\" for kfold_index in range(self.num_kfolds)],\n            'lgbm':[f\"./lgbm_{kfold_index}.pkl\" for kfold_index in range(self.num_kfolds)],\n            # 'cat':[f\"./cat_{kfold_index}.pkl\" for kfold_index in range(self.num_kfolds)]\n        }\n        \ntraining_config = TrainingConfig(\n    debug=False\n)\nif training_config.debug:\n    prob_df = prob_df.head(1000)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:42:08.340054Z","iopub.execute_input":"2021-05-30T10:42:08.340332Z","iopub.status.idle":"2021-05-30T10:42:08.348717Z","shell.execute_reply.started":"2021-05-30T10:42:08.340305Z","shell.execute_reply":"2021-05-30T10:42:08.347706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    def __init__(self):\n        self.num_kfolds:int = training_config.num_kfolds\n        self.num_spieces:int = training_config.num_spieces\n        self.num_candidates:int = training_config.num_candidates\n        self.max_distance:int = training_config.max_distance\n        self.nocall_threshold:float = training_config.nocall_threshold\n        self.num_prob:int = training_config.num_prob\n        # 是否要檢查在最適閾值下迅速切斷時的分數\n        self.check_baseline:bool = True\n        # 用於判定是否這隻鳥適合的模型文件路徑列表\n        self.weights_filepath_dict = training_config.weights_filepath_dict\n        # 用於插入 'nocall' 的閾值\n        self.nocall_threshold = BEST_NOCALL_THRESHOLD\n        # 用於預測每個幀中每種鳥的叫聲概率的模型權重\n        self.checkpoint_paths = [ \n            Path(\"../input/clefmodel/birdclef_resnest50_fold0_epoch_27_f1_val_05179_20210520120053.pth\"), # id36\n            Path(\"../input/clefmodel/birdclef_resnest50_fold0_epoch_13_f1_val_03502_20210522050604.pth\"), # id51\n            Path(\"../input/birdclef-groupby-author-05221040-728258/birdclef_resnest50_fold0_epoch_33_f1_val_03859_20210524151554.pth\"), # id58\n            Path(\"../input/birdclef-groupby-author-05221040-728258/birdclef_resnest50_fold1_epoch_34_f1_val_04757_20210524185455.pth\"), # id59\n            Path(\"../input/birdclef-groupby-author-05221040-728258/birdclef_resnest50_fold2_epoch_34_f1_val_05027_20210524223209.pth\"), # id60\n            Path(\"../input/birdclef-groupby-author-05221040-728258/birdclef_resnest50_fold3_epoch_20_f1_val_04299_20210525010703.pth\"), # id61\n            Path(\"../input/birdclef-groupby-author-05221040-728258/birdclef_resnest50_fold4_epoch_34_f1_val_05140_20210525074929.pth\"), # id62\n            Path(\"../input/clefmodel/resnest50_sr32000_d7_miixup-5.0_2ndlw-0.6_grouped-by-auther/birdclef_resnest50_fold0_epoch_78_f1_val_03658_20210528221629.pth\"), # id97\n            Path(\"../input/clefmodel/resnest50_sr32000_d7_miixup-5.0_2ndlw-0.6_grouped-by-auther/birdclef_resnest50_fold0_epoch_84_f1_val_03689_20210528225810.pth\"), # id97\n            Path(\"../input/clefmodel/resnest50_sr32000_d7_miixup-5.0_2ndlw-0.6_grouped-by-auther/birdclef_resnest50_fold1_epoch_27_f1_val_03942_20210529062427.pth\"), # id98\n        ]\n        # 用於候選樣本提取的每個樣本中每種鳥的叫聲概率（緩存）\n        self.pred_filepath_list = [\n            self.get_prob_filepath_from_checkpoint(path) for path in self.checkpoint_paths\n        ]\n        # 用於判定是否這隻鳥適合時的最佳閾值\n        self.threshold = BEST_THRESHOLD\n        \n    def get_prob_filepath_from_checkpoint(self, checkpoint_path:Path) -> str:\n        filename = f\"train_soundscape_labels_probabilitiy_%s.csv\" % checkpoint_path.stem\n        return filename\n\nconfig = Config()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:42:08.350359Z","iopub.execute_input":"2021-05-30T10:42:08.350729Z","iopub.status.idle":"2021-05-30T10:42:08.366862Z","shell.execute_reply.started":"2021-05-30T10:42:08.350689Z","shell.execute_reply":"2021-05-30T10:42:08.366029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsubmission_df = bird_recognition.evaluation.run(\n    training_config,\n    config,\n    prob_df,\n    model_dict=config.weights_filepath_dict,\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:42:08.368156Z","iopub.execute_input":"2021-05-30T10:42:08.368551Z","iopub.status.idle":"2021-05-30T10:43:59.728853Z","shell.execute_reply.started":"2021-05-30T10:42:08.368514Z","shell.execute_reply":"2021-05-30T10:43:59.728038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T10:43:59.730217Z","iopub.execute_input":"2021-05-30T10:43:59.730567Z","iopub.status.idle":"2021-05-30T10:43:59.74233Z","shell.execute_reply.started":"2021-05-30T10:43:59.730529Z","shell.execute_reply":"2021-05-30T10:43:59.741443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}