{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Details of the solution\n\nhttps://www.kaggle.com/competitions/birdclef-2022/discussion/327193","metadata":{}},{"cell_type":"code","source":"#set up\n!cp -r ../input/timm-pytorch-image-models /kaggle/working/\n!pip install /kaggle/working/timm-pytorch-image-models/pytorch-image-models-master/\n!mkdir /kaggle/working/logs\n!mkdir output","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:53:09.859455Z","iopub.execute_input":"2022-06-05T23:53:09.860193Z","iopub.status.idle":"2022-06-05T23:53:42.383104Z","shell.execute_reply.started":"2022-06-05T23:53:09.860158Z","shell.execute_reply":"2022-06-05T23:53:42.382305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/torchlibrosa/torchlibrosa-0.0.9-py2.py3-none-any.whl\n!pip install ../input/birdclef-2022-wheels/timm-0.4.12-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:53:42.386047Z","iopub.execute_input":"2022-06-05T23:53:42.386273Z","iopub.status.idle":"2022-06-05T23:54:31.510675Z","shell.execute_reply.started":"2022-06-05T23:53:42.386247Z","shell.execute_reply":"2022-06-05T23:54:31.509824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN part","metadata":{}},{"cell_type":"code","source":"## https://stackoverflow.com/questions/1868714/how-do-i-copy-an-entire-directory-of-files-into-an-existing-directory-using-pyth\ndef copytree(src, dst, symlinks=False, ignore=None):\n    for item in os.listdir(src):\n        s = os.path.join(src, item)\n        d = os.path.join(dst, item)\n        if os.path.isdir(s):\n            shutil.copytree(s, d, symlinks, ignore)\n        else:\n            shutil.copy2(s, d)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:54:31.513505Z","iopub.execute_input":"2022-06-05T23:54:31.514488Z","iopub.status.idle":"2022-06-05T23:54:31.52027Z","shell.execute_reply.started":"2022-06-05T23:54:31.514448Z","shell.execute_reply":"2022-06-05T23:54:31.519523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(array):\n    array = np.array(array)\n    num_classes = 21\n    num_chunks = 12\n    \n    predictions_per_file = num_classes * num_chunks\n    array = array.reshape(-1, predictions_per_file)\n    num_files, _ = array.shape\n    new_array = np.empty((num_files, num_chunks, num_classes), dtype=array.dtype)\n    for i, prediction in enumerate(array):\n        new_array[i] = array[i].reshape(-1, num_classes).T.reshape(num_classes, num_chunks).T\n    return new_array\n\ndef roll_with_pad(data, shift=0):\n    axis = 0\n    \n    abs_shift = abs(shift)\n    data = np.pad(data, [(abs_shift, abs_shift), (0, 0)])\n    return data if shift == 0 else np.roll(data, shift=shift, axis=axis)[abs_shift:-abs_shift]\n\ndef postprocess_probabilities(probs):\n    '''\n        input: :param: probs - np.array of shape (num_files, num_chunks, num_classes)\n        output: probs - np.array, postprocessed probabilities\n    '''\n    shifts = [-2, -1, 0, 1, 2]\n    weights = np.array([0.5, 1.0, 7.0, 1.0, 0.5])\n#     shifts = [-3, -2, -1, 0, 1, 2, 3]\n#     weights = np.array([0.5, 1.0, 2.0, 7.0, 2.0, 1.0, 0.5])\n    high_pass = 0.5\n    \n    num_files, num_classes, num_chunks = probs.shape\n    \n    for i, file_prediction in enumerate(probs):\n        ## time-smoothing\n        weighted_preds = []\n        for shift, weight in zip(shifts, weights):\n            weighted_preds.append(roll_with_pad(file_prediction, shift=shift) * weight / weights.sum())\n        probs[i] = np.sum(weighted_preds, axis=0)\n        \n        #Vlomme's PP OFF\n#         # https://github.com/vlomme/Birdcall-Identification-competition/blob/master/train.py postprocessing\n#         max_probs = np.amax(probs[i], axis=0)\n#         high_pass_mask = max_probs > high_pass\n#         max_probs = np.where(high_pass_mask, max_probs, 0)\n#         probs[i] += max_probs / 70\n    \n    return probs","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:54:31.522882Z","iopub.execute_input":"2022-06-05T23:54:31.523778Z","iopub.status.idle":"2022-06-05T23:54:31.537083Z","shell.execute_reply.started":"2022-06-05T23:54:31.523742Z","shell.execute_reply":"2022-06-05T23:54:31.536153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport shutil\nimport subprocess\nimport pandas as pd\nfrom os.path import join\n\n# First, load list of audio files. We could use 'test.csv' as well,\n# but for now, let's stick with parsing the test_soundscape folder.\ntest_audio_dir = '../input/birdclef-2022/test_soundscapes/'\nscored_birds = [\"akiapo\", \"aniani\", \"apapan\", \"barpet\", \"crehon\", \"elepai\", \"ercfra\", \\\n                \"hawama\", \"hawcre\", \"hawgoo\", \"hawhaw\", \"hawpet1\", \"houfin\", \"iiwi\", \"jabwar\", \"maupar\", \\\n                \"omao\", \"puaioh\", \"skylar\", \"warwhe1\", \"yefcan\"]\n\n## set high thresholds to birds which were predicted just fine on local validation\n## is needed also to create per-model predictions\ndefault_threshold = 0.04\nthresholds = {k:v for k, v in zip(scored_birds, np.ones_like(scored_birds).astype(np.float32) * default_threshold)}\nstr_thresholds = ' '.join([f'{x:.4f}' for x in thresholds.values()])\n\nthresholds[\"skylar\"] = 0.5\n# thresholds[\"elepai\"] = 0.2\n# thresholds[\"houfin\"] = 0.2\n# thresholds[\"jabwar\"] = 0.4\n# thresholds[\"yefcan\"] = 0.3\n# thresholds[\"maupar\"] = 0.01\n\nnum_workers = 2\nbatch_size = 32","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:54:31.538389Z","iopub.execute_input":"2022-06-05T23:54:31.538887Z","iopub.status.idle":"2022-06-05T23:54:33.534392Z","shell.execute_reply.started":"2022-06-05T23:54:31.538852Z","shell.execute_reply":"2022-06-05T23:54:33.533379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## best ensemble\n# experiment_names = ['exp-11-serv', 'exp6data', 'exp7serv', 'exp7v3', 'full-exp-5', 'start-fold4-exp10-full', 'start-fold4-exp13-full', 'start-fold4-exp15-full', 'start-fold4-exp17-full', 'new-metrics-exp-9-v2', 'new-metrics-exp-17', 'new-metrics-exp-18', 'new-metrics-exp-19']\n# dirs_names = ['exp-11-serv', 'exp-6-serv', 'exp-7-serv', 'exp7', 'full_exp5', 'start_fold4_exp10_full', 'start_fold4_exp13_full_r', 'start_fold4_exp15_full', 'start_fold4_exp17_full', 'new-metrics-exp-9', 'new-metrics-exp-17', 'new-metrics-exp-18', 'new-metrics-exp-19']\n# weights = np.array([1.0, 1.0, 1.0, 5.0, 1.0, 6.0, 7.0, 1.0, 2.0, 7.0, 8.0, 7.0, 7.0])\n# folds = [0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4]\n\n## remove models w/o augs, must improve the score due to reduction of FP:\nexperiment_names = ['start-fold4-exp13-full', 'start-fold4-exp15-full', 'start-fold4-exp17-full', 'new-metrics-exp-9-v2', 'new-metrics-exp-17', 'new-metrics-exp-18', 'new-metrics-exp-19']\ndirs_names = ['start_fold4_exp13_full_r', 'start_fold4_exp15_full', 'start_fold4_exp17_full', 'new-metrics-exp-9', 'new-metrics-exp-17', 'new-metrics-exp-18', 'new-metrics-exp-19']\nweights = [7.0, 1.0, 2.0, 7.0, 8.0, 7.0, 7.0]\nfolds = [0, 0, 0, 4, 4, 4, 4]\nconfigs = ['base_config.yaml'] * 7\n\n## 100-epochs 5-fold ensemble effnet_v2s model\nexperiment_names.extend(['new-metrics-exp-23'] * 1)\ndirs_names.extend(['new-metrics-exp-23'] * 1)\nweights.extend([12.0] * 1)\nfolds.extend([4])\nconfigs.extend(['tf_effnet_v2_s_in21k.yaml'] * 1)\n\nweights = np.array(weights)\ninference_on_dir_name = 'inferenceondirfastv16'","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:54:33.535978Z","iopub.execute_input":"2022-06-05T23:54:33.536222Z","iopub.status.idle":"2022-06-05T23:54:33.546559Z","shell.execute_reply.started":"2022-06-05T23:54:33.536187Z","shell.execute_reply":"2022-06-05T23:54:33.545861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for experiment_name in experiment_names:\n    try:\n        copytree(f'../input/{experiment_name}', './experiments/')\n    except:\n        print(\"Workadround about copy-tree file-exists behaviour\")\nshutil.copyfile(f'../input/{inference_on_dir_name}/inference_on_dir_different_thresholds.py', 'inference_on_dir.py')\n\nfor experiment_name, fold, config in zip(dirs_names, folds, configs):\n    print(f\"Running inference for experiment name : {experiment_name}\")\n    subprocess.run(f'python3 inference_on_dir.py --experiment-dir ./experiments \\\n                            --exp {experiment_name} \\\n                            --inference-dir ../input/birdclef-2022/test_soundscapes/ \\\n                            --thresholds {str_thresholds} \\\n                            --num-workers {num_workers} \\\n                            --batch-size {batch_size} \\\n                            --fold {fold} \\\n                            --cfg {config}', shell=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:54:33.548076Z","iopub.execute_input":"2022-06-05T23:54:33.548543Z","iopub.status.idle":"2022-06-05T23:56:14.378281Z","shell.execute_reply.started":"2022-06-05T23:54:33.548505Z","shell.execute_reply":"2022-06-05T23:56:14.376678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subs_with_probs = []\nfor exp_dir in dirs_names:\n    subs_with_probs.append(pd.read_csv(join('./experiments', exp_dir, 'submission_w_probs.csv')))","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:14.380058Z","iopub.execute_input":"2022-06-05T23:56:14.380373Z","iopub.status.idle":"2022-06-05T23:56:14.433014Z","shell.execute_reply.started":"2022-06-05T23:56:14.380304Z","shell.execute_reply":"2022-06-05T23:56:14.431915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for exp_name, sub in zip(experiment_names, subs_with_probs):\n    print(\"{} : {:.4f}\".format(exp_name, sub['score'].mean()))","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:14.438823Z","iopub.execute_input":"2022-06-05T23:56:14.43902Z","iopub.status.idle":"2022-06-05T23:56:14.451847Z","shell.execute_reply.started":"2022-06-05T23:56:14.438996Z","shell.execute_reply":"2022-06-05T23:56:14.451039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.rmtree('./experiments')\n\nprobs     = np.array([df['score'] for df in subs_with_probs])\nnew_probs = (probs * weights.reshape(-1, 1)).sum(axis=0) / weights.sum()\n\n#transform\ntransformed_rows = transform(subs_with_probs[0].row_id)\ntransformed_probs = transform(new_probs)\ntransformed_rows.shape, transformed_probs.shape, transformed_rows[0, :3, :3]\n\n#PP\nprint(new_probs.shape)\nnew_probs = postprocess_probabilities(transformed_probs.copy())\nnew_probs.mean(), new_probs.shape, new_probs[0, :, 0]\n\n#sub\nnew_sub = pd.DataFrame({\n    'row_id' : transformed_rows.flatten(),\n    'target' : -2,\n    'probability' : new_probs.flatten()\n})\nnew_sub['bird'] = new_sub['row_id'].apply(lambda x : x.split('_')[2])\n\nfor bird, threshold in thresholds.items():\n    print(f\"{bird} : {threshold:.2f}\")\n    mask = new_sub.bird == bird\n    new_sub.loc[mask, 'target'] = new_sub.loc[mask, 'probability'] > threshold\n    \nnew_sub_with_proba = new_sub.copy()\nnew_sub = new_sub.drop(columns=['bird', 'probability'])\n# new_sub.to_csv('submission.csv', index=False)\nnew_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:14.456047Z","iopub.execute_input":"2022-06-05T23:56:14.456836Z","iopub.status.idle":"2022-06-05T23:56:15.01104Z","shell.execute_reply.started":"2022-06-05T23:56:14.456799Z","shell.execute_reply":"2022-06-05T23:56:15.010274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#まとめ\ndf_ens_Slime   = new_sub_with_proba.copy()\ndf_ens_Slime.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:15.012254Z","iopub.execute_input":"2022-06-05T23:56:15.013671Z","iopub.status.idle":"2022-06-05T23:56:15.022748Z","shell.execute_reply.started":"2022-06-05T23:56:15.013628Z","shell.execute_reply":"2022-06-05T23:56:15.021882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SED part","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport pickle\nimport yaml\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:15.023882Z","iopub.execute_input":"2022-06-05T23:56:15.024294Z","iopub.status.idle":"2022-06-05T23:56:15.031859Z","shell.execute_reply.started":"2022-06-05T23:56:15.024243Z","shell.execute_reply":"2022-06-05T23:56:15.031107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ==Ens==\ndef ens_weightave(df_ens1,df_ens2,\n                  dict_thresh_short_ens1,dict_thresh_short_ens2,\n                  dict_thresh_long_ens1,dict_thresh_long_ens2,\n                  weight1,weight2):\n    df_ens1 = df_ens1.rename(columns={'target':'target1','score':'score1','score_long':'score_long1'})\n    df_ens2 = df_ens2.rename(columns={'target':'target2','score':'score2','score_long':'score_long2'})\n    df_ens  = df_ens2.merge(df_ens1,on='row_id')\n    row_id              = df_ens['row_id'].values\n    probas_ens1         = df_ens['score1'].values\n    probas_long_ens1    = df_ens['score_long1'].values\n    probas_ens2         = df_ens['score2'].values\n    probas_long_ens2    = df_ens['score_long2'].values\n    #proba\n    probas_ens        = weight1*probas_ens1       + weight2*probas_ens2\n    probas_long_ens   = weight1*probas_long_ens1  + weight2*probas_long_ens2\n    #thresh\n    dict_thresh_short_ens={}\n    dict_thresh_long_ens ={}\n    for bird_name in dict_thresh_short_ens1.keys():\n        thresh_ens1      = dict_thresh_short_ens1[bird_name]\n        thresh_long_ens1 = dict_thresh_long_ens1[bird_name]\n        thresh_ens2      = dict_thresh_short_ens2[bird_name]\n        thresh_long_ens2 = dict_thresh_long_ens2[bird_name]\n        thresh_ens       = weight1*thresh_ens1     +weight2*thresh_ens2\n        thresh_long_ens  = weight1*thresh_long_ens1+weight2*thresh_long_ens2\n        dict_thresh_short_ens.update({bird_name:thresh_ens})\n        dict_thresh_long_ens.update({bird_name:thresh_long_ens})\n    #get_sub\n    pred        = {'row_id' : [], 'target' : [], 'score' : [], 'score_long' : []}\n    row_ids     = list(row_id)\n    for idx,row_id in enumerate(row_ids):\n        soundscape_name = row_id.split('_')[0]+'_'+row_id.split('_')[1]\n        bird_name       = row_id.split('_')[2]\n        time_name       = row_id.split('_')[3]\n        #proba\n        proba_ens       = probas_ens[idx]\n        proba_long_ens  = probas_long_ens[idx]\n        #thresh\n        thresh_ens      = dict_thresh_short_ens[bird_name]\n        thresh_long_ens = dict_thresh_long_ens[bird_name]\n        #judge\n        event           = ((proba_ens >= thresh_ens).astype(int) \\\n                          + (proba_long_ens >= thresh_long_ens).astype(int)) >= 2#long,short\n        pred['row_id'].append(row_id)\n        pred['target'].append(event)\n        pred['score'].append(proba_ens)\n        pred['score_long'].append(proba_long_ens)\n    #df\n    df_ens          = pd.DataFrame(pred, columns=['row_id', 'target', 'score', 'score_long'])\n    df_sub_ens      = df_ens[['row_id','target']]\n    return df_ens,df_sub_ens,dict_thresh_short_ens,dict_thresh_long_ens\n\n# ==PP==\ndef Slime_PP(df_ens,dict_thresh_short_ens,dict_thresh_long_ens):\n    row_id          = df_ens['row_id'].values\n    proba_ens       = df_ens['score'].values\n    proba_long_ens  = df_ens['score_long'].values\n    #\n    transformed_row_id         = transform(row_id)\n    transformed_proba_ens      = transform(proba_ens)\n    transformed_proba_long_ens = transform(proba_long_ens)\n    #pp\n    pp_proba_ens       = postprocess_probabilities(transformed_proba_ens.copy())\n    pp_proba_long_ens  = postprocess_probabilities(transformed_proba_long_ens.copy())\n    #flatten\n    pp_row_id          = transformed_row_id.flatten()\n    pp_proba_ens       = pp_proba_ens.flatten()\n    pp_proba_long_ens  = pp_proba_long_ens.flatten()\n    #get_sub\n    pred        = {'row_id' : [], 'target' : [], 'score' : [], 'score_long' : []}\n    row_ids     = list(pp_row_id)\n    for idx,row_id in enumerate(row_ids):\n        soundscape_name = row_id.split('_')[0]+'_'+row_id.split('_')[1]\n        bird_name       = row_id.split('_')[2]\n        time_name       = row_id.split('_')[3]\n        #thresh\n        thresh_ens      = dict_thresh_short_ens[bird_name]\n        thresh_long_ens = dict_thresh_long_ens[bird_name]\n        #proba\n        proba_ens       = pp_proba_ens[idx]\n        proba_long_ens  = pp_proba_long_ens[idx]\n        #judge\n    #         event           = ((proba_ens >= thresh_ens).astype(int) \\\n    #                           + (proba_long_ens >= thresh_long_ens).astype(int)) >= 2#long and short\n        event          = (proba_ens >= thresh_ens).astype(int) >= 1#short only\n        pred['row_id'].append(row_id)\n        pred['target'].append(event)\n        pred['score'].append(proba_ens)\n        pred['score_long'].append(proba_long_ens)\n\n    #df\n    df_pp_ens          = pd.DataFrame(pred, columns=['row_id', 'target', 'score', 'score_long'])\n    df_sub_pp_ens      = df_pp_ens[['row_id','target']]\n    return df_pp_ens,df_sub_pp_ens","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:15.033381Z","iopub.execute_input":"2022-06-05T23:56:15.033636Z","iopub.status.idle":"2022-06-05T23:56:15.18711Z","shell.execute_reply.started":"2022-06-05T23:56:15.033604Z","shell.execute_reply":"2022-06-05T23:56:15.186385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SED Frontend1","metadata":{}},{"cell_type":"code","source":"# ==================================================================================================================================\n# Frontend1  = config_audio_inference_mel1\n# ==========================Part1_1 : Frontend1 BCE Loss==========================#\nmodels_path_nonfocal1  = {\n                'exp178_fold0':'../input/bird3-infrence-data/exp178_seres26_exp124base_ep35_StepLR_seed1701_1fold_labelsmoothOFF_BCE2WayLoss/model/fold0/best.pt',\n                'exp179_fold0':'../input/bird3-infrence-data/exp179_res34_exp124base_ep35_StepLR_seed1703_1fold_labelsmoothOFF_BCE2WayLoss/model/fold0/best.pt',\n                'exp180_fold0':'../input/bird3-infrence-data/exp180_res50_exp124base_ep35_StepLR_seed1706_1fold_labelsmoothOFF_BCE2WayLoss/model/fold0/best.pt',\n                'exp181_fold0':'../input/bird3-infrence-data/exp181_effv2s_exp124base_ep35_StepLR_seed1709_1fold_labelsmoothOFF_BCE2WayLoss/model/fold0/best.pt',\n                }\nconfigs_path_nonfocal1 = {\n                 'exp178_fold0':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                 'exp179_fold0':'../input/bird3-infrence-data/exp142_resnet34_exp124base_seed115/yaml/config_audio.yaml',\n                 'exp180_fold0':'../input/bird3-infrence-data/exp145_resnest50_exp124base_seed333/yaml/config_audio.yaml',\n                 'exp181_fold0':'../input/bird3-infrence-data/exp143_effv2s_exp124base_seed55/yaml/config_audio2.yaml',\n                }\nens_weights_nonfocal1  = {\n                 'exp178_fold0':1,\n                 'exp179_fold0':1,\n                 'exp180_fold0':1,\n                 'exp181_fold0':1,\n                }\n#==========================Part1_2 : Frontend1 Focalloss==========================#\nmodels_path_focal1  = {\n                    'exp124_fold0':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/model/fold0/best.pt',\n                    'exp124_fold1':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/model/fold1/best.pt',\n                    'exp124_fold2':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/model/fold2/best.pt',\n                    'exp124_fold3':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/model/fold3/best.pt',\n                    'exp148_fold0':'../input/bird3-infrence-data/exp148_seres26_exp124base_seed1411_1fold_ep35_StepLR/model/fold0/best.pt',\n                    'exp153_fold0':'../input/bird3-infrence-data/exp153_effv2s_exp124base_seed4515_1fold_ep35_StepLR/model/fold0/best.pt',\n                    'exp155_fold0':'../input/bird3-infrence-data/exp155_res50_exp124base_seed1995_1fold_ep35_StepLR/model/fold0/best.pt',\n                    'exp159_fold0':'../input/bird3-infrence-data/exp159_res34_exp124base_seed4655_1fold_ep35_StepLR/model/fold0/best.pt',\n                    }\nconfigs_path_focal1 = { \n                     'exp124_fold0':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                     'exp124_fold1':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                     'exp124_fold2':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                     'exp124_fold3':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                     'exp148_fold0':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                     'exp153_fold0':'../input/bird3-infrence-data/exp143_effv2s_exp124base_seed55/yaml/config_audio2.yaml',\n                     'exp155_fold0':'../input/bird3-infrence-data/exp145_resnest50_exp124base_seed333/yaml/config_audio.yaml',\n                     'exp159_fold0':'../input/bird3-infrence-data/exp142_resnet34_exp124base_seed115/yaml/config_audio.yaml',\n                   }\nens_weights_focal1  = { \n                     'exp124_fold0':1,\n                     'exp124_fold1':1,\n                     'exp124_fold2':1,\n                     'exp124_fold3':1,\n                     'exp148_fold0':1,\n                     'exp153_fold0':1,\n                     'exp155_fold0':1,\n                     'exp159_fold0':1,\n                    }\n#==========================save==========================#\nmodels_path              = dict()\nmodels_path['focal']     = models_path_focal1\nmodels_path['nonfocal']  = models_path_nonfocal1\n\nconfigs_path             = dict()\nconfigs_path['focal']    = configs_path_focal1\nconfigs_path['nonfocal'] = configs_path_nonfocal1\n\nens_weights              = dict()\nens_weights['focal']     = ens_weights_focal1\nens_weights['nonfocal']  = ens_weights_nonfocal1\n\nwith open('./output/models_path.pickle', mode='wb') as f:\n    pickle.dump(models_path,f)\nwith open('./output/configs_path.pickle', mode='wb') as f:\n    pickle.dump(configs_path,f)\nwith open('./output/ens_weights.pickle', mode='wb') as f:\n    pickle.dump(ens_weights,f)\n    \n#==========================inference==========================#\n!python ../input/bird3-infrence-data-ens/bird3_ens/train_cl_birds3_inference_fast_v3.py --config 'config_audio_inference_mel1' --num-workers 4 --batch-size 32  --coeff-thresh 1.0 --mode-ens Average --testmode OFF\n\n#==========================BCE Loss==========================#\ndf_ens1_nonfocal              = pd.read_csv('./output/sub_ens_nonfocal.csv')\ndf_sub_ens1_nonfocal          = df_ens1_nonfocal[['row_id','target']]\ndf_sub_ens1_nonfocal.to_csv(\"submission.csv\", index=False)\n#thresh\nwith open('./output/dict_thresh_short_ens_nonfocal.pickle', 'rb') as pickle_file:\n    dict_thresh_short_ens1_nonfocal  = pickle.load(pickle_file)\nwith open('./output/dict_thresh_long_ens_nonfocal.pickle', 'rb') as pickle_file:\n    dict_thresh_long_ens1_nonfocal   = pickle.load(pickle_file)\n\nprint('-------------BCE Loss------------------------')\nprint('pred_mean      :',np.round(df_ens1_nonfocal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens1_nonfocal['score_long'].values.mean(),5))  \nprint('--------------------------------------------------')\n\n\n#==========================Focal Loss==========================#\ndf_ens1_focal                 = pd.read_csv('./output/sub_ens_focal.csv')\ndf_sub_ens1_focal             = df_ens1_focal[['row_id','target']]\ndf_sub_ens1_focal.to_csv(\"submission.csv\", index=False)\n#thresh\nwith open('./output/dict_thresh_short_ens_focal.pickle', 'rb') as pickle_file:\n    dict_thresh_short_ens1_focal  = pickle.load(pickle_file)\nwith open('./output/dict_thresh_long_ens_focal.pickle', 'rb') as pickle_file:\n    dict_thresh_long_ens1_focal   = pickle.load(pickle_file)\n\nprint('-------------Focal Loss------------------------')\nprint('pred_mean      :',np.round(df_ens1_focal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens1_focal['score_long'].values.mean(),5))  \nprint('--------------------------------------------------')\n\ngc.collect()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:15.188672Z","iopub.execute_input":"2022-06-05T23:56:15.189207Z","iopub.status.idle":"2022-06-05T23:56:48.186655Z","shell.execute_reply.started":"2022-06-05T23:56:15.189168Z","shell.execute_reply":"2022-06-05T23:56:48.185883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SED Frontend2","metadata":{}},{"cell_type":"code","source":"# ====================================================================================#\n# Frontend2  = config_audio_inference_mel3  \n# ==========================Part2_1 : Frontend2 BCE LOSS==========================#\nmodels_path_nonfocal2  = {\n                    'exp185_fold0':'../input/bird3-infrence-data/exp185_seres26_exp124base_seed2016_1fold_labelsmoothOFF_BCE2WayLoss_chunk15_changemel/model/fold0/best.pt', \n                    'exp186_fold0':'../input/bird3-infrence-data/exp186_effv2s_exp124base_seed2031_1fold_labelsmoothOFF_BCE2WayLoss_chunk15_changemel/model/fold0/best.pt', \n                    'exp187_fold0':'../input/bird3-infrence-data/exp187_res34_exp124base_seed2032_1fold_labelsmoothOFF_BCE2WayLoss_chunk15_changemel/model/fold0/best.pt', \n                    'exp188_fold0':'../input/bird3-infrence-data/exp188_res50_exp124base_seed2034_1fold_labelsmoothOFF_BCE2WayLoss_chunk15_changemel/model/fold0/best.pt', \n                }\n\nconfigs_path_nonfocal2 = {\n                    'exp185_fold0':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml', \n                    'exp186_fold0':'../input/bird3-infrence-data/exp143_effv2s_exp124base_seed55/yaml/config_audio2.yaml',    \n                    'exp187_fold0':'../input/bird3-infrence-data/exp142_resnet34_exp124base_seed115/yaml/config_audio.yaml',  \n                    'exp188_fold0':'../input/bird3-infrence-data/exp145_resnest50_exp124base_seed333/yaml/config_audio.yaml', \n                }\n\nens_weights_nonfocal2  = {\n                     'exp185_fold0':1, \n                     'exp186_fold0':1, \n                     'exp187_fold0':1, \n                     'exp188_fold0':1, \n                }\n\n#==========================Part1_2 : Frontend1 Focalloss==========================#\nmodels_path_focal2  = {\n                    'exp166_fold0':'../input/bird3-infrence-data/exp166_effv2s_exp124base_seed1341_1fold_ep35_StepLR_chunk15_changemel/model/fold0/best.pt',\n                    'exp167_fold0':'../input/bird3-infrence-data/exp167_seres26_exp124base_seed1351_1fold_ep35_StepLR_chunk15_changemel/model/fold0/best.pt',\n                    'exp170_fold0':'../input/bird3-infrence-data/exp170_res50_exp124base_seed1451_1fold_ep35_StepLR_chunk15_changemel/model/fold0/best.pt',\n                    'exp171_fold0':'../input/bird3-infrence-data/exp171_res34_exp124base_seed1461_1fold_ep35_StepLR_chunk15_changemel/model/fold0/best.pt',\n                }\n\nconfigs_path_focal2 = { \n                    'exp166_fold0':'../input/bird3-infrence-data/exp143_effv2s_exp124base_seed55/yaml/config_audio2.yaml',\n                    'exp167_fold0':'../input/bird3-infrence-data/exp124_seres26_chunk10_pseudoexp044_strong2_ep30_cutmix05_pseudo2ndmaskON_randlowpassOFF/yaml/config_audio4.yaml',\n                    'exp170_fold0':'../input/bird3-infrence-data/exp145_resnest50_exp124base_seed333/yaml/config_audio.yaml',\n                    'exp171_fold0':'../input/bird3-infrence-data/exp142_resnet34_exp124base_seed115/yaml/config_audio.yaml',\n               }\n\nens_weights_focal2  = { \n                     'exp166_fold0':1,\n                     'exp167_fold0':1,\n                     'exp170_fold0':1,\n                     'exp171_fold0':1,\n                }\n\n#==========================合体==========================#\nmodels_path              = dict()\nmodels_path['focal']     = models_path_focal2\nmodels_path['nonfocal']  = models_path_nonfocal2\n\nconfigs_path             = dict()\nconfigs_path['focal']    = configs_path_focal2\nconfigs_path['nonfocal'] = configs_path_nonfocal2\n\nens_weights              = dict()\nens_weights['focal']     = ens_weights_focal2\nens_weights['nonfocal']  = ens_weights_nonfocal2\n\nwith open('./output/models_path.pickle', mode='wb') as f:\n    pickle.dump(models_path,f)\nwith open('./output/configs_path.pickle', mode='wb') as f:\n    pickle.dump(configs_path,f)\nwith open('./output/ens_weights.pickle', mode='wb') as f:\n    pickle.dump(ens_weights,f)\n\n#==========================inference==========================#\n!python ../input/bird3-infrence-data-ens/bird3_ens/train_cl_birds3_inference_fast_v3.py --config 'config_audio_inference_mel3' --num-workers 4 --batch-size 32  --coeff-thresh 1.0 --mode-ens Average --testmode OFF\n\n#==========================BCE LOSS==========================#\ndf_ens2_nonfocal              = pd.read_csv('./output/sub_ens_nonfocal.csv')\ndf_sub_ens2_nonfocal          = df_ens2_nonfocal[['row_id','target']]\ndf_sub_ens2_nonfocal.to_csv(\"submission.csv\", index=False)\n\n#thresh\nwith open('./output/dict_thresh_short_ens_nonfocal.pickle', 'rb') as pickle_file:\n    dict_thresh_short_ens2_nonfocal  = pickle.load(pickle_file)\nwith open('./output/dict_thresh_long_ens_nonfocal.pickle', 'rb') as pickle_file:\n    dict_thresh_long_ens2_nonfocal   = pickle.load(pickle_file)\n\nprint('-------------BCE LOSS------------------------')\nprint('pred_mean      :',np.round(df_ens2_nonfocal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens2_nonfocal['score_long'].values.mean(),5))  \nprint('--------------------------------------------------')\n\n#==========================Focal LOSS==========================#\ndf_ens2_focal                 = pd.read_csv('./output/sub_ens_focal.csv')\ndf_sub_ens2_focal             = df_ens2_focal[['row_id','target']]\ndf_sub_ens2_focal.to_csv(\"submission.csv\", index=False)\n\n#thresh\nwith open('./output/dict_thresh_short_ens_focal.pickle', 'rb') as pickle_file:\n    dict_thresh_short_ens2_focal  = pickle.load(pickle_file)\nwith open('./output/dict_thresh_long_ens_focal.pickle', 'rb') as pickle_file:\n    dict_thresh_long_ens2_focal   = pickle.load(pickle_file)\n\nprint('-------------Focal LOSS------------------------')\nprint('pred_mean      :',np.round(df_ens2_focal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens2_focal['score_long'].values.mean(),5))  \nprint('--------------------------------------------------')\n\ngc.collect()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:56:48.188394Z","iopub.execute_input":"2022-06-05T23:56:48.188645Z","iopub.status.idle":"2022-06-05T23:57:13.948142Z","shell.execute_reply.started":"2022-06-05T23:56:48.18861Z","shell.execute_reply":"2022-06-05T23:57:13.947386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SED ensemble","metadata":{}},{"cell_type":"code","source":"#BCE LOSS\nweight_nonfocal_mel1=0.7\nweight_nonfocal_mel2=0.3\ndf_ens_nonfocal,df_sub_ens_nonfocal,dict_thresh_short_ens_nonfocal,dict_thresh_long_ens_nonfocal = ens_weightave(df_ens1_nonfocal,df_ens2_nonfocal,\n                                                                                                      dict_thresh_short_ens1_nonfocal,dict_thresh_short_ens2_nonfocal,\n                                                                                                      dict_thresh_long_ens1_nonfocal,dict_thresh_long_ens2_nonfocal,\n                                                                                                      weight1=weight_nonfocal_mel1,weight2=weight_nonfocal_mel2)\n#Focal LOSS\nweight_focal_mel1=0.7\nweight_focal_mel2=0.3\ndf_ens_focal,df_sub_ens_focal,dict_thresh_short_ens_focal,dict_thresh_long_ens_focal =  ens_weightave(df_ens1_focal,df_ens2_focal,\n                                                                                                      dict_thresh_short_ens1_focal,dict_thresh_short_ens2_focal,\n                                                                                                      dict_thresh_long_ens1_focal,dict_thresh_long_ens2_focal,\n                                                                                                      weight1=weight_focal_mel1,weight2=weight_focal_mel2)\n\nprint('------------------------BCE LOSS-------------------------------')\nprint('pred_mean      :',np.round(df_ens_nonfocal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens_nonfocal['score_long'].values.mean(),5))   \nprint('-------------------------------------------------------------------')\n\nprint('------------------------Focal LOSS-------------------------------')\nprint('pred_mean      :',np.round(df_ens_focal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens_focal['score_long'].values.mean(),5))   \nprint('-------------------------------------------------------------------')","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:13.949938Z","iopub.execute_input":"2022-06-05T23:57:13.950206Z","iopub.status.idle":"2022-06-05T23:57:13.983606Z","shell.execute_reply.started":"2022-06-05T23:57:13.950171Z","shell.execute_reply":"2022-06-05T23:57:13.982949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SED PP for BCEloss","metadata":{}},{"cell_type":"code","source":"df_ens_nonfocal,df_sub_ens_nonfocal = Slime_PP(df_ens_nonfocal,dict_thresh_short_ens_nonfocal,dict_thresh_long_ens_nonfocal)\n\nprint('------------------------BCE LOSS Afte PP-------------------------------')\nprint('pred_mean      :',np.round(df_ens_nonfocal['score'].values.mean(),5))    \nprint('pred_long_mean :',np.round(df_ens_nonfocal['score_long'].values.mean(),5))   \nprint('-------------------------------------------------------------------')\n","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:13.984591Z","iopub.execute_input":"2022-06-05T23:57:13.985345Z","iopub.status.idle":"2022-06-05T23:57:13.998896Z","shell.execute_reply.started":"2022-06-05T23:57:13.985296Z","shell.execute_reply":"2022-06-05T23:57:13.997999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SED Output","metadata":{}},{"cell_type":"code","source":"df_ens_UEMU_NonFocal = df_ens_nonfocal.copy()\ndf_ens_UEMU_Focal    = df_ens_focal.copy()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.000274Z","iopub.execute_input":"2022-06-05T23:57:14.001059Z","iopub.status.idle":"2022-06-05T23:57:14.005405Z","shell.execute_reply.started":"2022-06-05T23:57:14.001022Z","shell.execute_reply":"2022-06-05T23:57:14.004632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Team Ensembling part","metadata":{}},{"cell_type":"markdown","source":"Group1:14Birds (CNN + SED ,BCE loss)","metadata":{}},{"cell_type":"code","source":"# =======Params============================\n#thresh\ndefault_threshold              = 0.05 #0.05\nthresholds_ens                 = {k:v for k, v in zip(scored_birds, np.ones_like(scored_birds).astype(np.float32) * default_threshold)}\nthresholds_ens[\"skylar\"]       = 0.35\n\n#weight\nweight_Slime                   = 0.35\nweight_UEMU                    = 0.65\n\n# =========================================\n#Ensemble\ndf_ens_UEMU_NonFocal           = df_ens_UEMU_NonFocal.sort_values(['row_id'])\ndf_ens_Slime                   = df_ens_Slime.sort_values(['row_id'])\n\ndf_ens_NonFocal                = df_ens_Slime.copy()\ndf_ens_NonFocal['probability'] = weight_Slime*df_ens_Slime['probability'] + weight_UEMU*df_ens_UEMU_NonFocal['score'] \n\nfor bird, threshold in thresholds_ens.items():\n    print(f\"{bird} : {threshold:.2f}\")\n    mask = df_ens_NonFocal.bird == bird\n    df_ens_NonFocal.loc[mask, 'target'] = df_ens_NonFocal.loc[mask, 'probability'] > threshold\n\n#sub\ndf_ens_sub = df_ens_NonFocal.drop(columns=['probability', 'bird'])\ndf_ens_sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.006807Z","iopub.execute_input":"2022-06-05T23:57:14.007307Z","iopub.status.idle":"2022-06-05T23:57:14.050642Z","shell.execute_reply.started":"2022-06-05T23:57:14.007272Z","shell.execute_reply":"2022-06-05T23:57:14.049983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group2:7Birds (SED ,Focal loss)","metadata":{}},{"cell_type":"code","source":"# =======Params============================\n#Select bird\nbird_Focal      = ['crehon','ercfra','hawgoo','hawhaw','hawpet1','maupar','puaioh']\n#     bird_Focal      = ['crehon','ercfra','hawama','hawcre','hawgoo','hawhaw','hawpet1','maupar','puaioh']\n\n# =========================================\ndf_ens_Focal    = df_ens_UEMU_Focal.copy()\n\nbird_NonFocal   = list(set(scored_birds)-set(bird_Focal))\nprint('bird_Focal',len(bird_Focal),bird_Focal)\nprint('bird_NonFocal',len(bird_NonFocal),bird_NonFocal)\n\ndf_ens_NonFocal = df_ens_NonFocal.rename(columns={'target':'target_NonFocal'})\ndf_ens_Focal    = df_ens_Focal.rename(columns={'target':'target_Focal'})\ndf_ens_Team     = df_ens_NonFocal[['row_id','target_NonFocal','bird']].copy()\ndf_ens_Team     = df_ens_Team.merge(df_ens_Focal[['row_id','target_Focal']],on='row_id')\n\ntargets    = []\nfor idx in range(len(df_ens_Team)):\n    if df_ens_Team['bird'][idx] in bird_Focal:\n        targets.append(df_ens_Team['target_Focal'][idx])\n    else:\n        targets.append(df_ens_Team['target_NonFocal'][idx])\ndf_ens_Team['target'] = targets\n\n#sub\ndf_ens_sub = df_ens_Team.drop(columns=['target_NonFocal','target_Focal', 'bird'])\ndf_ens_sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.05191Z","iopub.execute_input":"2022-06-05T23:57:14.052147Z","iopub.status.idle":"2022-06-05T23:57:14.07718Z","shell.execute_reply.started":"2022-06-05T23:57:14.052116Z","shell.execute_reply":"2022-06-05T23:57:14.076518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ens_Team['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.078265Z","iopub.execute_input":"2022-06-05T23:57:14.078644Z","iopub.status.idle":"2022-06-05T23:57:14.085988Z","shell.execute_reply.started":"2022-06-05T23:57:14.078609Z","shell.execute_reply":"2022-06-05T23:57:14.085055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hist BCE loss Part","metadata":{}},{"cell_type":"code","source":"df_ens_Slime['probability'].hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.087471Z","iopub.execute_input":"2022-06-05T23:57:14.087752Z","iopub.status.idle":"2022-06-05T23:57:14.504075Z","shell.execute_reply.started":"2022-06-05T23:57:14.087716Z","shell.execute_reply":"2022-06-05T23:57:14.503425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ens_UEMU_NonFocal['score'].hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.505257Z","iopub.execute_input":"2022-06-05T23:57:14.505508Z","iopub.status.idle":"2022-06-05T23:57:14.874076Z","shell.execute_reply.started":"2022-06-05T23:57:14.505475Z","shell.execute_reply":"2022-06-05T23:57:14.873375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hist Focal loss Part","metadata":{}},{"cell_type":"code","source":"df_ens_Focal['score'].hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:14.875217Z","iopub.execute_input":"2022-06-05T23:57:14.876089Z","iopub.status.idle":"2022-06-05T23:57:15.234924Z","shell.execute_reply.started":"2022-06-05T23:57:14.87604Z","shell.execute_reply":"2022-06-05T23:57:15.234167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ens_Focal['score_long'].hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:15.236227Z","iopub.execute_input":"2022-06-05T23:57:15.236484Z","iopub.status.idle":"2022-06-05T23:57:15.589811Z","shell.execute_reply.started":"2022-06-05T23:57:15.236451Z","shell.execute_reply":"2022-06-05T23:57:15.58912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_ens_Team['probability'].hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-06-05T23:57:15.590945Z","iopub.execute_input":"2022-06-05T23:57:15.591207Z","iopub.status.idle":"2022-06-05T23:57:15.595098Z","shell.execute_reply.started":"2022-06-05T23:57:15.591174Z","shell.execute_reply":"2022-06-05T23:57:15.594262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}