{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":3776.310275,"end_time":"2023-03-20T18:10:05.730581","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-03-20T17:07:09.420306","version":"2.4.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0212a7caf58345cd9711a4ddfabe83c7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0e4cac39c70347cd81cb2cbb9440515d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1117dd321a5a40958afc859aaa68210c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"172ab16b2c194786bf3738f722f89a81":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1c5ae63deeeb452193bad2b55bb7f97e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2492442e63be4f70b2426167a66b5844":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2aea947b81824166a36a356e83c64c5a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f3a856662c294621b5e105e4ed606b97","IPY_MODEL_42d73832b1ab471b92bbe1403c3d5509","IPY_MODEL_3b7988fa862146b9bcffc07e3778c576"],"layout":"IPY_MODEL_1c5ae63deeeb452193bad2b55bb7f97e"}},"307386f8d5dc4be1847271524734ca80":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6c9a5af2c6244d72b11ca4ff597a5a43","placeholder":"​","style":"IPY_MODEL_0212a7caf58345cd9711a4ddfabe83c7","value":" 16941/16941 [12:06&lt;00:00, 26.42it/s]"}},"31f86b59ff034c99ab1bd5db5c489599":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d3d78009138d47c59000f3c8d3b00497","placeholder":"​","style":"IPY_MODEL_e25336a1f16447c69ba1693b5e06250e","value":" 105535/105535 [02:25&lt;00:00, 791.76it/s]"}},"3b7988fa862146b9bcffc07e3778c576":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_1117dd321a5a40958afc859aaa68210c","placeholder":"​","style":"IPY_MODEL_0e4cac39c70347cd81cb2cbb9440515d","value":" 16941/16941 [45:41&lt;00:00,  7.91it/s]"}},"3ff58045068f48d3807f4faad80cebd8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c468f2c578e945d1a3c6f6140b2d4045","IPY_MODEL_7ee96ab6bc50439c80faee6030631d19","IPY_MODEL_31f86b59ff034c99ab1bd5db5c489599"],"layout":"IPY_MODEL_e043fbcb94f54a24961fc2d5a4055c9d"}},"42d73832b1ab471b92bbe1403c3d5509":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_56ad04c310b4463ba0f49f12412a7fab","max":16941,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2492442e63be4f70b2426167a66b5844","value":16941}},"52236b375cbc444f825de53701db2f95":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"56ad04c310b4463ba0f49f12412a7fab":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5d632b68eaaa4d3b98b19cd846d37447":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6c9a5af2c6244d72b11ca4ff597a5a43":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7ee96ab6bc50439c80faee6030631d19":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8532fe34e923407ca41e88024273819b","max":105535,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b905b2f331d3459697c87ca127c57ae7","value":105535}},"8202c30acb454392ac63503cdd055b37":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8532fe34e923407ca41e88024273819b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8faf3acbc2b046b1aab7a509bf4e1c5b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"90e9ca7899744bba9f207af94ef215b0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ab10a0dbd5224f95a42009072845eeaf":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"af2cb9bf3344439da110f1e04787f131":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8faf3acbc2b046b1aab7a509bf4e1c5b","max":16941,"min":0,"orientation":"horizontal","style":"IPY_MODEL_ab10a0dbd5224f95a42009072845eeaf","value":16941}},"b905b2f331d3459697c87ca127c57ae7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ba8712baa9bc4ae092116851566e40e9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_cf16bbfde0f54cc6bc028397cdf25b1d","IPY_MODEL_af2cb9bf3344439da110f1e04787f131","IPY_MODEL_307386f8d5dc4be1847271524734ca80"],"layout":"IPY_MODEL_52236b375cbc444f825de53701db2f95"}},"c468f2c578e945d1a3c6f6140b2d4045":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_172ab16b2c194786bf3738f722f89a81","placeholder":"​","style":"IPY_MODEL_8202c30acb454392ac63503cdd055b37","value":"100%"}},"cf16bbfde0f54cc6bc028397cdf25b1d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_90e9ca7899744bba9f207af94ef215b0","placeholder":"​","style":"IPY_MODEL_ea841e93c62f4a669a4b4187b9132771","value":"100%"}},"d3d78009138d47c59000f3c8d3b00497":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e043fbcb94f54a24961fc2d5a4055c9d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e25336a1f16447c69ba1693b5e06250e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ea841e93c62f4a669a4b4187b9132771":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f3a856662c294621b5e105e4ed606b97":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f7ff6f3fb3f64c23924800760d115f6a","placeholder":"​","style":"IPY_MODEL_5d632b68eaaa4d3b98b19cd846d37447","value":"100%"}},"f7ff6f3fb3f64c23924800760d115f6a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <h><center><b>🐧BIRDCIEF🐧2024</b></center></h>\n\n<img src = 'https://cdn.iconscout.com/wordpress/2021/10/01.png?f=webp&w=1440' class=\"center\">\n\n## ***I am trying to audio to image then classify the prediciton.***","metadata":{"papermill":{"duration":0.005322,"end_time":"2023-03-20T17:07:22.314736","exception":false,"start_time":"2023-03-20T17:07:22.309414","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# **Import necessary library**","metadata":{"papermill":{"duration":0.004132,"end_time":"2023-03-20T17:07:22.323147","exception":false,"start_time":"2023-03-20T17:07:22.319015","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q noisereduce\n!pip wheel -q https://github.com/Borda/kaggle_image-classify/archive/refs/heads/main.zip --wheel-dir frozen_packages\n!pip wheel -q https://github.com/PyTorchLightning/lightning-flash/archive/refs/heads/feature/soft_targets.zip --wheel-dir frozen_packages\n!rm frozen_packages/torch*\n!ls -l frozen_packages | grep -e kaggle -e flash","metadata":{"papermill":{"duration":119.36097,"end_time":"2023-03-20T17:09:21.688611","exception":false,"start_time":"2023-03-20T17:07:22.327641","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-10T13:34:30.754656Z","iopub.execute_input":"2024-04-10T13:34:30.755137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\n\nimport torchaudio\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel,delayed\nimport matplotlib.pyplot as plt\n\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nfrom functools import partial\nfrom joblib import Parallel, delayed\nimport glob\nfrom pprint import pprint\n\n\n\nimport torch\nimport torchaudio\nimport noisereduce\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport librosa\nfrom math import ceil\nfrom pprint import pprint\nfrom torch import Tensor\nfrom torch.utils.data import DataLoader\n","metadata":{"papermill":{"duration":5.454376,"end_time":"2023-03-20T17:09:27.150447","exception":false,"start_time":"2023-03-20T17:09:21.696071","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load the data**","metadata":{"papermill":{"duration":0.004271,"end_time":"2023-03-20T17:09:27.159596","exception":false,"start_time":"2023-03-20T17:09:27.155325","status":"completed"},"tags":[]}},{"cell_type":"code","source":"Path = \"/kaggle/input/birdclef-2024\"\npath_csv = os.path.join(Path, \"train_metadata.csv\")\ntrain_meta = pd.read_csv(path_csv).sample(frac=1)\ndisplay(train_meta.head())","metadata":{"papermill":{"duration":0.19245,"end_time":"2023-03-20T17:09:27.357627","exception":false,"start_time":"2023-03-20T17:09:27.165177","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Let check length of particular recording**","metadata":{"papermill":{"duration":0.005592,"end_time":"2023-03-20T17:09:27.369529","exception":false,"start_time":"2023-03-20T17:09:27.363937","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#explore audio data lengths\n\ndef get_length(fn):\n    fp = os.path.join(Path,\"train_audio\",fn)\n    waveform,sample_rate = torchaudio.load(fp)\n    return waveform.size()[-1]\n\nsizes = Parallel(n_jobs = os.cpu_count())(delayed(get_length)(fn) for fn in tqdm(train_meta[\"filename\"]))","metadata":{"papermill":{"duration":727.17696,"end_time":"2023-03-20T17:21:34.552131","exception":false,"start_time":"2023-03-20T17:09:27.375171","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(sizes, bins=150,color='red')\nplt.gca().set_xscale('log')\nplt.gca().set_yscale('log')\nplt.grid()","metadata":{"papermill":{"duration":1.27881,"end_time":"2023-03-20T17:21:35.836550","exception":false,"start_time":"2023-03-20T17:21:34.557740","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Convert audio to set of spectrogram**","metadata":{"papermill":{"duration":0.004843,"end_time":"2023-03-20T17:21:35.846776","exception":false,"start_time":"2023-03-20T17:21:35.841933","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SPECTROGRAM_PARAMS = dict(\n    sample_rate=32_000,\n    hop_length=640,\n    n_fft=800,\n    n_mels=128,\n    fmin=20,\n    fmax=16_000,\n    win_length=512\n)\nPCEN_PARAS = dict(\n    time_constant=0.06,\n    eps=1e-6,\n    gain=0.8,\n    power=0.25,\n    bias=10,\n)\n\n\n@torch.no_grad()\ndef create_spectrogram(\n    fname: str,\n    reduce_noise: bool = False,\n    frame_size: int = 5,\n    frame_step: int = 2,\n    spec_params: dict = SPECTROGRAM_PARAMS,\n) -> list:\n    waveform, sample_rate = librosa.core.load(fname, sr=spec_params[\"sample_rate\"], mono=True)\n    if reduce_noise:\n        waveform = noisereduce.reduce_noise(\n            y=waveform,\n            sr=sample_rate,\n            time_constant_s=float(frame_size),\n            time_mask_smooth_ms=250,\n            n_fft=spec_params[\"n_fft\"],\n            use_tqdm=False,\n            n_jobs=2,\n        )\n\n    step = int(frame_step * sample_rate)\n    size = int(frame_size * sample_rate)\n    count = ceil((len(waveform) - size) / float(step))\n    frames = []\n    for i in range(max(1, count)):\n        begin = i * step\n        frame = waveform[begin:begin + size]\n        if len(frame) < size:\n            if i == 0:\n                rep = round(float(size) / len(frame))\n                frame = frame.repeat(int(rep))\n            elif len(frame) < (size * 0.33):\n                continue\n            else:\n                frame = waveform[-size:]\n        frames.append(frame)\n\n    spectrograms = []\n    for frm in frames:\n        sg = librosa.feature.melspectrogram(\n            y=frm,\n            sr=sample_rate,\n            n_fft=spec_params[\"n_fft\"],\n            win_length=spec_params[\"win_length\"],\n            hop_length=spec_params[\"hop_length\"],\n            n_mels=spec_params[\"n_mels\"],\n            fmin=spec_params[\"fmin\"],\n            fmax=spec_params[\"fmax\"],\n            power=1,\n        )\n#         sg = librosa.pcen(sg, sr=sample_rate, hop_length=spec_params[\"hop_length\"], **PCEN_PARAS)\n        sg = librosa.amplitude_to_db(sg, ref=np.max)\n        spectrograms.append(np.nan_to_num(sg))\n    return spectrograms","metadata":{"papermill":{"duration":0.020707,"end_time":"2023-03-20T17:21:35.872585","exception":false,"start_time":"2023-03-20T17:21:35.851878","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_audio = os.path.join(Path, \"train_audio\", \"asbfly/XC134896.ogg\")\n# path_audio = os.path.join(PATH_DATASET, \"train_audio\", \"elepai/XC27344.ogg\")\n# path_audio = os.path.join(PATH_DATASET, \"train_audio\", \"hawgoo/XC210217.ogg\")\nprint(path_audio)\nsgs = create_spectrogram(path_audio, reduce_noise=False)\n\n\nfig, axarr = plt.subplots(nrows=len(sgs), figsize=(8, 3 * len(sgs)))\nfor i, sg in enumerate(sgs):\n    print(np.min(sg), np.max(sg))\n    im = axarr[i].imshow(sg)  # librosa\n    plt.colorbar(im, ax=axarr[i])\n","metadata":{"papermill":{"duration":15.825771,"end_time":"2023-03-20T17:21:51.703238","exception":false,"start_time":"2023-03-20T17:21:35.877467","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Export only frames form the recording beginning and ending**","metadata":{"papermill":{"duration":0.05616,"end_time":"2023-03-20T17:21:51.818156","exception":false,"start_time":"2023-03-20T17:21:51.761996","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def convert_and_export(\n    fn: str, path_in: str, path_out: str,\n    reduce_noise: bool = False,\n    frame_size: int = 5, frame_step: int = 2,\n    img_extension=\".png\",\n) -> None:\n    path_audio = os.path.join(path_in, fn)\n    try:\n        sgs = create_spectrogram(\n            path_audio,\n            reduce_noise=reduce_noise,\n            frame_size=frame_size,\n            frame_step=frame_step,\n        )\n    except Exception as ex:\n        print(f\"Failed conversion for audio: {path_audio}\")\n        return\n    if not sgs:\n        print(f\"Too short audio for: {path_audio}\")\n        return\n    # see: https://www.kaggle.com/c/birdclef-2022/discussion/308861\n    # this is adjustment for window 5s and step 2s\n    nb = ceil((10 - frame_size) / frame_step) + 1\n    if len(sgs) > 2 * nb:\n        sgs = sgs[:nb] + sgs[-nb:]\n    path_npz = os.path.join(path_out, fn + '.npz')\n    os.makedirs(os.path.dirname(path_npz), exist_ok=True)\n    # np.savez_compressed(path_npz, np.array(sgs, dtype=np.float16))\n    for i, sg in enumerate(sgs):\n        path_img = os.path.join(path_out, fn + f\".{i:03}\" + img_extension)\n        try:\n            # plt.imsave(path_img, sg, vmin=-70, vmax=20)\n            sg = (sg + 80) / 80.0\n            sg = np.clip(sg, a_min=0, a_max=1) * 255\n            img = Image.fromarray(sg.astype(np.uint8))\n            img.resize((256,256)).save(path_img)\n        except Exception as ex:\n            print(f\"Failed exporting for image: {path_img}\")\n            continue\n\n","metadata":{"papermill":{"duration":0.071493,"end_time":"2023-03-20T17:21:51.944598","exception":false,"start_time":"2023-03-20T17:21:51.873105","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Conversion of full dataset**","metadata":{"papermill":{"duration":0.054982,"end_time":"2023-03-20T17:21:52.055525","exception":false,"start_time":"2023-03-20T17:21:52.000543","status":"completed"},"tags":[]}},{"cell_type":"code","source":"_convert_and_export = partial(\n    convert_and_export,\n    path_in=os.path.join(Path, \"train_audio\"),\n    path_out=\"train_images\",\n)\n\n_= Parallel(n_jobs=3)(delayed(_convert_and_export)(fn) for fn in tqdm(train_meta[\"filename\"]))\n# _= list(map(_convert_and_export, tqdm(train_meta[\"filename\"])))\n\n","metadata":{"papermill":{"duration":2742.850554,"end_time":"2023-03-20T18:07:34.961034","exception":false,"start_time":"2023-03-20T17:21:52.110480","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_meta[\"filename\"][1])\nimgs = glob.glob(os.path.join(\"train_images\", train_meta[\"filename\"][1] + \".*.png\"))\npprint(sorted(imgs))\n\npath_img = imgs[0]\nprint(path_img)\nimg = plt.imread(path_img)\nprint(img.shape)\nplt.imshow(img)\n\n","metadata":{"papermill":{"duration":0.376246,"end_time":"2023-03-20T18:07:35.394664","exception":false,"start_time":"2023-03-20T18:07:35.018418","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Color normalization**","metadata":{"papermill":{"duration":0.05891,"end_time":"2023-03-20T18:07:35.516461","exception":false,"start_time":"2023-03-20T18:07:35.457551","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _color_means(img_path):\n    img = plt.imread(img_path)\n    if np.max(img) > 1.5:\n        img = img / 255.0\n    clr_mean = np.mean(img) if img.ndim == 2 else {i: np.mean(img[..., i]) for i in range(3)}\n    clr_std = np.std(img) if img.ndim == 2 else {i: np.std(img[..., i]) for i in range(3)}\n    return clr_mean, clr_std\n\nimages = glob.glob(os.path.join(\"train_images\", \"*\", \"*.png\"))\nclr_mean_std = Parallel(n_jobs=os.cpu_count())(delayed(_color_means)(fn) for fn in tqdm(images))","metadata":{"papermill":{"duration":146.677477,"end_time":"2023-03-20T18:10:02.254690","exception":false,"start_time":"2023-03-20T18:07:35.577213","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_mean = pd.DataFrame([c[0] for c in clr_mean_std]).describe()\ndisplay(img_color_mean.T)","metadata":{"papermill":{"duration":0.145022,"end_time":"2023-03-20T18:10:02.467808","exception":false,"start_time":"2023-03-20T18:10:02.322786","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_std = pd.DataFrame([c[1] for c in clr_mean_std]).describe()\ndisplay(img_color_std.T)","metadata":{"papermill":{"duration":0.118549,"end_time":"2023-03-20T18:10:02.648887","exception":false,"start_time":"2023-03-20T18:10:02.530338","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_mean = list(img_color_mean.T[\"mean\"])\nimg_color_std = list(img_color_std.T[\"mean\"])\nprint(img_color_mean, img_color_std)","metadata":{"papermill":{"duration":0.077099,"end_time":"2023-03-20T18:10:02.789360","exception":false,"start_time":"2023-03-20T18:10:02.712261","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}