{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":179165407,"sourceType":"kernelVersion"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"papermill":{"default_parameters":{},"duration":1759.895161,"end_time":"2024-08-30T11:09:43.802608","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-30T10:40:23.907447","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"016fa0439e614957ae1d00f8bd4c2198":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0787305da5854215a5382f793d62c419":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"093f632d8eaf421abe5458ff9acd44e4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"09f903828a9142939b58fffd17168142":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_093f632d8eaf421abe5458ff9acd44e4","max":33569,"min":0,"orientation":"horizontal","style":"IPY_MODEL_555a5166873246ef93b4070f3a166ee6","value":33569}},"0b191f112f8c4023ad181264831a896a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"0ea4efabc67e438e8a5b2d226e516dcb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e481f53b82d3467bb9f2f6f3964ab381","max":33670,"min":0,"orientation":"horizontal","style":"IPY_MODEL_40f33116f1d4447e9d8703fc1f896c02","value":33670}},"1256a8f5d1a34740a05e581bd80a874e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1284a9c7292a4145ac31d38c7a26919c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_58e3d114b0724f8a88c734e94d1908cb","IPY_MODEL_cc3ae62338554853b9b0417e16f5f0fd","IPY_MODEL_163883dad31f48a7ba1474150a08207f"],"layout":"IPY_MODEL_016fa0439e614957ae1d00f8bd4c2198"}},"14fddccbd18c421e9dc16f8771d0f02d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_540b1866696a4119b8fc1cfbb510b84b","placeholder":"​","style":"IPY_MODEL_9979e573871546b780678d880feea05e","value":"100%"}},"1527e933c0444a9199c4031aec7ff853":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"163883dad31f48a7ba1474150a08207f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_17d54c05beb844bb91a28722ffdfbedd","placeholder":"​","style":"IPY_MODEL_442f5d00f6b548edaf3f1bb7ef21d773","value":" 97/97 [00:00&lt;00:00, 203.77it/s]"}},"17d54c05beb844bb91a28722ffdfbedd":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1901140c342248e7853568c7cd963e4d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1932921d06d642e6a46fb096886693ff":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1a8d0647e7fd4c65826fddc1a98feff6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1aa5f551a1a945759c21c47d43dc0517":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1c828fe0f7f44bfc8f4115adc9d54f46":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"218c58b31b8b4b7cbd2224a92ec22807":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_5c458a33e1b747fcadf0ac702d2ae9b4","max":147218,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8417b4c832cb425d8ef553941fdaa0e5","value":147218}},"243e34a7720e4dfbb0719b83d0a270bc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5d335510cb894d09a8b1cd75dcb6a9f0","placeholder":"​","style":"IPY_MODEL_940c2aed6cd84d699582b58f7a21b5ff","value":" 147218/147218 [16:37&lt;00:00, 169.86it/s]"}},"30df670ec6fd42ce909c471161e5e274":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ce54abb554ad427596c25f20e1b0aadd","IPY_MODEL_218c58b31b8b4b7cbd2224a92ec22807","IPY_MODEL_687a59cf4dc343169ee07265e31959f2"],"layout":"IPY_MODEL_b7dc5a5d3c16482ca1251a50b23bd6f4"}},"31d262ece0d64a0cafb78bad35ccacc7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3517a7272a064de7ab1be06b4e9b3fe6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"40f33116f1d4447e9d8703fc1f896c02":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"442f5d00f6b548edaf3f1bb7ef21d773":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4fd09261692f4244b582f4032fa48ed6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"540b1866696a4119b8fc1cfbb510b84b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"555a5166873246ef93b4070f3a166ee6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"58e3d114b0724f8a88c734e94d1908cb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a005ca32f9514bb2aca574880a0768a9","placeholder":"​","style":"IPY_MODEL_ae9ad4a415df44d3879eb48e4d593920","value":"100%"}},"59a031626da74f34a23205e066db5d94":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"59f457bdfc18482ba0dc26298ddabe06":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5b1bdd4255b547428d958d4623b75589":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1a8d0647e7fd4c65826fddc1a98feff6","max":79979,"min":0,"orientation":"horizontal","style":"IPY_MODEL_1527e933c0444a9199c4031aec7ff853","value":79979}},"5c458a33e1b747fcadf0ac702d2ae9b4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5d335510cb894d09a8b1cd75dcb6a9f0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6433db07ef20465984834d1d5a63a177":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"687a59cf4dc343169ee07265e31959f2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d05eebd779284645b06dfcbce22d579f","placeholder":"​","style":"IPY_MODEL_4fd09261692f4244b582f4032fa48ed6","value":" 147218/147218 [05:58&lt;00:00, 382.27it/s]"}},"698f9ade77ca416c94d56ce7b1c11d69":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6c0f86fbdc004c20919b5615c0377a89":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_59a031626da74f34a23205e066db5d94","placeholder":"​","style":"IPY_MODEL_ffb785288ebb4155bf7b47b0e4d0201e","value":"100%"}},"6f964668a22a46a587a667eeff4911ea":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7c0bead91ced49d588c1f8fed5085832":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8417b4c832cb425d8ef553941fdaa0e5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"873a1fc6a1454eba9b24ea19f3bef45e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b44721ca37464b2da6a1808719a280bd","placeholder":"​","style":"IPY_MODEL_e90737905cb647e6aa107bfd8ad1c55a","value":" 79979/79979 [02:40&lt;00:00, 527.92it/s]"}},"8afc7458156746d981aebdfec4981c35":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_698f9ade77ca416c94d56ce7b1c11d69","placeholder":"​","style":"IPY_MODEL_7c0bead91ced49d588c1f8fed5085832","value":"100%"}},"8b789a3d3922408e8aeeee950943b47c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8c47931328de491493be53506c6e65fe":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_3517a7272a064de7ab1be06b4e9b3fe6","placeholder":"​","style":"IPY_MODEL_0787305da5854215a5382f793d62c419","value":" 33670/33670 [01:04&lt;00:00, 636.44it/s]"}},"8d5b041393124c9db873e85fe5b4d127":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_14fddccbd18c421e9dc16f8771d0f02d","IPY_MODEL_5b1bdd4255b547428d958d4623b75589","IPY_MODEL_873a1fc6a1454eba9b24ea19f3bef45e"],"layout":"IPY_MODEL_adfcd8632a674a57877712725e846b65"}},"940c2aed6cd84d699582b58f7a21b5ff":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9979e573871546b780678d880feea05e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a005ca32f9514bb2aca574880a0768a9":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a8781d0c800144eb82dcf7e42f9e4010":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_fcc4082a02724033992859c1709c7d46","IPY_MODEL_f71ce5a38125420dba1677f7b7d49fb3","IPY_MODEL_243e34a7720e4dfbb0719b83d0a270bc"],"layout":"IPY_MODEL_59f457bdfc18482ba0dc26298ddabe06"}},"adfcd8632a674a57877712725e846b65":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ae9ad4a415df44d3879eb48e4d593920":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b44721ca37464b2da6a1808719a280bd":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b7dc5a5d3c16482ca1251a50b23bd6f4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c9073dbe004d494b9332cb38e8806d5d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_8afc7458156746d981aebdfec4981c35","IPY_MODEL_09f903828a9142939b58fffd17168142","IPY_MODEL_edebade91c4b410aa764827231811fea"],"layout":"IPY_MODEL_1256a8f5d1a34740a05e581bd80a874e"}},"cc3ae62338554853b9b0417e16f5f0fd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1aa5f551a1a945759c21c47d43dc0517","max":97,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0b191f112f8c4023ad181264831a896a","value":97}},"ce54abb554ad427596c25f20e1b0aadd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6433db07ef20465984834d1d5a63a177","placeholder":"​","style":"IPY_MODEL_31d262ece0d64a0cafb78bad35ccacc7","value":"100%"}},"d05eebd779284645b06dfcbce22d579f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e481f53b82d3467bb9f2f6f3964ab381":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e90737905cb647e6aa107bfd8ad1c55a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ec8a2b01dd6c41b5a434969d0205b170":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec8e7c1b4ff04e7ab8c556ce4e71440c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"edebade91c4b410aa764827231811fea":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_1901140c342248e7853568c7cd963e4d","placeholder":"​","style":"IPY_MODEL_8b789a3d3922408e8aeeee950943b47c","value":" 33569/33569 [00:56&lt;00:00, 1745.07it/s]"}},"f71ce5a38125420dba1677f7b7d49fb3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1932921d06d642e6a46fb096886693ff","max":147218,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6f964668a22a46a587a667eeff4911ea","value":147218}},"fcc4082a02724033992859c1709c7d46":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ec8e7c1b4ff04e7ab8c556ce4e71440c","placeholder":"​","style":"IPY_MODEL_1c828fe0f7f44bfc8f4115adc9d54f46","value":"100%"}},"fe9eadb19eab47728311364a0141b034":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_6c0f86fbdc004c20919b5615c0377a89","IPY_MODEL_0ea4efabc67e438e8a5b2d226e516dcb","IPY_MODEL_8c47931328de491493be53506c6e65fe"],"layout":"IPY_MODEL_ec8a2b01dd6c41b5a434969d0205b170"}},"ffb785288ebb4155bf7b47b0e4d0201e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Analysis - Kaggle - RSNA 2024 Lumbar Spine Degenerative Classification\n\nThis notebook explores the competition data presented for [Kaggle - RSNA 2024 Lumbar Spine Degenerative Classification](https://www.kaggle.com/competitions/rsna-2024-lumbar-spine-degenerative-classification) competition\n\n> _The goal of this competition is to create models that can be used to aid in the detection and classification of degenerative spine conditions using lumbar spine MR images. Competitors will develop models that simulate a radiologist's performance in diagnosing spine conditions._\n\nAuthor: `@cyshin971`  \nDate: `10/6/2024`","metadata":{"papermill":{"duration":0.026221,"end_time":"2024-08-30T10:40:26.680258","exception":false,"start_time":"2024-08-30T10:40:26.654037","status":"completed"},"tags":[]}},{"cell_type":"code","source":"version = '2.0'\nisLocal = False # Set to True when running in local environment","metadata":{"papermill":{"duration":0.037509,"end_time":"2024-08-30T10:40:26.742199","exception":false,"start_time":"2024-08-30T10:40:26.704690","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:37.278154Z","iopub.execute_input":"2024-10-06T06:39:37.278467Z","iopub.status.idle":"2024-10-06T06:39:37.288254Z","shell.execute_reply.started":"2024-10-06T06:39:37.278427Z","shell.execute_reply":"2024-10-06T06:39:37.287395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 🤔 | What this notebook contains","metadata":{"papermill":{"duration":0.024264,"end_time":"2024-08-30T10:40:26.791146","exception":false,"start_time":"2024-08-30T10:40:26.766882","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"This notebook expands upon the `complete_train.csv` metadata provided by Muhammad Tariq Pervez in [Refined Train Data for Lumber Spine Degenerative](https://www.kaggle.com/datasets/tariqcp/train-data-for-rsna-2024-lsdc) [2] and adds information regarding the DICOM data including the size and location of the image into three metadata files.\n- `analyzed_train.csv` : metadata of each image instance\n- `analyzed_labeled_train.csv` : metadata of each label\n- `analyzed_train_series_description.csv`: metadata of each series\n\n> The notebook also provides analyzed metadata for the sample test competition data (`analyzed_test.csv`, `analyzed_test_series_description.csv`). However if any of the added features in this notebook is to be used for model prediction, the code to add these features to the test data needs to be added to the submission notebook as the test data to calculate the score is different from the sample test data provided by the competition data\n\nas well as provide code to visualize images with annotation of the labels in 2D and 3D MRI slides.\n\n\n> For a clinical overview of Lumbar Spine Degenerative diseases see [Anatomy & Image Visualization Overview-RSNA RAIDS - Abhinav Suri](https://www.kaggle.com/code/abhinavsuri/anatomy-image-visualization-overview-rsna-raids) [3]\n\n\n### Competition data analysis  \n- Compile metadata for all images provided by competition (`analyzed_train.csv`)  \n\n- Adds the series description information to each label and instance  \n\n- Adds information regarding number of labels for each instance and series  \n\n- Adds the total number of instances in each series  \n\n- Adds the diagnosis information for each label, instance, and series  \n  - Specific diagnosis (`condition`) -> Diagnosis (`sup_condition`) E.G. L/R Subarticular Stenosis -> Subarticular Stenosis  \n  - Number of conditions associated with each instance and series  \n \n \n- Analyzes and adds information regarding the size of each image for each series type  \n  - Height/width ratio  \n  - Area of image  \n  - Most common, smallest, and largest sizes for each series type  \n  \n  \n- Adds information found in DICOM data  \n  - Slice location -> normalized slice location  \n  - Slice thickness  \n  - Minimum, maximum, and range of pixel values of each image (for processing)  \n  \n  \n### Data Visualization\n  - 2D image function that annotates all labels\n  - 3D MRI Slides function that annotates all labels\n  \n### A Dictionary of Edge Cases\n  - `t1_scs` : Saggital T1 images for `Spinal Canal Stenosis`\n  - `unlabeled_series` : images in series that are not included in train_series_description.csv\n  - `unlabeled_study` : images in study that is not included in train.csv\n  - `three_conditions_series` : images for series associated with 3 conditions\n  - `mixed_sup_condition_series` : images for the study that has more than one diagnosis  \n  - `multiple_labels_instances` : images of instances with > 5 labels\n  - `fifteen_labels_series` : images of series with the maximum number of labels (15)\n  - `two_thickness_ss_series` : images in series where there are labels for Subarticular Stenosis and 2 slice thicknesses in the instances\n  - `four_spacings_ss_series` : images in series where there are labels for Subarticular Stenosis and 4 slice spacings in the instances","metadata":{"papermill":{"duration":0.023987,"end_time":"2024-08-30T10:40:26.839346","exception":false,"start_time":"2024-08-30T10:40:26.815359","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 📜 | Version History","metadata":{"papermill":{"duration":0.023811,"end_time":"2024-08-30T10:40:26.887419","exception":false,"start_time":"2024-08-30T10:40:26.863608","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- `version 1.0` : initial release version  \n- `version 2.0` : Minor text fixes and added more edge cases\n  - Removed `slice_loc` after normalizing slice location\n  - Added `spacing` between instances in a series to the metadata\n  - Added the following information from DICOM files to the metadata\n    - `range_val` : Range of values in a series\n    - `sizes` : Unique sizes of images in a series\n    - `areas` : Unique areas of images in a series\n    - `ratios` : Unique ratios of images in a series\n    - `spacings` : Unique spacing between images in a series\n    - `slice_thicks` : Unique thickness of image slices in a series\n  - Added `find_edge_series(edge_case_key, verbose=True)` returns a list of `series_id` in an edge case\n  - Added edge cases: `multiple_labels_instances`, `fifteen_labels_series`, `two_thickness_ss_series`, `four_spacings_ss_series`","metadata":{"papermill":{"duration":0.023648,"end_time":"2024-08-30T10:40:26.934855","exception":false,"start_time":"2024-08-30T10:40:26.911207","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 📌 | References","metadata":{"papermill":{"duration":0.023691,"end_time":"2024-08-30T10:40:26.983076","exception":false,"start_time":"2024-08-30T10:40:26.959385","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"1. [RSNA 2024 Lumbar Spine Degenrative Classification - Kaggle Competition][1]\n2. [Consolidated Train Data - Muhammad Tariq Pervez][2]\n3. [Anatomy & Image Visualization Overview-RSNA RAIDS - Abhinav Suri][3]\n4. [Lumbar RSNA 2024: Visualizing + EDA + Sub - Allegich][4]\n5. [RSNA Lumbar Spine Analysis - Satya][5]\n\n[1]: https://www.kaggle.com/competitions/rsna-2024-lumbar-spine-degenerative-classification\n[2]: https://www.kaggle.com/competitions/rsna-2024-lumbar-spine-degenerative-classification/discussion/505929\n[3]: https://www.kaggle.com/code/abhinavsuri/anatomy-image-visualization-overview-rsna-raids\n[4]: https://www.kaggle.com/code/allegich/lumbar-rsna-2024-visualizing-eda-sub\n[5]: https://www.kaggle.com/code/satyaprakashshukl/rsna-lumbar-spine-analysis","metadata":{"papermill":{"duration":0.023483,"end_time":"2024-08-30T10:40:27.030382","exception":false,"start_time":"2024-08-30T10:40:27.006899","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 📚 | Import Libraries ","metadata":{"papermill":{"duration":0.023489,"end_time":"2024-08-30T10:40:27.077821","exception":false,"start_time":"2024-08-30T10:40:27.054332","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('display.precision', 3) # Set display precisions\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom matplotlib import animation, rc\nimport warnings\n\nimport pydicom\nfrom tqdm.notebook import tqdm\nimport joblib","metadata":{"papermill":{"duration":1.222829,"end_time":"2024-08-30T10:40:28.324827","exception":false,"start_time":"2024-08-30T10:40:27.101998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:37.304806Z","iopub.execute_input":"2024-10-06T06:39:37.305162Z","iopub.status.idle":"2024-10-06T06:39:38.181053Z","shell.execute_reply.started":"2024-10-06T06:39:37.305122Z","shell.execute_reply":"2024-10-06T06:39:38.180058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nprint(\"Python: \" + sys.version)\nprint(\"---------------------------\")\nprint(\"Pandas:\", pd.__version__)\nprint(\"Pydicom:\", pydicom.__version__)\nprint(\"Open CV:\", cv2.__version__)\nprint(\"---------------------------\")","metadata":{"papermill":{"duration":0.033766,"end_time":"2024-08-30T10:40:28.383070","exception":false,"start_time":"2024-08-30T10:40:28.349304","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.183094Z","iopub.execute_input":"2024-10-06T06:39:38.183642Z","iopub.status.idle":"2024-10-06T06:39:38.190762Z","shell.execute_reply.started":"2024-10-06T06:39:38.183595Z","shell.execute_reply":"2024-10-06T06:39:38.189630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🌐 | Global Variables","metadata":{"papermill":{"duration":0.023616,"end_time":"2024-08-30T10:40:28.430862","exception":false,"start_time":"2024-08-30T10:40:28.407246","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### ⚙️ | Configuration","metadata":{"papermill":{"duration":0.023767,"end_time":"2024-08-30T10:40:28.478840","exception":false,"start_time":"2024-08-30T10:40:28.455073","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    color_map = plt.cm.bone # 'viridis', 'binary', 'CMRmap', plt.cm.bone\n    \n    version_str = (version.split(\".\")[0] if version.split(\".\")[1] == '0' else version.replace(\".\", \"_\"))\n    analysis_version = f'analysis_v{version_str}'\n\nprint(f'Analysis Version: {CFG.analysis_version}')","metadata":{"papermill":{"duration":0.032552,"end_time":"2024-08-30T10:40:28.535068","exception":false,"start_time":"2024-08-30T10:40:28.502516","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.191761Z","iopub.execute_input":"2024-10-06T06:39:38.192157Z","iopub.status.idle":"2024-10-06T06:39:38.202563Z","shell.execute_reply.started":"2024-10-06T06:39:38.192106Z","shell.execute_reply":"2024-10-06T06:39:38.201427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 🔎 | Data Features","metadata":{"papermill":{"duration":0.023926,"end_time":"2024-08-30T10:40:28.583290","exception":false,"start_time":"2024-08-30T10:40:28.559364","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CONDITION:\n    num_classes = 5\n    class_names = ['Spinal Canal Stenosis',\n                   'Left Neural Foraminal Narrowing','Right Neural Foraminal Narrowing',\n                   'Left Subarticular Stenosis','Right Subarticular Stenosis']\n    label2name = dict(enumerate(class_names))\n    name2label = {v: k for k, v in label2name.items()}\n    \n    short_names = ['SCS',\n                   'l-FN', 'r-FN',\n                   'l-SS', 'r-SS']\n    label2short = dict(enumerate(short_names))\n    short2label = {v: k for k, v in label2short.items()}\n    def short2name(short_name):\n        return CONDITION.label2name[CONDITION.short2label[short_name]]\n    def name2short(class_name):\n        return CONDITION.label2short[CONDITION.name2label[class_name]]\n    \n    sub_names = ['spinal_canal_stenosis',\n                 'left_neural_foraminal_narrowing','right_neural_foraminal_narrowing',\n                 'left_subarticular_stenosis','right_subarticular_stenosis']\n    label2sub = dict(enumerate(sub_names))\n    sub2label = {v: k for k, v in label2sub.items()}\n    def sub2name(sub_name):\n        return CONDITION.label2name[CONDITION.sub2label[sub_name]]\n    def name2sub(class_name):\n        return CONDITION.label2sub[CONDITION.name2label[class_name]]\n    \n    num_sup_classes = 3\n    sup_class_names = ['Mixed', 'Spinal Canal Stenosis', 'Neural Foraminal Narrowing', 'Subarticular Stenosis']\n    sup_label2name = dict(enumerate(sup_class_names))\n    sup_name2label = {v: k for k, v in sup_label2name.items()}\n    name2sup_label = {0: 1, 1: 2, 2: 2, 3: 3, 4: 3}\n    sup2name_label = {0: [], 1: [0], 2: [1, 2], 3: [3, 4]}\n    def name2sup(sub_label):\n        return CONDITION.name2sup_label[sub_label]\n    def sup2name(sup_label):\n        return CONDITION.sup2name_label[sup_label]","metadata":{"papermill":{"duration":0.041895,"end_time":"2024-08-30T10:40:28.649632","exception":false,"start_time":"2024-08-30T10:40:28.607737","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.205402Z","iopub.execute_input":"2024-10-06T06:39:38.205798Z","iopub.status.idle":"2024-10-06T06:39:38.218245Z","shell.execute_reply.started":"2024-10-06T06:39:38.205756Z","shell.execute_reply":"2024-10-06T06:39:38.217524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SERIES:\n    num_classes = 3\n    class_names = ['Sagittal T2/STIR', 'Sagittal T1', 'Axial T2']\n    label2name = dict(enumerate(class_names))\n    name2label = {v: k for k, v in label2name.items()}\n    \n    short_names = ['saggital_t2', 'saggital_t1', 'axial_t2']\n    label2short = dict(enumerate(short_names))\n    short2label = {v: k for k, v in enumerate(short_names)}\n    def short2name(short_name):\n        return SERIES.label2name[SERIES.short2label[short_name]]\n    def name2short(class_name):\n        return SERIES.label2short[SERIES.name2label[class_name]]","metadata":{"papermill":{"duration":0.034144,"end_time":"2024-08-30T10:40:28.708006","exception":false,"start_time":"2024-08-30T10:40:28.673862","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.219430Z","iopub.execute_input":"2024-10-06T06:39:38.219814Z","iopub.status.idle":"2024-10-06T06:39:38.230586Z","shell.execute_reply.started":"2024-10-06T06:39:38.219732Z","shell.execute_reply":"2024-10-06T06:39:38.229686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LEVEL:\n    num_classes = 5\n    class_names = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n    label2name = dict(enumerate(class_names))\n    name2label = {v: k for k, v in label2name.items()}\n    \n    sub_names = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n    label2sub = dict(enumerate(sub_names))\n    sub2label = {v: k for k, v in enumerate(sub_names)}\n    def sub2name(sub_name):\n        return LEVEL.label2name[LEVEL.sub2label[sub_name]]\n    def name2sub(class_name):\n        return LEVEL.label2sub[LEVEL.name2label[class_name]]","metadata":{"papermill":{"duration":0.034896,"end_time":"2024-08-30T10:40:28.814193","exception":false,"start_time":"2024-08-30T10:40:28.779297","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.232028Z","iopub.execute_input":"2024-10-06T06:39:38.232381Z","iopub.status.idle":"2024-10-06T06:39:38.244837Z","shell.execute_reply.started":"2024-10-06T06:39:38.232341Z","shell.execute_reply":"2024-10-06T06:39:38.243919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OUTPUT:\n    num_classes = 3\n    class_names = ['Normal/Mild', 'Moderate', 'Severe']\n    label2name = dict(enumerate(class_names))\n    name2label = {v: k for k, v in label2name.items()}\n    \n    colors = {'Normal/Mild': 'green',\n              'Moderate': 'yellow',\n              'Severe': 'red'}","metadata":{"papermill":{"duration":0.033498,"end_time":"2024-08-30T10:40:28.872427","exception":false,"start_time":"2024-08-30T10:40:28.838929","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.246029Z","iopub.execute_input":"2024-10-06T06:39:38.246388Z","iopub.status.idle":"2024-10-06T06:39:38.254406Z","shell.execute_reply.started":"2024-10-06T06:39:38.246347Z","shell.execute_reply":"2024-10-06T06:39:38.253572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📁 | Dataset Path ","metadata":{"papermill":{"duration":0.024304,"end_time":"2024-08-30T10:40:28.920905","exception":false,"start_time":"2024-08-30T10:40:28.896601","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\n\nif isLocal: # if running locally or JupyterHub\n    BASE_PATH = \"../data\" # set data path to local (repository) (change for release rev)\n    ANALYSIS_PATH = f\"../data/analysis/{CFG.analysis_version}\"\n    COMPLETE_TRAIN_PATH = BASE_PATH\nelse: # set data path to Kaggle input\n    COMPLETE_TRAIN_PATH = \"/kaggle/input/consolidating-train-label-coordinates-and-images\"\n    BASE_PATH = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification\"\n    ANALYSIS_PATH = \"/kaggle/working/\"","metadata":{"papermill":{"duration":0.033049,"end_time":"2024-08-30T10:40:28.978088","exception":false,"start_time":"2024-08-30T10:40:28.945039","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.257302Z","iopub.execute_input":"2024-10-06T06:39:38.257600Z","iopub.status.idle":"2024-10-06T06:39:38.265157Z","shell.execute_reply.started":"2024-10-06T06:39:38.257569Z","shell.execute_reply":"2024-10-06T06:39:38.264310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📖 | Meta Data ","metadata":{"papermill":{"duration":0.023913,"end_time":"2024-08-30T10:40:29.026151","exception":false,"start_time":"2024-08-30T10:40:29.002238","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 🤝 | Helper Functions","metadata":{"papermill":{"duration":0.023875,"end_time":"2024-08-30T10:40:29.074042","exception":false,"start_time":"2024-08-30T10:40:29.050167","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Move a column to the last position in a DataFrame\ndef move_column_to_last(df, column_name):\n    cols = df.columns.tolist()\n    cols.remove(column_name)\n    cols.append(column_name)\n    return df[cols]\n\n# Merge two DataFrames and drop the columns to be added before adding them\ndef merge_dfs(df_main, df_tmp, cols_to_add, on_col):\n    if not isinstance(df_tmp, pd.DataFrame): # if df_tmp is not a DataFrame, convert it to a DataFrame\n        print(f'Converting [{type(df_tmp)}] to DataFrame')\n        df_tmp = pd.DataFrame(df_tmp)\n    df_main = df_main.drop(columns=cols_to_add, errors='ignore')\n    col_list = [on_col] + cols_to_add\n    for col in col_list:\n        if col not in df_tmp.columns.tolist():\n            print(f'{col} not in DataFrame')\n            return df_main\n    df_main = pd.merge(df_main, df_tmp[col_list], on=on_col, how='left')\n    return df_main\n\n# Filter a DataFrame for a specific feature\ndef filter_df(df, filter, feature, verbose=False):\n    if verbose:\n        print(f'Original: {df.shape}')\n        display(df.head(3))\n    filt_df = df[df[feature].isin(filter)]\n    if verbose:\n        print(f'Filtered: {df.shape}')\n        display(df.head(3))\n    return filt_df","metadata":{"papermill":{"duration":0.035682,"end_time":"2024-08-30T10:40:29.133569","exception":false,"start_time":"2024-08-30T10:40:29.097887","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.267895Z","iopub.execute_input":"2024-10-06T06:39:38.268176Z","iopub.status.idle":"2024-10-06T06:39:38.278877Z","shell.execute_reply.started":"2024-10-06T06:39:38.268145Z","shell.execute_reply":"2024-10-06T06:39:38.278047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_histogram(df, column, title='', labels_on=True, colors=None, size=(10, 5)):\n    \"\"\"\n    Plots a histogram of a dataframe column\n    Args:\n        df (DataFrame): dataframe containing column of interest\n        column (str): column name in str\n        labels_on (bool, optional): whether to display count labels with percentages (default to True)\n        colors (dict, optional): dictionary mapping column values to colors (default to empty dictionary)\n        size (tuple, optional): size of the resulting plot (default to (10, 5))\n    Returns:\n        colors: dictionary mapping column values to colors used in the plot\n    \"\"\"\n    # Count occurrences of each category\n    counts = df[column].value_counts()\n    total_count = counts.sum()\n\n    # Use the tab10 colormap for consistent colors if colors are not provided\n    if colors is None:\n        tab10_colors = plt.cm.tab10.colors\n        colors = {val: tab10_colors[i % len(tab10_colors)] for i, val in enumerate(counts.index)}\n\n    # Create a bar plot\n    plt.figure(figsize=size)\n    \n    # Determine if the counts.index can be converted to an integer or float and sort the index accordingly\n    if all(isinstance(val, int) for val in counts.index):\n        counts = counts.sort_index()\n    elif all(isinstance(val, float) for val in counts.index):\n        counts = counts.round(4).sort_index()\n    bars = plt.bar(counts.index.astype(str), counts.values, color=[colors[val] for val in counts.index])\n\n    # Add count labels to the histogram with percentage if labels_on is True\n    if labels_on:\n        for bar, count in zip(bars, counts.values):\n            count_text = f'{count} ({count / total_count * 100:.1f}%)'\n            plt.text(bar.get_x() + bar.get_width() / 2, count + max(count * 0.01, 0.1), count_text, \n                     ha='center', va='bottom')\n\n    # Set labels and title\n    plt.xlabel(column.title().replace('_', ' '), fontweight='bold')\n        \n    plt.ylabel('Count', fontweight='bold')\n    if title == '':\n        title = column.title().replace('_', ' ')\n    plt.title(title, fontsize=13, fontweight='bold')\n\n    # Remove top and right spines\n    plt.gca().spines['top'].set_visible(False)\n    plt.gca().spines['right'].set_visible(False)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return colors","metadata":{"papermill":{"duration":0.039652,"end_time":"2024-08-30T10:40:29.197258","exception":false,"start_time":"2024-08-30T10:40:29.157606","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.283323Z","iopub.execute_input":"2024-10-06T06:39:38.283602Z","iopub.status.idle":"2024-10-06T06:39:38.297644Z","shell.execute_reply.started":"2024-10-06T06:39:38.283573Z","shell.execute_reply":"2024-10-06T06:39:38.296577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Labeled Data","metadata":{"papermill":{"duration":0.023814,"end_time":"2024-08-30T10:40:29.245279","exception":false,"start_time":"2024-08-30T10:40:29.221465","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"This notebook uses the metadata from [Refined Train Data for Lumber Spine Degenerative](https://www.kaggle.com/datasets/tariqcp/train-data-for-rsna-2024-lsdc) [2] by Muhammad Tariq Pervez `complete_train.csv`","metadata":{"papermill":{"duration":0.023731,"end_time":"2024-08-30T10:40:29.293541","exception":false,"start_time":"2024-08-30T10:40:29.269810","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Kaggle Data\nkaggle_train_df = pd.read_csv(os.path.join(BASE_PATH, 'train.csv'))\ntrain_series_desc_df = pd.read_csv(os.path.join(BASE_PATH, 'train_series_descriptions.csv'))\ntest_series_desc_df = pd.read_csv(os.path.join(BASE_PATH, 'test_series_descriptions.csv'))\nsample_submission_df = pd.read_csv(os.path.join(BASE_PATH, 'sample_submission.csv'))\n\n# Refined Train Data for Lumber Spine Degenerative\ncomplete_train_df = pd.read_csv(os.path.join(COMPLETE_TRAIN_PATH, 'complete_train.csv'))\n\n# Add series description to the complete train dataframe\nlabeled_train_df = pd.merge(complete_train_df, train_series_desc_df[['series_id', 'series_description']], on='series_id', how='left')\n# Move output column to the end of the dataframe\nlabeled_train_df = move_column_to_last(labeled_train_df, 'output')\n\n# Redorder the rows w.r.t. instance number and level\nlabeled_train_df['level_label'] = labeled_train_df.level.map(LEVEL.name2label)\nlabeled_train_df = labeled_train_df.sort_values(['study_id', 'series_id', 'instance_number', 'level_label']).reset_index(drop=True)\nlabeled_train_df.drop(columns=['level_label'], inplace=True)\n\nlabeled_study_ids = labeled_train_df['study_id'].unique()\nlabeled_series_ids = labeled_train_df['series_id'].unique()\nlabeled_test_study_ids = test_series_desc_df['study_id'].unique()\nlabeled_test_series_ids = test_series_desc_df['series_id'].unique()\n\nprint(\"-------- LABELED ----------\")\nprint(f'No. of labeled data: {complete_train_df.shape[0]}')\nprint(f'unique study_ids:    {labeled_study_ids.shape[0]}')\nprint(f'unique series_ids:   {labeled_series_ids.shape[0]}')\nprint(\"---------- TEST -----------\")\nprint(f'unique study_ids:  {labeled_test_study_ids.shape[0]}')\nprint(f'unique series_ids: {labeled_test_series_ids.shape[0]}')\nprint(\"---------------------------\\n\")\n\ndel complete_train_df\n\nprint(f\"Labeled Train Data {labeled_train_df.shape}\")\ndisplay(labeled_train_df.head(3))\nprint(f\"Train Series Description {train_series_desc_df.shape}\")\ndisplay(train_series_desc_df.head(3))","metadata":{"papermill":{"duration":0.37675,"end_time":"2024-08-30T10:40:29.694182","exception":false,"start_time":"2024-08-30T10:40:29.317432","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.299083Z","iopub.execute_input":"2024-10-06T06:39:38.299438Z","iopub.status.idle":"2024-10-06T06:39:38.672758Z","shell.execute_reply.started":"2024-10-06T06:39:38.299399Z","shell.execute_reply":"2024-10-06T06:39:38.671884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Code Below showing the distribution of output data is derived from [Anatomy & Image Visualization Overview-RSNA RAIDS - Abhinav Suri](https://www.kaggle.com/code/abhinavsuri/anatomy-image-visualization-overview-rsna-raids) [3]\n> _\"There are missing data as some of the images (particularly subarticular stenosis category) do not have regions visualized (most superior vertebral bodies were less likely to make it into the imaging field)\"_","metadata":{"papermill":{"duration":0.024583,"end_time":"2024-08-30T10:40:29.743618","exception":false,"start_time":"2024-08-30T10:40:29.719035","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"> The majority of labeled data are `Normal/Mild` (77.3%) and only a few `Severe`","metadata":{"papermill":{"duration":0.024416,"end_time":"2024-08-30T10:40:29.792937","exception":false,"start_time":"2024-08-30T10:40:29.768521","status":"completed"},"tags":[]}},{"cell_type":"code","source":"figure, axis = plt.subplots(1, 3, figsize=(14, 3))\n\nfilter_dict = {'foraminal': CONDITION.sup_name2label['Neural Foraminal Narrowing'],\n               'subarticular': CONDITION.sup_name2label['Subarticular Stenosis'],\n               'canal': CONDITION.sup_name2label['Spinal Canal Stenosis']}\n\nfor idx, d in enumerate(filter_dict.keys()):\n    diagnosis = list(filter(lambda x: x.find(d) > -1, kaggle_train_df.columns))\n    dff = kaggle_train_df[diagnosis]\n    with warnings.catch_warnings():\n        warnings.simplefilter(action='ignore', category=FutureWarning)\n        value_counts = dff.apply(pd.value_counts).fillna(0).T\n    value_counts.plot(kind='bar', stacked=True, ax=axis[idx], color=[OUTPUT.colors[val] for val in value_counts.columns])\n    axis[idx].set_title(f'{CONDITION.sup_label2name[filter_dict[d]]} Distribution')\n\n# TODO: Match the colors of the output labels with the output colors\noutput_colors = plot_histogram(labeled_train_df, 'output', title='\\nOutput Distribution of Labels\\n', size=(7, 3))\n\ndel figure, axis, d, diagnosis, dff, value_counts, filter_dict","metadata":{"papermill":{"duration":1.340158,"end_time":"2024-08-30T10:40:31.158420","exception":false,"start_time":"2024-08-30T10:40:29.818262","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:38.673928Z","iopub.execute_input":"2024-10-06T06:39:38.674221Z","iopub.status.idle":"2024-10-06T06:39:40.005006Z","shell.execute_reply.started":"2024-10-06T06:39:38.674188Z","shell.execute_reply":"2024-10-06T06:39:40.003913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> There is a relatively even distribution of each condition in the labeled data\n> - each condition has a certain series type that most clearly shows the condition (`Spinal Canal Stenosis` -> `Sagittal T2/STIR`)\n> - There are 5 labels for Spinal Canal Stenosis in Saggital T1 MRIs","metadata":{"papermill":{"duration":0.026077,"end_time":"2024-08-30T10:40:31.211901","exception":false,"start_time":"2024-08-30T10:40:31.185824","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pd.crosstab(labeled_train_df['condition'], labeled_train_df['series_description'])","metadata":{"papermill":{"duration":0.068135,"end_time":"2024-08-30T10:40:31.306313","exception":false,"start_time":"2024-08-30T10:40:31.238178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:40.006389Z","iopub.execute_input":"2024-10-06T06:39:40.006774Z","iopub.status.idle":"2024-10-06T06:39:40.038161Z","shell.execute_reply.started":"2024-10-06T06:39:40.006730Z","shell.execute_reply":"2024-10-06T06:39:40.037297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> There is a relative even distribution of labels for each level and condition","metadata":{"papermill":{"duration":0.02711,"end_time":"2024-08-30T10:40:31.360337","exception":false,"start_time":"2024-08-30T10:40:31.333227","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pd.crosstab(labeled_train_df['condition'], labeled_train_df['level'])","metadata":{"papermill":{"duration":0.058428,"end_time":"2024-08-30T10:40:31.445192","exception":false,"start_time":"2024-08-30T10:40:31.386764","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:40.039678Z","iopub.execute_input":"2024-10-06T06:39:40.040388Z","iopub.status.idle":"2024-10-06T06:39:40.067615Z","shell.execute_reply.started":"2024-10-06T06:39:40.040344Z","shell.execute_reply":"2024-10-06T06:39:40.066720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unlabeled Data","metadata":{"papermill":{"duration":0.026923,"end_time":"2024-08-30T10:40:31.500460","exception":false,"start_time":"2024-08-30T10:40:31.473537","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Construct meta data for all images (labeled & unlabeled)","metadata":{"papermill":{"duration":0.02662,"end_time":"2024-08-30T10:40:31.554034","exception":false,"start_time":"2024-08-30T10:40:31.527414","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import re\n\n# Extract the last three numerical values from each file path of the train images\ndef extract_last_three_values(file_path):\n    # Split the path into parts\n    img_file_numbers = re.findall(r'\\d+', file_path)\n    # Extract the last three values before the file extension\n    last_3_values = img_file_numbers[-3:]\n    return last_3_values","metadata":{"execution":{"iopub.status.busy":"2024-10-06T06:39:40.068584Z","iopub.execute_input":"2024-10-06T06:39:40.068861Z","iopub.status.idle":"2024-10-06T06:39:40.074170Z","shell.execute_reply.started":"2024-10-06T06:39:40.068829Z","shell.execute_reply":"2024-10-06T06:39:40.073148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\n# loading images\ntrain_images = glob.glob(BASE_PATH + '/train_images/*/*/*.dcm')\ntest_images = glob.glob(BASE_PATH + '/test_images/*/*/*.dcm')\nprint(\"Total train images: \", len(train_images))\nprint(\"Total test images:  \", len(test_images))\n\n# Create a list of dictionaries containing the extracted values\ntrain_data = [{'study_id': int(values[0]), 'series_id': int(values[1]), 'instance_number': int(values[2]), 'image_file_path':str(values[0])+'_'+str(values[1])+'_'+str(values[2])+'.dcm'} for file_path in train_images for values in [extract_last_three_values(file_path)]]\ntest_data = [{'study_id': int(values[0]), 'series_id': int(values[1]), 'instance_number': int(values[2]), 'image_file_path':str(values[0])+'_'+str(values[1])+'_'+str(values[2])+'.dcm'} for file_path in test_images for values in [extract_last_three_values(file_path)]]\n# Create a DataFrame from the list of dictionaries\ntrain_df = pd.DataFrame(train_data)\ntest_df = pd.DataFrame(test_data)\n# Merge the train and test DataFrames with the series descriptions\ntrain_df = pd.merge(train_df, train_series_desc_df[['series_id', 'series_description']], on='series_id', how='left')\ntest_df = pd.merge(test_df, test_series_desc_df[['series_id', 'series_description']], on='series_id', how='left')\n\n# Redorder the rows w.r.t. instance number\ntrain_df = train_df.sort_values(['study_id', 'series_id', 'instance_number']).reset_index(drop=True)\ntest_df = test_df.sort_values(['study_id', 'series_id', 'instance_number']).reset_index(drop=True)\n\nstudy_ids = train_df['study_id'].unique()\nseries_ids = train_df['series_id'].unique()\n\nprint('-------- TRAIN ----------')\nprint(f'unique study_ids:    {labeled_study_ids.shape[0]} / {study_ids.shape[0]}')\nprint(f'unique series_ids:   {labeled_series_ids.shape[0]} / {series_ids.shape[0]}\\n')\n\ndel train_images, test_images, train_data, test_data\n\nPRINT_DATAFRAME = True\nif PRINT_DATAFRAME:\n    print(f'train_df shape: {train_df.shape}')\n    display(train_df.head(3))\n    print(f'test_df shape: {test_df.shape}')\n    display(test_df.head(3))","metadata":{"papermill":{"duration":63.738119,"end_time":"2024-08-30T10:41:35.319497","exception":false,"start_time":"2024-08-30T10:40:31.581378","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:39:40.075602Z","iopub.execute_input":"2024-10-06T06:39:40.075977Z","iopub.status.idle":"2024-10-06T06:40:13.543209Z","shell.execute_reply.started":"2024-10-06T06:39:40.075937Z","shell.execute_reply":"2024-10-06T06:40:13.542335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Missing IDs","metadata":{"papermill":{"duration":0.028046,"end_time":"2024-08-30T10:41:35.376015","exception":false,"start_time":"2024-08-30T10:41:35.347969","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- There is a study that is missing from the metadata data (`3008676218`)\n- There are `3` series with no series description (`3636216534`  `542282425` `3892989905`)","metadata":{"papermill":{"duration":0.027002,"end_time":"2024-08-30T10:41:35.430399","exception":false,"start_time":"2024-08-30T10:41:35.403397","status":"completed"},"tags":[]}},{"cell_type":"code","source":"missing_study_ids = np.array(train_df[~train_df['study_id'].isin(labeled_study_ids)]['study_id'].unique())\nmissing_series_ids = np.array(train_df[~train_df['series_id'].isin(labeled_series_ids)]['series_id'].unique())\nprint(f'unique study_ids:     {labeled_study_ids.shape[0]} / {study_ids.shape[0]}')\nprint(f\"> missing study_ids:  {missing_study_ids}\")\nprint(f'unique series_ids:    {labeled_series_ids.shape[0]} / {series_ids.shape[0]}')\nprint(f\"> missing series_ids: {missing_series_ids}\")","metadata":{"papermill":{"duration":0.042966,"end_time":"2024-08-30T10:41:35.500858","exception":false,"start_time":"2024-08-30T10:41:35.457892","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:13.544540Z","iopub.execute_input":"2024-10-06T06:40:13.544942Z","iopub.status.idle":"2024-10-06T06:40:13.557552Z","shell.execute_reply.started":"2024-10-06T06:40:13.544898Z","shell.execute_reply":"2024-10-06T06:40:13.556579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Added Features","metadata":{}},{"cell_type":"markdown","source":"### Total Labels in an Instance","metadata":{"papermill":{"duration":0.027286,"end_time":"2024-08-30T10:41:35.556096","exception":false,"start_time":"2024-08-30T10:41:35.528810","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"> There can be single OR multiple OR no data points associated with each instance","metadata":{"papermill":{"duration":0.02736,"end_time":"2024-08-30T10:41:35.610977","exception":false,"start_time":"2024-08-30T10:41:35.583617","status":"completed"},"tags":[]}},{"cell_type":"code","source":"num_labels = labeled_train_df.groupby(['series_id', 'instance_number', 'image_file_path']).size().reset_index(name='num_labels')\ntrain_df = merge_dfs(train_df, num_labels, ['num_labels'], 'image_file_path')\ntrain_df['num_labels'] = train_df['num_labels'].fillna(0).astype(int)\n\nPRINT_DATAFRAME = True\nif PRINT_DATAFRAME:\n    filtered_train_df = train_df[train_df['num_labels'] != 0]\n    print(f'{filtered_train_df.shape[0]} instances ({filtered_train_df.shape[0]*100/train_df.shape[0]:.2f}%) have labeled data out of {train_df.shape[0]} instances.')\n    display(filtered_train_df.head(3))\n    del filtered_train_df\n\ndel num_labels","metadata":{"papermill":{"duration":0.187881,"end_time":"2024-08-30T10:41:35.826668","exception":false,"start_time":"2024-08-30T10:41:35.638787","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:13.558771Z","iopub.execute_input":"2024-10-06T06:40:13.559173Z","iopub.status.idle":"2024-10-06T06:40:13.700050Z","shell.execute_reply.started":"2024-10-06T06:40:13.559139Z","shell.execute_reply":"2024-10-06T06:40:13.699110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Most instances have 1 label (46.2%) associated with each instance (up to 5 being relatively common)  ","metadata":{"papermill":{"duration":0.02906,"end_time":"2024-08-30T10:41:35.883834","exception":false,"start_time":"2024-08-30T10:41:35.854774","status":"completed"},"tags":[]}},{"cell_type":"code","source":"filtered_train_df = train_df[train_df['num_labels'] > 0]\nplot_histogram(filtered_train_df, 'num_labels', title='Distribution of Number of Labels for Labeled Instances', size=(12, 3))\ndel filtered_train_df","metadata":{"papermill":{"duration":0.394412,"end_time":"2024-08-30T10:41:36.306690","exception":false,"start_time":"2024-08-30T10:41:35.912278","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:13.701313Z","iopub.execute_input":"2024-10-06T06:40:13.701681Z","iopub.status.idle":"2024-10-06T06:40:14.041037Z","shell.execute_reply.started":"2024-10-06T06:40:13.701647Z","shell.execute_reply":"2024-10-06T06:40:14.040139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Total Instances in a Series","metadata":{"papermill":{"duration":0.029746,"end_time":"2024-08-30T10:41:36.366252","exception":false,"start_time":"2024-08-30T10:41:36.336506","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Calculate the total number of instances in each series and add to the series description DataFrames\ntrain_total_instances = train_df.groupby('series_id', observed=False).size().reset_index(name='total_instances')\ntrain_series_desc_df = merge_dfs(train_series_desc_df, train_total_instances, ['total_instances'], 'series_id')\ntest_total_instances = test_df.groupby('series_id', observed=False).size().reset_index(name='total_instances')\ntest_series_desc_df = merge_dfs(test_series_desc_df, test_total_instances, ['total_instances'], 'series_id')\n\ndel test_total_instances, train_total_instances\n\nPRINT_DATAFRAME = True\nif PRINT_DATAFRAME:\n    print(f'train_series_desc_df shape: {train_series_desc_df.shape}')\n    display(train_series_desc_df.head(3))","metadata":{"papermill":{"duration":0.059525,"end_time":"2024-08-30T10:41:36.456312","exception":false,"start_time":"2024-08-30T10:41:36.396787","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:14.042434Z","iopub.execute_input":"2024-10-06T06:40:14.043345Z","iopub.status.idle":"2024-10-06T06:40:14.069355Z","shell.execute_reply.started":"2024-10-06T06:40:14.043294Z","shell.execute_reply":"2024-10-06T06:40:14.068408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Total Labels in a Series","metadata":{"papermill":{"duration":0.029966,"end_time":"2024-08-30T10:41:36.519010","exception":false,"start_time":"2024-08-30T10:41:36.489044","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Find the total number of labels in each series\nnum_labels = train_df.groupby('series_id')['num_labels'].sum().reset_index(name='num_labels')\ntrain_series_desc_df = merge_dfs(train_series_desc_df, num_labels, ['num_labels'], 'series_id')\ntrain_series_desc_df['num_labels'] = train_series_desc_df['num_labels'].fillna(0)\ntrain_series_desc_df['num_labels'] = train_series_desc_df['num_labels'].astype(int)\n\n# Find the number of labeled instances in each series\nlabeled_instances = labeled_train_df.groupby('series_id')['instance_number'].agg(lambda x: list(set(x))).reset_index(name='labeled_instances')\nlabeled_instances['num_labeled'] = labeled_instances['labeled_instances'].apply(len)\n# TODO: Consider adding the list of labeled instances to the series description DataFrame\ntrain_series_desc_df = merge_dfs(train_series_desc_df, labeled_instances, ['num_labeled'], 'series_id')\ntrain_series_desc_df['num_labeled'] = train_series_desc_df['num_labeled'].fillna(0).astype(int)\n\ndisplay(train_series_desc_df.head(3))\n\ndel num_labels","metadata":{"papermill":{"duration":0.236178,"end_time":"2024-08-30T10:41:36.785181","exception":false,"start_time":"2024-08-30T10:41:36.549003","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:14.070397Z","iopub.execute_input":"2024-10-06T06:40:14.070669Z","iopub.status.idle":"2024-10-06T06:40:14.269845Z","shell.execute_reply.started":"2024-10-06T06:40:14.070639Z","shell.execute_reply":"2024-10-06T06:40:14.269042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors=plot_histogram(train_series_desc_df, 'num_labels', title='Distribution of Number of Labels in a Series\\n', labels_on=True,size=(15, 3))\nplot_histogram(train_series_desc_df, 'num_labeled', title='Distribution of Number of Labeled Instances in a Series', labels_on=True,size=(13, 3))\n\ndel colors","metadata":{"papermill":{"duration":0.78823,"end_time":"2024-08-30T10:41:37.605281","exception":false,"start_time":"2024-08-30T10:41:36.817051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:14.270928Z","iopub.execute_input":"2024-10-06T06:40:14.271227Z","iopub.status.idle":"2024-10-06T06:40:14.803325Z","shell.execute_reply.started":"2024-10-06T06:40:14.271195Z","shell.execute_reply":"2024-10-06T06:40:14.802490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conditions Associated with each Instance","metadata":{"papermill":{"duration":0.032226,"end_time":"2024-08-30T10:41:37.670335","exception":false,"start_time":"2024-08-30T10:41:37.638109","status":"completed"},"tags":[]}},{"cell_type":"code","source":"conditions = labeled_train_df.groupby('image_file_path')['condition'].agg(lambda x: list(set(x))).reset_index(name='conditions')\nconditions['num_conditions'] = conditions['conditions'].apply(lambda x: len(x) if isinstance(x, list) else 0)\nconditions['labels'] = conditions['conditions'].apply(lambda x: [CONDITION.name2label[c] for c in x])\nconditions['sup_labels'] = conditions['labels'].apply(lambda x: [CONDITION.name2sup_label[l] for l in x])\nconditions['sup_label'] = conditions['sup_labels'].apply(lambda x:\n      x[0] if len(x) == 1\n            else ( 0 if len(x) > 2\n                  else ( x[0] if x[0] == x[1]\n                        else 0)))\nconditions['sup_condition'] = conditions['sup_label'].map(CONDITION.sup_label2name)\ntrain_df = merge_dfs(train_df, conditions, ['num_conditions','sup_condition'], 'image_file_path')\nlabeled_train_df = merge_dfs(labeled_train_df, conditions, ['sup_condition'], 'image_file_path')\nlabeled_train_df = move_column_to_last(labeled_train_df, 'output')\ntrain_df = move_column_to_last(train_df, 'num_conditions')\ntrain_df['num_conditions'] = train_df['num_conditions'].fillna(0).astype(int)\ntrain_df['sup_condition'] = train_df['sup_condition'].fillna('None')\n\nPRINT_DATAFRAME = True\nif PRINT_DATAFRAME:\n    print(f'labeled_train_df shape: {labeled_train_df.shape}')\n    display(labeled_train_df.head(3))\n    print(f'train_df shape: {train_df.shape}')\n    filtered_train_df = train_df[train_df['sup_condition'] != \"None\"]\n    display(filtered_train_df.head(3))\n    \ndel conditions","metadata":{"papermill":{"duration":1.039852,"end_time":"2024-08-30T10:41:38.741816","exception":false,"start_time":"2024-08-30T10:41:37.701964","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:14.804542Z","iopub.execute_input":"2024-10-06T06:40:14.804837Z","iopub.status.idle":"2024-10-06T06:40:15.789209Z","shell.execute_reply.started":"2024-10-06T06:40:14.804805Z","shell.execute_reply":"2024-10-06T06:40:15.788329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> - Most instances do not have any labels (83.3%)\n> - Most labeled instances are associated with 1 condition (77.6%) while the rest have 2 conditions (22.4%)\n> - Instances with 2 conditions are either `Neural Foraminal Narrowing` (L/R) (99.7%) or `Subarticular Stenosis` (L/R) (0.3%) (no multiple diagnosis -> no `Mixed` diagnosis)","metadata":{"papermill":{"duration":0.031948,"end_time":"2024-08-30T10:41:38.806501","exception":false,"start_time":"2024-08-30T10:41:38.774553","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plot_histogram(train_df, 'num_conditions', title='Distribution of Number of Conditions for each Instance\\n', size=(6, 3))\n\nfiltered_train_df = train_df[train_df['num_labels'] != 0]\nplot_histogram(filtered_train_df, 'num_conditions', title='Distribution of Number of Conditions for Labeled Instances\\n', size=(7, 2.5))\nplot_histogram(filtered_train_df, 'sup_condition', title='Distribution of Diagnosis for Labeled Instances\\n', size=(7, 3))\n\nfiltered_train_df = filtered_train_df[filtered_train_df['num_conditions'] > 1]\nplot_histogram(filtered_train_df, 'sup_condition', title='Distribution of Diagnosis with > 1 Associated Conditions for Labeled Instances\\n', size=(8, 3))\ndel filtered_train_df","metadata":{"papermill":{"duration":1.111067,"end_time":"2024-08-30T10:41:39.949827","exception":false,"start_time":"2024-08-30T10:41:38.838760","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:15.790463Z","iopub.execute_input":"2024-10-06T06:40:15.790851Z","iopub.status.idle":"2024-10-06T06:40:16.819440Z","shell.execute_reply.started":"2024-10-06T06:40:15.790806Z","shell.execute_reply":"2024-10-06T06:40:16.818575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conditions Associated with each Series","metadata":{"papermill":{"duration":0.03465,"end_time":"2024-08-30T10:41:40.019870","exception":false,"start_time":"2024-08-30T10:41:39.985220","status":"completed"},"tags":[]}},{"cell_type":"code","source":"series_conditions = labeled_train_df.groupby('series_id')['condition'].agg(lambda x: list(set(x))).reset_index(name='conditions')\nseries_conditions['num_conditions'] = series_conditions['conditions'].apply(lambda x: len(x) if isinstance(x, list) else 0)\n\n\nseries_conditions['labels'] = series_conditions['conditions'].apply(lambda x: [CONDITION.name2label[c] for c in x])\nseries_conditions['sup_labels'] = series_conditions['labels'].apply(lambda x: [CONDITION.name2sup_label[l] for l in x])\nseries_conditions['sup_label'] = series_conditions['sup_labels'].apply(lambda x:\n      x[0] if len(x) == 1\n            else ( 0 if len(x) > 2\n                  else ( x[0] if x[0] == x[1]\n                        else 0)))\nseries_conditions['sup_condition'] = series_conditions['sup_label'].map(CONDITION.sup_label2name)\n\ntrain_series_desc_df = merge_dfs(train_series_desc_df, series_conditions, ['conditions','num_conditions', 'sup_condition'], 'series_id')\ntrain_series_desc_df['num_conditions'] = train_series_desc_df['num_conditions'].fillna(0).astype(int)\ntrain_series_desc_df['sup_condition'] = train_series_desc_df['sup_condition'].fillna('None')\n\nPRINT_DATAFRAME = True\nif PRINT_DATAFRAME:\n    filtered_train_series_desc_df = train_series_desc_df[train_series_desc_df['num_conditions'] > 0]\n    display(filtered_train_series_desc_df.head(3))\n    del filtered_train_series_desc_df\n    \ndel series_conditions","metadata":{"papermill":{"duration":0.321387,"end_time":"2024-08-30T10:41:40.376518","exception":false,"start_time":"2024-08-30T10:41:40.055131","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:16.820697Z","iopub.execute_input":"2024-10-06T06:40:16.821000Z","iopub.status.idle":"2024-10-06T06:40:17.085910Z","shell.execute_reply.started":"2024-10-06T06:40:16.820967Z","shell.execute_reply":"2024-10-06T06:40:17.085007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is one series that is associated with multiple diagnosis (`Mixed`)","metadata":{}},{"cell_type":"code","source":"colors=plot_histogram(train_series_desc_df, 'num_conditions', title='Distribution of Number of Conditions in a Series\\n', labels_on=True,size=(8, 3))\nplot_histogram(train_series_desc_df, 'sup_condition', title='Distribution of Diagnosis for each Series', size=(11, 3))\n\ndel colors","metadata":{"papermill":{"duration":0.423434,"end_time":"2024-08-30T10:41:40.835397","exception":false,"start_time":"2024-08-30T10:41:40.411963","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:17.087179Z","iopub.execute_input":"2024-10-06T06:40:17.087510Z","iopub.status.idle":"2024-10-06T06:40:17.456837Z","shell.execute_reply.started":"2024-10-06T06:40:17.087474Z","shell.execute_reply":"2024-10-06T06:40:17.455750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🩻 | Analyze DICOM Files","metadata":{"papermill":{"duration":0.037721,"end_time":"2024-08-30T10:41:40.910108","exception":false,"start_time":"2024-08-30T10:41:40.872387","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 🌐 | Analysis Variables","metadata":{}},{"cell_type":"code","source":"class ANALYSIS:\n    count = 0\n    tag_count = 0\n    tag_series_count = 0","metadata":{"execution":{"iopub.status.busy":"2024-10-06T06:40:17.458176Z","iopub.execute_input":"2024-10-06T06:40:17.459090Z","iopub.status.idle":"2024-10-06T06:40:17.464197Z","shell.execute_reply.started":"2024-10-06T06:40:17.459043Z","shell.execute_reply":"2024-10-06T06:40:17.463219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 🤝 | Helper Functions","metadata":{"papermill":{"duration":0.036751,"end_time":"2024-08-30T10:41:40.983329","exception":false,"start_time":"2024-08-30T10:41:40.946578","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Format derived from [Refined Train Data for Lumber Spine Degenerative](https://www.kaggle.com/datasets/tariqcp/train-data-for-rsna-2024-lsdc) [2]\n\n- Image file path (`image_file_path`): the MRI image file path given in the format found in complete_train.csv [[2]](https://www.kaggle.com/competitions/rsna-2024-lumbar-spine-degenerative-classification/discussion/505929) `{study_id}_{series_id}_{instance_number}.dcm`\n  > `4003253_702807833_8.dcm`\n\n- DICOM file path (`dcm_file_path`): the file path of the DICOM file in the kaggle dataset `{BASE_PATH}/train_images/{study_id}/{series_id}/{instance_number}.dcm`\n  > `/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/4003253/702807833/8.dcm`","metadata":{"papermill":{"duration":0.036173,"end_time":"2024-08-30T10:41:41.056366","exception":false,"start_time":"2024-08-30T10:41:41.020193","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Find the study_id given the series_id\ndef find_study_id(series_id, split='train'):\n    if split == 'test':\n        if series_id not in test_series_desc_df['series_id'].values:\n            print(f\"series_id {series_id} not found in test_series_descriptions.csv\")\n            return -1\n        return test_series_desc_df[test_series_desc_df['series_id'] == series_id]['study_id'].values[0]\n    if series_id not in train_series_desc_df['series_id'].values:\n        print(f\"series_id {series_id} not found in train_series_descriptions.csv\")\n        return -1\n    return train_series_desc_df[train_series_desc_df['series_id'] == series_id]['study_id'].values[0]\n\n# Find the series description given the series_id\ndef find_series_desc(series_id, split='train'):\n    if split == 'test':\n        if series_id not in test_series_desc_df['series_id'].values:\n            print(f\"series_id {series_id} not found in test_series_descriptions.csv\")\n            return -1\n        return test_series_desc_df[test_series_desc_df['series_id'] == series_id]['series_description'].values[0]\n    if series_id not in train_series_desc_df['series_id'].values:\n        print(f\"series_id {series_id} not found in train_series_descriptions.csv\")\n        return -1\n    return train_series_desc_df[train_series_desc_df['series_id'] == series_id]['series_description'].values[0]\n\n# Find the image file path given the series_id and instance number\ndef find_img_file_path(series_id, instance, split='train'):\n    return str(find_study_id(series_id, split))+'_'+str(series_id)+'_'+str(instance)+'.dcm'\n\n# Convert the image file path to the DICOM file path\ndef img_to_dcm_file_path(img_file_path, split='train', processed=False):\n    if split == 'test':\n        return os.path.join(BASE_PATH, 'test_images', img_file_path.replace('_', '/'))\n    return os.path.join(BASE_PATH, 'train_images', img_file_path.replace('_', '/'))\n\n# Find the study_id, series_id and instance number given the image file path\ndef decode_img_file_path(img_file_path):\n    study_id, series_id, instance = img_file_path.split('_')\n    return int(study_id), int(series_id), int(instance.split('.')[0])","metadata":{"papermill":{"duration":0.051888,"end_time":"2024-08-30T10:41:41.144756","exception":false,"start_time":"2024-08-30T10:41:41.092868","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:17.471549Z","iopub.execute_input":"2024-10-06T06:40:17.471840Z","iopub.status.idle":"2024-10-06T06:40:17.483203Z","shell.execute_reply.started":"2024-10-06T06:40:17.471808Z","shell.execute_reply":"2024-10-06T06:40:17.482501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Loading Functions","metadata":{"papermill":{"duration":0.036452,"end_time":"2024-08-30T10:41:41.217743","exception":false,"start_time":"2024-08-30T10:41:41.181291","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load MRI image only\ndef load_mri_image(img_file_path, split='train', processed=False):\n    dcm_file_path = img_to_dcm_file_path(img_file_path, split=split, processed=processed)\n    if processed:\n        return np.load(dcm_file_path)\n    return pydicom.dcmread(dcm_file_path).pixel_array\n\n# Load DICOM file (image + metadata)\ndef load_dicom_file(img_file_path, split='train'):\n    dcm_file_path = img_to_dcm_file_path(img_file_path, split=split)\n    dcm_file = pydicom.dcmread(dcm_file_path)\n    \n    # Convert the DICOM dataset to a dictionary to easily access metadata\n    metadata = {tag: dcm_file[tag].value for tag in dcm_file.dir()}\n    tags = metadata.keys()\n    \n    return dcm_file.pixel_array, metadata, tags","metadata":{"papermill":{"duration":0.046785,"end_time":"2024-08-30T10:41:41.301275","exception":false,"start_time":"2024-08-30T10:41:41.254490","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:17.484111Z","iopub.execute_input":"2024-10-06T06:40:17.484428Z","iopub.status.idle":"2024-10-06T06:40:17.496426Z","shell.execute_reply.started":"2024-10-06T06:40:17.484396Z","shell.execute_reply":"2024-10-06T06:40:17.495630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analyze DICOM Image Size","metadata":{"papermill":{"duration":0.036268,"end_time":"2024-08-30T10:41:41.528797","exception":false,"start_time":"2024-08-30T10:41:41.492529","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TODO: Find a dcm file that has 30 tags\nimfp = train_df['image_file_path'].iloc[0]\n_, metadata, tags = load_dicom_file(imfp)\nprint(f'Number of tags: {len(tags)} ({imfp})')\nprint('---------------------------')\nfor tag in tags:\n    if tag == \"PixelData\":\n        print(f'{tag} : ---------')\n        continue\n    print(f'{tag} : {metadata[tag]}')\ndel imfp, metadata, tags","metadata":{"papermill":{"duration":0.099094,"end_time":"2024-08-30T10:41:41.663244","exception":false,"start_time":"2024-08-30T10:41:41.564150","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:17.497571Z","iopub.execute_input":"2024-10-06T06:40:17.498096Z","iopub.status.idle":"2024-10-06T06:40:17.548023Z","shell.execute_reply.started":"2024-10-06T06:40:17.498052Z","shell.execute_reply":"2024-10-06T06:40:17.547202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyze_tags_data(img_file_path):\n    dcm_file_path = img_to_dcm_file_path(img_file_path, split='train')\n    dcm_file = pydicom.dcmread(dcm_file_path)\n    tags = dcm_file.dir()\n    size = (dcm_file['Rows'].value, dcm_file['Columns'].value)\n    return {'n_tags':len(tags), 'size':size, 'tags':tags}","metadata":{"papermill":{"duration":0.044318,"end_time":"2024-08-30T10:41:41.745066","exception":false,"start_time":"2024-08-30T10:41:41.700748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:17.549054Z","iopub.execute_input":"2024-10-06T06:40:17.549352Z","iopub.status.idle":"2024-10-06T06:40:17.554427Z","shell.execute_reply.started":"2024-10-06T06:40:17.549321Z","shell.execute_reply":"2024-10-06T06:40:17.553562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Increment Analysis count limit to rerun the analysis\ntags_analysis_limit = 1\nif ANALYSIS.tag_count < tags_analysis_limit:\n    train_file_paths = train_df['image_file_path'].values\n\n    # Parallelize the processing using joblib for analysis\n    analyzed_data = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n        joblib.delayed(analyze_tags_data)(tfp)\n        for tfp in tqdm(train_file_paths, total=len(train_file_paths))\n    )\n    del train_file_paths\n    \n    ANALYSIS.tag_count += 1\nelse:\n    print(f\"Analysis already done! [{ANALYSIS.tag_count}] Skipping...\")\n\ndel tags_analysis_limit","metadata":{"papermill":{"duration":358.655758,"end_time":"2024-08-30T10:47:40.437111","exception":false,"start_time":"2024-08-30T10:41:41.781353","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:40:17.555533Z","iopub.execute_input":"2024-10-06T06:40:17.556149Z","iopub.status.idle":"2024-10-06T06:44:16.178175Z","shell.execute_reply.started":"2024-10-06T06:40:17.556107Z","shell.execute_reply":"2024-10-06T06:44:16.177326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_tags = [data['tags'] for data in analyzed_data]\ndata_tags_analyzed = {}\nfor tags in data_tags:\n    for tag in tags:\n        if tag not in data_tags_analyzed: data_tags_analyzed[tag] = 1\n        else: data_tags_analyzed[tag] += 1\n\ncommon_tags = []\nuncommon_tags = []\nfor tag in data_tags_analyzed.keys():\n    if data_tags_analyzed[tag] == 147218:\n        print(tag)\n        common_tags.append(tag)\n    else:\n        uncommon_tags.append(tag)\n\ndel data_tags, data_tags_analyzed","metadata":{"execution":{"iopub.status.busy":"2024-10-06T06:44:16.179452Z","iopub.execute_input":"2024-10-06T06:44:16.179770Z","iopub.status.idle":"2024-10-06T06:44:17.611157Z","shell.execute_reply.started":"2024-10-06T06:44:16.179737Z","shell.execute_reply":"2024-10-06T06:44:17.610232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'\\nNumber of common tags: {len(common_tags)}')\nprint(f'Uncommon tags: {uncommon_tags}')","metadata":{"execution":{"iopub.status.busy":"2024-10-06T06:44:17.612227Z","iopub.execute_input":"2024-10-06T06:44:17.612558Z","iopub.status.idle":"2024-10-06T06:44:17.617310Z","shell.execute_reply.started":"2024-10-06T06:44:17.612524Z","shell.execute_reply":"2024-10-06T06:44:17.616414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Image Size Analysis","metadata":{"papermill":{"duration":0.043036,"end_time":"2024-08-30T10:47:40.518513","exception":false,"start_time":"2024-08-30T10:47:40.475477","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def tag_data_stat(tag_data):\n    print(f'Presenting statistics for {len(tag_data)} images\\n')\n    data_n_tags = [data['n_tags'] for data in tag_data]\n    data_size = [data['size'] for data in tag_data]\n    data_size_df = pd.DataFrame(data_size, columns=['height', 'width'])\n    data_size_df['tuple'] = list(zip(data_size_df['height'], data_size_df['width']))\n    data_size_df['area'] = data_size_df['width'] * data_size_df['height']\n    data_size_df['ratio'] = data_size_df['height'] / data_size_df['width']\n\n    min_size = data_size_df['area'].min()\n    max_size = data_size_df['area'].max()\n    most_common_size = data_size_df['area'].mode().values[0]\n\n    min_size_p = data_size_df['area'].value_counts(normalize=True).loc[min_size] * 100\n    max_size_p = data_size_df['area'].value_counts(normalize=True).loc[max_size] * 100\n    most_common_size_p = data_size_df['area'].value_counts(normalize=True).loc[most_common_size] * 100\n    \n    print(f'Unique number of tags:    {len(set(data_n_tags))}            = {sorted(set(data_n_tags))}')\n    print(f'Unique number of sizes:   {len(set(data_size))}')\n    print(f'  Smallest size:          {data_size_df.loc[data_size_df[\"area\"] == min_size, \"tuple\"].unique()[0]} = {data_size_df[\"area\"].value_counts().loc[min_size]} times ({min_size_p:.2f}%)')\n    print(f'  Largest size:           {data_size_df.loc[data_size_df[\"area\"] == max_size, \"tuple\"].unique()[0]} = {data_size_df[\"area\"].value_counts().loc[max_size]} times ({max_size_p:.2f}%)')\n    print(f'  Most common size:       {data_size_df.loc[data_size_df[\"area\"] == most_common_size, \"tuple\"].unique()[0]} = {data_size_df[\"area\"].value_counts().loc[most_common_size]} times ({most_common_size_p:.2f}%)')\n    print(f'Unique number of ratios:  {len(set(data_size_df[\"ratio\"]))} (height / width)')\n    print(f'  Smallest ratio:         {data_size_df[\"ratio\"].min()}')\n    print(f'  Largest ratio:          {data_size_df[\"ratio\"].max()}')\n    top_10_common_sizes = data_size_df['area'].value_counts().head(10)\n    top_10_common_ratios = data_size_df['ratio'].value_counts().head(10)\n    print('\\nTop 10 common sizes:')\n    plot_df = data_size_df[data_size_df['area'].isin(top_10_common_sizes.index.tolist())]\n    plot_histogram(plot_df, 'tuple', title='Size Distribution', labels_on=True, size=(14, 3))\n    print('Top 10 common ratios:')\n    plot_df = data_size_df[data_size_df['ratio'].isin(top_10_common_ratios.index.tolist())]\n    plot_histogram(plot_df, 'ratio', title='Ratio Distribution', labels_on=True, size=(12, 2.5))\n\ntag_data_stat(analyzed_data)","metadata":{"papermill":{"duration":1.176426,"end_time":"2024-08-30T10:47:41.731565","exception":false,"start_time":"2024-08-30T10:47:40.555139","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:44:17.618520Z","iopub.execute_input":"2024-10-06T06:44:17.618802Z","iopub.status.idle":"2024-10-06T06:44:18.683124Z","shell.execute_reply.started":"2024-10-06T06:44:17.618744Z","shell.execute_reply":"2024-10-06T06:44:18.682237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> - Most images have a ratio of 1 (height/width)  \n> - Most images are of size `(512,512)` and `(320,320) ` ","metadata":{"papermill":{"duration":0.037729,"end_time":"2024-08-30T10:47:41.808032","exception":false,"start_time":"2024-08-30T10:47:41.770303","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"#### Image Analysis for each Series Type","metadata":{"papermill":{"duration":0.038641,"end_time":"2024-08-30T10:47:43.379892","exception":false,"start_time":"2024-08-30T10:47:43.341251","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Increment Analysis count limit to rerun the analysis\ntags_analysis_limit = 1\nif ANALYSIS.tag_series_count < tags_analysis_limit:\n    for series_desc in SERIES.class_names:\n        filtered_df = filter_df(train_df, [series_desc], 'series_description')\n        train_file_paths = filtered_df['image_file_path'].values\n        # Parallelize the processing using joblib for analysis\n        analyzed_data = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n            joblib.delayed(analyze_tags_data)(fp)\n            for fp in tqdm(train_file_paths, total=len(train_file_paths))\n        )\n        print(f\"-------------- {series_desc} -------------\")\n        tag_data_stat(analyzed_data)\n        del train_file_paths, filtered_df\n        \n        ANALYSIS.tag_series_count += 1\nelse:\n    print(f\"Analysis already done! [{ANALYSIS.tag_count}] Skipping...\")\n\ndel tags_analysis_limit","metadata":{"papermill":{"duration":285.355494,"end_time":"2024-08-30T10:52:28.774061","exception":false,"start_time":"2024-08-30T10:47:43.418567","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:44:18.684236Z","iopub.execute_input":"2024-10-06T06:44:18.684525Z","iopub.status.idle":"2024-10-06T06:48:01.265049Z","shell.execute_reply.started":"2024-10-06T06:44:18.684494Z","shell.execute_reply":"2024-10-06T06:48:01.264046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Analyze DICOM File Image","metadata":{"papermill":{"duration":0.043654,"end_time":"2024-08-30T10:52:28.862437","exception":false,"start_time":"2024-08-30T10:52:28.818783","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Retrieve the image data from the DICOM file\n# TODO: Consider adding patient position and orientation for processing\ndef get_dicom_data(img_file_path, img, metadata):\n    data = {}\n    data['image_file_path'] = img_file_path\n    data['size'] = (metadata['Rows'], metadata['Columns']) # (height, width)\n    data['area'] = metadata['Rows'] * metadata['Columns']\n    data['spacing'] = metadata['SpacingBetweenSlices'] # TODO: Delete\n    data['slice_thick'] = metadata['SliceThickness'] # TODO: Delete (?)\n    data['slice_loc'] = metadata['SliceLocation']\n    data['ratio'] = metadata['Rows']/metadata['Columns']\n    data['min_val'] = np.min(img)\n    data['max_val'] = np.max(img)\n    data['range_val'] = np.max(img) - np.min(img)\n    \n    return data","metadata":{"papermill":{"duration":0.054688,"end_time":"2024-08-30T10:52:28.961378","exception":false,"start_time":"2024-08-30T10:52:28.906690","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:48:01.266391Z","iopub.execute_input":"2024-10-06T06:48:01.266700Z","iopub.status.idle":"2024-10-06T06:48:01.273646Z","shell.execute_reply.started":"2024-10-06T06:48:01.266667Z","shell.execute_reply":"2024-10-06T06:48:01.272666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyze_dicom(img_file_path, split='train'):\n    img, metadata, _ = load_dicom_file(img_file_path, split=split)\n    data = get_dicom_data(img_file_path, img, metadata)\n    data['image_file_path'] = img_file_path\n    return data\n\n# Increment Analysis count limit to rerun the analysis\nanalysis_limit = 1\nif ANALYSIS.count < analysis_limit:\n    print(\"Analyzing Train DICOM files...\")\n    train_file_paths = train_df['image_file_path'].values\n    # Parallelize the processing using joblib\n    analyzed_train_data = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n        joblib.delayed(analyze_dicom)(trainfp)\n        for trainfp in tqdm(train_file_paths, total=len(train_file_paths))\n    )\n\n    print(\"Analyzing Test DICOM files...\")\n    test_file_paths = test_df['image_file_path'].values\n    # Parallelize the processing using joblib\n    analyzed_test_data = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n        joblib.delayed(analyze_dicom)(testfp, split='test')\n        for testfp in tqdm(test_file_paths, total=len(test_file_paths))\n    )\n\n    # Add analyzed data to unlabeld train and test DataFrames\n    analyzed_train_data = pd.DataFrame(analyzed_train_data)\n    analyzed_test_data = pd.DataFrame(analyzed_test_data)\n    train_df = pd.merge(train_df, analyzed_train_data, on='image_file_path', how='left')\n    test_df = pd.merge(test_df, analyzed_test_data, on='image_file_path', how='left')\n\n    del analyzed_train_data, analyzed_test_data, train_file_paths, test_file_paths\n    ANALYSIS.count += 1\nelse:\n    print(f\"Analysis already done! [{ANALYSIS.count}] Skipping...\")\n\nPRINT_ANALYSIS = False\nif PRINT_ANALYSIS:\n    print(f'train_df shape: {train_df.shape}')\n    display(train_df.head(3))\n    print(f'test_df shape: {test_df.shape}')\n    display(test_df.head(3))","metadata":{"papermill":{"duration":1001.105367,"end_time":"2024-08-30T11:09:10.111216","exception":false,"start_time":"2024-08-30T10:52:29.005849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T06:48:01.274699Z","iopub.execute_input":"2024-10-06T06:48:01.274962Z","iopub.status.idle":"2024-10-06T07:02:49.284643Z","shell.execute_reply.started":"2024-10-06T06:48:01.274933Z","shell.execute_reply":"2024-10-06T07:02:49.283833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process the DICOM data\n# TODO: Take into consideration cases where there are 2 protocols in a single series\ndef process_dicom_data(unlabeled_df, series_desc_df):\n    min_vals = unlabeled_df.groupby('series_id', observed=False)['min_val'].min().reset_index(name='min_val')\n    max_vals = unlabeled_df.groupby('series_id', observed=False)['max_val'].max().reset_index(name='max_val')\n    range_val = unlabeled_df.groupby('series_id', observed=False)['range_val'].max().reset_index(name='range_val') # TODO: Delete\n    sizes = unlabeled_df.groupby('series_id', observed=False)['size'].unique().reset_index(name='sizes') # TODO: Delete\n    areas = unlabeled_df.groupby('series_id', observed=False)['area'].unique().reset_index(name='areas') # TODO: Delete\n    ratios = unlabeled_df.groupby('series_id', observed=False)['ratio'].unique().reset_index(name='ratios') # TODO: Delete\n    spacings = unlabeled_df.groupby('series_id', observed=False)['spacing'].unique().reset_index(name='spacings') # TODO: Delete\n    slice_thicks = unlabeled_df.groupby('series_id', observed=False)['slice_thick'].unique().reset_index(name='slice_thicks') # TODO: Delete\n    slice_locs_min = unlabeled_df.groupby('series_id', observed=False)['slice_loc'].min().reset_index(name='slice_locs_min')\n    slice_locs_max = unlabeled_df.groupby('series_id', observed=False)['slice_loc'].max().reset_index(name='slice_locs_max')\n    \n    series_desc_df = merge_dfs(series_desc_df, min_vals, ['min_val'], 'series_id')\n    series_desc_df = merge_dfs(series_desc_df, max_vals, ['max_val'], 'series_id')\n    series_desc_df = merge_dfs(series_desc_df, range_val, ['range_val'], 'series_id') # TODO: Delete\n    series_desc_df = merge_dfs(series_desc_df, sizes, ['sizes'], 'series_id') # TODO: Delete\n    series_desc_df = merge_dfs(series_desc_df, areas, ['areas'], 'series_id') # TODO: Delete\n    series_desc_df = merge_dfs(series_desc_df, ratios, ['ratios'], 'series_id') # TODO: Delete\n    series_desc_df = merge_dfs(series_desc_df, spacings, ['spacings'], 'series_id') # TODO: Delete\n    series_desc_df = merge_dfs(series_desc_df, slice_thicks, ['slice_thicks'], 'series_id') # TODO: Delete\n    series_desc_df = merge_dfs(series_desc_df, slice_locs_min, ['slice_locs_min'], 'series_id')\n    series_desc_df = merge_dfs(series_desc_df, slice_locs_max, ['slice_locs_max'], 'series_id')\n    \n    return series_desc_df","metadata":{"papermill":{"duration":0.056102,"end_time":"2024-08-30T11:09:10.212470","exception":false,"start_time":"2024-08-30T11:09:10.156368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:49.286205Z","iopub.execute_input":"2024-10-06T07:02:49.286532Z","iopub.status.idle":"2024-10-06T07:02:49.298606Z","shell.execute_reply.started":"2024-10-06T07:02:49.286498Z","shell.execute_reply":"2024-10-06T07:02:49.297659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aggregate the DICOM data and process the data\ntrain_series_desc_df = process_dicom_data(train_df, train_series_desc_df)\ntest_series_desc_df = process_dicom_data(test_df, test_series_desc_df)\n\nprint(f'train_df shape: {train_df.shape}')\ndisplay(train_df.head(3))\nprint(f'test_df shape: {test_df.shape}')\ndisplay(test_df.head(3))\nprint(f'train_series_desc_df shape: {train_series_desc_df.shape}')\ndisplay(train_series_desc_df.head(3))\nprint(f'labeled_train_df shape: {labeled_train_df.shape}')\ndisplay(labeled_train_df.head(3))","metadata":{"papermill":{"duration":1.023241,"end_time":"2024-08-30T11:09:11.280706","exception":false,"start_time":"2024-08-30T11:09:10.257465","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:49.299912Z","iopub.execute_input":"2024-10-06T07:02:49.300229Z","iopub.status.idle":"2024-10-06T07:02:52.534368Z","shell.execute_reply.started":"2024-10-06T07:02:49.300196Z","shell.execute_reply":"2024-10-06T07:02:52.533474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Slice Location","metadata":{"papermill":{"duration":0.04881,"end_time":"2024-08-30T11:09:11.379148","exception":false,"start_time":"2024-08-30T11:09:11.330338","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Normalize the slice location\ndef normalize_slice_loc(unlabeled_df, series_desc_df):\n    unlabeled_df = unlabeled_df.drop(columns=['slice_locs_min', 'slice_locs_max'], errors='ignore')\n    unlabeled_df = pd.merge(unlabeled_df, series_desc_df[['series_id', 'slice_locs_min', 'slice_locs_max']], on='series_id', how='left')\n    unlabeled_df['norm_loc'] = (unlabeled_df['slice_loc'] - unlabeled_df['slice_locs_min']) / (unlabeled_df['slice_locs_max'] - unlabeled_df['slice_locs_min'])\n    unlabeled_df.drop(columns=['slice_locs_min', 'slice_locs_max', 'slice_loc'], inplace=True)\n    series_desc_df.drop(columns=['slice_locs_min', 'slice_locs_max'], inplace=True)\n    \n    return unlabeled_df, series_desc_df","metadata":{"papermill":{"duration":0.057272,"end_time":"2024-08-30T11:09:11.484262","exception":false,"start_time":"2024-08-30T11:09:11.426990","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:52.535633Z","iopub.execute_input":"2024-10-06T07:02:52.536007Z","iopub.status.idle":"2024-10-06T07:02:52.543054Z","shell.execute_reply.started":"2024-10-06T07:02:52.535962Z","shell.execute_reply":"2024-10-06T07:02:52.542141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize the slice location\ntrain_df, train_series_desc_df = normalize_slice_loc(train_df, train_series_desc_df)\ntest_df, test_series_desc_df = normalize_slice_loc(test_df, test_series_desc_df)\n\n# add normalized location from train_df to labeled_train_df on image_file_path\nlabeled_train_df = merge_dfs(labeled_train_df, train_df[['image_file_path', 'norm_loc']], ['norm_loc'], 'image_file_path')\n# Move output column to the end of the dataframe\nlabeled_train_df = move_column_to_last(labeled_train_df, 'output')\n\nPRINT_ANALYSIS = True\nif PRINT_ANALYSIS:\n    max_instances_series = train_series_desc_df[train_series_desc_df['total_instances'] == 10]['series_id']\n    filtered_train_df = train_df[train_df['series_id'] == max_instances_series.values[0]]\n    print(f'Maximum number of instances in a series: {10}')\n    display(filtered_train_df.head(10))","metadata":{"papermill":{"duration":0.344623,"end_time":"2024-08-30T11:09:11.875830","exception":false,"start_time":"2024-08-30T11:09:11.531207","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:52.544129Z","iopub.execute_input":"2024-10-06T07:02:52.544423Z","iopub.status.idle":"2024-10-06T07:02:52.829446Z","shell.execute_reply.started":"2024-10-06T07:02:52.544391Z","shell.execute_reply":"2024-10-06T07:02:52.828544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DICOM Analysis","metadata":{"papermill":{"duration":0.045889,"end_time":"2024-08-30T11:09:11.968572","exception":false,"start_time":"2024-08-30T11:09:11.922683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ANALYZE_DATA = True\nPRINT_ANALYSIS = True\nFILTER_ANALYSIS = False\n\nif ANALYZE_DATA:\n    analyzed_features = ['slice_thick', 'range_val', 'ratio', 'area']\n    for series_desc in SERIES.class_names:\n        filtered_df = filter_df(train_df, [series_desc], 'series_description')\n        print(f'---------------- {series_desc} ----------------')\n        for feature in analyzed_features:\n            values = filtered_df[feature].unique()\n            values.sort()\n            print(f'{feature}: min={min(values)} max={max(values)}')\n        print()\n        \n    del filtered_df, values\n    \nif FILTER_ANALYSIS:\n    filter_feature = 'range_val'\n    filter_val = 18700\n    \n    filttered_df = train_df[train_df[filter_feature] > filter_val].copy()\n    print(f'{filter_feature} ({filter_val}): {filttered_df.shape}')\n    display(filttered_df.head(10))\n    \n    del filttered_df, filter_feature, filter_val","metadata":{"papermill":{"duration":0.493693,"end_time":"2024-08-30T11:09:12.508807","exception":false,"start_time":"2024-08-30T11:09:12.015114","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:52.830682Z","iopub.execute_input":"2024-10-06T07:02:52.830985Z","iopub.status.idle":"2024-10-06T07:02:53.211784Z","shell.execute_reply.started":"2024-10-06T07:02:52.830951Z","shell.execute_reply":"2024-10-06T07:02:53.210850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only `Subarticular Stenosis` series have more than 1 number of spacings, number of thicknesses, number of sizes  \n\n> i.e. only `Subarticular Stenosis` have images run on different protocols within the same imaging series (`Axial T2`)","metadata":{}},{"cell_type":"code","source":"ANALYZE_DATA = True\nPRINT_ANALYSIS = True\nFILTER_ANALYSIS = False\n\nanalysis_df = train_series_desc_df.copy()\nanalysis_df['num_sizes'] = analysis_df['sizes'].apply(lambda x: len(x))\nanalysis_df['num_spacings'] = analysis_df['spacings'].apply(lambda x: len(x))\nanalysis_df['num_thicks'] = analysis_df['slice_thicks'].apply(lambda x: len(x))\n\nif ANALYZE_DATA:\n    \n    unique_num_sizes = analysis_df['num_sizes'].unique()\n    unique_num_spacings = analysis_df['num_spacings'].unique()\n    unique_num_thicks = analysis_df['num_thicks'].unique()\n\n    unique_num_sizes.sort()\n    unique_num_spacings.sort()\n    unique_num_thicks.sort()\n    \n    if PRINT_ANALYSIS:\n        print(f'unique num_sizes:\\n{unique_num_sizes}\\n')\n        print(f'unique num_spacings:\\n{unique_num_spacings}\\n')\n        print(f'unique num_thicks:\\n{unique_num_thicks}\\n')\n\n    del unique_num_sizes, unique_num_spacings, unique_num_thicks\n    \n    # 35 series with 2 unique sizes (no series with > 2 sizes)\n    # 374 series with more than 1 unique spacings (217 with 2 spacings, 147 with 3 spacings, 10 with 4 spacings)\n    # 1 series with more than 1 unique slice_thickness\n    # Only Subarticular Stenosis series have more than 1 number of spacings, number of thicknesses, number of sizes\n    # i.e. only Subarticular Stenosis have images run on different protocoles within the same imaging series\n    if FILTER_ANALYSIS:\n        filter_feature = 'num_spacings'\n        filter_val = 4\n        \n        filtered_df = analysis_df[analysis_df[filter_feature] == filter_val].copy()\n        # filtered_df = filtered_df[filtered_df['sup_condition'] == 'Neural Foraminal Narrowing']\n        print(f'{filter_feature} ({filter_val}): {filtered_df.shape}')\n        display(filtered_df.head(10))\n        \n        del filtered_df, filter_feature, filter_val","metadata":{"execution":{"iopub.status.busy":"2024-10-06T07:02:53.213080Z","iopub.execute_input":"2024-10-06T07:02:53.213472Z","iopub.status.idle":"2024-10-06T07:02:53.242656Z","shell.execute_reply.started":"2024-10-06T07:02:53.213428Z","shell.execute_reply":"2024-10-06T07:02:53.241742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🖼️ | Data Visualization","metadata":{"papermill":{"duration":0.046375,"end_time":"2024-08-30T11:09:12.602470","exception":false,"start_time":"2024-08-30T11:09:12.556095","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 🤝 | Helper Functions","metadata":{"papermill":{"duration":0.046908,"end_time":"2024-08-30T11:09:12.698159","exception":false,"start_time":"2024-08-30T11:09:12.651251","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(labeled_train_df.columns.values)","metadata":{"papermill":{"duration":0.062849,"end_time":"2024-08-30T11:09:12.818104","exception":false,"start_time":"2024-08-30T11:09:12.755255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:53.243842Z","iopub.execute_input":"2024-10-06T07:02:53.244203Z","iopub.status.idle":"2024-10-06T07:02:53.257793Z","shell.execute_reply.started":"2024-10-06T07:02:53.244161Z","shell.execute_reply":"2024-10-06T07:02:53.256924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the label information for a given series_id and instance_number\ndef get_coord_info(series_id, instance_number, processed=False):\n    features = ['condition',\n                'x', 'y',\n                'output',\n                'level']\n    if instance_number in labeled_train_df.loc[labeled_train_df['series_id'] == series_id, 'instance_number'].values:\n        values = labeled_train_df.loc[(labeled_train_df['series_id'] == series_id) &\n                                      (labeled_train_df['instance_number'] == instance_number),\n                                      features].values.T\n        coord_info = {feat: value for feat, value in zip(features, values)}\n        if processed:\n            coord_info['x'] = labeled_train_df.loc[(labeled_train_df['series_id'] == series_id) &\n                                                    (labeled_train_df['instance_number'] == instance_number),\n                                                    'proc_x'].values\n            coord_info['y'] = labeled_train_df.loc[(labeled_train_df['series_id'] == series_id) &\n                                                    (labeled_train_df['instance_number'] == instance_number),\n                                                    'proc_y'].values\n    else: coord_info = {}\n    \n    return coord_info\n\n# Get the window parameters for a given img\ndef find_label_param(img, markersize, series_desc):\n    ms = img.shape[0] * markersize / 100\n    # Offset the annotation for the different series\n    offset = (-ms,0)\n    if SERIES.name2label[series_desc] == SERIES.name2label['Axial T2']: offset = (0,-ms)\n    return ms, offset","metadata":{"papermill":{"duration":0.05874,"end_time":"2024-08-30T11:09:12.923631","exception":false,"start_time":"2024-08-30T11:09:12.864891","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:53.259005Z","iopub.execute_input":"2024-10-06T07:02:53.259644Z","iopub.status.idle":"2024-10-06T07:02:53.268893Z","shell.execute_reply.started":"2024-10-06T07:02:53.259601Z","shell.execute_reply":"2024-10-06T07:02:53.268084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Annotate the image with the label information\ndef plot_coord(ax, coord_info, ms, figsize, show_info=True, label_offset=(0,0)):\n    rectangles = []\n    labels = []\n    # if coord_info is not empty\n    if coord_info:\n        output_colors = [OUTPUT.colors[out] for out in coord_info['output']]\n        # plot the rectangles\n        for cond, x, y, output_color, level in zip(coord_info['condition'], coord_info['x'], coord_info['y'], output_colors, coord_info['level']):            \n            rectangle = patches.Rectangle((x - ms / 2, y - ms / 2), ms, ms,\n                                               linewidth=1, edgecolor=output_color, facecolor='none')\n            ax.add_patch(rectangle)\n            rectangles.append(rectangle)\n            if show_info:\n                scn = CONDITION.label2short[CONDITION.name2label[cond]]\n                label = ax.text(x+label_offset[0], y+label_offset[1], scn, fontsize=figsize*1.7, color='pink', ha='center', va='bottom', fontweight='bold')\n                labels.append(label)\n                label = ax.text(x+label_offset[0], y+label_offset[1], level, fontsize=figsize*1.6, color='white', ha='center', va='top')\n                labels.append(label)\n    return rectangles, labels","metadata":{"papermill":{"duration":0.059593,"end_time":"2024-08-30T11:09:13.030283","exception":false,"start_time":"2024-08-30T11:09:12.970690","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:53.269790Z","iopub.execute_input":"2024-10-06T07:02:53.270036Z","iopub.status.idle":"2024-10-06T07:02:53.282592Z","shell.execute_reply.started":"2024-10-06T07:02:53.270007Z","shell.execute_reply":"2024-10-06T07:02:53.281659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 🩻 | DICOM plot","metadata":{"papermill":{"duration":0.045963,"end_time":"2024-08-30T11:09:13.122974","exception":false,"start_time":"2024-08-30T11:09:13.077011","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Function derived from [Anatomy & Image Visualization Overview-RSNA RAIDS - Abhinav Suri](https://www.kaggle.com/code/abhinavsuri/anatomy-image-visualization-overview-rsna-raids) [3]","metadata":{"papermill":{"duration":0.04641,"end_time":"2024-08-30T11:09:13.215477","exception":false,"start_time":"2024-08-30T11:09:13.169067","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Create subplots for displaying images\ndef create_subplots(images, max_images_per_row=1, figsize=4, show_info=False):\n    # Calculate the number of rows needed\n    num_images = len(images)\n    num_rows = (num_images + max_images_per_row - 1) // max_images_per_row  # Ceiling division\n    # Adjust the number of images per row if there are fewer images than the default\n    if num_images < max_images_per_row: max_images_per_row = num_images\n    # Calculate the figure size\n    fs = (max_images_per_row * figsize, num_rows * figsize)\n    if show_info: fs = (max_images_per_row*figsize, num_rows*figsize*1.1)\n    # Create a subplot grid\n    fig, axes = plt.subplots(num_rows, max_images_per_row, figsize=fs)\n    # Flatten axes array for easier looping if there are multiple rows\n    if num_rows > 1 or max_images_per_row > 1: axes = axes.flatten()\n    else: axes = [axes]  # Make it iterable for consistency\n    # Turn off unused subplots\n    for idx in range(num_images, len(axes)):\n        axes[idx].axis('off')\n\n    return fig, axes\n\n# display any images\ndef display_images(images, title='', max_images_per_row=1, figsize=4):\n    fig, axes = create_subplots(images, max_images_per_row=max_images_per_row, figsize=figsize)\n    \n    for i, img in enumerate(images):\n        ax = axes[i]\n        ax.imshow(img, cmap=CFG.color_map)\n        ax.axis('off')\n    \n    if title!='': fig.suptitle(title, fontsize=16)\n    plt.tight_layout()\n    plt.show()\n\n# Plot the raw DICOM images\ndef plot_dicom_images(img_file_paths, title='', max_images_per_row=1, figsize=4, markersize=10, show_label=False, show_info=False, split='train', processed=False):   \n    images = []\n    series_ids = []\n    instance_numbers = []\n    coord_infos = []\n    series_descs = []\n    for img_file_path in img_file_paths:\n        images.append(load_mri_image(img_file_path, split=split, processed=processed))\n        _, series_id, instance_number = decode_img_file_path(img_file_path)\n        series_ids.append(series_id)\n        instance_numbers.append(instance_number)\n        coord_infos.append(get_coord_info(series_id, instance_number, processed=processed))\n        series_descs.append(find_series_desc(series_id, split=split))\n    \n    fig, axes = create_subplots(images, max_images_per_row=max_images_per_row, figsize=figsize, show_info=show_info)\n\n    for i, img in enumerate(images):\n        ax = axes[i]\n        \n        if show_label:\n            ms, offset = find_label_param(img, markersize, series_descs[i])\n            plot_coord(ax, coord_infos[i], ms, figsize, show_info, offset)\n\n        ax.title.set_fontsize(figsize*3)\n        ax.title.set_text(f'{series_descs[i]} {series_ids[i]} ({instance_numbers[i]})')\n        ax.imshow(img, cmap=CFG.color_map)\n        if not show_info: ax.axis('off')\n    \n    if title!='': fig.suptitle(title, fontsize=16)\n    fig.tight_layout(rect=[0, 0, 0.98, 0.98])\n    plt.show()","metadata":{"papermill":{"duration":0.067565,"end_time":"2024-08-30T11:09:13.330211","exception":false,"start_time":"2024-08-30T11:09:13.262646","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:53.283892Z","iopub.execute_input":"2024-10-06T07:02:53.284225Z","iopub.status.idle":"2024-10-06T07:02:53.301064Z","shell.execute_reply.started":"2024-10-06T07:02:53.284193Z","shell.execute_reply":"2024-10-06T07:02:53.300318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_file_paths=labeled_train_df['image_file_path'].sample(6).values\nplot_dicom_images(img_file_paths, title='Sample Images', max_images_per_row=3, show_label=True, show_info=True)\ndel img_file_paths","metadata":{"papermill":{"duration":2.134404,"end_time":"2024-08-30T11:09:15.512211","exception":false,"start_time":"2024-08-30T11:09:13.377807","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:53.302019Z","iopub.execute_input":"2024-10-06T07:02:53.302325Z","iopub.status.idle":"2024-10-06T07:02:55.294138Z","shell.execute_reply.started":"2024-10-06T07:02:53.302285Z","shell.execute_reply":"2024-10-06T07:02:55.293217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 🧊 | 3D MRI Slides","metadata":{"papermill":{"duration":0.065802,"end_time":"2024-08-30T11:09:15.646073","exception":false,"start_time":"2024-08-30T11:09:15.580271","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Function derived from:\n- [Lumbar RSNA 2024: Visualizing + EDA + Sub - Allegich](https://www.kaggle.com/code/allegich/lumbar-rsna-2024-visualizing-eda-sub) [4]\n- [RSNA Lumbar Spine Analysis - Satya](https://www.kaggle.com/code/satyaprakashshukl/rsna-lumbar-spine-analysis) [5]","metadata":{"papermill":{"duration":0.067373,"end_time":"2024-08-30T11:09:15.778960","exception":false,"start_time":"2024-08-30T11:09:15.711587","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_3D_slides(series_id, markersize=10, figsize=6, show_label=True, show_info=True, split='train', processed = False):\n    if split == 'train':\n        img_file_paths = train_df[train_df['series_id'] == series_id]['image_file_path'].values\n    else:  # split == 'test'\n        img_file_paths = test_df[test_df['series_id'] == series_id]['image_file_path'].values\n    series_desc = find_series_desc(series_id, split=split)\n\n    images = []\n    for img_file_path in img_file_paths:\n        img = load_mri_image(img_file_path, split=split, processed=processed)\n        if img.max() == 0: continue  # Skip empty images\n        \n        # Get the instance number from the image file path and find the coordinate info\n        if show_label:\n            instance_number = int(img_file_path.split('_')[2].split('.')[0])\n            coord_info = get_coord_info(series_id, instance_number, processed=processed)\n        else: coord_info = {}\n\n        if coord_info:\n            images.append((img, coord_info))  # Store the image with its coordinate info\n        else:\n            images.append((img, None))  # No valid coordinates, append None\n\n    rc('animation', html='jshtml')\n\n    def create_animation(ims):\n        fig, ax = plt.subplots(figsize=(figsize, figsize))\n        plt.axis('off')\n        \n        im = ax.imshow(ims[0][0], cmap=CFG.color_map)  # Display the first image\n        ms, offset = find_label_param(ims[0][0], markersize, series_desc)\n        rects, labels = plot_coord(ax, ims[0][1], ms, figsize, show_info=show_info, label_offset=offset) # Plot the first rectangles\n        text = plt.text(0.05, 0.05, f'Slide {1}', transform=fig.transFigure, fontsize=16, color='darkblue')\n\n        def animate_func(i):\n            im.set_array(ims[i][0])  # Update the image in the animation\n            text.set_text(f'Slide {i + 1}')\n\n            # Remove the previous rectangles and labels\n            for rect in rects:\n                rect.remove()\n            rects.clear()\n            for label in labels:\n                label.remove()\n            labels.clear()\n\n            # Plot the new rectangles and labels\n            ms, offset = find_label_param(ims[i][0], markersize, series_desc)\n            rectangles, labs = plot_coord(ax, ims[i][1], ms, figsize, show_info=show_info, label_offset=offset)\n            for rect in rectangles:\n                rects.append(rect)\n            for label in labs:\n                labels.append(label)\n\n            return [im, text] + rects  # Ensure all rectangles are returned to be updated\n\n        plt.title(f'{str(find_study_id(series_id))}, {str(series_desc)} ({str(series_id)})')\n        plt.close()\n\n        return animation.FuncAnimation(fig, animate_func, frames=len(ims), interval=1000 // 10)\n\n    return create_animation(images)","metadata":{"papermill":{"duration":0.087713,"end_time":"2024-08-30T11:09:15.932754","exception":false,"start_time":"2024-08-30T11:09:15.845041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:55.295737Z","iopub.execute_input":"2024-10-06T07:02:55.296086Z","iopub.status.idle":"2024-10-06T07:02:55.315588Z","shell.execute_reply.started":"2024-10-06T07:02:55.296047Z","shell.execute_reply":"2024-10-06T07:02:55.314537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_3D_slides(847344271, split='train', show_label=True, show_info=True)","metadata":{"papermill":{"duration":9.325742,"end_time":"2024-08-30T11:09:25.323599","exception":false,"start_time":"2024-08-30T11:09:15.997857","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:02:55.316799Z","iopub.execute_input":"2024-10-06T07:02:55.317100Z","iopub.status.idle":"2024-10-06T07:03:04.084548Z","shell.execute_reply.started":"2024-10-06T07:02:55.317068Z","shell.execute_reply":"2024-10-06T07:03:04.082861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 💾 | Save Dataframes","metadata":{"papermill":{"duration":0.606202,"end_time":"2024-08-30T11:09:26.561393","exception":false,"start_time":"2024-08-30T11:09:25.955191","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f'train_df shape: {train_df.shape}')\ndisplay(train_df.head(3))\nprint(f'test_df shape: {test_df.shape}')\ndisplay(test_df.head(3))\nprint(f'labeled_train_df shape: {labeled_train_df.shape}')\ndisplay(labeled_train_df.head(3))\nprint(f'train_series_desc_df shape: {train_series_desc_df.shape}')\ndisplay(train_series_desc_df.head(3))\nprint(f'test_series_desc_df shape: {test_series_desc_df.shape}')\ndisplay(test_series_desc_df.head(3))","metadata":{"papermill":{"duration":0.666096,"end_time":"2024-08-30T11:09:27.819051","exception":false,"start_time":"2024-08-30T11:09:27.152955","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:04.086302Z","iopub.execute_input":"2024-10-06T07:03:04.086920Z","iopub.status.idle":"2024-10-06T07:03:04.186162Z","shell.execute_reply.started":"2024-10-06T07:03:04.086879Z","shell.execute_reply":"2024-10-06T07:03:04.185261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **NOTE** : If any of the added features in this notebook is to be used for model precition, the code to add these features to the test data will need to added to the submission notebook as the test data to calculate the score is different from the sample test data provided by the competition data","metadata":{"papermill":{"duration":0.603744,"end_time":"2024-08-30T11:09:29.011617","exception":false,"start_time":"2024-08-30T11:09:28.407873","status":"completed"},"tags":[]}},{"cell_type":"code","source":"os.makedirs(ANALYSIS_PATH, exist_ok=True)\n\ntrain_df.to_csv(os.path.join(ANALYSIS_PATH, 'analyzed_train.csv'), index=False)\ntest_df.to_csv(os.path.join(ANALYSIS_PATH, 'analyzed_test.csv'), index=False)\nlabeled_train_df.to_csv(os.path.join(ANALYSIS_PATH, 'analyzed_labeled_train.csv'), index=False)\ntrain_series_desc_df.to_csv(os.path.join(ANALYSIS_PATH, 'analyzed_train_series_descriptions.csv'), index=False)\ntest_series_desc_df.to_csv(os.path.join(ANALYSIS_PATH, 'analyzed_test_series_descriptions.csv'), index=False)\ntest_df.to_csv(os.path.join(ANALYSIS_PATH, 'analyzed_test.csv'), index=False)\n\nprint(os.listdir(ANALYSIS_PATH))","metadata":{"papermill":{"duration":3.363903,"end_time":"2024-08-30T11:09:32.970784","exception":false,"start_time":"2024-08-30T11:09:29.606881","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:04.187374Z","iopub.execute_input":"2024-10-06T07:03:04.187702Z","iopub.status.idle":"2024-10-06T07:03:08.651570Z","shell.execute_reply.started":"2024-10-06T07:03:04.187666Z","shell.execute_reply":"2024-10-06T07:03:08.650575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df = pd.read_csv(os.path.join(ANALYSIS_PATH, 'analyzed_train.csv'))\ndisplay(check_df.head(3))","metadata":{"papermill":{"duration":0.998043,"end_time":"2024-08-30T11:09:34.572944","exception":false,"start_time":"2024-08-30T11:09:33.574901","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:08.652869Z","iopub.execute_input":"2024-10-06T07:03:08.653197Z","iopub.status.idle":"2024-10-06T07:03:09.028281Z","shell.execute_reply.started":"2024-10-06T07:03:08.653162Z","shell.execute_reply":"2024-10-06T07:03:09.027385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📐 | Edge Cases","metadata":{"papermill":{"duration":0.600706,"end_time":"2024-08-30T11:09:35.782843","exception":false,"start_time":"2024-08-30T11:09:35.182137","status":"completed"},"tags":[]}},{"cell_type":"code","source":"edge_cases= {}\n\n# Print the Edge Cases\ndef print_edge_cases(e_cases):\n    print(f'Edge Cases: {len(e_cases)}')\n    print(\"---------------------------\")\n    for key in e_cases.keys():\n        print(f'{key}: {len(e_cases[key])}')","metadata":{"papermill":{"duration":0.616846,"end_time":"2024-08-30T11:09:36.990488","exception":false,"start_time":"2024-08-30T11:09:36.373642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:09.029566Z","iopub.execute_input":"2024-10-06T07:03:09.029876Z","iopub.status.idle":"2024-10-06T07:03:09.034896Z","shell.execute_reply.started":"2024-10-06T07:03:09.029842Z","shell.execute_reply":"2024-10-06T07:03:09.033988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"edge_cases['t1_scs'] = labeled_train_df[(labeled_train_df.condition == 'Spinal Canal Stenosis') &\n                                (labeled_train_df.series_description == 'Sagittal T1')]['image_file_path'].values\n\nedge_cases['unlabeled_series'] = train_df[train_df['series_id'].isin(missing_series_ids)]['image_file_path'].values\nedge_cases['unlabeled_study'] = train_df[train_df['study_id'].isin(missing_study_ids)]['image_file_path'].values\n\nedge_cases['multiple_labels_instances'] = train_df[train_df['num_labels'] > 5]['image_file_path'].values\nfifteen_label_series = train_series_desc_df[train_series_desc_df['num_labels'] == 15]['series_id'].values\nedge_cases['fifteen_labels_series'] = train_df[train_df['series_id'].isin(fifteen_label_series)]['image_file_path'].values\n\nthree_condition_series = train_series_desc_df[train_series_desc_df['num_conditions'] == 3]['series_id'].values\nedge_cases['three_conditions_series'] = train_df[train_df['series_id'].isin(three_condition_series)]['image_file_path'].values\nmixed_sup_condition_series = train_series_desc_df[train_series_desc_df['sup_condition'] == 'Mixed']['series_id'].values\nedge_cases['mixed_sup_condition_series'] = train_df[train_df['series_id'].isin(mixed_sup_condition_series)]['image_file_path'].values\n\nmultiple_thickness_series = analysis_df[analysis_df['num_thicks'] > 1]['series_id'].values\nedge_cases['two_thickness_ss_series'] = train_df[train_df['series_id'].isin(multiple_thickness_series)]['image_file_path'].values\nfour_spacing_series_df = analysis_df[analysis_df['num_spacings'] == 4].copy()\nfour_spacing_series = four_spacing_series_df[four_spacing_series_df['num_labeled'] > 7]['series_id'].values\nedge_cases['four_spacings_ss_series'] = train_df[train_df['series_id'].isin(four_spacing_series)]['image_file_path'].values\n\nprint_edge_cases(edge_cases)","metadata":{"papermill":{"duration":0.624292,"end_time":"2024-08-30T11:09:38.205008","exception":false,"start_time":"2024-08-30T11:09:37.580716","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:09.036018Z","iopub.execute_input":"2024-10-06T07:03:09.036287Z","iopub.status.idle":"2024-10-06T07:03:09.082061Z","shell.execute_reply.started":"2024-10-06T07:03:09.036240Z","shell.execute_reply":"2024-10-06T07:03:09.081225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\ndef save_edge_cases(edge_cases, file_path):\n    with open(file_path, 'wb') as f:\n        pickle.dump(edge_cases, f)\n\ndef load_edge_cases(file_path):\n    with open(file_path, 'rb') as f:\n        edge_cases = pickle.load(f)\n    return edge_cases\n\ndef find_edge_series(edge_case_key, verbose=True):\n    series_ids = []\n    for img_file_path in edge_cases[edge_case_key]:\n        _, series_id, _ = decode_img_file_path(img_file_path)\n        if series_id not in series_ids: series_ids.append(series_id)\n    if verbose: print(f'{edge_case_key}: {len(series_ids)}')\n    return series_ids\n\nsave_edge_cases(edge_cases, os.path.join(ANALYSIS_PATH, 'edge_cases.pkl'))","metadata":{"papermill":{"duration":0.602526,"end_time":"2024-08-30T11:09:39.400578","exception":false,"start_time":"2024-08-30T11:09:38.798052","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:09.083234Z","iopub.execute_input":"2024-10-06T07:03:09.083922Z","iopub.status.idle":"2024-10-06T07:03:09.091352Z","shell.execute_reply.started":"2024-10-06T07:03:09.083875Z","shell.execute_reply":"2024-10-06T07:03:09.090465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_edge_cases = load_edge_cases(os.path.join(ANALYSIS_PATH, 'edge_cases.pkl'))\nprint_edge_cases(check_edge_cases)","metadata":{"papermill":{"duration":0.593931,"end_time":"2024-08-30T11:09:40.586436","exception":false,"start_time":"2024-08-30T11:09:39.992505","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-06T07:03:09.092455Z","iopub.execute_input":"2024-10-06T07:03:09.092777Z","iopub.status.idle":"2024-10-06T07:03:09.106825Z","shell.execute_reply.started":"2024-10-06T07:03:09.092742Z","shell.execute_reply":"2024-10-06T07:03:09.105909Z"},"trusted":true},"execution_count":null,"outputs":[]}]}