{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hi everyone,\n\nI noticed that there are several corrupted files that causes errors when doing the training process. \n\nThis is a code to check image files and remove some corrupted ones from the metadata.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom PIL import Image\ndf_train=pd.read_csv('../input/snakeclef2022/SnakeCLEF2022-TrainMetadata.csv')\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:51:39.973783Z","iopub.execute_input":"2022-03-19T19:51:39.974775Z","iopub.status.idle":"2022-03-19T19:51:40.434341Z","shell.execute_reply.started":"2022-03-19T19:51:39.974728Z","shell.execute_reply":"2022-03-19T19:51:40.433255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"froot='../input/snakeclef2022/SnakeCLEF2022-small_size/SnakeCLEF2022-small_size'\nls_imgs=[os.path.join(froot,fp) for fp in df_train.file_path]\nls_imgs[:10]","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:52:03.07502Z","iopub.execute_input":"2022-03-19T19:52:03.075309Z","iopub.status.idle":"2022-03-19T19:52:03.662524Z","shell.execute_reply.started":"2022-03-19T19:52:03.075276Z","shell.execute_reply":"2022-03-19T19:52:03.661566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://github.com/python-pillow/Pillow/issues/3012","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check images if you can open them. \nerror_fn=list()\nN=len(ls_imgs)\ni=0\nfor filename in ls_imgs:\n        i=i+1\n        try:\n            #img = Image.open(filename)  # open the image file\n            #img.verify()  # verify that it is, in fact an image\n            #See https://github.com/python-pillow/Pillow/issues/3012\n            #img verify only checksum, do not detect truncated image case.\n            Image.open(filename).load()\n        except (IOError, SyntaxError) as e:\n            print(filename)\n            error_fn.append(filename)\n        if i%10000==0:\n            print('Reading {} out of {} total files...'.format(i,N))","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:53:10.714529Z","iopub.execute_input":"2022-03-19T19:53:10.715317Z","iopub.status.idle":"2022-03-19T19:58:52.034639Z","shell.execute_reply.started":"2022-03-19T19:53:10.71528Z","shell.execute_reply":"2022-03-19T19:58:52.033264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path_to_remove=[fn.split(\"\\\\\")[-1] for fn in error_fn]\ndf_train.query('file_path not in @file_path_to_remove').to_csv('SnakeCLEF2022-TrainMetadata-fixed.csv')\n\n","metadata":{},"execution_count":null,"outputs":[]}]}