{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Duplicates finding: https://www.kaggle.com/kwk100/siim-covid-19-duplicate-training-images","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.403821Z","iopub.execute_input":"2021-05-29T12:27:09.404225Z","iopub.status.idle":"2021-05-29T12:27:09.409688Z","shell.execute_reply.started":"2021-05-29T12:27:09.404194Z","shell.execute_reply":"2021-05-29T12:27:09.408103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ndf_study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ndf_dupicates=pd.read_csv('../input/covidduplicates/duplicates.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.411890Z","iopub.execute_input":"2021-05-29T12:27:09.412375Z","iopub.status.idle":"2021-05-29T12:27:09.468216Z","shell.execute_reply.started":"2021-05-29T12:27:09.412330Z","shell.execute_reply":"2021-05-29T12:27:09.467095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dupicates.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.470618Z","iopub.execute_input":"2021-05-29T12:27:09.471083Z","iopub.status.idle":"2021-05-29T12:27:09.487663Z","shell.execute_reply.started":"2021-05-29T12:27:09.471040Z","shell.execute_reply":"2021-05-29T12:27:09.486468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lduplicates=[]\nfor i in range(2,10):\n    for image in df_dupicates.loc[:,str(i)]:\n        if str(image)!='nan':\n            lduplicates.append(image.replace('.dcm','_image'))","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.489586Z","iopub.execute_input":"2021-05-29T12:27:09.490143Z","iopub.status.idle":"2021-05-29T12:27:09.500509Z","shell.execute_reply.started":"2021-05-29T12:27:09.490096Z","shell.execute_reply":"2021-05-29T12:27:09.499516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_duplicates=df_image[df_image.id.isin(lduplicates)]","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.501621Z","iopub.execute_input":"2021-05-29T12:27:09.501927Z","iopub.status.idle":"2021-05-29T12:27:09.519570Z","shell.execute_reply.started":"2021-05-29T12:27:09.501897Z","shell.execute_reply":"2021-05-29T12:27:09.518324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lduplicatess=image_duplicates.StudyInstanceUID+'_study'","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.520973Z","iopub.execute_input":"2021-05-29T12:27:09.521482Z","iopub.status.idle":"2021-05-29T12:27:09.530994Z","shell.execute_reply.started":"2021-05-29T12:27:09.521443Z","shell.execute_reply":"2021-05-29T12:27:09.530028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image.count()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.532356Z","iopub.execute_input":"2021-05-29T12:27:09.532828Z","iopub.status.idle":"2021-05-29T12:27:09.552188Z","shell.execute_reply.started":"2021-05-29T12:27:09.532783Z","shell.execute_reply":"2021-05-29T12:27:09.551206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image=df_image.query(\"id not in @lduplicates\")","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.553547Z","iopub.execute_input":"2021-05-29T12:27:09.554030Z","iopub.status.idle":"2021-05-29T12:27:09.572632Z","shell.execute_reply.started":"2021-05-29T12:27:09.553963Z","shell.execute_reply":"2021-05-29T12:27:09.571658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image.count()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.574863Z","iopub.execute_input":"2021-05-29T12:27:09.575384Z","iopub.status.idle":"2021-05-29T12:27:09.591035Z","shell.execute_reply.started":"2021-05-29T12:27:09.575333Z","shell.execute_reply":"2021-05-29T12:27:09.589805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_duplicates=df_study[df_study.id.isin(lduplicatess)]","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.593158Z","iopub.execute_input":"2021-05-29T12:27:09.593475Z","iopub.status.idle":"2021-05-29T12:27:09.606247Z","shell.execute_reply.started":"2021-05-29T12:27:09.593436Z","shell.execute_reply":"2021-05-29T12:27:09.604945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study.count()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.607777Z","iopub.execute_input":"2021-05-29T12:27:09.608136Z","iopub.status.idle":"2021-05-29T12:27:09.624883Z","shell.execute_reply.started":"2021-05-29T12:27:09.608102Z","shell.execute_reply":"2021-05-29T12:27:09.624063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study=df_study.query(\"id not in @lduplicatess\")","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.626167Z","iopub.execute_input":"2021-05-29T12:27:09.626606Z","iopub.status.idle":"2021-05-29T12:27:09.645663Z","shell.execute_reply.started":"2021-05-29T12:27:09.626574Z","shell.execute_reply":"2021-05-29T12:27:09.643957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study.count()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.647303Z","iopub.execute_input":"2021-05-29T12:27:09.647730Z","iopub.status.idle":"2021-05-29T12:27:09.667559Z","shell.execute_reply.started":"2021-05-29T12:27:09.647694Z","shell.execute_reply":"2021-05-29T12:27:09.666308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_duplicates.to_csv(\"train_image_level_deleted.csv\",index=False)\nstudy_duplicates.to_csv(\"train_study_level_deleted.csv\",index=False)\ndf_image.to_csv(\"train_image_level_wndupls.csv\",index=False)\ndf_study.to_csv(\"train_study_level_wndupls.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T12:27:09.669209Z","iopub.execute_input":"2021-05-29T12:27:09.669814Z","iopub.status.idle":"2021-05-29T12:27:09.770817Z","shell.execute_reply.started":"2021-05-29T12:27:09.669777Z","shell.execute_reply":"2021-05-29T12:27:09.769418Z"},"trusted":true},"execution_count":null,"outputs":[]}]}