{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Importing Required packages\nimport os\nimport torch\nimport numpy as np\nimport pandas as pd\n\nimport ast \nfrom glob import glob\nfrom tqdm.auto import tqdm\nimport shutil as sh\n\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, clear_output\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-10T20:28:48.263869Z","iopub.execute_input":"2021-06-10T20:28:48.264243Z","iopub.status.idle":"2021-06-10T20:28:48.272586Z","shell.execute_reply.started":"2021-06-10T20:28:48.264196Z","shell.execute_reply":"2021-06-10T20:28:48.271648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading the dataframe\nIMAGE_ROOT = '../input/siim-covid19-detection'\ntrain_image = pd.read_csv(f'{IMAGE_ROOT}/train_image_level.csv')\ntrain_study = pd.read_csv(f'{IMAGE_ROOT}/train_study_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:47:42.29127Z","iopub.execute_input":"2021-06-10T20:47:42.291704Z","iopub.status.idle":"2021-06-10T20:47:42.340117Z","shell.execute_reply.started":"2021-06-10T20:47:42.291667Z","shell.execute_reply":"2021-06-10T20:47:42.338846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study['StudyInstanceUID'] = train_study['id'].apply(lambda x: x.replace('_study', ''))\ndel train_study['id']\ntrain = train_image.merge(train_study, on='StudyInstanceUID')\ntrain['ImageID'] = train['id'].apply(lambda x: x.replace('_image', ''))\ndel train['id']\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:47:45.015668Z","iopub.execute_input":"2021-06-10T20:47:45.016065Z","iopub.status.idle":"2021-06-10T20:47:45.052071Z","shell.execute_reply.started":"2021-06-10T20:47:45.016024Z","shell.execute_reply":"2021-06-10T20:47:45.05133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = ['Typical Appearance','Negative for Pneumonia', 'Indeterminate Appearance', 'Atypical Appearance']\nrows = []\ncolumns = train.columns\ndf = train.copy()\nfor i, row in df.iterrows():\n    row['label'] = row[class_names][row[class_names]==1].index[0]  # set class label\n    if pd.isna(row['boxes']):\n        rows.append(row.values)\n    else:\n        list_of_boxes = ast.literal_eval(row['boxes']) \n        [rows.append([box]+list(row.values[1:])) for box in list_of_boxes]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:47:48.708674Z","iopub.execute_input":"2021-06-10T20:47:48.709087Z","iopub.status.idle":"2021-06-10T20:47:57.448834Z","shell.execute_reply.started":"2021-06-10T20:47:48.70905Z","shell.execute_reply":"2021-06-10T20:47:57.447785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame(rows, columns = columns)\ncols = ['ImageID', 'boxes', 'label']\ntrain_df = train_df[cols]\nboxes = ['x', 'y', 'width', 'height']\nbboxes = []\nfor i, row in train_df.iterrows():\n    if not pd.isna(row['boxes']):\n        bboxes.append(list(row['boxes'].values()))\n    else:\n        bboxes.append([np.nan, np.nan, np.nan, np.nan])\n\ntrain_df = train_df.join(pd.DataFrame(bboxes, columns = boxes))\ndel train_df['boxes']\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:47:57.45086Z","iopub.execute_input":"2021-06-10T20:47:57.451177Z","iopub.status.idle":"2021-06-10T20:47:58.406697Z","shell.execute_reply.started":"2021-06-10T20:47:57.451146Z","shell.execute_reply":"2021-06-10T20:47:58.40534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['x_center'] = (train_df['x'] + train_df['width'])/2\ntrain_df['y_center'] = (train_df['y'] + train_df['height'])/2\ntrain_df.columns = ['image_id', 'classes', 'x', 'y', 'w', 'h','x_center','y_center']\n\n# Making new dataframe, suitable to make suitable dataset for yolov5\ndf = train_df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:47:58.408328Z","iopub.execute_input":"2021-06-10T20:47:58.40874Z","iopub.status.idle":"2021-06-10T20:47:58.4322Z","shell.execute_reply.started":"2021-06-10T20:47:58.408689Z","shell.execute_reply":"2021-06-10T20:47:58.431105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = list(set(df.image_id))\nprint(\"Total Images: \",len(index))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:48:44.575715Z","iopub.execute_input":"2021-06-10T20:48:44.576135Z","iopub.status.idle":"2021-06-10T20:48:44.585873Z","shell.execute_reply.started":"2021-06-10T20:48:44.576103Z","shell.execute_reply":"2021-06-10T20:48:44.584403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert classes to numerical targets\nlabel2target = {l:t for t,l in enumerate(df['classes'].unique())}\ntarget2label = {t:l for l,t in label2target.items()}\ntargets = []\ntargets.append([label2target[c] for c in df['classes']])\ndf['classes'] = targets[0]\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:31:56.825055Z","iopub.execute_input":"2021-06-10T21:31:56.825457Z","iopub.status.idle":"2021-06-10T21:31:56.831263Z","shell.execute_reply.started":"2021-06-10T21:31:56.825425Z","shell.execute_reply":"2021-06-10T21:31:56.830359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find(name, path):\n    for root, dirs, files in os.walk(path):\n        if name in files:\n            return os.path.join(root, name)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:44:49.749515Z","iopub.execute_input":"2021-06-10T21:44:49.749922Z","iopub.status.idle":"2021-06-10T21:44:49.755851Z","shell.execute_reply.started":"2021-06-10T21:44:49.749887Z","shell.execute_reply":"2021-06-10T21:44:49.754804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ***SLOWWWWWWW***","metadata":{}},{"cell_type":"code","source":"source = 'train'\nif True:\n    for fold in [0]:\n        val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]\n        for name,mini in tqdm(df.groupby('image_id')):\n            if name in val_index:\n                path2save = 'val/'\n            else:\n                path2save = 'train/'\n            if not os.path.exists('convertor/fold{}/labels/'.format(fold)+path2save):\n                os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save)\n            with open('convertor/fold{}/labels/'.format(fold)+path2save+name+\".txt\", 'w+') as f:\n                row = mini[['classes','x_center','y_center','w','h']].astype(float).values\n                row = row/1024\n                row = row.astype(str)\n                for j in range(len(row)):\n                    text = ' '.join(row[j])\n                    f.write(text)\n                    f.write(\"\\n\")\n            if not os.path.exists('convertor/fold{}/images/{}'.format(fold,path2save)):\n                os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save))\n            filepath = find(\"{}.dcm\".format(name), \"../input/siim-covid19-detection/{}\".format(source))\n            sh.copy(filepath,'convertor/fold{}/images/{}/{}.dcm'.format(fold,path2save,name))\n    \n# Bases on this notebook: https://www.kaggle.com/orkatz2/yolov5-train","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:57:20.819355Z","iopub.execute_input":"2021-06-10T21:57:20.819705Z","iopub.status.idle":"2021-06-10T21:59:28.394887Z","shell.execute_reply.started":"2021-06-10T21:57:20.819676Z","shell.execute_reply":"2021-06-10T21:59:28.392827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -ltr ../input/siim-covid19-detection/train/*/*/000a312787f2.dcm","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:35:32.16882Z","iopub.execute_input":"2021-06-10T21:35:32.169206Z","iopub.status.idle":"2021-06-10T21:35:39.582064Z","shell.execute_reply.started":"2021-06-10T21:35:32.169172Z","shell.execute_reply":"2021-06-10T21:35:39.580839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm","metadata":{},"execution_count":null,"outputs":[]}]}