{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9066996,"sourceType":"datasetVersion","datasetId":5468565},{"sourceId":9074064,"sourceType":"datasetVersion","datasetId":5473695},{"sourceId":9074196,"sourceType":"datasetVersion","datasetId":5473802},{"sourceId":9074271,"sourceType":"datasetVersion","datasetId":5473861}],"dockerImageVersionId":30746,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":104.251541,"end_time":"2024-08-02T18:37:00.909603","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-02T18:35:16.658062","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phase 1: Data preprocessing","metadata":{"papermill":{"duration":0.014862,"end_time":"2024-08-02T18:35:19.498953","exception":false,"start_time":"2024-08-02T18:35:19.484091","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Train and Test Data Preprocessing**\n","metadata":{"papermill":{"duration":0.014523,"end_time":"2024-08-02T18:35:19.528062","exception":false,"start_time":"2024-08-02T18:35:19.513539","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Importing Libraries:**","metadata":{"papermill":{"duration":0.014328,"end_time":"2024-08-02T18:35:19.557781","exception":false,"start_time":"2024-08-02T18:35:19.543453","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\nimport glob\nimport pandas as pd\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt \nimport pydicom\nimport pandas as pd\nimport shutil\nfrom IPython.display import FileLink\nimport re\nimport numpy as np\nimport gc\nimport os\nimport zipfile\n%matplotlib inline\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.590092Z","iopub.status.busy":"2024-08-02T18:35:19.589601Z","iopub.status.idle":"2024-08-02T18:35:19.602961Z","shell.execute_reply":"2024-08-02T18:35:19.601944Z"},"papermill":{"duration":0.032251,"end_time":"2024-08-02T18:35:19.605183","exception":false,"start_time":"2024-08-02T18:35:19.572932","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Path and Uploading Files:**","metadata":{"papermill":{"duration":0.014199,"end_time":"2024-08-02T18:35:19.634031","exception":false,"start_time":"2024-08-02T18:35:19.619832","status":"completed"},"tags":[]}},{"cell_type":"code","source":"base_path='/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\ntrain_images = glob.glob(base_path+'train_images/*/*/*.dcm')\ntrain_label_coord_df = pd.read_csv(base_path+'train_label_coordinates.csv')\ntrain_label_coord_df.shape\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.664684Z","iopub.status.busy":"2024-08-02T18:35:19.664285Z","iopub.status.idle":"2024-08-02T18:35:19.670827Z","shell.execute_reply":"2024-08-02T18:35:19.669722Z"},"papermill":{"duration":0.024614,"end_time":"2024-08-02T18:35:19.673238","exception":false,"start_time":"2024-08-02T18:35:19.648624","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_df = pd.read_csv(base_path+'train.csv')\ntrain_df.shape\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.704468Z","iopub.status.busy":"2024-08-02T18:35:19.704095Z","iopub.status.idle":"2024-08-02T18:35:19.710264Z","shell.execute_reply":"2024-08-02T18:35:19.709310Z"},"papermill":{"duration":0.024393,"end_time":"2024-08-02T18:35:19.712631","exception":false,"start_time":"2024-08-02T18:35:19.688238","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating New Columns and Adding Target Values:**","metadata":{"papermill":{"duration":0.014487,"end_time":"2024-08-02T18:35:19.742412","exception":false,"start_time":"2024-08-02T18:35:19.727925","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ntrain_label_coord_df['new_col'] = train_label_coord_df['condition'].str.lower().str.replace(' ', '_') +  '_' + train_label_coord_df['level'].str.lower().str.replace('/', '_')\n\ndef get_target_value(row):\n    study_id = row['study_id']\n    new_col = row['new_col']\n    return train_df[train_df['study_id'] == study_id][new_col].values[0]\n\ntrain_label_coord_df['target'] = train_label_coord_df.apply(get_target_value, axis=1)\n\ntrain_label_coord_df.drop(columns=['new_col'], inplace=True)\n\ntrain_train_label_marged_df = train_label_coord_df.copy()\n\ntrain_label_coord_df.drop(columns=['target'], inplace=True)\n\ntrain_train_label_marged_df.head()\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.773563Z","iopub.status.busy":"2024-08-02T18:35:19.773177Z","iopub.status.idle":"2024-08-02T18:35:19.779894Z","shell.execute_reply":"2024-08-02T18:35:19.778904Z"},"papermill":{"duration":0.024893,"end_time":"2024-08-02T18:35:19.782042","exception":false,"start_time":"2024-08-02T18:35:19.757149","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Adding File Path Column and Creating Target Index:**","metadata":{"papermill":{"duration":0.015582,"end_time":"2024-08-02T18:35:19.812692","exception":false,"start_time":"2024-08-02T18:35:19.797110","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ntrain_train_label_marged_df.shape\ntrain_train_label_marged_df['file_path'] = None\ntrain_train_label_marged_df.head()\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.844783Z","iopub.status.busy":"2024-08-02T18:35:19.843965Z","iopub.status.idle":"2024-08-02T18:35:19.850706Z","shell.execute_reply":"2024-08-02T18:35:19.849712Z"},"papermill":{"duration":0.025214,"end_time":"2024-08-02T18:35:19.852947","exception":false,"start_time":"2024-08-02T18:35:19.827733","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\noutput_dir = 'june/'\nfolder = 'cleaned_images'\n\ncleaned_images_path = os.path.join(output_dir, folder)\n\nos.makedirs(cleaned_images_path, exist_ok=True)\n\nprint(f\"Folder created at: {cleaned_images_path}\")\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.885091Z","iopub.status.busy":"2024-08-02T18:35:19.884676Z","iopub.status.idle":"2024-08-02T18:35:19.890771Z","shell.execute_reply":"2024-08-02T18:35:19.889845Z"},"papermill":{"duration":0.024398,"end_time":"2024-08-02T18:35:19.892809","exception":false,"start_time":"2024-08-02T18:35:19.868411","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Function that Extracts Last Values from File Path:**","metadata":{"papermill":{"duration":0.01673,"end_time":"2024-08-02T18:35:19.924969","exception":false,"start_time":"2024-08-02T18:35:19.908239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ndef extract_last_values(file_path):\n    img_file_numbers = re.findall(r'\\d+', file_path)\n    last_values = img_file_numbers[-3:]\n    return last_values\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:19.957158Z","iopub.status.busy":"2024-08-02T18:35:19.956427Z","iopub.status.idle":"2024-08-02T18:35:19.962692Z","shell.execute_reply":"2024-08-02T18:35:19.961679Z"},"papermill":{"duration":0.024694,"end_time":"2024-08-02T18:35:19.964851","exception":false,"start_time":"2024-08-02T18:35:19.940157","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Processing and Copying of Files:**","metadata":{"papermill":{"duration":0.016137,"end_time":"2024-08-02T18:35:19.996874","exception":false,"start_time":"2024-08-02T18:35:19.980737","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ncounter=0\nvalues_found=0\nstudy_ids = train_train_label_marged_df['study_id'].values\nseries_ids = train_train_label_marged_df['series_id'].values\ninstance_numbers = train_train_label_marged_df['instance_number'].values\nfor file_path in train_images:\n    last_values = extract_last_values(file_path)\n    study_id, series_id, instance_number = last_values\n    cleaned_images_path = os.path.join(output_dir, folder)\n    mask = (study_ids == int(study_id)) & (series_ids == int(series_id)) & (instance_numbers == int(instance_number))\n    if np.any(mask):\n        train_train_label_marged_df.loc[mask, 'file_path'] = study_id+'_'+series_id+'_'+instance_number+'.dcm'#file_path\n        destination_file_path =os.path.join('june/cleaned_images', study_id+'_'+series_id+'_'+instance_number+'.dcm')\n        shutil.copy(file_path, destination_file_path)","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:20.029865Z","iopub.status.busy":"2024-08-02T18:35:20.029471Z","iopub.status.idle":"2024-08-02T18:35:20.036163Z","shell.execute_reply":"2024-08-02T18:35:20.035209Z"},"papermill":{"duration":0.025942,"end_time":"2024-08-02T18:35:20.038432","exception":false,"start_time":"2024-08-02T18:35:20.012490","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating a New Train DataFrame and Clearing Memory:**","metadata":{"papermill":{"duration":0.015474,"end_time":"2024-08-02T18:35:20.069509","exception":false,"start_time":"2024-08-02T18:35:20.054035","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\nnew_train_df = train_train_label_marged_df\n\ndel train_train_label_marged_df\ngc.collect()\nnew_train_df.head()\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:20.103151Z","iopub.status.busy":"2024-08-02T18:35:20.102056Z","iopub.status.idle":"2024-08-02T18:35:20.108616Z","shell.execute_reply":"2024-08-02T18:35:20.107567Z"},"papermill":{"duration":0.025759,"end_time":"2024-08-02T18:35:20.110844","exception":false,"start_time":"2024-08-02T18:35:20.085085","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reorganizing Columns and Saving Data:**","metadata":{"papermill":{"duration":0.015027,"end_time":"2024-08-02T18:35:20.141370","exception":false,"start_time":"2024-08-02T18:35:20.126343","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ncolumns = new_train_df.columns.tolist()\ncolumns.remove('target')\ncolumns.append('target')\nnew_train_df = new_train_df[columns]\nnew_train_df.head()","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:20.174356Z","iopub.status.busy":"2024-08-02T18:35:20.173456Z","iopub.status.idle":"2024-08-02T18:35:20.180217Z","shell.execute_reply":"2024-08-02T18:35:20.179207Z"},"papermill":{"duration":0.02581,"end_time":"2024-08-02T18:35:20.182677","exception":false,"start_time":"2024-08-02T18:35:20.156867","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_train_df.to_csv('june/new_train.csv',index=False)","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:20.215947Z","iopub.status.busy":"2024-08-02T18:35:20.215251Z","iopub.status.idle":"2024-08-02T18:35:20.219929Z","shell.execute_reply":"2024-08-02T18:35:20.218899Z"},"papermill":{"duration":0.023804,"end_time":"2024-08-02T18:35:20.222142","exception":false,"start_time":"2024-08-02T18:35:20.198338","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncomplete_train_df=pd.read_csv('june/new_train.csv')\ncleaned_train_images = glob.glob('june/cleaned_images/*.dcm')\nprint('cleaned_train_images:\\n', len(cleaned_train_images))\n","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:20.255787Z","iopub.status.busy":"2024-08-02T18:35:20.254805Z","iopub.status.idle":"2024-08-02T18:35:20.261249Z","shell.execute_reply":"2024-08-02T18:35:20.260210Z"},"papermill":{"duration":0.025574,"end_time":"2024-08-02T18:35:20.263550","exception":false,"start_time":"2024-08-02T18:35:20.237976","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_dir = 'june'\noutput_folder_zip = 'cleaned_images_zip'\n\n# Creating path for the new folder\ncleaned_images_path = os.path.join(output_dir, output_folder_zip)\n\n# Create the folder\nos.makedirs(cleaned_images_path, exist_ok=True)\n\nprint(f\"Folder created at: {cleaned_images_path}\")","metadata":{"execution":{"iopub.execute_input":"2024-08-02T18:35:20.296899Z","iopub.status.busy":"2024-08-02T18:35:20.296521Z","iopub.status.idle":"2024-08-02T18:35:20.302856Z","shell.execute_reply":"2024-08-02T18:35:20.301825Z"},"papermill":{"duration":0.025592,"end_time":"2024-08-02T18:35:20.304952","exception":false,"start_time":"2024-08-02T18:35:20.279360","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#####################################################################################","metadata":{"execution":{"iopub.status.busy":"2024-08-04T05:58:55.933333Z","iopub.execute_input":"2024-08-04T05:58:55.933725Z","iopub.status.idle":"2024-08-04T05:58:55.966794Z","shell.execute_reply.started":"2024-08-04T05:58:55.933689Z","shell.execute_reply":"2024-08-04T05:58:55.965647Z"},"trusted":true},"execution_count":null,"outputs":[]}]}