{"cells":[{"metadata":{"_uuid":"bbcb140474872f5de0aed322229964632dddc2eb"},"cell_type":"markdown","source":"# Introduction\n\nIn this kernel, I wants to create a single csv file for all the training data. "},{"metadata":{"trusted":true,"_uuid":"be1496865e0398cd7e5c134d744c9e6653d62c4a"},"cell_type":"code","source":"import numpy as np\nimport pydicom\nimport cv2\nimport os\nfrom tqdm import tqdm\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d65a47a9a31b2fbc4f75607eab6750aad8255435"},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e191e4d99293b700e0395d6adb6a3bfaeff8c8c0"},"cell_type":"code","source":"test_path = '../input/stage_1_test_images/'\ntrain_path = '../input/stage_1_train_images/'\ntest_save_path = '../working/test_image/'\ntrain_save_path = '../working/train_image/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9646372758e175a40ade269172bdd3b257dafeab"},"cell_type":"markdown","source":"* test_path - path for the test images\n* train_path - path of the training image\n* test_save_path - path for saving the test images in png format\n* train_save_path - path for saving the training images in png format"},{"metadata":{"trusted":true,"_uuid":"7614b9c72f9615d982a1d69daa8c599e5ef3961c"},"cell_type":"code","source":"def save_image(img, file_path):\n    \"\"\"\n    :param img - numpy array (image)\n    :param file_path - path where to save file\n    \"\"\"\n    ## You can even add a cv2.resize() function to resize the image\n    cv2.imwrite(file_path, img)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d85af9f24f0f16fac78f5ed2b52e3473bb6f56bc"},"cell_type":"markdown","source":"Save the numpy array as the image"},{"metadata":{"trusted":true,"_uuid":"958eaa0aed85fcb0da95ab098d06097f56df1f5d"},"cell_type":"code","source":"def read_dcm_file(file_path):\n    \"\"\"\n    :param file_path - path of the DCM image file\n    \"\"\"\n    dcm_file = pydicom.read_file(file_path)\n    return dcm_file","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fcc850d04a5746fb92367a04d64fdd37612d2305"},"cell_type":"markdown","source":"Reading the DCM file"},{"metadata":{"trusted":true,"_uuid":"1f562b97961bf2a654af765066f239223965af49"},"cell_type":"code","source":"def main(path, name, save_path):\n    \"\"\"\n    :param path - path for the DCM image files\n    :param name - name of the csv file\n    :param save_path - path where to save the image\n    \"\"\"\n\n    list_files = os.listdir(path)\n    #['PatientAge', 'PatientBirthDate', 'PatientID', 'PatientName', 'PatientOrientation', 'PatientSex']\n\n    p_age_list = []\n    p_birthdate_list = []\n    p_id_list = []\n    p_name_list = []\n    p_orientation_list = []\n    p_sex_list = []\n\n    for idx, file_name in tqdm(enumerate(list_files)):\n        file_id = file_name.split('.')[0]\n        file_path = path + file_name\n        dcm_data = read_dcm_file(file_path)\n        #dcm_attr = dcm_data.dir('pat')\n        image = dcm_data.pixel_array\n        image_path = save_path + file_id + '.png'\n        save_image(image, image_path)\n\n        p_id = dcm_data.PatientID\n        p_name = dcm_data.PatientName\n        p_age = dcm_data.PatientAge\n        p_sex = dcm_data.PatientSex\n        p_orientation = dcm_data.PatientOrientation\n\n        p_id_list.append(p_id)\n        p_name_list.append(p_name)\n        p_age_list.append(p_age)\n        p_sex_list.append(p_sex)\n        p_orientation_list.append(p_orientation)\n\n    df = pd.DataFrame()\n    df['patientId'] = p_id_list\n    df['patientName'] = p_name_list\n    df['patientAge'] = p_age_list\n    df['patientSex'] = p_sex_list\n    df['patientOrientation'] = p_orientation_list\n\n    df.to_csv(name)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"079850d038ca08d504bbf52cfc49593a027af740"},"cell_type":"markdown","source":"# main()\nThis function save the information present in DCM file into a csv format"},{"metadata":{"trusted":true,"_uuid":"009bdb3f330d3fd250bb9bef5cf2ff4773c15750"},"cell_type":"code","source":"def parse_train_label(df, file_name):\n    \"\"\"\n    Method to read a CSV file (Pandas dataframe) and parse the\n    data into the following nested dictionary:\n\n      parsed = {\n\n        'patientId-00': {\n            'dicom': path/to/dicom/file,\n            'label': either 0 or 1 for normal or pnuemonia,\n            'boxes': list of box(es)\n        },\n        'patientId-01': {\n            'dicom': path/to/dicom/file,\n            'label': either 0 or 1 for normal or pnuemonia,\n            'boxes': list of box(es)\n        }, ...\n\n      }\n\n    \"\"\"\n    # --- Define lambda to extract coords in list [y, x, height, width]\n    extract_box = lambda row: [row['y'], row['x'], row['height'], row['width']]\n\n    parsed = {}\n    for n, row in df.iterrows():\n        # --- Initialize patient entry into parsed\n        pid = row['patientId']\n        if pid not in parsed:\n            parsed[pid] = {\n                'patientId': pid,\n                'label': row['Target'],\n                'boxes': []}\n\n        # --- Add box if opacity is present\n        if parsed[pid]['label'] == 1:\n            parsed[pid]['boxes'].append(extract_box(row))\n\n    s = pd.DataFrame.from_dict(parsed, orient='index', columns=['patientId', 'label', 'boxes'])\n    s.to_csv(file_name)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb74ff45b40c642ac36852a86c2d14d4e93000da"},"cell_type":"markdown","source":"This function is taken from [https://www.kaggle.com/peterchang77/exploratory-data-analysis](http://)<br/>\nI just add a few lines in it. This function takes stage_1_train_labels.csv process it and create a dictionary with all the patientId and realted information, then save it in a csv file\n"},{"metadata":{"trusted":true,"_uuid":"3fce9651e0fd5d113c03f1d54b7a89d9ad5518ac"},"cell_type":"code","source":"def parse_class_info(df, file_name):\n    parsed = {}\n    for n, row in df.iterrows():\n        pid = row['patientId']\n\n        if pid not in parsed:\n            parsed[pid]  = {\n                'patientId': pid,\n                'class': [row['class']]\n            }\n        else:\n            parsed[pid]['class'].append(row['class'])\n\n    s = pd.DataFrame.from_dict(parsed, orient='index', columns=['patientId', 'class'])\n    s.to_csv(file_name)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cec15c830326a484ff9d5bb30efe1c8fd9b04560"},"cell_type":"markdown","source":"This function is also same as above but takes the stage_1_detailed_class_info.csv file"},{"metadata":{"trusted":true,"_uuid":"ff2846a5cec9161c85d8615111b14edfb36f35d7","scrolled":true},"cell_type":"code","source":"## Train DCM Images\nmain(train_path, '../working/train_image.csv', train_save_path)\n## Test DCM Images\nmain(test_path, '../working/test_image.csv', test_save_path)\n\n## Combining all the data\ntrain_label = pd.read_csv('../input/stage_1_train_labels.csv')\nparse_train_label(train_label, '../working/train_labels.csv')\n\nclass_info = pd.read_csv('../input/stage_1_detailed_class_info.csv')\nparse_class_info(class_info, '../working/class_info.csv')\n\ntrain_label = pd.read_csv('../working/train_labels.csv', index_col='patientId')\nclass_info = pd.read_csv('../working/class_info.csv', index_col='patientId')\ntrain_image = pd.read_csv('../working/train_image.csv', index_col='patientId')\n\ntrain_label['class'] = class_info['class']\ntrain_label['age'] = train_image['patientAge']\ntrain_label['sex'] = train_image['patientSex']\ntrain_label.to_csv('final_train_data.csv')\n\nimport base64\nimport pandas as pd\nfrom IPython.display import HTML\n\ndef create_download_link( df, title = \"Download CSV file\", filename = \"download_final_train_data.csv\"):\n    csv = df.to_csv()\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(train_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18f8195be277d3a9c88c8ea66c96986d73910775"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}