{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Gender Bias\n\n- Is there a gender difference in Covid19 infection? Early news reports indicated that women are less likely to be severely infected than men. Does this show up in the competition data set?","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:37:05.831449Z","iopub.execute_input":"2021-07-03T00:37:05.831869Z","iopub.status.idle":"2021-07-03T00:37:06.109776Z","shell.execute_reply.started":"2021-07-03T00:37:05.831785Z","shell.execute_reply":"2021-07-03T00:37:06.108676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DICOM Meta tag\n\n- The patient's name and other data are hashed, but we can see that the gender metadata is available (I wish age was also available!).","metadata":{}},{"cell_type":"code","source":"path = \"../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm\"\ndicom = pydicom.read_file(path)\nprint('\\n'.join(str(dicom).split('\\n')[14:17]))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:37:06.111776Z","iopub.execute_input":"2021-07-03T00:37:06.112379Z","iopub.status.idle":"2021-07-03T00:37:06.660950Z","shell.execute_reply.started":"2021-07-03T00:37:06.112335Z","shell.execute_reply":"2021-07-03T00:37:06.659898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:37:06.662414Z","iopub.execute_input":"2021-07-03T00:37:06.662778Z","iopub.status.idle":"2021-07-03T00:37:06.687067Z","shell.execute_reply.started":"2021-07-03T00:37:06.662733Z","shell.execute_reply":"2021-07-03T00:37:06.685894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map labels and gender","metadata":{}},{"cell_type":"code","source":"ids, genders, labels = [], [], []\ntrainfiles = [(dirname, filenames) for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/train/')) if len(filenames) > 0]\nfor dirname, filenames in tqdm(trainfiles):\n    for file in filenames:\n        sid = dirname.split(\"/\")[-2]+'_study'\n        id = file.replace('.dcm','')+'_image'\n        ids.append(id)\n        label = np.argmax(df_study[df_study.id==sid][df_study.columns[1:5]].values[0])\n        labels.append(df_study.columns[1:5][label])\n        path = os.path.join(dirname, file)\n        dicom = pydicom.read_file(path)\n        genders.append(str(dicom.get_item('00100040').value.decode('utf8'))[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:37:06.688569Z","iopub.execute_input":"2021-07-03T00:37:06.688978Z","iopub.status.idle":"2021-07-03T00:56:17.236847Z","shell.execute_reply.started":"2021-07-03T00:37:06.688943Z","shell.execute_reply":"2021-07-03T00:56:17.235853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Gender is F or M. There is no DICOM that does not contain gender metadata.","metadata":{}},{"cell_type":"code","source":"set(genders)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:56:17.240254Z","iopub.execute_input":"2021-07-03T00:56:17.240580Z","iopub.status.idle":"2021-07-03T00:56:17.248005Z","shell.execute_reply.started":"2021-07-03T00:56:17.240541Z","shell.execute_reply":"2021-07-03T00:56:17.247071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\"id\":ids,\"gender\":genders,\"label\":labels}).to_csv(\"genders.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:56:17.249501Z","iopub.execute_input":"2021-07-03T00:56:17.249807Z","iopub.status.idle":"2021-07-03T00:56:17.298813Z","shell.execute_reply.started":"2021-07-03T00:56:17.249754Z","shell.execute_reply":"2021-07-03T00:56:17.297739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number in the dataset \n\n- Men and women are evenly represented in the data set.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndf = pd.read_csv('genders.csv')\nplt.hist(df.gender)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:58:13.576932Z","iopub.execute_input":"2021-07-03T00:58:13.577300Z","iopub.status.idle":"2021-07-03T00:58:13.775128Z","shell.execute_reply.started":"2021-07-03T00:58:13.577269Z","shell.execute_reply":"2021-07-03T00:58:13.774054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.hist(column=\"label\", by=\"gender\")","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:58:17.576789Z","iopub.execute_input":"2021-07-03T00:58:17.577378Z","iopub.status.idle":"2021-07-03T00:58:17.996264Z","shell.execute_reply.started":"2021-07-03T00:58:17.577327Z","shell.execute_reply":"2021-07-03T00:58:17.995329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gender Bias in Study\n\n- Obviously, there are more 'negatives' among women. WoW! As in the news, Women appear to be resistant to COVID-19.","metadata":{}},{"cell_type":"code","source":"ids, genders = [], []\ntestfiles = [(dirname, filenames) for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/test/')) if len(filenames) > 0]\nfor dirname, filenames in tqdm(testfiles):\n    for file in filenames:\n        sid = dirname.split(\"/\")[-2]+'_study'\n        id = file.replace('.dcm','')+'_image'\n        ids.append(id)\n        path = os.path.join(dirname, file)\n        dicom = pydicom.read_file(path)\n        genders.append(str(dicom.get_item('00100040').value.decode('utf8'))[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-03T00:58:50.768527Z","iopub.execute_input":"2021-07-03T00:58:50.768904Z","iopub.status.idle":"2021-07-03T01:02:13.798100Z","shell.execute_reply.started":"2021-07-03T00:58:50.768871Z","shell.execute_reply":"2021-07-03T01:02:13.797072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### In the Test set\n\n- The same tag for the test data set.","metadata":{}},{"cell_type":"code","source":"set(genders)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T01:05:58.711053Z","iopub.execute_input":"2021-07-03T01:05:58.711449Z","iopub.status.idle":"2021-07-03T01:05:58.718008Z","shell.execute_reply.started":"2021-07-03T01:05:58.711416Z","shell.execute_reply":"2021-07-03T01:05:58.716747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\"id\":ids,\"gender\":genders}).to_csv(\"genders_test.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T01:06:01.876577Z","iopub.execute_input":"2021-07-03T01:06:01.876954Z","iopub.status.idle":"2021-07-03T01:06:01.888785Z","shell.execute_reply.started":"2021-07-03T01:06:01.876922Z","shell.execute_reply":"2021-07-03T01:06:01.887899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('genders_test.csv')\nplt.hist(df.gender)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T01:06:05.777202Z","iopub.execute_input":"2021-07-03T01:06:05.777569Z","iopub.status.idle":"2021-07-03T01:06:05.911614Z","shell.execute_reply.started":"2021-07-03T01:06:05.777539Z","shell.execute_reply":"2021-07-03T01:06:05.910425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We might be able to use this for something...","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}