{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# make dataframe\n\n#### I got the image information by [[siim-step1]get_imginfo](https://www.kaggle.com/kunihikofurugori/siim-step1-get-imginfo).\n\n#### Next step, I will make the dataframe based on image information.\n\n## my public notebook \n\n### [step1 get_imageinformation](https://www.kaggle.com/kunihikofurugori/siim-step1-get-imginfo).\n\n### [step2 make_dataframe](https://www.kaggle.com/kunihikofurugori/step2-make-dataframe/edit/run/69201903).\n\n### [step3-1 renew-imglev_ds](https://www.kaggle.com/kunihikofurugori/siim-step3-1-renew-imglev-ds)\n\n### [step3-2 renew-studylev_ds](https://www.kaggle.com/kunihikofurugori/siim-step3-1-renew-studylev-ds)\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n#!pip install python-gdcm\n#!pip install pylibjpeg-libjpeg\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\n#import pydicom\n#import gdcm\n#from pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nimport glob\nimport os\nfrom PIL import Image\nimport ast","metadata":{"execution":{"iopub.status.busy":"2021-07-30T12:32:20.861144Z","iopub.execute_input":"2021-07-30T12:32:20.861492Z","iopub.status.idle":"2021-07-30T12:32:21.014802Z","shell.execute_reply.started":"2021-07-30T12:32:20.861418Z","shell.execute_reply":"2021-07-30T12:32:21.014101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = [\"394536fc0dca_image\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-30T12:32:21.049656Z","iopub.execute_input":"2021-07-30T12:32:21.049921Z","iopub.status.idle":"2021-07-30T12:32:21.053781Z","shell.execute_reply.started":"2021-07-30T12:32:21.049897Z","shell.execute_reply":"2021-07-30T12:32:21.052905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make df_image","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/cov19detection-dataset/imginfo.csv\")\ndf = df[~df.id_x.isin(ids)]\ndf[\"augflag\"] = 0\nindex = df[(df.crop_fxmax > 0.7) & (df.crop_fymax > 0.7) & (df.crop_fxmin < 0.3) &(df.crop_fymin < 0.3)].index\ndf.loc[index,\"augflag\"] = 1\ndf = df.dropna(how='any')\ndf['boxes'] = df.boxes.apply(lambda x: ast.literal_eval(x))\ndf['boxnum'] = df.boxes.apply(lambda x: len(x))\ndf= df[df.boxnum.isin([1,2,3])]\ndf['box1'] = df.boxes.apply(lambda x: x[0])\ndf['box2'] = df.boxes.apply(lambda x: x[1] if len(x)>1 else 0)\ndf['box3'] = df.boxes.apply(lambda x: x[2] if len(x)>2 else 0)\ndf['xmin1'] = df.box1.apply(lambda x: x['x'])\ndf['ymin1'] = df.box1.apply(lambda x: x['y'])\ndf['xmax1'] = df.box1.apply(lambda x: x['x']+x['width'])\ndf['ymax1'] = df.box1.apply(lambda x: x['y']+x['height'])\ndf['area1'] = df.box1.apply(lambda x: (x['width']*x['height']))\n\ndf['xmin2'] = df.box2.apply(lambda x: x['x'] if x !=0 else 0)\ndf['ymin2'] = df.box2.apply(lambda x: x['y'] if x !=0 else 0)\ndf['xmax2'] = df.box2.apply(lambda x: x['x']+x['width'] if x !=0 else 0)\ndf['ymax2'] = df.box2.apply(lambda x: x['y']+x['height'] if x !=0 else 0)\ndf['area2'] = df.box2.apply(lambda x: x['width']*x['height'] if x !=0 else 0)\n\ndf['xmin3'] = df.box3.apply(lambda x: x['x'] if x !=0 else 0)\ndf['ymin3'] = df.box3.apply(lambda x: x['y'] if x !=0 else 0)\ndf['xmax3'] = df.box3.apply(lambda x: x['x']+x['width'] if x !=0 else 0)\ndf['ymax3'] = df.box3.apply(lambda x: x['y']+x['height'] if x !=0 else 0)\ndf['area3'] = df.box3.apply(lambda x: x['width']*x['height'] if x !=0 else 0)\n\ndf = df[df.dim2 > df.xmax1]\ndf = df[df.dim1 > df.ymax1]\ndf = df[df.dim2 > df.xmax2]\ndf = df[df.dim1 > df.ymax2]\ndf = df[df.dim2 > df.xmax3]\ndf = df[df.dim1 > df.ymax3]\ndf = df[df.xmin1>=0]\ndf = df[df.xmin2>=0]\ndf = df[df.xmin3>=0]\ndf = df[df.ymin1>=0]\ndf = df[df.ymin2>=0]\ndf = df[df.ymin3>=0]\n\n\ndf[['fracxmin1','fracxmax1','fracxmin2','fracxmax2','fracxmin3','fracxmax3']] = df.apply(lambda x: x[['xmin1','xmax1','xmin2','xmax2','xmin3','xmax3']]/x.dim2, axis=1)\ndf[['fracymin1','fracymax1','fracymin2','fracymax2','fracymin3','fracymax3']] = df.apply(lambda x: x[['ymin1','ymax1','ymin2','ymax2','ymin3','ymax3']]/x.dim1, axis=1)\n\ndf[\"fracarea1\"] = df.apply(lambda x: (x.fracxmax1-x.fracxmin1)*(x.fracymax1-x.fracymin1),axis=1)\ndf[\"fracarea2\"] = df.apply(lambda x: (x.fracxmax2-x.fracxmin2)*(x.fracymax2-x.fracymin2),axis=1)\ndf[\"fracarea3\"] = df.apply(lambda x: (x.fracxmax3-x.fracxmin3)*(x.fracymax3-x.fracymin3),axis=1)\n\nprint(len(df))\ndf = df[(df.fracarea1>5e-3) | (df.fracarea1==0)]\ndf = df[(df.fracarea2>5e-3) | (df.fracarea2==0)]\ndf = df[(df.fracarea3>5e-3) | (df.fracarea3==0)]\nprint(len(df))\n\ndel df[\"Unnamed: 0\"]\ndel df[\"label\"]\ndel df[\"boxes\"]\ndel df[\"StudyInstanceUID\"]\ndf = df.rename(columns={'id_x': 'id'})\ndf.columns","metadata":{"execution":{"iopub.status.busy":"2021-07-30T12:32:45.898883Z","iopub.execute_input":"2021-07-30T12:32:45.899319Z","iopub.status.idle":"2021-07-30T12:32:53.081939Z","shell.execute_reply.started":"2021-07-30T12:32:45.899276Z","shell.execute_reply":"2021-07-30T12:32:53.081071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.fracxmin1.mean(),df.fracymin1.mean(),df.fracxmax1.mean(),df.fracymax1.mean())\nprint(df.fracxmin2.mean(),df.fracymin2.mean(),df.fracxmax2.mean(),df.fracymax2.mean())\nprint(df.fracxmin3.mean(),df.fracymin3.mean(),df.fracxmax3.mean(),df.fracymax3.mean())","metadata":{"execution":{"iopub.status.busy":"2021-07-30T12:34:02.959664Z","iopub.execute_input":"2021-07-30T12:34:02.959995Z","iopub.status.idle":"2021-07-30T12:34:02.971650Z","shell.execute_reply.started":"2021-07-30T12:34:02.959969Z","shell.execute_reply":"2021-07-30T12:34:02.970795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# safety cropping","metadata":{}},{"cell_type":"code","source":"\"\"\"\ncnd1 = df.fracxmax1 > df.crop_fxmax\ndf.loc[cnd1,\"crop_fxmax\"] = df.fracxmax1*1.01\n\ncnd2 = df.fracxmax2 > df.crop_fxmax\ndf.loc[cnd2,\"crop_fxmax\"] = df.fracxmax2*1.01\n\ncnd3 = df.fracxmax3 > df.crop_fxmax\ndf.loc[cnd3,\"crop_fxmax\"] = df.fracxmax3*1.01\n\ncnd4 = df.fracxmin1 < df.crop_fxmin\ndf.loc[cnd4,\"crop_fxmin\"] = df.fracxmin1*0.99\n\ncnd5 = (df.fracxmin2 < df.crop_fxmin) | (df.fracxmin2 == 0)\ndf.loc[cnd5,\"crop_fxmin\"] = df.fracxmin2*0.99\n\ncnd6 = (df.fracxmin3 < df.crop_fxmin) | (df.fracxmin3 == 0)\ndf.loc[cnd6,\"crop_fxmin\"] = df.fracxmin3*0.99\n\ncnd1 = df.fracymax1 > df.crop_fymax\ndf.loc[cnd1,\"crop_fymax\"] = df.fracymax1*1.01\n\ncnd2 = df.fracymax2 > df.crop_fymax\ndf.loc[cnd2,\"crop_fymax\"] = df.fracymax2*1.01\n\ncnd3 = df.fracymax3 > df.crop_fymax\ndf.loc[cnd3,\"crop_fymax\"] = df.fracymax3*1.01\n\ncnd4 = df.fracymin1 < df.crop_fymin\ndf.loc[cnd4,\"crop_fymin\"] = df.fracymin1*0.99\n\ncnd5 = (df.fracymin2 < df.crop_fymin) | (df.fracymin2 == 0)\ndf.loc[cnd5,\"crop_fymin\"] = df.fracymin2*0.99\n\ncnd6 = (df.fracymin3 < df.crop_fymin) | (df.fracymin3 == 0)\ndf.loc[cnd6,\"crop_fymin\"] = df.fracymin3*0.99\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"area = []\narea1 = df.fracarea1.values\narea2 = df.fracarea2.values\narea3 = df.fracarea3.values\narea1 = area1[area1>0]\narea2 = area2[area2>0]\narea3 = area3[area3>0]\narea= np.append(area,area1)\narea= np.append(area,area2)\narea= np.append(area,area3)\nplt.hist(area,bins=np.logspace(np.log10(area.min()), np.log10(area.max()), 100),range=[area.min(),area.max()])\nplt.xscale('log')\nplt.title(f\"areamax:{area.max():.3g},areamin:{area.min():.3g}\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:35:49.112348Z","iopub.execute_input":"2021-07-26T13:35:49.112708Z","iopub.status.idle":"2021-07-26T13:35:49.604574Z","shell.execute_reply.started":"2021-07-26T13:35:49.112684Z","shell.execute_reply":"2021-07-26T13:35:49.604108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_image = df[df.columns[[0,5,6,7,8,9,10,11,30,31,32,33,34,35,36,37,38,39,40,41]]]\ndf_image= df[['id', 'path', 'fold', 'mean', 'var', 'dim1', 'dim2', 'boxnum',\n       'fracxmin1', 'fracxmax1', 'fracxmin2', 'fracxmax2', 'fracxmin3',\n       'fracxmax3', 'fracymin1', 'fracymax1', 'fracymin2', 'fracymax2',\n       'fracymin3', 'fracymax3', 'crop_xmin', 'crop_ymin', 'crop_xmax',\n       'crop_ymax', 'crop_fxmin','crop_fymin', 'crop_fxmax','crop_fymax','augflag']]\nmeanmax = df_image['mean'].values.max()\nvarmax = df_image['var'].values.max()\ndf_image['norm_mean'] = df_image['mean'].apply(lambda x: x/meanmax)\ndf_image['norm_var'] = df_image['var'].apply(lambda x: x/varmax)\ndf_image = df_image.reset_index(drop=True)\ndisplay(df_image)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:39:58.655496Z","iopub.execute_input":"2021-07-26T13:39:58.65576Z","iopub.status.idle":"2021-07-26T13:39:58.699283Z","shell.execute_reply.started":"2021-07-26T13:39:58.655737Z","shell.execute_reply":"2021-07-26T13:39:58.697961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, StratifiedKFold\n\nRANDOM_STATE = 35\nfold = 1\n\nkfold = KFold(n_splits=5, random_state=RANDOM_STATE, shuffle=True)\ndf_image['test_fold'] = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:19:07.770576Z","iopub.execute_input":"2021-07-26T13:19:07.7708Z","iopub.status.idle":"2021-07-26T13:19:08.624678Z","shell.execute_reply.started":"2021-07-26T13:19:07.770776Z","shell.execute_reply":"2021-07-26T13:19:08.623831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splits= kfold.split(df_image)\ntrain_indexs = []\ntest_indexs = []\nfor i,(train_index, test_index) in enumerate(splits):\n    print(train_index.shape,test_index.shape)\n    train_indexs.append(train_index)\n    test_indexs.append(test_index)\n    df_image.loc[test_index,'test_fold'] = i","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:19:08.625643Z","iopub.execute_input":"2021-07-26T13:19:08.625841Z","iopub.status.idle":"2021-07-26T13:19:08.637186Z","shell.execute_reply.started":"2021-07-26T13:19:08.62582Z","shell.execute_reply":"2021-07-26T13:19:08.635785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image.to_csv(\"train_image.csv\")\ndf_image.columns","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:19:08.638881Z","iopub.execute_input":"2021-07-26T13:19:08.639111Z","iopub.status.idle":"2021-07-26T13:19:08.726704Z","shell.execute_reply.started":"2021-07-26T13:19:08.63909Z","shell.execute_reply":"2021-07-26T13:19:08.725882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make df_study","metadata":{}},{"cell_type":"code","source":"ids = [\"0b858129adb4_image\",\n       \"3d12cb6aad8b_image\",\n       \"41e9a794b342_image\",\n       \"681ed0b5dff2_image\",\n       \"0eb641cb0dcd_image\",\n       \"0f9709784c19_image\",\n       \"3b982073ec16_image\",\n       \"4f3d52d652dd_image\",\n       \"9c24e37a0ef5_image\",\n       \"1b92142f4362_image\",\n       \"6f749e2783e1_image\",\n       \"c3510a436bff_image\",\n       \"394536fc0dca_image\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:19:08.727832Z","iopub.execute_input":"2021-07-26T13:19:08.728092Z","iopub.status.idle":"2021-07-26T13:19:08.732086Z","shell.execute_reply.started":"2021-07-26T13:19:08.72807Z","shell.execute_reply":"2021-07-26T13:19:08.731207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/cov19detection-dataset/imginfo.csv\")\ndf = df.rename(columns={'id_x': 'id'})\ndf[\"augflag\"] = 0\nindex = df[(df.crop_fxmax > 0.7) & (df.crop_fymax > 0.7) & (df.crop_fxmin < 0.3) &(df.crop_fymin < 0.3)].index\ndf.loc[index,\"augflag\"] = 1\n\ndf_study = df#[df.columns[[1,4,5,6,7,8,9,11,12,13,14,-1]]]\nmeanmax = df_study['mean'].values.max()\nvarmax = df_study['var'].values.max()\ndf_study['norm_mean'] = df_study['mean'].apply(lambda x: x/meanmax)\ndf_study['norm_var'] = df_study['var'].apply(lambda x: x/varmax)\nprint(len(df_study),len(ids))\ndf_study = df_study[~df_study.id.isin(ids)]\nprint(len(df_study))\ndf_study = df_study[~df_study.duplicated(keep='last',subset=['mean', 'var'])]\nprint(len(df))\ndf_study.to_csv(\"train_study.csv\")\ndf_study","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:42:32.115552Z","iopub.execute_input":"2021-07-26T13:42:32.115841Z","iopub.status.idle":"2021-07-26T13:42:32.309793Z","shell.execute_reply.started":"2021-07-26T13:42:32.11581Z","shell.execute_reply":"2021-07-26T13:42:32.308965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make detection or non-detection image","metadata":{}},{"cell_type":"code","source":"ids = [\"0b858129adb4_image\",\n       \"3d12cb6aad8b_image\",\n       \"41e9a794b342_image\",\n       \"681ed0b5dff2_image\",\n       \"0eb641cb0dcd_image\",\n       \"0f9709784c19_image\",\n       \"3b982073ec16_image\",\n       \"4f3d52d652dd_image\",\n       \"394536fc0dca_image\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:19:08.879457Z","iopub.execute_input":"2021-07-26T13:19:08.879734Z","iopub.status.idle":"2021-07-26T13:19:08.884218Z","shell.execute_reply.started":"2021-07-26T13:19:08.879701Z","shell.execute_reply":"2021-07-26T13:19:08.882993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/cov19detection-dataset/imginfo.csv\")\ndf[\"augflag\"] = 0\nindex = df[(df.crop_fxmax > 0.7) & (df.crop_fymax > 0.7) & (df.crop_fxmin < 0.3) &(df.crop_fymin < 0.3)].index\ndf.loc[index,\"augflag\"] = 1\ndf[\"none\"] = 0\nindex = df[df.label=='none 1 0 0 1 1'].index\ndf.loc[index,\"none\"] = 1\nindex = df[(df.none==1) & (df[\"Negative for Pneumonia\"]==0)].index\ndf = df.drop(index).reset_index(drop=True)\ndf = df.rename(columns={'id_x': 'id'})\ndf = df[~df.id.isin(ids)]\ndf = df[~df.duplicated(keep='last',subset=['mean', 'var'])].reset_index(drop=True)\ndf.to_csv(\"train_none.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-26T13:42:42.045879Z","iopub.execute_input":"2021-07-26T13:42:42.046238Z","iopub.status.idle":"2021-07-26T13:42:42.310235Z","shell.execute_reply.started":"2021-07-26T13:42:42.046206Z","shell.execute_reply":"2021-07-26T13:42:42.309449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}