{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold\nfrom glob import glob","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:29.999158Z","iopub.execute_input":"2022-08-16T07:56:29.999875Z","iopub.status.idle":"2022-08-16T07:56:30.006978Z","shell.execute_reply.started":"2022-08-16T07:56:29.999840Z","shell.execute_reply":"2022-08-16T07:56:30.005979Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nstudy_df = study_df.rename(columns={'id': 'study_id'})\nstudy_df","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:31.662604Z","iopub.execute_input":"2022-08-16T07:56:31.663578Z","iopub.status.idle":"2022-08-16T07:56:31.688772Z","shell.execute_reply.started":"2022-08-16T07:56:31.663539Z","shell.execute_reply":"2022-08-16T07:56:31.687610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\n\n\nimage_df = image_df.rename(columns={'id': 'image_id'})\nimage_df = image_df.rename(columns={'StudyInstanceUID': 'study_id'})\nimage_df=image_df.drop(labels=[\"boxes\",\"label\"], axis=1) \n\nimage_df\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:33.580392Z","iopub.execute_input":"2022-08-16T07:56:33.581013Z","iopub.status.idle":"2022-08-16T07:56:33.620510Z","shell.execute_reply.started":"2022-08-16T07:56:33.580976Z","shell.execute_reply":"2022-08-16T07:56:33.619343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df[\"study_id\"] = image_df[\"study_id\"] + \"_study\"\n\nimage_df\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:35.582882Z","iopub.execute_input":"2022-08-16T07:56:35.583260Z","iopub.status.idle":"2022-08-16T07:56:35.596754Z","shell.execute_reply.started":"2022-08-16T07:56:35.583226Z","shell.execute_reply":"2022-08-16T07:56:35.595697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(study_df,image_df,how='inner',on=['study_id'])\ndf.reset_index(inplace=True, drop=True)\n\n#df = pd.merge(study_df,image_df, on=\"study_id\") #關聯合併\n#df = study_df.join(image_df.set_index('study_id'), on='study_id')\n\n\ndf  #原本全部的檔案\ndf.to_csv(\"original_file.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:46.252819Z","iopub.execute_input":"2022-08-16T07:56:46.253567Z","iopub.status.idle":"2022-08-16T07:56:46.282845Z","shell.execute_reply.started":"2022-08-16T07:56:46.253526Z","shell.execute_reply":"2022-08-16T07:56:46.281905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask1 = (df[\"Negative for Pneumonia\"]!=1) #陰性\nmask2 = (df[\"Typical Appearance\"]!=1)  #典型\ndf1=df[(mask1) & (mask2)] \n#df.drop(\"Indeterminate Appearance\",axis=1)\ndf1=df1.drop(labels=[\"Negative for Pneumonia\",\"Typical Appearance\"], axis=1) \ndf1.reset_index(inplace=True, drop=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:48.312888Z","iopub.execute_input":"2022-08-16T07:56:48.313900Z","iopub.status.idle":"2022-08-16T07:56:48.324894Z","shell.execute_reply.started":"2022-08-16T07:56:48.313852Z","shell.execute_reply":"2022-08-16T07:56:48.323680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:50.302388Z","iopub.execute_input":"2022-08-16T07:56:50.302796Z","iopub.status.idle":"2022-08-16T07:56:50.319131Z","shell.execute_reply.started":"2022-08-16T07:56:50.302761Z","shell.execute_reply":"2022-08-16T07:56:50.318024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nresult=df1.image_id.str.split('_').str.get(0)\n\n#result + \"_HSV\"\n\n#df1.shape[0]\n\ni=0\n\nwhile i<=1590:\n    \n    #df1=df1.append({'study_id' : df1[i][0] , 'Indeterminate Appearance' : df1[i][1], 'Atypical Appearance' : df1[i][2] , 'image_id' :  df1[i][3]} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_HSV\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_HorizontalFlip\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RGBShift\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RandomBrightness\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RandomContrast\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RandomCrop\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RandomFog\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RandomSnow\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_RandomSunFlare\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_Rotate\"} , ignore_index=True)\n    df1=df1.append({'study_id' : df1[\"study_id\"][i] , 'Indeterminate Appearance' : df1[\"Indeterminate Appearance\"][i], 'Atypical Appearance' : df1[\"Atypical Appearance\"][i] , 'image_id' : result[i] + \"_VerticalFlip\"} , ignore_index=True)\n    \n    #print(i)\n    \n    i=i+1","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:56:53.205965Z","iopub.execute_input":"2022-08-16T07:56:53.206689Z","iopub.status.idle":"2022-08-16T07:57:38.576005Z","shell.execute_reply.started":"2022-08-16T07:56:53.206650Z","shell.execute_reply":"2022-08-16T07:57:38.574980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = df1.drop(df1.index[0:1591])\ndf2.reset_index(inplace=True, drop=True)\ndf2 #過濾後\ndf2.to_csv(\"after_filter.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndf3 = pd.merge(df,df2, on='study_id')\ndf3.tail(100)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nmask1 = (df1[\"Negative for Pneumonia\"]!=1) #陰性\nmask2 = (df1[\"Typical Appearance\"]!=1)  #典型\ndf=df1[(mask1) & (mask2)] \n#df.drop(\"Indeterminate Appearance\",axis=1)\ndf=df.drop(labels=[\"Negative for Pneumonia\",\"Typical Appearance\"], axis=1) \ndf.reset_index(inplace=True, drop=True)\n\ndf\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}