{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70173,"databundleVersionId":7683204,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 予備知識","metadata":{}},{"cell_type":"code","source":"#BASIC\nimport numpy as np \nimport pandas as pd \nimport os\n\n# DATA visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\n\nimport PIL\nfrom IPython.display import Image, display\nfrom plotly import graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\n\nimport openslide","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-28T04:54:38.118314Z","iopub.execute_input":"2024-03-28T04:54:38.118749Z","iopub.status.idle":"2024-03-28T04:54:38.341161Z","shell.execute_reply.started":"2024-03-28T04:54:38.118698Z","shell.execute_reply":"2024-03-28T04:54:38.340195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_FOLDER = \"/kaggle/input/aipathology202403/\"\n!ls {BASE_FOLDER}","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:51:03.973167Z","iopub.execute_input":"2024-03-28T04:51:03.974345Z","iopub.status.idle":"2024-03-28T04:51:05.075537Z","shell.execute_reply.started":"2024-03-28T04:51:03.974285Z","shell.execute_reply":"2024-03-28T04:51:05.073959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(BASE_FOLDER+\"train.csv\")\ntest = pd.read_csv(BASE_FOLDER+\"unlabel.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:51:05.077905Z","iopub.execute_input":"2024-03-28T04:51:05.078270Z","iopub.status.idle":"2024-03-28T04:51:05.092172Z","shell.execute_reply.started":"2024-03-28T04:51:05.078236Z","shell.execute_reply":"2024-03-28T04:51:05.090986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.863711Z","iopub.execute_input":"2024-03-28T04:50:52.864194Z","iopub.status.idle":"2024-03-28T04:50:52.885018Z","shell.execute_reply.started":"2024-03-28T04:50:52.864151Z","shell.execute_reply":"2024-03-28T04:50:52.882977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**isup_gradeとgleason_scoreが何種類あるか確認する**","metadata":{}},{"cell_type":"code","source":"print(\"unique ids : \", len(train.ID.unique()))\nprint(\"unique isup_grade(target) : \", len(train.isup_grade.unique()))\nprint(\"unique gleason_score : \", len(train.gleason_score.unique()))","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.886126Z","iopub.execute_input":"2024-03-28T04:50:52.887065Z","iopub.status.idle":"2024-03-28T04:50:52.899645Z","shell.execute_reply.started":"2024-03-28T04:50:52.887022Z","shell.execute_reply":"2024-03-28T04:50:52.898794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**それぞれ何の値があるかを見ておく**","metadata":{}},{"cell_type":"code","source":"train.isup_grade.unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.900964Z","iopub.execute_input":"2024-03-28T04:50:52.902144Z","iopub.status.idle":"2024-03-28T04:50:52.910573Z","shell.execute_reply.started":"2024-03-28T04:50:52.902114Z","shell.execute_reply":"2024-03-28T04:50:52.909759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.gleason_score.unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.912028Z","iopub.execute_input":"2024-03-28T04:50:52.912376Z","iopub.status.idle":"2024-03-28T04:50:52.919268Z","shell.execute_reply.started":"2024-03-28T04:50:52.912340Z","shell.execute_reply":"2024-03-28T04:50:52.918464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**gleason_scoreとisup_gradeの対応関係が一対一対応ではなく、一対多であることを確認しておく**\n\n<img src=\"https://storage.googleapis.com/kaggle-media/competitions/PANDA/Screen%20Shot%202020-04-08%20at%202.03.53%20PM.png\" height=\"100px\">","metadata":{}},{"cell_type":"code","source":"print(train[train.gleason_score=='4+4'].isup_grade.unique())\nprint(train[train.isup_grade==4].gleason_score.unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.920413Z","iopub.execute_input":"2024-03-28T04:50:52.921060Z","iopub.status.idle":"2024-03-28T04:50:52.940837Z","shell.execute_reply.started":"2024-03-28T04:50:52.921028Z","shell.execute_reply":"2024-03-28T04:50:52.939507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**例外もあるのでちゃんと確認しておく**","metadata":{}},{"cell_type":"code","source":"for grade in range(1, 6):\n    print(f\"isup_grade {grade}\", train[train.isup_grade==grade].gleason_score.unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.942714Z","iopub.execute_input":"2024-03-28T04:50:52.943192Z","iopub.status.idle":"2024-03-28T04:50:52.955667Z","shell.execute_reply.started":"2024-03-28T04:50:52.943149Z","shell.execute_reply":"2024-03-28T04:50:52.954236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[((train.isup_grade==2)&(train.gleason_score=='4+3'))]","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.960012Z","iopub.execute_input":"2024-03-28T04:50:52.960543Z","iopub.status.idle":"2024-03-28T04:50:52.973975Z","shell.execute_reply.started":"2024-03-28T04:50:52.960502Z","shell.execute_reply":"2024-03-28T04:50:52.972783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**isup_gradeが2なのにも関わらず、gleason_scoreが4+3であるデータが一つ存在する。これは一旦削除してこの後のEDAに続く**","metadata":{}},{"cell_type":"code","source":"train.drop([1613],inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.975348Z","iopub.execute_input":"2024-03-28T04:50:52.975797Z","iopub.status.idle":"2024-03-28T04:50:52.985478Z","shell.execute_reply.started":"2024-03-28T04:50:52.975758Z","shell.execute_reply":"2024-03-28T04:50:52.982986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"※閑話休題","metadata":{}},{"cell_type":"markdown","source":"下記二つの記法があるが筆者は下の方が短くなるので好みである。好みの問題なので自分で使うときは好きな方を使うと良い。\n\nただ、下の記法は日本語がダメであったり、pandas特有の関数と同じ名前の列名には適用できないので、上の記法の方が確かである。","metadata":{}},{"cell_type":"code","source":"print(train[train['gleason_score']=='4+4'].isup_grade.unique())\nprint(train[train.gleason_score=='4+4'].isup_grade.unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.987606Z","iopub.execute_input":"2024-03-28T04:50:52.988231Z","iopub.status.idle":"2024-03-28T04:50:52.996970Z","shell.execute_reply.started":"2024-03-28T04:50:52.988196Z","shell.execute_reply":"2024-03-28T04:50:52.996217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"temp = train.groupby('isup_grade').count()['ID'].reset_index().sort_values(by='ID',ascending=False)\ntemp.style.background_gradient(cmap='Purples')","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:52.998108Z","iopub.execute_input":"2024-03-28T04:50:52.998585Z","iopub.status.idle":"2024-03-28T04:50:53.116727Z","shell.execute_reply.started":"2024-03-28T04:50:52.998557Z","shell.execute_reply":"2024-03-28T04:50:53.115775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure(go.Funnelarea(\n    text =temp.isup_grade,\n    values = temp.ID,\n    title = {\"position\": \"top center\", \"text\": \"Funnel-Chart of ISUP_grade Distribution\"}\n    ))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:53.117905Z","iopub.execute_input":"2024-03-28T04:50:53.118428Z","iopub.status.idle":"2024-03-28T04:50:53.379871Z","shell.execute_reply.started":"2024-03-28T04:50:53.118397Z","shell.execute_reply":"2024-03-28T04:50:53.378804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(temp, x='isup_grade', y='ID',\n             hover_data=['ID', 'isup_grade'], color='ID',\n             labels={'pop':'population of Canada'}, height=400)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:53.381406Z","iopub.execute_input":"2024-03-28T04:50:53.381930Z","iopub.status.idle":"2024-03-28T04:50:54.817058Z","shell.execute_reply.started":"2024-03-28T04:50:53.381891Z","shell.execute_reply":"2024-03-28T04:50:54.815951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train.groupby('gleason_score').count()['ID'].reset_index().sort_values(by='ID',ascending=False)\ntemp.style.background_gradient(cmap='Reds')","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:54.818543Z","iopub.execute_input":"2024-03-28T04:50:54.818974Z","iopub.status.idle":"2024-03-28T04:50:54.839261Z","shell.execute_reply.started":"2024-03-28T04:50:54.818936Z","shell.execute_reply":"2024-03-28T04:50:54.838132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(temp, x='gleason_score', y='ID',\n             hover_data=['ID', 'gleason_score'], color='ID',\n             labels={'pop':'population of Canada'}, height=400)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:50:54.840774Z","iopub.execute_input":"2024-03-28T04:50:54.841866Z","iopub.status.idle":"2024-03-28T04:50:54.918712Z","shell.execute_reply.started":"2024-03-28T04:50:54.841822Z","shell.execute_reply":"2024-03-28T04:50:54.917639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image EDA","metadata":{}},{"cell_type":"markdown","source":"画像ファイルの名前はIDが与えられて\n\n{ID}_{0~35}.png\n\nという名前でついています","metadata":{}},{"cell_type":"code","source":"def display_image(ID):\n    train_ = train[train.ID==ID]\n    isup_grade = train_[\"isup_grade\"].values[0]\n    gleason_score = train_[\"gleason_score\"].values[0]\n    target = train_[\"target\"].values[0]\n    \n    print()\n    print(\"\\tID : \", ID)\n    print(\"\\tisup_grade : \", isup_grade)\n    print(\"\\tgleason_score : \", gleason_score)\n    print(\"\\ttraget : \", target)\n    print()\n    \n    plt.figure(figsize=(10, 10))\n    for index in range(36):\n        img = cv2.imread(os.path.join(BASE_FOLDER, f'train/{ID}_{index}.png'))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        plt.subplot(6, 6, index+1)\n        plt.imshow(img)\n        plt.axis(\"off\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T05:17:50.660924Z","iopub.execute_input":"2024-03-28T05:17:50.661288Z","iopub.status.idle":"2024-03-28T05:17:50.670639Z","shell.execute_reply.started":"2024-03-28T05:17:50.661259Z","shell.execute_reply":"2024-03-28T05:17:50.669401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IDs = np.random.choice(train.ID.values, 3)\n\nfor ID in IDs:\n    display_image(ID)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T05:17:51.573947Z","iopub.execute_input":"2024-03-28T05:17:51.574320Z","iopub.status.idle":"2024-03-28T05:17:58.289539Z","shell.execute_reply.started":"2024-03-28T05:17:51.574291Z","shell.execute_reply":"2024-03-28T05:17:58.286624Z"},"trusted":true},"execution_count":null,"outputs":[]}]}