{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\njpg_list=os.listdir('../input/siim-covid19-resized-to-256px-jpg/train')\ntrain_data_df=pd.DataFrame(jpg_list)\ntrain_data_df=train_data_df.reset_index()\ntrain_data_df['image_id']=train_data_df[0]\ntrain_data_df['process_run']=train_data_df['index']+1\ndel train_data_df[0]\ndel train_data_df['index']\ntrain_data_df","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:33:55.890292Z","iopub.execute_input":"2022-04-25T09:33:55.890959Z","iopub.status.idle":"2022-04-25T09:33:55.925341Z","shell.execute_reply.started":"2022-04-25T09:33:55.890918Z","shell.execute_reply":"2022-04-25T09:33:55.924191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom IPython.display import clear_output\nimport time\nimport sys\ntrain_data_df['id']=train_data_df['image_id'].str.replace('.jpg','_image')\ntrain_image_level=pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrain_study=pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_study['StudyInstanceUID']=train_study['id'].str.replace(\"_study\", \"\")\ndel train_study['id']\ntrain_data_df=pd.merge(train_data_df, train_image_level, how='inner')\ntrain_data_df=pd.merge(train_data_df, train_study, how='inner')\ntrain_data_df\ndel train_image_level\ndel train_study\n\n\nvar_dict={'Negative for Pneumonia': 1, 'Typical Appearance': 2, 'Indeterminate Appearance': 3, 'Atypical Appearance': 4}\ntrain_data_df['categorical_train']=0\nfor key, value in var_dict.items():\n    train_data_df['categorical_train']=train_data_df[[key, 'categorical_train']].apply(\n        lambda x:value if x[key]==1 else x['categorical_train'], axis=1)\n\nprint(train_data_df[['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance', 'categorical_train']].drop_duplicates())\ny_train=train_data_df['categorical_train'].to_numpy()\nprint(y_train.shape)\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:33:56.347735Z","iopub.execute_input":"2022-04-25T09:33:56.348142Z","iopub.status.idle":"2022-04-25T09:33:56.758974Z","shell.execute_reply.started":"2022-04-25T09:33:56.348106Z","shell.execute_reply":"2022-04-25T09:33:56.758172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nfrom PIL import Image\nfrom numpy import asarray\njpg_list_train=train_data_df['image_id'].to_list()\nbatchsize = 100\nbatch_num=0\nnumberf=0\nX_train_full=np.empty([256, 256])\nfor i in range(0, len(jpg_list_train), batchsize):\n    batch = jpg_list_train[i:i+batchsize]\n    batch_num=1+batch_num\n    print(\"Batch Num:\", batch_num)\n    start=time.time()\n    X_train_batch=np.empty([256, 256])\n    for i in batch:\n        image = Image.open('../input/siim-covid19-resized-to-256px-jpg/train/{}'.format(i))\n        data=asarray(image)\n        X_train_batch=np.dstack([X_train_batch, data])\n        numberf = numberf+1\n        clear_output(wait=True)        \n    X_train_batch=X_train_batch[0:, 0:, 1:]\n    print(X_train_batch.shape)\n    X_train_full=np.dstack([X_train_full, X_train_batch])\ndel X_train_batch","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:34:09.331007Z","iopub.execute_input":"2022-04-25T09:34:09.331632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_full=X_train_full[0:, 0:, 1:]\nX_train_full=np.transpose(X_train_full, (2, 0, 1))\nX_train_full.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:28:38.189491Z","iopub.execute_input":"2022-04-25T09:28:38.189889Z","iopub.status.idle":"2022-04-25T09:31:12.701867Z","shell.execute_reply.started":"2022-04-25T09:28:38.189844Z","shell.execute_reply":"2022-04-25T09:31:12.700589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_full=X_train_full[0:, 0:, 1:]\nX_train_full=np.transpose(X_train_full, (2, 0, 1))\nX_train_full.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:19.499230Z","iopub.execute_input":"2022-04-25T09:27:19.499552Z","iopub.status.idle":"2022-04-25T09:27:19.512208Z","shell.execute_reply.started":"2022-04-25T09:27:19.499520Z","shell.execute_reply":"2022-04-25T09:27:19.511111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:19.515835Z","iopub.status.idle":"2022-04-25T09:27:19.516299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"impimport random\ntrain_data_df['categorical_train']=pd.to_numeric(train_data_df['categorical_train'])\nvar_dict2={'Negative_Pneumonia': 1, 'Typical_Appearance': 2, 'Indeterminate_Appearance': 3, 'Atypical_Appearance': 4}\nvar_dict3={}\ncount = 1\nNegative_Pneumonia=random.sample(train_data_df['process_run'][train_data_df['categorical_train']==1].tolist(), 5)\nTypical_Appearance=random.sample(train_data_df['process_run'][train_data_df['categorical_train']==2].tolist(), 5)\nIndeterminate_Appearance=random.sample(train_data_df['process_run'][train_data_df['categorical_train']==3].tolist(), 5)\nAtypical_Appearance=random.sample(train_data_df['process_run'][train_data_df['categorical_train']==4].tolist(), 5)\nfig1, [(ax0, ax1, ax2, ax3, ax4), (ax5, ax6, ax7, ax8, ax9), (ax10, ax11, ax12, ax13, ax14), (ax15, ax16, ax17, ax18, ax19)] = plt.subplots(4,5, figsize=(20,15))\nfig1.suptitle('Train Selected Results', size='xx-large', y=.92)\nax0.set_ylabel('Negative_Pneumonia', rotation=90, size='large')\nax5.set_ylabel('Typical_Appearance', rotation=90, size='large')\nax10.set_ylabel('Indeterminate_Appearance', rotation=90, size='large')\nax15.set_ylabel('Atypical_Appearance', rotation=90, size='large')\naxes1={ax0: Negative_Pneumonia, ax1: Negative_Pneumonia, ax2: Negative_Pneumonia, ax3: Negative_Pneumonia, ax4:Negative_Pneumonia,ax5:Typical_Appearance, ax6:Typical_Appearance, ax7:Typical_Appearance, ax8:Typical_Appearance, ax9:Typical_Appearance,\\\n       ax10:Indeterminate_Appearance, ax11:Indeterminate_Appearance, ax12:Indeterminate_Appearance, ax13:Indeterminate_Appearance, ax14:Indeterminate_Appearance,ax15:Atypical_Appearance, ax16:Atypical_Appearance, ax17:Atypical_Appearance, ax18:Atypical_Appearance, ax19:Atypical_Appearance}\nloopvar=0\nfor key, value in axes1.items():\n    if loopvar==4:\n        loopvar=0\n    else:\n        loopvar=loopvar+1\n    key.imshow(Image.fromarray(X_train_full[value[loopvar]]))\n\nplt.show(fig1)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:33:02.309585Z","iopub.execute_input":"2022-04-25T09:33:02.310340Z","iopub.status.idle":"2022-04-25T09:33:02.322641Z","shell.execute_reply.started":"2022-04-25T09:33:02.310293Z","shell.execute_reply":"2022-04-25T09:33:02.321478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport subprocess\nimport psutil\ny_train=np.array(y_train)\n\nX, X_valid, y_train, y_valid = train_test_split(X,y_train,test_size=0.15)\nprint(X.shape, X_valid.shape, y_train.shape, y_valid.shape)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-04-25T09:27:19.538035Z","iopub.status.idle":"2022-04-25T09:27:19.538812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\ntest1=os.listdir('../input/siim-covid19-resized-to-256px-jpg/train')\nImage.open('../input/siim-covid19-resized-to-256px-jpg/train/{}'.format(test1[0]))","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:19.556334Z","iopub.status.idle":"2022-04-25T09:27:19.556870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.open('../input/siim-covid19-resized-to-256px-jpg/train/{}'.format(test1[1]))","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:19.573265Z","iopub.status.idle":"2022-04-25T09:27:19.573775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### this updated model is the basic way to address the issue while keeping the same simple CONV2D framework\nimport tensorflow as tf\nfrom tensorflow import keras\nimport matplotlib.pyplot as plt\nfrom matplotlib import ticker\nimport numpy as np\n# from keras.preprocessing.image import ImageDataGenerator\n# from keras.preprocessing.image import img_to_array, load_img\n# from keras import layers, models, optimizers\n# from keras import backend as K\n# from numpy import asarray\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Input, Dropout, Flatten, Convolution2D, MaxPooling2D, Dense, Activation\nfrom keras.optimizers import RMSprop, Adam\n\nkeras.backend.clear_session()\nmodel=keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(64,3,3,padding = 'same', activation ='relu', input_shape=(150,150,1)))\nmodel.add(keras.layers.Conv2D(64,3,3,padding = \"same\", activation = \"relu\"))\nmodel.add(keras.layers.Dropout(0.2))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2), padding = \"same\"))\nmodel.add(keras.layers.Conv2D(128,3,3,padding = 'same', activation = 'relu'))\nmodel.add(keras.layers.Conv2D(128,3,3,padding = 'same', activation = 'relu'))\nmodel.add(keras.layers.Dropout(0.2))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2), padding = \"same\"))\nmodel.add(keras.layers.Conv2D(256,3,3,padding = 'same', activation = 'relu'))\nmodel.add(keras.layers.Conv2D(256,3,3,padding = 'same', activation = 'relu'))\nmodel.add(keras.layers.Dropout(0.2))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2), padding = \"same\"))\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(256,activation='relu'))\nmodel.add(keras.layers.Dense(128,activation='relu'))\nmodel.add(keras.layers.Dense(64,activation='relu'))\nmodel.add(keras.layers.Dense(32,activation='relu'))\nmodel.add(keras.layers.Dense(4))\nmodel.add(keras.layers.Activation('softmax'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:29.752592Z","iopub.execute_input":"2022-04-25T09:27:29.752954Z","iopub.status.idle":"2022-04-25T09:27:35.793454Z","shell.execute_reply.started":"2022-04-25T09:27:29.752919Z","shell.execute_reply":"2022-04-25T09:27:35.792420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install livelossplot\nfrom keras.optimizers import Adam\nfrom livelossplot import PlotLossesKeras\nmodel.compile(loss=\"categorical_crossentropy\", optimizer='Adam', metrics=[tensorflow.keras.metrics.categorical_accuracy])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:35.796530Z","iopub.execute_input":"2022-04-25T09:27:35.796820Z","iopub.status.idle":"2022-04-25T09:27:36.055756Z","shell.execute_reply.started":"2022-04-25T09:27:35.796794Z","shell.execute_reply":"2022-04-25T09:27:36.054276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(np.load('./X.npy'), np.load('./y_train.npy'), epochs=60, validation_data=(np.load('./X_valid.npy'), np.load('./y_valid.npy')), callbacks=[PlotLossesKeras(), WandbCallback()])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:36.056799Z","iopub.status.idle":"2022-04-25T09:27:36.057193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(150*150)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T09:27:36.057994Z","iopub.status.idle":"2022-04-25T09:27:36.058349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:44:44.330986Z","iopub.status.idle":"2022-04-25T08:44:44.331732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history).plot(marker='|',markersize=12, figsize=(10,10), linewidth=4)\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:44:44.333563Z","iopub.status.idle":"2022-04-25T08:44:44.334331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import asarray\njpg_list_test=os.listdir('../input/siim-covid19-resized-to-256px-jpg/test')\nX_test = []\ny_name = []\nend1=0\nfor image in jpg_list_test:\n    X_test.append(cv2.resize(cv2.imread('../input/siim-covid19-resized-to-256px-jpg/test/{}'.format(image)), (150,150), interpolation=cv2.INTER_CUBIC))\n    clear_output(wait=True)\n    end1=end1+1\n    y_name.append(image)\n    print('Images Processed:', end1)\n    print('Percent Complete:', round((100*(end1/len(jpg_list_train))),2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test=asarray(X_test)\n# y=asarray(y, dtype='float32')\n# x=np.transpose(x, (2, 1, 3, 0))\nX_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_test=X_test[0:, 0:, 1:]\n# X_test=np.transpose(X_test, (2, 0, 1))\nX_test = X_test / 255\nX_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pred=model.predict_classes(X_test)\ny_predicition=pd.DataFrame(x_test)\ny_predicition['y_predicition']=y_test_pred\n# test=y_predicition[y_prediction].groupby(y_prediction).count()\n# y4_predicition.to_csv(./y4_prediction.csv, index=False)\n# example_test=pd.read_csv(./y4_prediction.csv)\ny_predicition","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ny_predicition=y_predicition.reset_index()\ny_predicition['y_predicition']=pd.to_numeric(y_predicition['y_predicition'])\nvar_dict2={'Negative_Pneumonia': 1, 'Typical_Appearance': 2, 'Indeterminate_Appearance': 3, 'Atypical_Appearance': 4}\nvar_dict3={}\ncount = 1\nfor key, value in var_dict2.items():\n    list1=y_predicition[y_predicition['y_predicition']==value]\n    print(key, list1['index'].to_list())\n    print()\n    print()\nNegative_Pneumonia=random.sample(y_predicition['index'][y_predicition['y_predicition']==1].tolist(), 5)\nTypical_Appearance=random.sample(y_predicition['index'][y_predicition['y_predicition']==2].tolist(), 5)\nIndeterminate_Appearance=random.sample(y_predicition['index'][y_predicition['y_predicition']==3].tolist(), 5)\nAtypical_Appearance=random.sample(y_predicition['index'][y_predicition['y_predicition']==4].tolist(), 5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, [(ax0, ax1, ax2, ax3, ax4), (ax5, ax6, ax7, ax8, ax9), (ax10, ax11, ax12, ax13, ax14), (ax15, ax16, ax17, ax18, ax19)] = plt.subplots(4,5, figsize=(20,15))\nfig.suptitle('Test Selected Results', size='xx-large', y=.92)\nax0.set_ylabel('Negative_Pneumonia', rotation=90, size='large')\nax5.set_ylabel('Typical_Appearance', rotation=90, size='large')\nax10.set_ylabel('Indeterminate_Appearance', rotation=90, size='large')\nax15.set_ylabel('Atypical_Appearance', rotation=90, size='large')\naxes1={ax0: Negative_Pneumonia, ax1: Negative_Pneumonia, ax2: Negative_Pneumonia, ax3: Negative_Pneumonia, ax4:Negative_Pneumonia,ax5:Typical_Appearance, ax6:Typical_Appearance, ax7:Typical_Appearance, ax8:Typical_Appearance, ax9:Typical_Appearance,\\\n       ax10:Indeterminate_Appearance, ax11:Indeterminate_Appearance, ax12:Indeterminate_Appearance, ax13:Indeterminate_Appearance, ax14:Indeterminate_Appearance,ax15:Atypical_Appearance, ax16:Atypical_Appearance, ax17:Atypical_Appearance, ax18:Atypical_Appearance, ax19:Atypical_Appearance}\nloopvar=0\nfor key, value in axes1.items():\n    if loopvar==4:\n        loopvar=0\n    else:\n        loopvar=loopvar+1\n    key.imshow(Image.fromarray(X_test[value[loopvar]]*255))\nrows = ['Row {}'.format(row) for row in ['A', 'B', 'C', 'D']]\n\nplt.show(fig)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.show(fig1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}