{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n# #http://ix.io/3rxn\n# t2 = r'''\n# id,PredictionString\n# abe007e3fc08_image,None 1 0 0 1 1\n# b92e6f1b6c1b_image,opacity 0.448915 36 100 37 169 opacity 0.514129 146 204 55 183\n# 948e9f366810_image,opacity 0.600151 68 136 78 222 opacity 0.615949 179 253 67 211\n# 3cccbd897c39_image,None 1 0 0 1 1\n# 3bb8935d30d9_image,None 1 0 0 1 1\n# 37d6750b5282_image,opacity 0.289108 52 108 85 197 opacity 0.450337 153 209 87 187\n# 67f192ad1a53_image,opacity 0.42714 147 199 50 164 opacity 0.579659 45 105 36 140\n# 7f090fbd8e7c_image,None 1 0 0 1 1\n# 898d24f3cda6_image,opacity 0.479425 17 105 35 171\n# a0f6e42396fc_image,opacity 0.37193 171 231 106 182 opacity 0.431259 67 127 61 179\n# 422c0e145066_image,None 1 0 0 1 1\n# 46719b856de1_image,None 1 0 0 1 1\n# 31c07523a69a_image,opacity 0.427264 53 113 78 200 opacity 0.565582 168 238 58 194\n# f77d7d1aebab_image,None 1 0 0 1 1\n# ccc5b63ca96d_image,None 1 0 0 1 1\n# 5e8ac1fe2b82_image,opacity 0.318599 166 232 126 242 opacity 0.427661 48 120 78 234\n# '''\n","metadata":{"_uuid":"a96e68b7-c621-4367-aa6a-fefd5be705cc","_cell_guid":"dfcdd8da-e20f-4961-b23e-2859979c8e4d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-19T11:33:23.323247Z","iopub.execute_input":"2021-07-19T11:33:23.323625Z","iopub.status.idle":"2021-07-19T11:33:23.328315Z","shell.execute_reply.started":"2021-07-19T11:33:23.323536Z","shell.execute_reply":"2021-07-19T11:33:23.327249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import io\n# import pandas\n# import numpy\n# with io.StringIO(t2) as f:\n#     t3 = pandas.read_csv(f)\n# t3.PredictionString = t3.PredictionString.apply(lambda x: x.replace('Negative', 'negative').replace('None', 'none'))\n# for i in range(t3.shape[0]):\n#     t4 = t3.iloc[i].PredictionString\n#     if 'opacity' in t4:\n#         t5 = t4.split()\n#         assert len(t5) % 6 == 0, 'error'\n#         t6 = numpy.array(t5).reshape(-1, 6)\n#         assert numpy.all(t6[:, 0] == 'opacity'), 'error 2'\n#         #print(t6)\n#         assert numpy.all(t6[:, 2].astype(numpy.int32) < t6[:, 3].astype(numpy.int32)), 'error 3'\n#         assert numpy.all(t6[:, 4].astype(numpy.int32) < t6[:, 5].astype(numpy.int32)), 'error 4'\n#         t7 = numpy.stack([\n#             t6[:, 0],\n#             t6[:, 1],\n#             t6[:, 2], t6[:, 4],\n#             t6[:, 3], t6[:, 5]\n#         ]).T\n#         #print(t7)\n#         #print(i)\n#         t8 = ' '.join(t7.flatten().tolist())\n#         #print(t8)\n#         t3.iloc[i].PredictionString = t8\n# assert (submission_df['id'].values == t3['id'].values).all()\n\n# display(t3)\n\n\n# t3.to_csv('submission.csv', index=False)\n\n# #https://i.imgur.com/3ARpdM0.png\n# #incorrect format of labels, big letters, etc\n    \n# t9 = t3.copy()\n# t9.PredictionString = submission_df.PredictionString\n# t9.iloc[2476].values[1] = 'opacity 0.318599 166 126 232 242 opacity 0.427661 48 78 120 234'\n# display(t9)\n# t9.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.334271Z","iopub.execute_input":"2021-07-19T11:33:23.334754Z","iopub.status.idle":"2021-07-19T11:33:23.342997Z","shell.execute_reply.started":"2021-07-19T11:33:23.334718Z","shell.execute_reply":"2021-07-19T11:33:23.341489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom path import Path\nimport pprint\n\n# def display(*args, **kwargs):\n#     pprint.pprint([args, kwargs])\n    \ndataset_path = Path('../input/siim-covid19-detection')\nsubmission_df = pd.read_csv(dataset_path/'sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.348874Z","iopub.execute_input":"2021-07-19T11:33:23.349709Z","iopub.status.idle":"2021-07-19T11:33:23.401891Z","shell.execute_reply.started":"2021-07-19T11:33:23.349665Z","shell.execute_reply":"2021-07-19T11:33:23.400732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.406706Z","iopub.execute_input":"2021-07-19T11:33:23.408787Z","iopub.status.idle":"2021-07-19T11:33:23.447859Z","shell.execute_reply.started":"2021-07-19T11:33:23.408743Z","shell.execute_reply":"2021-07-19T11:33:23.439320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.tail(20)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.453326Z","iopub.execute_input":"2021-07-19T11:33:23.453976Z","iopub.status.idle":"2021-07-19T11:33:23.475046Z","shell.execute_reply.started":"2021-07-19T11:33:23.453934Z","shell.execute_reply":"2021-07-19T11:33:23.474019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.477410Z","iopub.execute_input":"2021-07-19T11:33:23.477883Z","iopub.status.idle":"2021-07-19T11:33:23.492582Z","shell.execute_reply.started":"2021-07-19T11:33:23.477846Z","shell.execute_reply":"2021-07-19T11:33:23.490752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.495813Z","iopub.execute_input":"2021-07-19T11:33:23.497213Z","iopub.status.idle":"2021-07-19T11:33:23.863397Z","shell.execute_reply.started":"2021-07-19T11:33:23.497174Z","shell.execute_reply":"2021-07-19T11:33:23.862464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### default sumbimission gets a score of 0.1 - hehe","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### use info from study level classifier now\n\nhttps://www.kaggle.com/rbhambri/image-clf-chexnet-vanilla-cnn\n\nhttps://www.kaggle.com/awsaf49/siim-covid-19-study-level-infer","metadata":{}},{"cell_type":"markdown","source":"### sample\n\n898d24f3cda6_image,opacity 0.479425 17 105 35 171\n","metadata":{}},{"cell_type":"code","source":"image_id = '898d24f3cda6_image'","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.865530Z","iopub.execute_input":"2021-07-19T11:33:23.865809Z","iopub.status.idle":"2021-07-19T11:33:23.872980Z","shell.execute_reply.started":"2021-07-19T11:33:23.865783Z","shell.execute_reply":"2021-07-19T11:33:23.871991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[submission_df.id == image_id]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.875977Z","iopub.execute_input":"2021-07-19T11:33:23.876286Z","iopub.status.idle":"2021-07-19T11:33:23.889680Z","shell.execute_reply.started":"2021-07-19T11:33:23.876261Z","shell.execute_reply":"2021-07-19T11:33:23.888692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls ../input/covid-jpg-512/test | grep cda6.jpg","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:23.892864Z","iopub.execute_input":"2021-07-19T11:33:23.893218Z","iopub.status.idle":"2021-07-19T11:33:24.787454Z","shell.execute_reply.started":"2021-07-19T11:33:23.893184Z","shell.execute_reply":"2021-07-19T11:33:24.786560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls ../input/siim-covid19-detection/test/*/* | grep 898d24f3cda6","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:24.789709Z","iopub.execute_input":"2021-07-19T11:33:24.789964Z","iopub.status.idle":"2021-07-19T11:33:29.642704Z","shell.execute_reply.started":"2021-07-19T11:33:24.789938Z","shell.execute_reply":"2021-07-19T11:33:29.641773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### load model and generator","metadata":{}},{"cell_type":"code","source":"img_size = 512","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:29.644309Z","iopub.execute_input":"2021-07-19T11:33:29.644668Z","iopub.status.idle":"2021-07-19T11:33:29.651804Z","shell.execute_reply.started":"2021-07-19T11:33:29.644615Z","shell.execute_reply":"2021-07-19T11:33:29.650835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 16","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:29.655246Z","iopub.execute_input":"2021-07-19T11:33:29.655525Z","iopub.status.idle":"2021-07-19T11:33:29.661614Z","shell.execute_reply.started":"2021-07-19T11:33:29.655499Z","shell.execute_reply":"2021-07-19T11:33:29.660731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:29.663064Z","iopub.execute_input":"2021-07-19T11:33:29.663816Z","iopub.status.idle":"2021-07-19T11:33:35.117107Z","shell.execute_reply.started":"2021-07-19T11:33:29.663776Z","shell.execute_reply":"2021-07-19T11:33:35.116152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_generator_test = ImageDataGenerator()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.118487Z","iopub.execute_input":"2021-07-19T11:33:35.118901Z","iopub.status.idle":"2021-07-19T11:33:35.122786Z","shell.execute_reply.started":"2021-07-19T11:33:35.118858Z","shell.execute_reply":"2021-07-19T11:33:35.121900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test = submission_df[submission_df.id.str.endswith('_image')]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.124203Z","iopub.execute_input":"2021-07-19T11:33:35.124774Z","iopub.status.idle":"2021-07-19T11:33:35.138202Z","shell.execute_reply.started":"2021-07-19T11:33:35.124735Z","shell.execute_reply":"2021-07-19T11:33:35.137158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.139363Z","iopub.execute_input":"2021-07-19T11:33:35.139732Z","iopub.status.idle":"2021-07-19T11:33:35.148852Z","shell.execute_reply.started":"2021-07-19T11:33:35.139694Z","shell.execute_reply":"2021-07-19T11:33:35.147964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.150182Z","iopub.execute_input":"2021-07-19T11:33:35.150598Z","iopub.status.idle":"2021-07-19T11:33:35.159075Z","shell.execute_reply.started":"2021-07-19T11:33:35.150563Z","shell.execute_reply":"2021-07-19T11:33:35.158228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.160285Z","iopub.execute_input":"2021-07-19T11:33:35.160561Z","iopub.status.idle":"2021-07-19T11:33:35.173947Z","shell.execute_reply.started":"2021-07-19T11:33:35.160528Z","shell.execute_reply":"2021-07-19T11:33:35.172883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test = submission_df[submission_df.id.str.endswith('_study')]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.175276Z","iopub.execute_input":"2021-07-19T11:33:35.175717Z","iopub.status.idle":"2021-07-19T11:33:35.183947Z","shell.execute_reply.started":"2021-07-19T11:33:35.175684Z","shell.execute_reply":"2021-07-19T11:33:35.183162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.187365Z","iopub.execute_input":"2021-07-19T11:33:35.187751Z","iopub.status.idle":"2021-07-19T11:33:35.195343Z","shell.execute_reply.started":"2021-07-19T11:33:35.187724Z","shell.execute_reply":"2021-07-19T11:33:35.194324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test['id_image'] = image_df_test['id'].str.replace('_image', '.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.196781Z","iopub.execute_input":"2021-07-19T11:33:35.197222Z","iopub.status.idle":"2021-07-19T11:33:35.208770Z","shell.execute_reply.started":"2021-07-19T11:33:35.197183Z","shell.execute_reply":"2021-07-19T11:33:35.207816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.210554Z","iopub.execute_input":"2021-07-19T11:33:35.211412Z","iopub.status.idle":"2021-07-19T11:33:35.223852Z","shell.execute_reply.started":"2021-07-19T11:33:35.211232Z","shell.execute_reply":"2021-07-19T11:33:35.222801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir = '../input/covid-jpg-512/test'","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.225363Z","iopub.execute_input":"2021-07-19T11:33:35.225776Z","iopub.status.idle":"2021-07-19T11:33:35.230250Z","shell.execute_reply.started":"2021-07-19T11:33:35.225734Z","shell.execute_reply":"2021-07-19T11:33:35.229119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest_generator = image_generator_test.flow_from_dataframe(\n    dataframe = image_df_test,\n    directory = test_dir,\n    x_col = 'id_image',\n#     y_col = 'study_label',\n    target_size=(img_size, img_size),\n    batch_size=batch_size,\n    class_mode=None,\n#     subset='validation', \n    shuffle=False, \n#     seed=23\n) ","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:35.236980Z","iopub.execute_input":"2021-07-19T11:33:35.237258Z","iopub.status.idle":"2021-07-19T11:33:36.314483Z","shell.execute_reply.started":"2021-07-19T11:33:35.237232Z","shell.execute_reply":"2021-07-19T11:33:36.313029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.applications.vgg16 import  VGG16\nfrom tensorflow.keras.applications import DenseNet121\n\nfrom tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout\nfrom tensorflow.keras.layers import GlobalAveragePooling2D,  BatchNormalization, Activation\n# from tensorflow.keras import models\n\nfrom tensorflow.keras.models import Model, Sequential","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:36.317446Z","iopub.execute_input":"2021-07-19T11:33:36.317725Z","iopub.status.idle":"2021-07-19T11:33:36.322695Z","shell.execute_reply.started":"2021-07-19T11:33:36.317700Z","shell.execute_reply":"2021-07-19T11:33:36.321680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_chextnet_model_v1():\n    \"\"\"\n    v1 - uses densenet + chextnet weights\n    \"\"\"\n    # load model design\n    pre_model = DenseNet121(weights=None,\n                        include_top=False,\n                        input_shape=(img_size,img_size,3)\n                        )\n    out = Dense(14, activation='sigmoid')(pre_model.output)\n    pre_model = Model(inputs=pre_model.input, outputs=out) \n    \n    # load model wieghts\n    chex_weights_path = '../input/chexnet-weights/brucechou1983_CheXNet_Keras_0.3.0_weights.h5'\n    pre_model.load_weights(chex_weights_path)\n\n    # make layers trainable?\n    # pre_model.trainable = False\n    pre_model.trainable = True\n\n    # get summary/print\n    pre_model.summary()\n    \n    \n    # add future layers.\n    # last_layer = pre_model.get_layer('conv5_block16_concat')\n    last_layer = pre_model.layers[-2]\n\n    print('last layer output shape: ', last_layer.output_shape)\n    last_output = last_layer.output\n#     last_layer\n\n    # Flatten the output layer to 1 dimension\n    # x = Flatten()(last_output)\n    x = GlobalAveragePooling2D()(last_output)\n\n    # Add a fully connected layer with 512 hidden units and ReLU activation\n    # x = Dense(512, activation='relu')(x)\n    # Add a dropout rate of 0.2\n    # x = Dropout(0.2)(x)                  \n\n\n    # # Add a fully connected layer with 128 hidden units and ReLU activation\n    # x = Dense(128, activation='relu')(x)\n\n\n    # Add final classification layer\n    x = Dense(4, activation='softmax')(x)\n\n    # final model\n    model = Model( pre_model.input, x) \n\n    # model.summary()\n    # plot_model(model)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:36.324306Z","iopub.execute_input":"2021-07-19T11:33:36.325248Z","iopub.status.idle":"2021-07-19T11:33:36.344055Z","shell.execute_reply.started":"2021-07-19T11:33:36.325168Z","shell.execute_reply":"2021-07-19T11:33:36.343068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_chextnet_model_v1()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:36.349876Z","iopub.execute_input":"2021-07-19T11:33:36.350680Z","iopub.status.idle":"2021-07-19T11:33:42.229275Z","shell.execute_reply.started":"2021-07-19T11:33:36.350620Z","shell.execute_reply":"2021-07-19T11:33:42.228395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls ../input/image-clf-chexnet-vanilla-cnn","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:42.230466Z","iopub.execute_input":"2021-07-19T11:33:42.230811Z","iopub.status.idle":"2021-07-19T11:33:42.928267Z","shell.execute_reply.started":"2021-07-19T11:33:42.230768Z","shell.execute_reply":"2021-07-19T11:33:42.927255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('../input/image-clf-chexnet-vanilla-cnn/chextnet_model_512_july11.h5')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:42.929833Z","iopub.execute_input":"2021-07-19T11:33:42.930180Z","iopub.status.idle":"2021-07-19T11:33:44.377073Z","shell.execute_reply.started":"2021-07-19T11:33:42.930140Z","shell.execute_reply":"2021-07-19T11:33:44.376044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.379040Z","iopub.execute_input":"2021-07-19T11:33:44.379451Z","iopub.status.idle":"2021-07-19T11:33:44.383767Z","shell.execute_reply.started":"2021-07-19T11:33:44.379409Z","shell.execute_reply":"2021-07-19T11:33:44.382828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = np.argmax(model.predict(test_generator), axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.385437Z","iopub.execute_input":"2021-07-19T11:33:44.386117Z","iopub.status.idle":"2021-07-19T11:33:44.393578Z","shell.execute_reply.started":"2021-07-19T11:33:44.386065Z","shell.execute_reply":"2021-07-19T11:33:44.392631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# abc","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.395400Z","iopub.execute_input":"2021-07-19T11:33:44.395975Z","iopub.status.idle":"2021-07-19T11:33:44.402731Z","shell.execute_reply.started":"2021-07-19T11:33:44.395933Z","shell.execute_reply":"2021-07-19T11:33:44.401751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds[:25]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.404191Z","iopub.execute_input":"2021-07-19T11:33:44.404670Z","iopub.status.idle":"2021-07-19T11:33:44.411614Z","shell.execute_reply.started":"2021-07-19T11:33:44.404547Z","shell.execute_reply":"2021-07-19T11:33:44.410834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.412916Z","iopub.execute_input":"2021-07-19T11:33:44.413336Z","iopub.status.idle":"2021-07-19T11:33:44.419521Z","shell.execute_reply.started":"2021-07-19T11:33:44.413301Z","shell.execute_reply":"2021-07-19T11:33:44.418828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_df_test['pred_study_class'] = preds","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.420485Z","iopub.execute_input":"2021-07-19T11:33:44.420945Z","iopub.status.idle":"2021-07-19T11:33:44.428540Z","shell.execute_reply.started":"2021-07-19T11:33:44.420908Z","shell.execute_reply":"2021-07-19T11:33:44.427720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.429993Z","iopub.execute_input":"2021-07-19T11:33:44.430344Z","iopub.status.idle":"2021-07-19T11:33:44.438129Z","shell.execute_reply.started":"2021-07-19T11:33:44.430310Z","shell.execute_reply":"2021-07-19T11:33:44.437092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {0: 'atypical', 1: 'indeterminate', 2: 'negative', 3: 'typical'}","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.439548Z","iopub.execute_input":"2021-07-19T11:33:44.439945Z","iopub.status.idle":"2021-07-19T11:33:44.446388Z","shell.execute_reply.started":"2021-07-19T11:33:44.439867Z","shell.execute_reply":"2021-07-19T11:33:44.445511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = model.predict(test_generator)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:33:44.447763Z","iopub.execute_input":"2021-07-19T11:33:44.448168Z","iopub.status.idle":"2021-07-19T11:34:07.592909Z","shell.execute_reply.started":"2021-07-19T11:33:44.448132Z","shell.execute_reply":"2021-07-19T11:34:07.591729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.595031Z","iopub.execute_input":"2021-07-19T11:34:07.597159Z","iopub.status.idle":"2021-07-19T11:34:07.607503Z","shell.execute_reply.started":"2021-07-19T11:34:07.597112Z","shell.execute_reply":"2021-07-19T11:34:07.606744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ix = 0\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.609988Z","iopub.execute_input":"2021-07-19T11:34:07.611067Z","iopub.status.idle":"2021-07-19T11:34:07.617753Z","shell.execute_reply.started":"2021-07-19T11:34:07.611030Z","shell.execute_reply":"2021-07-19T11:34:07.616428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c[ix]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.619452Z","iopub.execute_input":"2021-07-19T11:34:07.620778Z","iopub.status.idle":"2021-07-19T11:34:07.641998Z","shell.execute_reply.started":"2021-07-19T11:34:07.620741Z","shell.execute_reply":"2021-07-19T11:34:07.641179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\n\nfor c_row in c:\n#     print(c_row)\n    c_row_best_ix = np.argmax(c_row)\n    c_row_best = c_row[c_row_best_ix]\n    \n    c_row_dict = {'c_row_best_ix': c_row_best_ix, 'c_row_best': c_row_best, 'c_row': list(c_row)}\n#     print(c_row_dict)\n    preds.append(c_row_dict)\n    \n#     break","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.647375Z","iopub.execute_input":"2021-07-19T11:34:07.650218Z","iopub.status.idle":"2021-07-19T11:34:07.691986Z","shell.execute_reply.started":"2021-07-19T11:34:07.650178Z","shell.execute_reply":"2021-07-19T11:34:07.691220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [ max(c[ix] for ix in range(1263))]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.695836Z","iopub.execute_input":"2021-07-19T11:34:07.698916Z","iopub.status.idle":"2021-07-19T11:34:07.707928Z","shell.execute_reply.started":"2021-07-19T11:34:07.698879Z","shell.execute_reply":"2021-07-19T11:34:07.706274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds[:10]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.712141Z","iopub.execute_input":"2021-07-19T11:34:07.712520Z","iopub.status.idle":"2021-07-19T11:34:07.726231Z","shell.execute_reply.started":"2021-07-19T11:34:07.712486Z","shell.execute_reply":"2021-07-19T11:34:07.725348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test['predictions'] = preds","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.730905Z","iopub.execute_input":"2021-07-19T11:34:07.732961Z","iopub.status.idle":"2021-07-19T11:34:07.743564Z","shell.execute_reply.started":"2021-07-19T11:34:07.732922Z","shell.execute_reply":"2021-07-19T11:34:07.742523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.747681Z","iopub.execute_input":"2021-07-19T11:34:07.749764Z","iopub.status.idle":"2021-07-19T11:34:07.773857Z","shell.execute_reply.started":"2021-07-19T11:34:07.749726Z","shell.execute_reply":"2021-07-19T11:34:07.773006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_study_class_name(row):\n    predictions = row.predictions\n    \n    pred_study_class = predictions['c_row_best_ix']\n    \n    pred_study_class_name = label_map[pred_study_class]\n    return pred_study_class_name\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.777869Z","iopub.execute_input":"2021-07-19T11:34:07.780212Z","iopub.status.idle":"2021-07-19T11:34:07.787277Z","shell.execute_reply.started":"2021-07-19T11:34:07.780172Z","shell.execute_reply":"2021-07-19T11:34:07.786359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_study_class_proba(row):\n    predictions = row.predictions\n    c_row_best_proba = predictions['c_row_best']\n    return c_row_best_proba","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.792120Z","iopub.execute_input":"2021-07-19T11:34:07.794566Z","iopub.status.idle":"2021-07-19T11:34:07.800607Z","shell.execute_reply.started":"2021-07-19T11:34:07.794529Z","shell.execute_reply":"2021-07-19T11:34:07.799793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test['study_class_name'] = image_df_test.apply(lambda row:get_study_class_name(row) , axis=1)\nimage_df_test['study_class_proba'] = image_df_test.apply(lambda row:get_study_class_proba(row) , axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.805299Z","iopub.execute_input":"2021-07-19T11:34:07.807635Z","iopub.status.idle":"2021-07-19T11:34:07.887862Z","shell.execute_reply.started":"2021-07-19T11:34:07.807599Z","shell.execute_reply":"2021-07-19T11:34:07.886984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.892070Z","iopub.execute_input":"2021-07-19T11:34:07.894345Z","iopub.status.idle":"2021-07-19T11:34:07.921698Z","shell.execute_reply.started":"2021-07-19T11:34:07.894304Z","shell.execute_reply":"2021-07-19T11:34:07.920846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### now link image back to study","metadata":{}},{"cell_type":"code","source":"study_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.925797Z","iopub.execute_input":"2021-07-19T11:34:07.928003Z","iopub.status.idle":"2021-07-19T11:34:07.944250Z","shell.execute_reply.started":"2021-07-19T11:34:07.927965Z","shell.execute_reply":"2021-07-19T11:34:07.943383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test['id'] = study_df_test['id'].str.replace('_study',\"\")","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.948219Z","iopub.execute_input":"2021-07-19T11:34:07.950390Z","iopub.status.idle":"2021-07-19T11:34:07.963122Z","shell.execute_reply.started":"2021-07-19T11:34:07.950352Z","shell.execute_reply":"2021-07-19T11:34:07.962134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.966922Z","iopub.execute_input":"2021-07-19T11:34:07.969060Z","iopub.status.idle":"2021-07-19T11:34:07.984341Z","shell.execute_reply.started":"2021-07-19T11:34:07.969025Z","shell.execute_reply":"2021-07-19T11:34:07.983428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_absolute_file_paths(directory):\n    all_abs_file_paths = []\n    for dirpath,_,filenames in os.walk(directory):\n        for f in filenames:\n            all_abs_file_paths.append(os.path.abspath(os.path.join(dirpath, f)))\n    return all_abs_file_paths","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:07.988533Z","iopub.execute_input":"2021-07-19T11:34:07.990697Z","iopub.status.idle":"2021-07-19T11:34:07.998507Z","shell.execute_reply.started":"2021-07-19T11:34:07.990657Z","shell.execute_reply":"2021-07-19T11:34:07.997328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm; tqdm.pandas();","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:08.003809Z","iopub.execute_input":"2021-07-19T11:34:08.005953Z","iopub.status.idle":"2021-07-19T11:34:08.014858Z","shell.execute_reply.started":"2021-07-19T11:34:08.005915Z","shell.execute_reply":"2021-07-19T11:34:08.013773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:08.018972Z","iopub.execute_input":"2021-07-19T11:34:08.021147Z","iopub.status.idle":"2021-07-19T11:34:08.027307Z","shell.execute_reply.started":"2021-07-19T11:34:08.021108Z","shell.execute_reply":"2021-07-19T11:34:08.026365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test[\"study_dir\"] = \"/kaggle/input/siim-covid19-detection/test/\" + study_df_test[\"id\"]\nstudy_df_test[\"images_per_study\"] = study_df_test.study_dir.progress_apply(lambda x: len(get_absolute_file_paths(x)))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:08.031380Z","iopub.execute_input":"2021-07-19T11:34:08.033369Z","iopub.status.idle":"2021-07-19T11:34:09.482529Z","shell.execute_reply.started":"2021-07-19T11:34:08.033335Z","shell.execute_reply":"2021-07-19T11:34:09.481711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:09.485257Z","iopub.execute_input":"2021-07-19T11:34:09.485508Z","iopub.status.idle":"2021-07-19T11:34:09.499439Z","shell.execute_reply.started":"2021-07-19T11:34:09.485481Z","shell.execute_reply":"2021-07-19T11:34:09.498546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_test.images_per_study.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:09.502323Z","iopub.execute_input":"2021-07-19T11:34:09.502563Z","iopub.status.idle":"2021-07-19T11:34:09.512601Z","shell.execute_reply.started":"2021-07-19T11:34:09.502540Z","shell.execute_reply":"2021-07-19T11:34:09.511544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:09.515247Z","iopub.execute_input":"2021-07-19T11:34:09.515488Z","iopub.status.idle":"2021-07-19T11:34:09.521069Z","shell.execute_reply.started":"2021-07-19T11:34:09.515465Z","shell.execute_reply":"2021-07-19T11:34:09.520138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepaths = glob.glob('/kaggle/input/siim-covid19-detection/test/**/*dcm',recursive=True)\ntest_df = pd.DataFrame({'filepath':filepaths,})\ntest_df['image_id'] = test_df.filepath.map(lambda x: x.split('/')[-1].replace('.dcm', '')+'_image')\ntest_df['study_id'] = test_df.filepath.map(lambda x: x.split('/')[-3].replace('.dcm', '')+'_study')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:09.523420Z","iopub.execute_input":"2021-07-19T11:34:09.523694Z","iopub.status.idle":"2021-07-19T11:34:11.292990Z","shell.execute_reply.started":"2021-07-19T11:34:09.523664Z","shell.execute_reply":"2021-07-19T11:34:11.292207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['id'] = test_df['image_id']","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.294222Z","iopub.execute_input":"2021-07-19T11:34:11.294575Z","iopub.status.idle":"2021-07-19T11:34:11.298545Z","shell.execute_reply.started":"2021-07-19T11:34:11.294542Z","shell.execute_reply":"2021-07-19T11:34:11.297674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.299923Z","iopub.execute_input":"2021-07-19T11:34:11.300543Z","iopub.status.idle":"2021-07-19T11:34:11.310751Z","shell.execute_reply.started":"2021-07-19T11:34:11.300508Z","shell.execute_reply":"2021-07-19T11:34:11.309582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.312315Z","iopub.execute_input":"2021-07-19T11:34:11.312687Z","iopub.status.idle":"2021-07-19T11:34:11.325799Z","shell.execute_reply.started":"2021-07-19T11:34:11.312629Z","shell.execute_reply":"2021-07-19T11:34:11.325053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.329129Z","iopub.execute_input":"2021-07-19T11:34:11.329368Z","iopub.status.idle":"2021-07-19T11:34:11.336435Z","shell.execute_reply.started":"2021-07-19T11:34:11.329346Z","shell.execute_reply":"2021-07-19T11:34:11.335386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.338015Z","iopub.execute_input":"2021-07-19T11:34:11.338421Z","iopub.status.idle":"2021-07-19T11:34:11.356382Z","shell.execute_reply.started":"2021-07-19T11:34:11.338387Z","shell.execute_reply":"2021-07-19T11:34:11.355682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test2 = image_df_test.merge(test_df, \n                                     on='id',\n                                 )","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.357658Z","iopub.execute_input":"2021-07-19T11:34:11.358146Z","iopub.status.idle":"2021-07-19T11:34:11.372348Z","shell.execute_reply.started":"2021-07-19T11:34:11.357957Z","shell.execute_reply":"2021-07-19T11:34:11.371498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test2.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.375515Z","iopub.execute_input":"2021-07-19T11:34:11.375781Z","iopub.status.idle":"2021-07-19T11:34:11.396114Z","shell.execute_reply.started":"2021-07-19T11:34:11.375758Z","shell.execute_reply":"2021-07-19T11:34:11.394999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test2.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.402300Z","iopub.execute_input":"2021-07-19T11:34:11.402543Z","iopub.status.idle":"2021-07-19T11:34:11.408687Z","shell.execute_reply.started":"2021-07-19T11:34:11.402520Z","shell.execute_reply":"2021-07-19T11:34:11.407712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test2.study_class_name.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.410825Z","iopub.execute_input":"2021-07-19T11:34:11.411377Z","iopub.status.idle":"2021-07-19T11:34:11.420492Z","shell.execute_reply.started":"2021-07-19T11:34:11.411340Z","shell.execute_reply":"2021-07-19T11:34:11.419329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dict(image_df_test2.study_id.value_counts())\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.421986Z","iopub.execute_input":"2021-07-19T11:34:11.422431Z","iopub.status.idle":"2021-07-19T11:34:11.428685Z","shell.execute_reply.started":"2021-07-19T11:34:11.422396Z","shell.execute_reply":"2021-07-19T11:34:11.427749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test2.study_id.describe()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.429993Z","iopub.execute_input":"2021-07-19T11:34:11.430419Z","iopub.status.idle":"2021-07-19T11:34:11.443247Z","shell.execute_reply.started":"2021-07-19T11:34:11.430329Z","shell.execute_reply":"2021-07-19T11:34:11.442214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_test2[image_df_test2.study_id == '342624fb2b11_study']","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.444795Z","iopub.execute_input":"2021-07-19T11:34:11.445141Z","iopub.status.idle":"2021-07-19T11:34:11.467254Z","shell.execute_reply.started":"2021-07-19T11:34:11.445108Z","shell.execute_reply":"2021-07-19T11:34:11.466225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_df_test2[image_df_test2.study_id == 'bc73883551cf_study']\nimage_df_test2[image_df_test2.study_id == 'bc73883551cf_study'].iloc[0].predictions\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.468862Z","iopub.execute_input":"2021-07-19T11:34:11.469243Z","iopub.status.idle":"2021-07-19T11:34:11.479002Z","shell.execute_reply.started":"2021-07-19T11:34:11.469210Z","shell.execute_reply":"2021-07-19T11:34:11.478207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### assign label to each study","metadata":{"execution":{"iopub.status.busy":"2021-07-11T19:06:15.931516Z","iopub.execute_input":"2021-07-11T19:06:15.931844Z","iopub.status.idle":"2021-07-11T19:06:15.935666Z","shell.execute_reply.started":"2021-07-11T19:06:15.931815Z","shell.execute_reply":"2021-07-11T19:06:15.93452Z"}}},{"cell_type":"code","source":"study_class_map = {}","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.480288Z","iopub.execute_input":"2021-07-19T11:34:11.480913Z","iopub.status.idle":"2021-07-19T11:34:11.487403Z","shell.execute_reply.started":"2021-07-19T11:34:11.480820Z","shell.execute_reply":"2021-07-19T11:34:11.486485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ix, row in image_df_test2.iterrows():\n#     print(ix)\n#     print(row)\n    \n    study_id = row.study_id                                           \n    study_class_name = row.study_class_name                                             \n    study_class_proba = row.study_class_proba\n    all_class_predictions = row.predictions['c_row']\n\n    if study_id not in study_class_map:\n        study_class_map[study_id] = (study_class_name, study_class_proba, all_class_predictions)\n    \n    \n#     break","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.488778Z","iopub.execute_input":"2021-07-19T11:34:11.489267Z","iopub.status.idle":"2021-07-19T11:34:11.632099Z","shell.execute_reply.started":"2021-07-19T11:34:11.489229Z","shell.execute_reply":"2021-07-19T11:34:11.631172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study_class_map","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.633404Z","iopub.execute_input":"2021-07-19T11:34:11.633770Z","iopub.status.idle":"2021-07-19T11:34:11.637311Z","shell.execute_reply.started":"2021-07-19T11:34:11.633734Z","shell.execute_reply":"2021-07-19T11:34:11.636472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:11.638698Z","iopub.execute_input":"2021-07-19T11:34:11.639221Z","iopub.status.idle":"2021-07-19T11:34:11.652338Z","shell.execute_reply.started":"2021-07-19T11:34:11.639186Z","shell.execute_reply":"2021-07-19T11:34:11.651394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def update_PredictionString(row):\n# #     print(row)\n#     row_id = row.id\n    \n#     if row_id in study_class_map:\n#         study_class, study_class_proba = study_class_map[row_id]\n#         study_class_proba = round(study_class_proba, 2)\n#         print(row_id, study_class, study_class_proba)\n    \n#         PredictionString = study_class + ' ' + str(study_class_proba) + ' 0 0 1 1'\n#         print(PredictionString)\n#         return PredictionString\n        \n#     else:\n#         print('warning for', row_id)\n#         return row.PredictionString\n    \n    \n#     print('===========\\n')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:40.024444Z","iopub.execute_input":"2021-07-19T11:34:40.024838Z","iopub.status.idle":"2021-07-19T11:34:40.029275Z","shell.execute_reply.started":"2021-07-19T11:34:40.024802Z","shell.execute_reply":"2021-07-19T11:34:40.028144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {0: 'atypical', 1: 'indeterminate', 2: 'negative', 3: 'typical'}","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:41.963284Z","iopub.execute_input":"2021-07-19T11:34:41.963674Z","iopub.status.idle":"2021-07-19T11:34:41.968834Z","shell.execute_reply.started":"2021-07-19T11:34:41.963620Z","shell.execute_reply":"2021-07-19T11:34:41.967487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_PredictionString_v2(row):\n#     print(row)\n    row_id = row.id\n    \n    if row_id in study_class_map:\n        study_class, study_class_proba, all_class_predictions = study_class_map[row_id]\n        study_class_proba = round(study_class_proba, 2)\n#         print(row_id, study_class, study_class_proba, all_class_predictions)\n    \n        negative_proba = all_class_predictions[2]\n        PredictionString = 'negative ' + str(negative_proba) + ' 0 0 1 1'\n        \n        typical_proba = all_class_predictions[3]\n        PredictionString += ' typical ' + str(typical_proba) + ' 0 0 1 1'\n\n        indeterminate_proba = all_class_predictions[1]\n        PredictionString += ' indeterminate ' + str(indeterminate_proba) + ' 0 0 1 1'\n        \n        atypical_proba = all_class_predictions[0]\n        PredictionString += ' atypical ' + str(atypical_proba) + ' 0 0 1 1'\n        \n        \n#         print(PredictionString)\n        return PredictionString\n        \n    else:\n#         print('warning for', row_id)\n        my_str = 'opacity 0.479425 17 105 35 171'\n#         return row.PredictionString\n        return my_str\n    \n    \n#     print('===========\\n')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:27.353012Z","iopub.execute_input":"2021-07-19T11:36:27.353461Z","iopub.status.idle":"2021-07-19T11:36:27.362300Z","shell.execute_reply.started":"2021-07-19T11:36:27.353429Z","shell.execute_reply":"2021-07-19T11:36:27.359669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def update_PredictionString_v3(row):\n# #     print(row)\n#     row_id = row.id\n    \n#     if row_id in study_class_map:\n#         study_class, study_class_proba, all_class_predictions = study_class_map[row_id]\n#         study_class_proba = round(study_class_proba, 2)\n# #         print(row_id, study_class, study_class_proba, all_class_predictions)\n    \n# #         negative_proba = all_class_predictions[2]\n#         negative_proba = 1\n#         PredictionString = 'negative ' + str(negative_proba) + ' 0 0 1 1'\n        \n# #         typical_proba = all_class_predictions[3]\n#         typical_proba = 1\n#         PredictionString += ' typical ' + str(typical_proba) + ' 0 0 1 1'\n\n# #         indeterminate_proba = all_class_predictions[1]\n#         indeterminate_proba = 1\n#         PredictionString += ' indeterminate ' + str(indeterminate_proba) + ' 0 0 1 1'\n        \n# #         atypical_proba = all_class_predictions[0]\n#         atypical_proba = 1\n#         PredictionString += ' atypical ' + str(atypical_proba) + ' 0 0 1 1'\n        \n        \n# #         print(PredictionString)\n#         return PredictionString\n        \n#     else:\n#         print('warning for', row_id)\n#         return row.PredictionString\n    \n    \n# #     print('===========\\n')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:30.446059Z","iopub.execute_input":"2021-07-19T11:36:30.446571Z","iopub.status.idle":"2021-07-19T11:36:30.451853Z","shell.execute_reply.started":"2021-07-19T11:36:30.446525Z","shell.execute_reply":"2021-07-19T11:36:30.450981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# abc","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:33.779202Z","iopub.execute_input":"2021-07-19T11:36:33.779588Z","iopub.status.idle":"2021-07-19T11:36:33.783892Z","shell.execute_reply.started":"2021-07-19T11:36:33.779555Z","shell.execute_reply":"2021-07-19T11:36:33.782625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df.head().apply(lambda row: update_PredictionString(row), axis=1)\nsubmission_df.head().apply(lambda row: update_PredictionString_v2(row), axis=1)\n# submission_df.head().apply(lambda row: update_PredictionString_v3(row), axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:33.977583Z","iopub.execute_input":"2021-07-19T11:36:33.977962Z","iopub.status.idle":"2021-07-19T11:36:33.986454Z","shell.execute_reply.started":"2021-07-19T11:36:33.977929Z","shell.execute_reply":"2021-07-19T11:36:33.985428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.tail().apply(lambda row: update_PredictionString_v2(row), axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:47.572923Z","iopub.execute_input":"2021-07-19T11:36:47.573266Z","iopub.status.idle":"2021-07-19T11:36:47.581250Z","shell.execute_reply.started":"2021-07-19T11:36:47.573236Z","shell.execute_reply":"2021-07-19T11:36:47.580228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df['PredictionString'] = submission_df.apply(lambda row: update_PredictionString(row), axis=1)\nsubmission_df['PredictionString'] = submission_df.apply(lambda row: update_PredictionString_v2(row), axis=1)\n# submission_df['PredictionString'] = submission_df.apply(lambda row: update_PredictionString_v3(row), axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:50.738558Z","iopub.execute_input":"2021-07-19T11:36:50.738927Z","iopub.status.idle":"2021-07-19T11:36:50.786330Z","shell.execute_reply.started":"2021-07-19T11:36:50.738897Z","shell.execute_reply":"2021-07-19T11:36:50.785432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:53.001193Z","iopub.execute_input":"2021-07-19T11:36:53.001515Z","iopub.status.idle":"2021-07-19T11:36:53.012277Z","shell.execute_reply.started":"2021-07-19T11:36:53.001486Z","shell.execute_reply":"2021-07-19T11:36:53.011011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.tail(20)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:53.582698Z","iopub.execute_input":"2021-07-19T11:36:53.583034Z","iopub.status.idle":"2021-07-19T11:36:53.593726Z","shell.execute_reply.started":"2021-07-19T11:36:53.583005Z","shell.execute_reply":"2021-07-19T11:36:53.592729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:36:59.491111Z","iopub.execute_input":"2021-07-19T11:36:59.491433Z","iopub.status.idle":"2021-07-19T11:36:59.512229Z","shell.execute_reply.started":"2021-07-19T11:36:59.491404Z","shell.execute_reply":"2021-07-19T11:36:59.511481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### now trigger image binary clf to detect opacity vs. none\n\nhttps://www.kaggle.com/rbhambri/chexnet-transfer-learning-binary-image-clf\n\n","metadata":{}},{"cell_type":"code","source":"# 7f090fbd8e7c_image,None 1 0 0 1 1\n# 898d24f3cda6_image,opacity 0.479425 17 105 35 171\n# a0f6e42396fc_image,opacity 0.37193 171 231 106 182 opacity 0.431259 67 127 61 179\n# 422c0e145066_image,None 1 0 0 1 1\n# 46719b856de1_image,None 1 0 0 1 1\n# 31c07523a69a_image,opacity 0.427264 53 113 78 200 opacity 0.565582 168 238 58 194\n# f77d7d1aebab_image,None 1 0 0 1 1\n# ccc5b63ca96d_image,None 1 0 0 1 1\n# 5e8ac1fe2b82_image,opacity 0.318599 166 232 126 242 opacity 0.427661 48 120 78 234\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T11:34:12.008480Z","iopub.status.idle":"2021-07-19T11:34:12.009189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}