{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is fifth and the last part of [RANZCR 1st Place Solution by TF](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-1-make-masks).\nThis notebook is based on [RANZCR 1st Place Soluiton Inference (small ver.)](https://www.kaggle.com/haqishen/ranzcr-1st-place-soluiton-inference-small-ver).\n\nThe last step is inference. The segmentation and classification model trained in the previous steps is used.\n\nThe original notebook uses 5 folds for both segmentation and classification models. This implementation uses 1 fold for the segmentation model and 5 folds for the classification model.\n\nThe structure of the inference model is as follows:\n\n* 1024x1024 size images are put into the segmentation model. It outputs 1024x1024 masks.\n* Masks from each folds are averaged.  This implementation has only 1 fold, so the averaging has no effect. The generated masks are resized to 512x512.\n* 512x512 size images and the resized masks are concatenated to make 5 channel inputs. They are put into the classification model.\n* The classification model has 3 outputs, ETT, other, and pred. For the inference model, only the pred output is used.","metadata":{}},{"cell_type":"code","source":"%env SM_FRAMEWORK=tf.keras\n!pip install ../input/segmentation-models-keras/Keras_Applications-1.0.8-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/image_classifiers-1.0.0-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/efficientnet-1.0.0-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/segmentation_models-1.0.1-py3-none-any.whl --quiet\n\nprint(\"Segmentation Models installed.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# libraries\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.layers as L\nimport segmentation_models as sm\n\nprint(tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/ranzcr-clip-catheter-line-classification'\nmodel_dir = '../input/ranzcr-1st-place-solution-by-tf-models'\nseg_image_size = 1024\ncls_image_size = 512\nbatch_size = 16 # original is 8","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))\n# df_sub = df_sub.iloc[:358] if df_sub.shape[0] == 3582 else df_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Weight file names for the segmentation and classification models.\n# For the segmentation model, use the only one weight for all folds.\nseg_model_names = [\n    'seg_model_V10_0.hdf5'\n]\ncls_model_names = [\n    'cls_model_V14_0.hdf5',\n    'cls_model_V15_1.hdf5',\n    'cls_model_V15_2.hdf5',\n    'cls_model_V16_3.hdf5',\n    'cls_model_V16_4.hdf5'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"tfrec_path = data_dir + '/test_tfrecords/*.tfrec'\ntfrec_file_names = sorted(tf.io.gfile.glob(tfrec_path))\ntfrec_file_names = \\\n    [ tfrec_file_names[0] ] if DEBUG else tfrec_file_names\n\ntfrec_file_names","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    return image\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'StudyInstanceUID': tf.io.FixedLenFeature([], tf.string),\n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    study_inst_id = example['StudyInstanceUID']\n    return image, study_inst_id\n\ndef load_dataset(filenames):\n    ds = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)\n    ds = ds.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test_ds = load_dataset(tfrec_file_names)\n\nraw_test_ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_inst_id_list = [\n    study_inst_id.numpy().decode('utf-8') for image, study_inst_id in raw_test_ds\n]\n\nprint(study_inst_id_list[ :10 ])\nprint(study_inst_id_list[ -10: ])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_df_sub_shape = df_sub.shape[0]\nn_study_inst_id = len(study_inst_id_list)\ndf_sub = df_sub.iloc[ :n_study_inst_id ]\n\nprint(\"original df_sub.shape[0]:\", orig_df_sub_shape)\nprint(\"updated df_sub.shape[0]: \", df_sub.shape[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_study_inst_id(image, study_inst_id):\n    return image\n\ndef preprocess_image(image):\n    # Range 0..1 for segmentation model\n    # tf.image.resize() returns float tensor.\n    image_seg = tf.image.resize(image, (seg_image_size, seg_image_size))\n    image_seg = image_seg / 255.0\n    # Range 0..255 for class model.\n    image_cls = tf.image.resize(image, (cls_image_size, cls_image_size))\n    return ((image_seg, image_cls), )\n\ndef make_test_dataset():\n    ds = load_dataset(tfrec_file_names)\n    ds = ds.map(drop_study_inst_id, num_parallel_calls=AUTOTUNE)\n    ds = ds.map(preprocess_image, num_parallel_calls=AUTOTUNE)\n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = make_test_dataset()\n\ntest_ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pylab import rcParams\nrcParams['figure.figsize'] = 20,10\n\nf, axarr = plt.subplots(1,5)\ntest_ds_iter = iter(test_ds.unbatch())\nimg512_list = []\nfor p in range(5):\n    items = next(test_ds_iter)\n    img1024, img512 = items[0]\n    axarr[p].imshow(img1024)\n    img512_list.append(img512)\n    \nf, axarr = plt.subplots(1,5)\nfor p in range(5):\n    axarr[p].imshow(img512_list[p] / 255.0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def load_model(weight_file_name):\n    weight_file_path = os.path.join(model_dir, weight_file_name)\n    model = tf.keras.models.load_model(weight_file_path)\n    return model\n\ndef make_seg_masks(x):\n    fold_seg_masks = tf.stack(x, axis=0)\n    # [ fold, batch, height, width, channel ]\n    average_seg_masks = \\\n        tf.math.reduce_mean(fold_seg_masks, axis=0)\n    return average_seg_masks\n\ndef make_cls_inputs(x):\n    cls_images = x[0] # (512, 512, 3), [0..255]\n    seg_masks = x[1]  # (1024, 1024, 2), [0..1]\n    \n    seg_masks = tf.image.resize(\n        seg_masks, (cls_image_size, cls_image_size))\n    seg_masks = seg_masks * 255.0\n    \n    cls_inputs = tf.concat([cls_images, seg_masks], axis=-1)\n    return cls_inputs\n\ndef make_model(cls_model_name):\n    seg_images = tf.keras.Input(\n        shape=(seg_image_size, seg_image_size, 3),\n        name=\"seg_images\")\n    seg_outputs = []\n    for seg_model_name in seg_model_names:\n        seg_model = load_model(seg_model_name)\n        seg_output = seg_model(seg_images)\n        seg_outputs.append(seg_output)\n    seg_masks = L.Lambda(\n        make_seg_masks, name=\"seg_masks\")(seg_outputs)\n    \n    cls_images = tf.keras.Input(\n        shape=(cls_image_size, cls_image_size, 3),\n        name=\"cls_images\")\n    cls_inputs = L.Lambda(\n        make_cls_inputs, name=\"cls_inputs\")(\n        [cls_images, seg_masks])\n    cls_model = load_model(cls_model_name)\n    ett, others, pred = cls_model(cls_inputs)\n\n    model = tf.keras.Model(\n        inputs=[seg_images, cls_images],\n        outputs=pred,\n        name=\"infer_model\")\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# default distribution strategy in Tensorflow. Works on CPU and single GPU.\nstrategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PROBS = []\nshow_summary = True\nfor cls_model_name in cls_model_names:\n    print(\"####################\")\n    print(\"# {0}\".format(cls_model_name))\n    with strategy.scope():\n        model = make_model(cls_model_name)\n    if show_summary:\n        model.summary()\n        show_summary = False\n    \n    pred = model.predict(test_ds, verbose=1)\n    PROBS.append(pred)\n    print()\n    \nPROBS = np.array(PROBS)\nPROBS.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Distribution","metadata":{}},{"cell_type":"code","source":"# Put study instance ID values from TFRecords to submission\ndf_sub['StudyInstanceUID'] = study_inst_id_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_cols = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put mean prediction values to submission\ndf_sub[target_cols] = PROBS.mean(0)\n\nsns.distplot(df_sub[[\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n]])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rank Prediction & Submit","metadata":{}},{"cell_type":"code","source":"# Calculate rank for each folds.\ndf_subs = [df_sub.copy() for _ in range(PROBS.shape[0])]\nfor i, this_sub in enumerate(df_subs):\n    this_sub[target_cols] = PROBS[i]\n    this_sub[target_cols] = \\\n        this_sub[target_cols].rank(pct=True)  # rank","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate mean rank values.\nrank_values = \\\n    [this_sub[target_cols].values for this_sub in df_subs]\ndf_sub[target_cols] = \\\n    np.stack(rank_values, 0).mean(0)  # mean","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit the result.\ndf_sub.to_csv('submission.csv', index=False)\n\n!head submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}