{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers\nimport os\nimport re\nimport math\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = \"../input/ranzcr-clip-catheter-line-classification\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(GCS_DS_PATH+\"/train.csv\")\ntrain_df.index = train_df[\"StudyInstanceUID\"]\ndel train_df[\"StudyInstanceUID\"]\n\ntrain_annot_df = pd.read_csv(GCS_DS_PATH+\"/train_annotations.csv\")\ntrain_annot_df.index = train_annot_df[\"StudyInstanceUID\"]\ndel train_annot_df[\"StudyInstanceUID\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_annot_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = list(train_df.columns[:-1])\nclasses_normal= [name for name in classes[:-1] if name.split(\" - \")[1] == \"Normal\"]\nclasses_abnormal= [name for name in classes[:-1] if name.split(\" - \")[1] == \"Abnormal\"]\nclasses_borderline = [name for name in classes[:-1] if name.split(\" - \")[1] == \"Borderline\"]\nclasses_count = train_df[classes].sum(axis = 0)\nnum_classes = len(classes_count)\n\n\nprint(\"Number of Classes: {}\".format(num_classes))\nclasses_count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = {}\nls = list(classes_count.values)\ntot_samples = sum(ls)\n\nfor i in range(num_classes):\n    class_weights[i] = tot_samples/(num_classes*ls[i])\n\nclass_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_ids = train_df[\"PatientID\"].unique()\npatientwise_count = train_df['PatientID'].value_counts()\nnum_patients = len(patientwise_count)\nprint(\"Number of patients: \",num_patients)\npatientwise_count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [600,600]\nAUTO = tf.data.experimental.AUTOTUNE\n\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/test_tfrecords/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0 \n    image = tf.image.resize(image, [*IMAGE_SIZE])\n    return image\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"StudyInstanceUID\"           : tf.io.FixedLenFeature([], tf.string),\n        \"image\"                      : tf.io.FixedLenFeature([], tf.string),\n        \"ETT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Incompletely Imaged\"  : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"Swan Ganz Catheter Present\" : tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = [example['ETT - Abnormal'],\n                 example['ETT - Borderline'],\n                 example['ETT - Normal'],\n                 example['NGT - Abnormal'],\n                 example['NGT - Borderline'],\n                 example['NGT - Incompletely Imaged'],\n                 example['NGT - Normal'],\n                 example['CVC - Abnormal'],\n                 example['CVC - Borderline'],\n                 example['CVC - Normal'],\n                 example['Swan Ganz Catheter Present']]\n    label = [tf.cast(i,tf.float32) for i in label]\n    return image, label\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"StudyInstanceUID\"           : tf.io.FixedLenFeature([], tf.string),\n        \"image\"                      : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['StudyInstanceUID']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    return image,label\n\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint('Dataset: {} unlabeled test images'.format(NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\ntest_ds = get_test_dataset()\nprint(\"Test:\", test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/kerasapplications -q\n!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.models.load_model(\"../input/ranzcr-clip-tpu/model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids=[]\ntest_pred = []\nj=0\nfor batch in test_ds:\n    images,ids_batch = batch\n    pred_batch = model.predict(images)\n    for i,ids in enumerate(ids_batch):\n        j+=1\n        if j%500 == 0:\n            print(str(j),\"Test Images Done\")\n        test_ids.append(ids)\n        test_pred.append(pred_batch[i])\n\ntest_ids = [np.array(i).astype(\"str\").tolist() for i in test_ids]\ntest_df = pd.DataFrame(test_pred,index=test_ids,columns=classes)\ntest_df.index.name = \"StudyInstanceUID\"\ntest_df.to_csv(\"./submission.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}