{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 📝<span style=\"font-family:cursive;\">Final Part From TPU Session</span>"},{"metadata":{},"cell_type":"markdown","source":"## 🏆 <a href='https://www.kaggle.com/fauzanalfariz/ranzcr-detection-with-xception'> [Part 1] -> [RANZCR]: Detection With Xception | TPU Session</a>"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-12-31T04:50:50.344102Z","iopub.status.busy":"2020-12-31T04:50:50.343592Z","iopub.status.idle":"2020-12-31T04:50:54.557002Z","shell.execute_reply":"2020-12-31T04:50:54.556056Z"},"papermill":{"duration":4.225145,"end_time":"2020-12-31T04:50:54.557118","exception":false,"start_time":"2020-12-31T04:50:50.331973","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Importing useful libraries\n\nimport pandas as pd \nimport numpy as np\nimport os\nimport tensorflow as tf\n\nimport math\nimport seaborn as sns\nimport albumentations as A \nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport os, gc, cv2, random, warnings, math, sys, json, pprint\nfrom glob import glob\nfrom pylab import rcParams\n\n# sklearn\nfrom sklearn.model_selection import  GroupKFold\nfrom sklearn.metrics import roc_auc_score\n\n\n#import efficientnet.tfkeras as efn\nfrom tensorflow.keras import backend as K\n\n# torch\nimport torch\nimport torchvision\nfrom torchvision import transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\n\ndata_dir ='../input/ranzcr-clip-catheter-line-classification/'\n\ntrain = os.path.join(data_dir, 'train.csv')\ntrain_df = pd.read_csv(train)  \n\nsub = os.path.join(data_dir,'sample_submission.csv')\nsub_df = pd.read_csv(sub)\n\ntest_images = data_dir + \"test/\" + sub_df['StudyInstanceUID'] + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def NeedleAugmentation(image, n_needles=2, dark_needles=False, p=0.5, needle_folder='../input/xray-needle-augmentation'):\n    \n    '''\n      Open CV - Based Custom Augmentation\n    '''\n    \n    aug_prob = random.random()\n    if aug_prob < p:\n        height, width, _ = image.shape  # target image width and height\n        needle_images = [im for im in os.listdir(needle_folder) if 'png' in im]\n\n        for _ in range(1, n_needles):\n            needle = cv2.cvtColor(cv2.imread(os.path.join(needle_folder, random.choice(needle_images))), cv2.COLOR_BGR2RGB)\n            needle = cv2.flip(needle, random.choice([-1, 0, 1]))\n            needle = cv2.rotate(needle, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = needle.shape  # needle image width and height\n            roi_ho = random.randint(0, abs(image.shape[0] - needle.shape[0]))\n            roi_wo = random.randint(0, abs(image.shape[1] - needle.shape[1]))\n            roi = image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask \n            img2gray = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of needle in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of insect from insect image.\n            if dark_needles:\n                img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n                needle_fg = cv2.bitwise_and(img_bg, img_bg, mask=mask)\n            else:\n                needle_fg = cv2.bitwise_and(needle, needle, mask=mask)\n\n            # Put needle in ROI and modify the target image\n            dst = cv2.add(img_bg, needle_fg, dtype=cv2.CV_64F)\n\n            image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n\n    return image","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-31T04:50:54.57664Z","iopub.status.busy":"2020-12-31T04:50:54.57469Z","iopub.status.idle":"2020-12-31T04:50:54.577337Z","shell.execute_reply":"2020-12-31T04:50:54.577772Z"},"papermill":{"duration":0.013673,"end_time":"2020-12-31T04:50:54.577877","exception":false,"start_time":"2020-12-31T04:50:54.564204","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TARGET_SIZE = 750","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-31T04:50:54.615601Z","iopub.status.busy":"2020-12-31T04:50:54.613911Z","iopub.status.idle":"2020-12-31T04:50:54.616613Z","shell.execute_reply":"2020-12-31T04:50:54.617025Z"},"papermill":{"duration":0.032566,"end_time":"2020-12-31T04:50:54.617134","exception":false,"start_time":"2020-12-31T04:50:54.584568","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def build_decoder(with_labels = True,\n                  target_size = (TARGET_SIZE, TARGET_SIZE), \n                  ext = 'jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels = 3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels = 3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels = True):\n    def augment(img):\n        img = NeedleAugmentation(img, n_needles=2, dark_needles=False, p=0.5)\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels = None, bsize = 32, cache = True,\n                  decode_fn = None, augment_fn = None,\n                  augment = True, repeat = True, shuffle = 1024, \n                  cache_dir = \"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls = AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls = AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"strategy = auto_select_accelerator()\nbatch_size = strategy.num_replicas_in_sync * 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = tf.keras.models.load_model('../input/ranzcr-detection-with-xception/Xception_750_TPU.h5')\n                                       #custom_objects={'FixedDropout': tf.keras.layers.Dropout})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_decoder = build_decoder(with_labels=False, target_size=(TARGET_SIZE, TARGET_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-31T04:51:03.839841Z","iopub.status.busy":"2020-12-31T04:51:03.839306Z","iopub.status.idle":"2020-12-31T04:51:04.020033Z","shell.execute_reply":"2020-12-31T04:51:04.019066Z"},"papermill":{"duration":0.200796,"end_time":"2020-12-31T04:51:04.020142","exception":false,"start_time":"2020-12-31T04:51:03.819346","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_df = build_dataset(\n    test_images, bsize=batch_size, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    #decode_fn=test_decoder \n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-31T04:51:04.042154Z","iopub.status.busy":"2020-12-31T04:51:04.041612Z","iopub.status.idle":"2020-12-31T04:55:51.209497Z","shell.execute_reply":"2020-12-31T04:55:51.208995Z"},"papermill":{"duration":287.180593,"end_time":"2020-12-31T04:55:51.209618","exception":false,"start_time":"2020-12-31T04:51:04.029025","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"y_preds = model.predict(test_df, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-31T04:55:51.363257Z","iopub.status.busy":"2020-12-31T04:55:51.361589Z","iopub.status.idle":"2020-12-31T04:55:51.400127Z","shell.execute_reply":"2020-12-31T04:55:51.399288Z"},"papermill":{"duration":0.118491,"end_time":"2020-12-31T04:55:51.400261","exception":false,"start_time":"2020-12-31T04:55:51.28177","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sub_df.iloc[:, 1:] = y_preds\ndisplay(sub_df)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-31T04:55:51.547299Z","iopub.status.busy":"2020-12-31T04:55:51.546791Z","iopub.status.idle":"2020-12-31T04:55:51.838794Z","shell.execute_reply":"2020-12-31T04:55:51.838008Z"},"papermill":{"duration":0.366821,"end_time":"2020-12-31T04:55:51.838899","exception":false,"start_time":"2020-12-31T04:55:51.472078","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 😄<span style=\"font-family:cursive;\">if this notebook,it is very useful for you let's to upvote :)</span>"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}