{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load Libraries","metadata":{"editable":false}},{"cell_type":"code","source":"import os\nimport glob\n\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\n\nimport random\nfrom tqdm.notebook import tqdm\nimport pydicom # Handle MRI images\n\nimport cv2  # OpenCV - https://docs.opencv.org/master/d6/d00/tutorial_py_root.html\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn import model_selection\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.initializers import RandomUniform\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:33:57.361777Z","iopub.execute_input":"2021-11-17T04:33:57.362104Z","iopub.status.idle":"2021-11-17T04:33:57.369527Z","shell.execute_reply.started":"2021-11-17T04:33:57.36206Z","shell.execute_reply":"2021-11-17T04:33:57.368426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Datasets","metadata":{"editable":false}},{"cell_type":"code","source":"data_dir = Path('../input/rsna-miccai-brain-tumor-radiogenomic-classification/')\n\nmri_types = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]\nexcluded_images = [109, 123, 709] # Bad images\n\ntrain_df = pd.read_csv(data_dir / \"train_labels.csv\")\ntest_df = pd.read_csv(data_dir / \"sample_submission.csv\")\nsample_submission = pd.read_csv(data_dir / \"sample_submission.csv\")\n\ntrain_df = train_df[~train_df.BraTS21ID.isin(excluded_images)].reset_index(drop=True)\n\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:33:57.371285Z","iopub.execute_input":"2021-11-17T04:33:57.371935Z","iopub.status.idle":"2021-11-17T04:33:57.418304Z","shell.execute_reply.started":"2021-11-17T04:33:57.371892Z","shell.execute_reply":"2021-11-17T04:33:57.417332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KFold - Future Features","metadata":{"editable":false}},{"cell_type":"code","source":"def create_folds(data, num_splits):\n    data[\"kfold\"] = -1\n    kf = model_selection.KFold(n_splits=num_splits, shuffle=True, random_state=42)\n    for f, (t, v) in enumerate(kf.split(X=data)):\n        data.loc[v, \"kfold\"] = f\n    return data","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:34:06.003424Z","iopub.execute_input":"2021-11-17T04:34:06.004448Z","iopub.status.idle":"2021-11-17T04:34:06.01164Z","shell.execute_reply.started":"2021-11-17T04:34:06.004406Z","shell.execute_reply":"2021-11-17T04:34:06.010708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EPOCHS=20\nk = 5\n\ntrain_df = create_folds(train_df, k)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:34:08.580698Z","iopub.execute_input":"2021-11-17T04:34:08.581012Z","iopub.status.idle":"2021-11-17T04:34:08.596971Z","shell.execute_reply.started":"2021-11-17T04:34:08.580984Z","shell.execute_reply":"2021-11-17T04:34:08.595957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:34:16.070516Z","iopub.execute_input":"2021-11-17T04:34:16.070858Z","iopub.status.idle":"2021-11-17T04:34:16.093512Z","shell.execute_reply.started":"2021-11-17T04:34:16.070816Z","shell.execute_reply":"2021-11-17T04:34:16.092667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{"editable":false}},{"cell_type":"code","source":"def load_dicom(path, size = 224):\n    ''' \n    Reads a DICOM image, standardizes so that the pixel values are between 0 and 1, then rescales to 0 and 255\n    \n    Not super sure if this kind of scaling is appropriate, but everyone seems to do it. \n    '''\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    # transform data into black and white scale / grayscale\n#     data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return cv2.resize(data, (size, size))\n\ndef get_all_image_paths(brats21id, image_type, folder='train'): \n    '''\n    Returns an arry of all the images of a particular type for a particular patient ID\n    '''\n    assert(image_type in mri_types)\n    \n    patient_path = os.path.join(\n        \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/%s/\" % folder, \n        str(brats21id).zfill(5),\n    )\n\n    paths = sorted(\n        glob.glob(os.path.join(patient_path, image_type, \"*\")), \n        key=lambda x: int(x[:-4].split(\"-\")[-1]),\n    )\n    \n    num_images = len(paths)\n    \n    start = int(num_images * 0.25)\n    end = int(num_images * 0.75)\n\n    interval = 3\n    \n    if num_images < 10: \n        interval = 1\n    \n    return np.array(paths[start:end:interval])\n\ndef get_all_images(brats21id, image_type, folder='train', size=225):\n    return [load_dicom(path, size) for path in get_all_image_paths(brats21id, image_type, folder)]\n\ndef get_all_data_for_train(image_type, image_size=32):\n    global train_df\n    \n    X = []\n    y = []\n    train_ids = []\n\n    for i in tqdm(train_df.index):\n        x = train_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'train', image_size)\n        label = x['MGMT_value']\n\n        X += images\n        y += [label] * len(images)\n        train_ids += [int(x['BraTS21ID'])] * len(images)\n        assert(len(X) == len(y))\n    return np.array(X), np.array(y), np.array(train_ids)\n\ndef get_all_data_for_test(image_type, image_size=32):\n    global test_df\n    \n    X = []\n    test_ids = []\n\n    for i in tqdm(test_df.index):\n        x = test_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'test', image_size)\n        X += images\n        test_ids += [int(x['BraTS21ID'])] * len(images)\n\n    return np.array(X), np.array(test_ids)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:34:25.115466Z","iopub.execute_input":"2021-11-17T04:34:25.116053Z","iopub.status.idle":"2021-11-17T04:34:25.135004Z","shell.execute_reply.started":"2021-11-17T04:34:25.116022Z","shell.execute_reply":"2021-11-17T04:34:25.133466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load all Images\n```\nX - contains all the images for each patient \ntrainidt - trainidt is a mask vector into X, y for training.  There's a patient id/BraTS21ID corresponding to each image (e.g. (0, 0, 0, 0, 2,2, 3,3,3,3,3,...) )\ntestidt - testidt is a mask vector into X_test for testing\n```","metadata":{"editable":false}},{"cell_type":"code","source":"X, y, trainidt = get_all_data_for_train('T1wCE', image_size=32)\nX_test, testidt = get_all_data_for_test('T1wCE', image_size=32)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T04:34:36.854892Z","iopub.execute_input":"2021-11-17T04:34:36.855207Z","iopub.status.idle":"2021-11-17T04:38:06.525156Z","shell.execute_reply.started":"2021-11-17T04:34:36.855163Z","shell.execute_reply":"2021-11-17T04:38:06.523907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y, trainidt = get_all_data_for_train('T2w', image_size=32)\nX_test, testidt = get_all_data_for_test('T2w', image_size=32)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:33:01.599396Z","iopub.execute_input":"2021-11-17T06:33:01.59973Z","iopub.status.idle":"2021-11-17T06:37:38.515727Z","shell.execute_reply.started":"2021-11-17T06:33:01.5997Z","shell.execute_reply":"2021-11-17T06:37:38.514735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train/Validation Split","metadata":{"editable":false}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid, trainidt_train, trainidt_valid = train_test_split(X, y, trainidt, test_size=0.2, random_state=42)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.518134Z","iopub.execute_input":"2021-11-17T06:37:38.518776Z","iopub.status.idle":"2021-11-17T06:37:38.537399Z","shell.execute_reply.started":"2021-11-17T06:37:38.518732Z","shell.execute_reply":"2021-11-17T06:37:38.536205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding a Dimension","metadata":{"editable":false}},{"cell_type":"code","source":"X_train = tf.expand_dims(X_train, axis=-1)\nX_valid = tf.expand_dims(X_valid, axis=-1)\nX_train.shape","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.539368Z","iopub.execute_input":"2021-11-17T06:37:38.539823Z","iopub.status.idle":"2021-11-17T06:37:38.565683Z","shell.execute_reply.started":"2021-11-17T06:37:38.539719Z","shell.execute_reply":"2021-11-17T06:37:38.564386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One-hot encode labels","metadata":{"editable":false}},{"cell_type":"code","source":"y_train = to_categorical(y_train)\ny_valid = to_categorical(y_valid)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.569301Z","iopub.execute_input":"2021-11-17T06:37:38.569681Z","iopub.status.idle":"2021-11-17T06:37:38.576525Z","shell.execute_reply.started":"2021-11-17T06:37:38.569613Z","shell.execute_reply":"2021-11-17T06:37:38.575119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tunable Model","metadata":{"editable":false}},{"cell_type":"markdown","source":"## Using the SIREN activation layer. Refer to https://vsitzmann.github.io/siren/ for more details.","metadata":{"editable":false}},{"cell_type":"code","source":"class SineDenseLayer(keras.layers.Layer):\n    # See paper sec. 3.2, final paragraph, and supplement Sec. 1.5 for discussion of omega_0.\n    \n    # If is_first=True, omega_0 is a frequency factor which simply multiplies the activations before the \n    # nonlinearity. Different signals may require different omega_0 in the first layer - this is a \n    # hyperparameter.\n    \n    # If is_first=False, then the weights will be divided by omega_0 so as to keep the magnitude of \n    # activations constant, but boost gradients to the weight matrix (see supplement Sec. 1.5)\n    \n    def __init__(self, features,\n                 is_first=False, omega_0=30):\n        super().__init__()\n        self.omega_0 = omega_0\n        self.is_first = is_first\n        \n        self.features = features\n        \n        if self.is_first:\n            initializer = RandomUniform(-1 / self.features, 1 / self.features)   \n            self.linear = keras.layers.Dense(features, kernel_initializer=initializer)\n    \n        else:\n            initializer = RandomUniform(-np.sqrt(6 / self.features) / self.omega_0, np.sqrt(6 / self.features) / self.omega_0)\n            self.linear = keras.layers.Dense(features, kernel_initializer=initializer)\n     \n\n    def call(self, input):\n        return tf.math.sin(self.omega_0 * self.linear(input))\n    \n#     def forward_with_intermediate(self, input): \n#         # For visualization of activation distributions\n#         intermediate = self.omega_0 * self.linear(input)\n#         return tf.math.sin(intermediate), intermediate\n\nclass SineConvLayer(keras.layers.Layer):\n    # See paper sec. 3.2, final paragraph, and supplement Sec. 1.5 for discussion of omega_0.\n    \n    # If is_first=True, omega_0 is a frequency factor which simply multiplies the activations before the \n    # nonlinearity. Different signals may require different omega_0 in the first layer - this is a \n    # hyperparameter.\n    \n    # If is_first=False, then the weights will be divided by omega_0 so as to keep the magnitude of \n    # activations constant, but boost gradients to the weight matrix (see supplement Sec. 1.5)\n    \n    def __init__(self, features, kernel_size,\n                 is_first=False, omega_0=30):\n        super().__init__()\n        self.omega_0 = omega_0\n        self.is_first = is_first\n        \n        self.features = features\n        \n        if self.is_first:\n            initializer = RandomUniform(-1 / self.features, 1 / self.features)            \n            self.conv = keras.layers.Conv2D(features, kernel_size, kernel_initializer=initializer)\n            \n        else:\n            initializer = RandomUniform(-np.sqrt(6 / self.features) / self.omega_0, np.sqrt(6 / self.features) / self.omega_0)\n            self.conv = keras.layers.Conv2D(features, kernel_size, kernel_initializer=initializer)\n            \n\n    def call(self, input):\n        return tf.math.sin(self.omega_0 * self.conv(input))\n    \n#     def forward_with_intermediate(self, input): \n#         # For visualization of activation distributions\n#         intermediate = self.omega_0 * self.linear(input)\n#         return tf.math.sin(intermediate), intermediate\n\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.578635Z","iopub.execute_input":"2021-11-17T06:37:38.57923Z","iopub.status.idle":"2021-11-17T06:37:38.597167Z","shell.execute_reply.started":"2021-11-17T06:37:38.579185Z","shell.execute_reply":"2021-11-17T06:37:38.596039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_tuner as kt\n\n\ndef make_model(hp):\n    inputs = keras.Input(shape=X_train.shape[1:])\n    \n    x = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(inputs)\n\n#     num_block = hp.Int('num_block', min_value=2, max_value=5, step=1)\n#     num_filters = hp.Int('num_filters', min_value=32, max_value=128, step=32)\n    \n#     x = keras.layers.Conv2D(64, kernel_size=(4, 4), activation=\"relu\", name=\"Conv_1\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_Conv_1_' + str(0),\n                                            min_value=64,\n                                            max_value=256,\n                                            step=32),\n                            kernel_size=(4, 4),\n                            activation=\"relu\", \n                            name=\"Conv_1\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_conv2_' + str(1),\n                                            min_value=16,\n                                            max_value=128,\n                                            step=16),\n                            kernel_size=(2, 2),\n                            activation=\"relu\",\n                            name=\"Conv_2\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n    \n#     for i in range(num_block):\n#         x = keras.layers.Conv2D(num_filters, \n#                                 kernel_size=(4, 4),\n#                                 activation=\"relu\",\n#                                 )(x)\n    \n#         x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n#     x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n\n#     h = keras.layers.Dropout(0.1)(h)\n    x = layers.Dropout(\n        hp.Float('dense_dropout', min_value=0., max_value=0.7)\n    )(x)\n    x = keras.layers.Flatten()(x)\n#     reduction_type = hp.Choice('reduction_type', ['flatten', 'avg'])\n#     if reduction_type == 'flatten':\n#         x = layers.Flatten()(x)\n#     else:\n#         x = layers.GlobalAveragePooling2D()(x)\n        \n#     x = keras.layers.Dense(32, activation=\"relu\")(x)\n    x = layers.Dense(\n        units=hp.Int('num_dense_units', min_value=16, max_value=64, step=8),\n        activation='relu'\n    )(x)\n\n    outputs = keras.layers.Dense(2, activation=\"softmax\")(x)\n\n    model = keras.Model(inputs, outputs)\n\n    roc_auc = tf.keras.metrics.AUC(name='roc_auc', curve='ROC')\n\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[roc_auc]\n    )\n    model.summary()\n    return model","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.599152Z","iopub.execute_input":"2021-11-17T06:37:38.59955Z","iopub.status.idle":"2021-11-17T06:37:38.61704Z","shell.execute_reply.started":"2021-11-17T06:37:38.599451Z","shell.execute_reply":"2021-11-17T06:37:38.615782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation\n\n- https://www.tensorflow.org/guide/keras/preprocessing_layers\n- https://keras.io/examples/vision/image_classification_from_scratch/","metadata":{"editable":false}},{"cell_type":"code","source":"def make_model_augmented(hp):\n    input_shape = (32, 32, 1)\n    classes = 10\n\n    # Create a data augmentation stage with horizontal flipping, rotations, zooms\n#     data_augmentation = keras.Sequential(\n#         [\n#             layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n#             layers.experimental.preprocessing.RandomRotation(0.1),\n#             layers.experimental.preprocessing.RandomZoom(0.1),\n#         ]\n#     )\n    \n    data_augmentation = keras.Sequential(\n    [\n        layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n        layers.experimental.preprocessing.RandomRotation(0.1),\n    ]\n)\n\n    shape=X_train.shape[1:]\n    print(f\"shape={shape}\") # shape=(32, 32, 1)\n    \n    inputs = keras.Input(shape=input_shape)\n    x = data_augmentation(inputs)\n\n    x = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(x)\n#     x = layers.experimental.preprocessing.RandomFlip(\"horizontal\")(x),\n#     x = layers.experimental.preprocessing.RandomRotation(0.1)(x),\n#     x = layers.experimental.preprocessing.RandomZoom(\n#         height_factor = 0.2,\n#         width_factor = -0.3,\n#         fill_mode = \"constant\",\n#         interpolation = \"bilinear\",\n#         seed = 42\n#     )(x),\n#     num_block = hp.Int('num_block', min_value=2, max_value=5, step=1)\n#     num_filters = hp.Int('num_filters', min_value=32, max_value=128, step=32)\n    \n#     x = keras.layers.Conv2D(64, kernel_size=(4, 4), activation=\"relu\", name=\"Conv_1\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_Conv_1_' + str(0),\n                                            min_value=64,\n                                            max_value=256,\n                                            step=32),\n                            kernel_size=(4, 4),\n                            activation=\"relu\", \n                            name=\"Conv_1\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_conv2_' + str(1),\n                                            min_value=16,\n                                            max_value=128,\n                                            step=16),\n                            kernel_size=(2, 2),\n                            activation=\"relu\",\n                            name=\"Conv_2\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n    \n#     for i in range(num_block):\n#         x = keras.layers.Conv2D(num_filters, \n#                                 kernel_size=(4, 4),\n#                                 activation=\"relu\",\n#                                 )(x)\n    \n#         x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n#     x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n\n#     h = keras.layers.Dropout(0.1)(h)\n    x = layers.Dropout(\n        hp.Float('dense_dropout', min_value=0., max_value=0.7)\n    )(x)\n    x = keras.layers.Flatten()(x)\n#     reduction_type = hp.Choice('reduction_type', ['flatten', 'avg'])\n#     if reduction_type == 'flatten':\n#         x = layers.Flatten()(x)\n#     else:\n#         x = layers.GlobalAveragePooling2D()(x)\n        \n#     x = keras.layers.Dense(32, activation=\"relu\")(x)\n    x = layers.Dense(\n        units=hp.Int('num_dense_units', min_value=16, max_value=64, step=8),\n        activation='relu'\n    )(x)\n\n    outputs = keras.layers.Dense(2, activation=\"softmax\")(x)\n\n    model = keras.Model(inputs, outputs)\n\n    roc_auc = tf.keras.metrics.AUC(name='roc_auc', curve='ROC')\n\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[roc_auc]\n    )\n    model.summary()\n    return model","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.618772Z","iopub.execute_input":"2021-11-17T06:37:38.619877Z","iopub.status.idle":"2021-11-17T06:37:38.638893Z","shell.execute_reply.started":"2021-11-17T06:37:38.619832Z","shell.execute_reply":"2021-11-17T06:37:38.637907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_tuner as kt\n\n\ndef make_model_siren(hp):\n    inputs = keras.Input(shape=X_train.shape[1:])\n    \n    x = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(inputs)\n\n    x = SineConvLayer(features=hp.Int('features_conv_1', min_value=64, max_value=256, step=32),\n                      kernel_size=hp.Int('kernel_conv_1', min_value=2, max_value=7, step=1),\n                      is_first=True, \n                      omega_0=hp.Int('omega_0_conv_1', min_value=10, max_value=50, step=5))(x)\n    \n    x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n    x = SineConvLayer(features=hp.Int('features_conv_2', min_value=16, max_value=128, step=16),\n                      kernel_size=hp.Int('kernel_conv_2', min_value=2, max_value=7, step=1),\n                      is_first=False, \n                      omega_0=hp.Int('omega_0_conv_2', min_value=10, max_value=50, step=5))(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n    \n    x = layers.Dropout(\n        hp.Float('dense_dropout', min_value=0., max_value=0.7)\n    )(x)\n    x = keras.layers.Flatten()(x)\n    x = SineDenseLayer(features=hp.Int('features_dense_1', min_value=64, max_value=256, step=32),\n                      is_first=False, \n                      omega_0=hp.Int('omega_0_dense_1', min_value=10, max_value=50, step=5))(x)\n\n    outputs = keras.layers.Dense(2, activation=\"softmax\")(x)\n\n    model = keras.Model(inputs, outputs)\n\n    roc_auc = tf.keras.metrics.AUC(name='roc_auc', curve='ROC')\n\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[roc_auc]\n    )\n    model.summary()\n    return model","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.640406Z","iopub.execute_input":"2021-11-17T06:37:38.64128Z","iopub.status.idle":"2021-11-17T06:37:38.658742Z","shell.execute_reply.started":"2021-11-17T06:37:38.641235Z","shell.execute_reply":"2021-11-17T06:37:38.657572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameter Search","metadata":{"editable":false}},{"cell_type":"code","source":"tuner = kt.tuners.BayesianOptimization(\n#     make_model_siren,\n#     make_model,\n    make_model_augmented,\n    objective='val_loss',\n    max_trials=5,  # Set to 5 to run quicker, but need 100+ for good results\n    overwrite=True)\n\ncallbacks=[keras.callbacks.EarlyStopping(monitor='val_roc_acc', mode='max', patience=3, baseline=0.9)]\n\ntuner.search(X_train, y_train, validation_split=0.2, callbacks=callbacks, verbose=1, epochs=20)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:37:38.662479Z","iopub.execute_input":"2021-11-17T06:37:38.662814Z","iopub.status.idle":"2021-11-17T06:42:15.805049Z","shell.execute_reply.started":"2021-11-17T06:37:38.662771Z","shell.execute_reply":"2021-11-17T06:42:15.803776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find the best epoch value","metadata":{"editable":false}},{"cell_type":"code","source":"best_hp = tuner.get_best_hyperparameters()[0]\nbest_model = make_model(best_hp)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:42:15.812678Z","iopub.execute_input":"2021-11-17T06:42:15.813338Z","iopub.status.idle":"2021-11-17T06:42:15.953816Z","shell.execute_reply.started":"2021-11-17T06:42:15.813293Z","shell.execute_reply":"2021-11-17T06:42:15.952748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Model","metadata":{"editable":false}},{"cell_type":"code","source":"best_model.save(\"best_model\")","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:42:15.958169Z","iopub.execute_input":"2021-11-17T06:42:15.95869Z","iopub.status.idle":"2021-11-17T06:42:17.15559Z","shell.execute_reply.started":"2021-11-17T06:42:15.958596Z","shell.execute_reply":"2021-11-17T06:42:17.154467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = best_model.fit(X_train, y_train, validation_split=0.2, epochs=50)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:42:17.162385Z","iopub.execute_input":"2021-11-17T06:42:17.16266Z","iopub.status.idle":"2021-11-17T06:44:07.15667Z","shell.execute_reply.started":"2021-11-17T06:42:17.162614Z","shell.execute_reply":"2021-11-17T06:44:07.155718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nloss = history.history['roc_auc']\nval_loss = history.history['val_roc_auc']\n\ne = range(50)\n\nplt.figure()\nplt.plot(e,loss, 'r', label='Train')\nplt.plot(e,val_loss, 'b', label='Validation')\n# plt.yscale(\"log\")\n# plt.title('ROC')\n# plt.xlabel('Epoch')\n# plt.ylabel('Loss Value')\n# plt.ylim([0.9, 0.7])\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:44:07.158946Z","iopub.execute_input":"2021-11-17T06:44:07.159307Z","iopub.status.idle":"2021-11-17T06:44:07.453089Z","shell.execute_reply.started":"2021-11-17T06:44:07.159251Z","shell.execute_reply":"2021-11-17T06:44:07.451878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner.results_summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:44:07.455211Z","iopub.execute_input":"2021-11-17T06:44:07.455527Z","iopub.status.idle":"2021-11-17T06:44:07.469451Z","shell.execute_reply.started":"2021-11-17T06:44:07.455479Z","shell.execute_reply":"2021-11-17T06:44:07.468538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on Validation Set","metadata":{"editable":false}},{"cell_type":"code","source":"y_pred = best_model.predict(X_valid)\n\npred = np.argmax(y_pred, axis=1)\n\nresult = pd.DataFrame(trainidt_valid)\nresult[1] = pred\n\nresult.columns = [\"BraTS21ID\", \"MGMT_value\"]\nresult2 = result.groupby(\"BraTS21ID\", as_index=False).mean()\nresult2","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:44:07.470978Z","iopub.execute_input":"2021-11-17T06:44:07.471347Z","iopub.status.idle":"2021-11-17T06:44:07.76645Z","shell.execute_reply.started":"2021-11-17T06:44:07.471294Z","shell.execute_reply":"2021-11-17T06:44:07.765196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = best_model.predict(X_valid)\n\npred = np.argmax(y_pred, axis=1)\n\nresult = pd.DataFrame(trainidt_valid)\nresult[1] = pred\n\nresult.columns = [\"BraTS21ID\", \"MGMT_value\"]\nresult2 = result.groupby(\"BraTS21ID\", as_index=False).mean()\nresult2","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:44:07.76874Z","iopub.execute_input":"2021-11-17T06:44:07.769449Z","iopub.status.idle":"2021-11-17T06:44:07.979139Z","shell.execute_reply.started":"2021-11-17T06:44:07.769394Z","shell.execute_reply":"2021-11-17T06:44:07.97793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve\ny_pred = best_model.predict_on_batch(X_valid)\n# print(y_pred)\npred = np.argmax(y_pred, axis=1)\nvalid = np.argmax(y_valid, axis=1)\nns_fpr, ns_tpr, _ = roc_curve(valid, pred)\nimport matplotlib.pyplot as plt\ny_pred_train = best_model.predict_on_batch(X_train)\npred_t = np.argmax(y_pred_train, axis=1)\ntrain_t = np.argmax(y_train, axis=1)\nfpr, tpr, _ = roc_curve(train_t, pred_t)\nplt.figure()\nplt.plot(fpr,tpr,'r',label='Train (AUC ={0:.2f})'.format(loss[-1]))\nplt.plot(ns_fpr,ns_tpr,'b', label='validation (AUC ={0:.2f})'.format(val_loss[-1]))\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T11:48:36.781401Z","iopub.execute_input":"2021-12-12T11:48:36.781696Z","iopub.status.idle":"2021-12-12T11:48:37.013379Z","shell.execute_reply.started":"2021-12-12T11:48:36.781667Z","shell.execute_reply":"2021-12-12T11:48:37.012525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result2 = result2.merge(train_df, on=\"BraTS21ID\")\nresult2","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:44:08.769926Z","iopub.execute_input":"2021-11-17T06:44:08.770691Z","iopub.status.idle":"2021-11-17T06:44:08.799465Z","shell.execute_reply.started":"2021-11-17T06:44:08.770511Z","shell.execute_reply":"2021-11-17T06:44:08.798358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc = roc_auc_score(\n    result2.MGMT_value_y,\n    result2.MGMT_value_x,\n)\nprint(f\"Validation AUC={auc}\")\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:44:08.80161Z","iopub.execute_input":"2021-11-17T06:44:08.802112Z","iopub.status.idle":"2021-11-17T06:44:08.814453Z","shell.execute_reply.started":"2021-11-17T06:44:08.802069Z","shell.execute_reply":"2021-11-17T06:44:08.812962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on the Test Set","metadata":{"editable":false}},{"cell_type":"code","source":"y_pred = best_model.predict(X_test)\n\npred = np.argmax(y_pred, axis=1) #\n\nresult = pd.DataFrame(testidt)\nresult[1] = pred\npred","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:44:08.816599Z","iopub.execute_input":"2021-11-17T06:44:08.817325Z","iopub.status.idle":"2021-11-17T06:44:09.074951Z","shell.execute_reply.started":"2021-11-17T06:44:08.817269Z","shell.execute_reply":"2021-11-17T06:44:09.073885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission File","metadata":{"editable":false}},{"cell_type":"code","source":"result.columns=['BraTS21ID','MGMT_value']\n\nresult2 = result.groupby('BraTS21ID',as_index=False).mean()\nresult2['BraTS21ID'] = sample_submission['BraTS21ID']\n\nresult2","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:25:03.185235Z","iopub.execute_input":"2021-11-17T06:25:03.185892Z","iopub.status.idle":"2021-11-17T06:25:03.226611Z","shell.execute_reply.started":"2021-11-17T06:25:03.185848Z","shell.execute_reply":"2021-11-17T06:25:03.22562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rounding... 0.907866 -> 0.9\nresult2['MGMT_value'] = result2['MGMT_value'].apply(lambda x:round(x*10)/10)\n# result2['MGMT_value'] = result2['MGMT_value'] # No rounding\nresult2.to_csv('submission.csv',index=False)\nresult2","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-17T06:25:03.230627Z","iopub.execute_input":"2021-11-17T06:25:03.230948Z","iopub.status.idle":"2021-11-17T06:25:03.252341Z","shell.execute_reply.started":"2021-11-17T06:25:03.230906Z","shell.execute_reply":"2021-11-17T06:25:03.251385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}