{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1322485,"sourceType":"datasetVersion","datasetId":758225},{"sourceId":1322494,"sourceType":"datasetVersion","datasetId":688574},{"sourceId":1322517,"sourceType":"datasetVersion","datasetId":689329},{"sourceId":1322552,"sourceType":"datasetVersion","datasetId":688719},{"sourceId":1322612,"sourceType":"datasetVersion","datasetId":689578},{"sourceId":1324333,"sourceType":"datasetVersion","datasetId":762256},{"sourceId":1324349,"sourceType":"datasetVersion","datasetId":762108},{"sourceId":1324366,"sourceType":"datasetVersion","datasetId":762176},{"sourceId":1324385,"sourceType":"datasetVersion","datasetId":762138},{"sourceId":1324412,"sourceType":"datasetVersion","datasetId":762168},{"sourceId":1324430,"sourceType":"datasetVersion","datasetId":768238},{"sourceId":1324483,"sourceType":"datasetVersion","datasetId":758409},{"sourceId":1339671,"sourceType":"datasetVersion","datasetId":758489}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Imports**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport seaborn as sns\nimport PIL\nimport time\nimport os\nimport torch\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom IPython.display import Image\nimport tensorflow.keras.backend as K\nfrom keras.losses import binary_crossentropy\nfrom tensorflow.keras.applications import ResNet50, ConvNeXtTiny, ConvNeXtSmall, ConvNeXtBase, ConvNeXtLarge, ConvNeXtXLarge\nfrom tensorflow.keras.losses import BinaryCrossentropy, BinaryFocalCrossentropy\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split, KFold\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\nfrom tensorflow.keras.layers import Flatten, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import optimizers\nimport tensorflow as tf, re, math\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nimport tempfile","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:06.044667Z","iopub.execute_input":"2024-03-09T13:49:06.045545Z","iopub.status.idle":"2024-03-09T13:49:25.720747Z","shell.execute_reply.started":"2024-03-09T13:49:06.045516Z","shell.execute_reply":"2024-03-09T13:49:25.719921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Versión de TensorFlow:\", tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:25.722475Z","iopub.execute_input":"2024-03-09T13:49:25.723029Z","iopub.status.idle":"2024-03-09T13:49:25.728784Z","shell.execute_reply.started":"2024-03-09T13:49:25.723003Z","shell.execute_reply":"2024-03-09T13:49:25.727710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dir(tf.keras.applications))","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:25.734292Z","iopub.execute_input":"2024-03-09T13:49:25.734651Z","iopub.status.idle":"2024-03-09T13:49:25.779017Z","shell.execute_reply.started":"2024-03-09T13:49:25.734608Z","shell.execute_reply":"2024-03-09T13:49:25.778079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Kaggle's SIIM-ISIC Melanoma Classification**","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_csv = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:25.779992Z","iopub.execute_input":"2024-03-09T13:49:25.780261Z","iopub.status.idle":"2024-03-09T13:49:25.905611Z","shell.execute_reply.started":"2024-03-09T13:49:25.780239Z","shell.execute_reply":"2024-03-09T13:49:25.904355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_img = train_csv['target'].size\nmalignant = np.count_nonzero(train_csv['target'])\nbenign = total_img - malignant\n\nprint('Total {} {:.2f}% \\nBenign {} {:.2f}% \\nMalignant {} {:.2f}%'.format(total_img,100 * total_img / total_img, benign, 100 * benign / total_img, malignant, 100 * malignant / total_img))","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:25.907056Z","iopub.execute_input":"2024-03-09T13:49:25.907365Z","iopub.status.idle":"2024-03-09T13:49:25.916072Z","shell.execute_reply.started":"2024-03-09T13:49:25.907340Z","shell.execute_reply":"2024-03-09T13:49:25.915015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2, pandas as pd, matplotlib.pyplot as plt\n\ntrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n\nprint('Images with Melanoma')\nimgs = train.loc[train.target==1].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n img = cv2.imread('../input/jpeg-melanoma-128x128/train/%s.jpg'%k)\n img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n plt.subplot(1,10,i+1); plt.axis('off')\n plt.imshow(img)\nplt.show()\n\nprint('Images without Melanoma')\nimgs = train.loc[train.target==0].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n img = cv2.imread('../input/jpeg-melanoma-128x128/train/%s.jpg'%k)\n img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n plt.subplot(1,10,i+1); plt.axis('off')\n plt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:25.917345Z","iopub.execute_input":"2024-03-09T13:49:25.917645Z","iopub.status.idle":"2024-03-09T13:49:27.190898Z","shell.execute_reply.started":"2024-03-09T13:49:25.917620Z","shell.execute_reply":"2024-03-09T13:49:27.190027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Parameters**\nIn order to be a proper cross validation with a meaningful overall CV score (aligned with LB score), **you need to choose the same** `IMG_SIZES`, `INC2019`, `INC2018`, and `EFF_NETS` **for each fold**. If your goal is to just run lots of experiments, then you can choose to have a different experiment in each fold. Then each fold is like a holdout validation experiment. When you find a configuration you like, you can use that configuration for all folds. \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DEVICEis GPU or TPU.
SEEDa different seed produces a different triple stratified kfold split.
FOLDSnumber of folds. Best set to 3, 5, or 15 but can be any number between 2 and 15.
IMG_SIZESis a Python list of length FOLDS. These are the image sizes to use each fold.
INC2019This includes the new half of the 2019 competition data.
INC2018This includes the second half of the 2019 competition data.
BATCH_SIZESis a list of length FOLDS. These are batch sizes for each fold.
EPOCHSis a list of length FOLDS. These are maximum epochs.
EFF_NETSis a list of length FOLDS. These are the ConvNeXt to use each fold.
EFF_NETS_NUMis a list of length FOLDS. These are the index for each item of EFF_NETS.
WGTSthis should be `1/FOLDS` for each fold.
TTAtest time augmentation.
\n","metadata":{}},{"cell_type":"code","source":"DEVICE = \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\n# NUMBER OF FOLDS. USE 3, 5, OR 15 \nFOLDS = 3\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD: 128, 192, 256, 384, 512, 768 \nIMG_SIZES = [128,128,128]\n\n# INCLUDE OLD COMP DATA? YES=1 NO=0\nINC2019 = [0,0,0]\nINC2018 = [0,0,0]\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [32]*FOLDS\nEPOCHS = [25]*FOLDS\n\n# WHICH CONVNEXT VARIANT TO USE: [Tiny, Small, Base, Large, XLarge]\nEFF_NETS = ['Tiny','Tiny','Tiny']\nEFF_NETS_NUM = [0,1,2]\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 11","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:27.192114Z","iopub.execute_input":"2024-03-09T13:49:27.192446Z","iopub.status.idle":"2024-03-09T13:49:27.200423Z","shell.execute_reply.started":"2024-03-09T13:49:27.192405Z","shell.execute_reply":"2024-03-09T13:49:27.199316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n print(\"connecting to TPU...\")\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Running on TPU ', tpu.master())\n except ValueError:\n print(\"Could not connect to TPU\")\n tpu = None\n\n if tpu:\n try:\n print(\"initializing TPU ...\")\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(\"TPU initialized\")\n except _:\n print(\"failed to initialize TPU\")\n else:\n DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n print(\"Using default strategy for CPU and single GPU\")\n strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n \n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:27.201534Z","iopub.execute_input":"2024-03-09T13:49:27.201810Z","iopub.status.idle":"2024-03-09T13:49:27.328025Z","shell.execute_reply.started":"2024-03-09T13:49:27.201787Z","shell.execute_reply":"2024-03-09T13:49:27.326795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 1: Preprocess Data**\nPreprocess has already been done and saved to TFRecords. Here we choose which size to load. We can use either 128x128, 192x192, 256x256, 384x384, 512x512, 768x768 by changing the `IMG_SIZES` variable in the preceeding code section. These TFRecords are discussed [here][1]. The advantage of using different input sizes is discussed [here][2]\n\n[1]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/155579\n[2]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/160147","metadata":{}},{"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS\n\nfor i,k in enumerate(IMG_SIZES):\n GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))\n GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\nfiles_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:27.331774Z","iopub.execute_input":"2024-03-09T13:49:27.332154Z","iopub.status.idle":"2024-03-09T13:49:29.340622Z","shell.execute_reply.started":"2024-03-09T13:49:27.332126Z","shell.execute_reply":"2024-03-09T13:49:29.339792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 2: Data Augmentation**\nThis notebook uses rotation, sheer, zoom, shift augmentation first shown in this notebook [here][1] and successfully used in Melanoma comp by AgentAuers [here][2]. This notebook also uses horizontal flip, hue, saturation, contrast, brightness augmentation similar to last years winner and also similar to AgentAuers' notebook.\n\nAdditionally we can decide to use external data by changing the variables `INC2019` and `INC2018` in the preceeding code section. These variables respectively indicate whether to load last year 2019 data and/or year 2018 + 2017 data. These datasets are discussed [here][3]\n\nConsider experimenting with different augmenation and/or external data. The code to load TFRecords is taken from AgentAuers' notebook [here][2]. Thank you AgentAuers, this is great work.\n\n[1]: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\n[2]: https://www.kaggle.com/agentauers/incredible-tpus-finetune-effnetb0-b6-at-once\n[3]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910","metadata":{}},{"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.341712Z","iopub.execute_input":"2024-03-09T13:49:29.341994Z","iopub.status.idle":"2024-03-09T13:49:29.346810Z","shell.execute_reply.started":"2024-03-09T13:49:29.341971Z","shell.execute_reply":"2024-03-09T13:49:29.345652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n # returns 3x3 transformmatrix which transforms indicies\n \n # CONVERT DEGREES TO RADIANS\n rotation = math.pi * rotation / 180.\n shear = math.pi * shear / 180.\n\n def get_3x3_mat(lst):\n return tf.reshape(tf.concat([lst],axis=0), [3,3])\n \n # ROTATION MATRIX\n c1 = tf.math.cos(rotation)\n s1 = tf.math.sin(rotation)\n one = tf.constant([1],dtype='float32')\n zero = tf.constant([0],dtype='float32')\n \n rotation_matrix = get_3x3_mat([c1, s1, zero, \n -s1, c1, zero, \n zero, zero, one]) \n # SHEAR MATRIX\n c2 = tf.math.cos(shear)\n s2 = tf.math.sin(shear) \n \n shear_matrix = get_3x3_mat([one, s2, zero, \n zero, c2, zero, \n zero, zero, one]) \n # ZOOM MATRIX\n zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero, \n zero, one/width_zoom, zero, \n zero, zero, one]) \n # SHIFT MATRIX\n shift_matrix = get_3x3_mat([one, zero, height_shift, \n zero, one, width_shift, \n zero, zero, one])\n \n return K.dot(K.dot(rotation_matrix, shear_matrix), \n K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, DIM=256): \n # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n # output - image randomly rotated, sheared, zoomed, and shifted\n XDIM = DIM%2 #fix for size 331\n \n rot = ROT_ * tf.random.normal([1], dtype='float32')\n shr = SHR_ * tf.random.normal([1], dtype='float32') \n h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n # GET TRANSFORMATION MATRIX\n m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n # LIST DESTINATION PIXEL INDICES\n x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n z = tf.ones([DIM*DIM], dtype='int32')\n idx = tf.stack( [x,y,z] )\n \n # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n idx2 = K.cast(idx2, dtype='int32')\n idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n \n # FIND ORIGIN PIXEL VALUES \n idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n d = tf.gather_nd(image, tf.transpose(idx3))\n \n return tf.reshape(d,[DIM, DIM,3])","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.348455Z","iopub.execute_input":"2024-03-09T13:49:29.348763Z","iopub.status.idle":"2024-03-09T13:49:29.376035Z","shell.execute_reply.started":"2024-03-09T13:49:29.348739Z","shell.execute_reply":"2024-03-09T13:49:29.375076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n tfrec_format = {\n 'image' : tf.io.FixedLenFeature([], tf.string),\n 'image_name' : tf.io.FixedLenFeature([], tf.string),\n 'patient_id' : tf.io.FixedLenFeature([], tf.int64),\n 'sex' : tf.io.FixedLenFeature([], tf.int64),\n 'age_approx' : tf.io.FixedLenFeature([], tf.int64),\n 'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n 'diagnosis' : tf.io.FixedLenFeature([], tf.int64),\n 'target' : tf.io.FixedLenFeature([], tf.int64)\n } \n example = tf.io.parse_single_example(example, tfrec_format)\n return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n tfrec_format = {\n 'image' : tf.io.FixedLenFeature([], tf.string),\n 'image_name' : tf.io.FixedLenFeature([], tf.string),\n }\n example = tf.io.parse_single_example(example, tfrec_format)\n return example['image'], example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True, dim=256): \n img = tf.image.decode_jpeg(img, channels=3)\n img = tf.cast(img, tf.float32) / 255.0\n \n if augment:\n img = transform(img,DIM=dim)\n img = tf.image.random_flip_left_right(img)\n #img = tf.image.random_hue(img, 0.01)\n img = tf.image.random_saturation(img, 0.7, 1.3)\n img = tf.image.random_contrast(img, 0.8, 1.2)\n img = tf.image.random_brightness(img, 0.1)\n \n img = tf.reshape(img, [dim,dim, 3])\n \n return img\n\ndef count_data_items(filenames):\n n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n for filename in filenames]\n return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.377378Z","iopub.execute_input":"2024-03-09T13:49:29.377758Z","iopub.status.idle":"2024-03-09T13:49:29.392105Z","shell.execute_reply.started":"2024-03-09T13:49:29.377733Z","shell.execute_reply":"2024-03-09T13:49:29.391290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n labeled=True, return_image_names=True, batch_size=16, dim=256):\n \n ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n ds = ds.cache()\n \n if repeat:\n ds = ds.repeat()\n \n if shuffle: \n ds = ds.shuffle(1024*8)\n opt= tf.data.Options()\n opt.experimental_deterministic = False\n ds = ds.with_options(opt)\n \n if labeled: \n ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n else:\n ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n num_parallel_calls=AUTO) \n \n ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim),imgname_or_label), num_parallel_calls=AUTO)\n \n ds = ds.batch(batch_size * REPLICAS)\n ds = ds.prefetch(AUTO)\n return ds","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.393400Z","iopub.execute_input":"2024-03-09T13:49:29.393771Z","iopub.status.idle":"2024-03-09T13:49:29.407336Z","shell.execute_reply.started":"2024-03-09T13:49:29.393734Z","shell.execute_reply":"2024-03-09T13:49:29.406373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 3: Build Model**\nThis is a common model architecute. Consider experimenting with different backbones, custom heads, losses, and optimizers. Also consider inputing meta features into your CNN.","metadata":{}},{"cell_type":"code","source":"EFNS = [ConvNeXtTiny, ConvNeXtTiny, ConvNeXtTiny, ConvNeXtTiny, ConvNeXtTiny]\n\ndef build_model(dim=128, ef=0):\n inp = tf.keras.layers.Input(shape=(dim,dim,3))\n base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)\n \n x = base(inp)\n x = tf.keras.layers.GlobalAveragePooling2D()(x)\n x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n \n model = tf.keras.Model(inputs=inp,outputs=x)\n opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n \n model.compile(optimizer=opt,loss=loss,metrics= ['AUC',\n 'accuracy',\n tf.keras.metrics.Recall(),\n tf.keras.metrics.Precision(),\n tf.keras.metrics.TruePositives(),\n tf.keras.metrics.TrueNegatives(),\n tf.keras.metrics.FalsePositives(),\n tf.keras.metrics.FalseNegatives()\n ])\n model.summary()\n return model","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.408583Z","iopub.execute_input":"2024-03-09T13:49:29.408959Z","iopub.status.idle":"2024-03-09T13:49:29.421650Z","shell.execute_reply.started":"2024-03-09T13:49:29.408934Z","shell.execute_reply":"2024-03-09T13:49:29.420894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 4: Train Schedule**\nThis is a common train schedule for transfer learning. The learning rate starts near zero, then increases to a maximum, then decays over time. Consider changing the schedule and/or learning rates. Note how the learning rate max is larger with larger batches sizes. This is a good practice to follow.","metadata":{}},{"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n lr_start = 0.000005\n lr_max = 0.00000125 * REPLICAS * batch_size\n lr_min = 0.000001\n lr_ramp_ep = 5\n lr_sus_ep = 0\n lr_decay = 0.8\n \n def lrfn(epoch):\n if epoch < lr_ramp_ep:\n lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n \n elif epoch < lr_ramp_ep + lr_sus_ep:\n lr = lr_max\n \n else:\n lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n \n return lr\n\n lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n return lr_callback","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.422664Z","iopub.execute_input":"2024-03-09T13:49:29.422948Z","iopub.status.idle":"2024-03-09T13:49:29.434647Z","shell.execute_reply.started":"2024-03-09T13:49:29.422925Z","shell.execute_reply":"2024-03-09T13:49:29.433925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model\nOur model will be trained for the number of FOLDS and EPOCHS you chose in the configuration above. Each fold the model with lowest validation loss will be saved and used to predict OOF and test. Adjust the variables `VERBOSE` and `DISPLOY_PLOT` below to determine what output you want displayed. The variable `VERBOSE=1 or 2` will display the training and validation loss and auc for each epoch as text. The variable `DISPLAY_PLOT` shows this information as a plot. ","metadata":{}},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\ntime_lst = []; oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n \n # DISPLAY FOLD INFO\n if DEVICE=='TPU':\n if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n print('#'*25); print('#### FOLD',fold+1)\n print('#### Image Size %i with ConvNeXt%s and batch_size %i'%\n (IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n \n # CREATE TRAIN AND VALIDATION SUBSETS\n files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])\n \n if INC2019[fold]:\n files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])\n print('#### Using 2019 external data')\n \n if INC2018[fold]:\n files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])\n print('#### Using 2018+2017 external data')\n \n np.random.shuffle(files_train); print('#'*25)\n files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])\n files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))\n \n # BUILD MODEL\n K.clear_session()\n with strategy.scope():\n print('#'*25)\n model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS_NUM[fold])\n print('#'*25)\n \n # SAVE BEST MODEL EACH FOLD\n sv = tf.keras.callbacks.ModelCheckpoint(\n 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n save_weights_only=True, mode='min', save_freq='epoch')\n \n # TRAIN\n print('Training...')\n t1 = time.time()\n history = model.fit(get_dataset(files_train,\n augment = True,\n shuffle = True,\n repeat = True,\n dim = IMG_SIZES[fold],\n batch_size = BATCH_SIZES[fold]),\n epochs = EPOCHS[fold],\n callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], \n steps_per_epoch = count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n validation_data = get_dataset(files_valid,augment=False,shuffle=False,\n repeat = False,\n dim = IMG_SIZES[fold]), \n #class_weight = {0:1,1:2},\n verbose=VERBOSE\n )\n t2 = time.time()\n time_lst.append(t2-t1)\n print('Loading best model...')\n model.load_weights('fold-%i.h5'%fold)\n \n # PREDICT OOF USING TTA\n print('Predicting OOF with TTA...')\n ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) ) \n #oof_pred.append(model.predict(get_dataset(files_valid,dim=IMG_SIZES[fold]),verbose=1))\n \n # GET OOF TARGETS AND NAMES\n ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n labeled=True, return_image_names=True)\n oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n labeled=False, return_image_names=True)\n oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n \n # PREDICT TEST USING TTA\n print('Predicting Test with TTA...')\n ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,\n repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[fold]\n \n # REPORT RESULTS\n auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n oof_val.append(np.max( history.history['val_auc'] ))\n print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n \n # PLOT TRAINING\n if DISPLAY_PLOT:\n plt.figure(figsize=(15,5))\n plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n plt.legend(loc=2)\n plt2 = plt.gca().twinx()\n plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n plt.ylabel('Loss',size=14)\n plt.title('FOLD %i - Image Size %i, ConvNeXt%s, inc2019=%i, inc2018=%i'%\n (fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)\n plt.legend(loc=3)\n plt.show()\n \n for metric_name in history.history.keys():\n print(f\"{metric_name}: {history.history[metric_name]}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:49:29.435835Z","iopub.execute_input":"2024-03-09T13:49:29.436158Z","iopub.status.idle":"2024-03-09T13:50:57.274811Z","shell.execute_reply.started":"2024-03-09T13:49:29.436135Z","shell.execute_reply":"2024-03-09T13:50:57.273317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Calculate OOF AUC**\nThe OOF (out of fold) predictions are saved to disk. If you wish to ensemble multiple models, use the OOF to determine what are the best weights to blend your models with. Choose weights that maximize OOF CV score when used to blend OOF. Then use those same weights to blend your test predictions.","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\nnames = np.concatenate(oof_names); folds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.3f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(\n image_name = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:50:57.275874Z","iopub.status.idle":"2024-03-09T13:50:57.276259Z","shell.execute_reply.started":"2024-03-09T13:50:57.276087Z","shell.execute_reply":"2024-03-09T13:50:57.276103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 5: Post process**\nThere are ways to modify predictions based on patient information to increase CV LB. You can experiment with that here on your OOF.","metadata":{}},{"cell_type":"markdown","source":"# **Submit To Kaggle**","metadata":{}},{"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],\n labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n for img, img_name in iter(ds.unbatch())])","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:50:57.277845Z","iopub.status.idle":"2024-03-09T13:50:57.278292Z","shell.execute_reply.started":"2024-03-09T13:50:57.278084Z","shell.execute_reply":"2024-03-09T13:50:57.278104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('image_name') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:50:57.279400Z","iopub.status.idle":"2024-03-09T13:50:57.279827Z","shell.execute_reply.started":"2024-03-09T13:50:57.279627Z","shell.execute_reply":"2024-03-09T13:50:57.279648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T13:50:57.281169Z","iopub.status.idle":"2024-03-09T13:50:57.281629Z","shell.execute_reply.started":"2024-03-09T13:50:57.281384Z","shell.execute_reply":"2024-03-09T13:50:57.281405Z"},"trusted":true},"execution_count":null,"outputs":[]}]}