{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1322485,"sourceType":"datasetVersion","datasetId":758225},{"sourceId":1322494,"sourceType":"datasetVersion","datasetId":688574},{"sourceId":1322517,"sourceType":"datasetVersion","datasetId":689329},{"sourceId":1322552,"sourceType":"datasetVersion","datasetId":688719},{"sourceId":1322612,"sourceType":"datasetVersion","datasetId":689578},{"sourceId":1324333,"sourceType":"datasetVersion","datasetId":762256},{"sourceId":1324349,"sourceType":"datasetVersion","datasetId":762108},{"sourceId":1324366,"sourceType":"datasetVersion","datasetId":762176},{"sourceId":1324385,"sourceType":"datasetVersion","datasetId":762138},{"sourceId":1324412,"sourceType":"datasetVersion","datasetId":762168},{"sourceId":1324430,"sourceType":"datasetVersion","datasetId":768238},{"sourceId":1324483,"sourceType":"datasetVersion","datasetId":758409},{"sourceId":1339671,"sourceType":"datasetVersion","datasetId":758489},{"sourceId":1362814,"sourceType":"datasetVersion","datasetId":791650},{"sourceId":1362826,"sourceType":"datasetVersion","datasetId":791647},{"sourceId":1362834,"sourceType":"datasetVersion","datasetId":791645},{"sourceId":1362843,"sourceType":"datasetVersion","datasetId":791643},{"sourceId":1362855,"sourceType":"datasetVersion","datasetId":791638},{"sourceId":1362877,"sourceType":"datasetVersion","datasetId":791635}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Imports**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport seaborn as sns\nimport PIL\nimport time\nimport os\nimport torch\nimport tensorflow as tf, re, math\nimport random\n\nfrom tensorflow import keras\nfrom IPython.display import Image\nimport tensorflow.keras.backend as K\nfrom keras.losses import binary_crossentropy\nfrom tensorflow.keras.applications import ResNet50, ResNet101, ResNet152 \nfrom tensorflow.keras.losses import BinaryCrossentropy, BinaryFocalCrossentropy\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split, KFold\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\nfrom tensorflow.keras.layers import Flatten, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import optimizers\nimport tensorflow as tf, re, math\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nimport tempfile\n\nimport pandas as pd, numpy as np, gc\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-06-16T22:33:01.789589Z","iopub.execute_input":"2024-06-16T22:33:01.790601Z","iopub.status.idle":"2024-06-16T22:33:25.690593Z","shell.execute_reply.started":"2024-06-16T22:33:01.790562Z","shell.execute_reply":"2024-06-16T22:33:25.689234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Versión de TensorFlow:\", tf.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dir(tf.keras.applications))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport keras\nimport sys\nimport os\n\n# Verificar la versión de TensorFlow\nprint(\"Versión de TensorFlow:\", tf.__version__)\n\n# Verificar la versión de Keras\nprint(\"Versión de Keras:\", keras.__version__)\n\n# Verificar la versión de CUDA\ncuda_version = os.popen('nvcc --version').read()\nprint(\"Versión de CUDA:\", cuda_version)\n\n# Verificar la versión de Python\nprint(\"Versión de Python:\", sys.version)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n \"\"\"Sets the seed of the entire notebook so results are the same every time we run. This is for reproducibility\"\"\"\n np.random.seed(seed)\n random.seed(seed)\n tf.random.set_seed(seed)\n\n # Disable GPU parallelism to ensure reproducibility on GPU\n tf.config.threading.set_inter_op_parallelism_threads(1)\n tf.config.threading.set_intra_op_parallelism_threads(1)\n\n # Disable CuDNN auto-tuning to ensure reproducibility on GPU\n tf.config.set_soft_device_placement(True)\n tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": False})\n\n print('SEEDING DONE')\n\nset_seed(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Kaggle's SIIM-ISIC Melanoma Classification**","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_csv = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_img = train_csv['target'].size\nmalignant = np.count_nonzero(train_csv['target'])\nbenign = total_img - malignant\n\nprint('Total {} {:.2f}% \\nBenign {} {:.2f}% \\nMalignant {} {:.2f}%'.format(total_img,100 * total_img / total_img, benign, 100 * benign / total_img, malignant, 100 * malignant / total_img))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_for_0 = 1.02 #(1 / (benign / total_img))\nweight_for_1 = 85.08 #(1 / (malignant / total_img))\nclass_weight = {0: weight_for_0, 1: weight_for_1}\n\nprint('Weight for class 0: {}'.format(weight_for_0))\nprint('Weight for class 1: {}'.format(weight_for_1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2, pandas as pd, matplotlib.pyplot as plt\n\ntrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n\nprint('Images with Melanoma')\nimgs = train.loc[train.target==1].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n img = cv2.imread('../input/jpeg-melanoma-128x128/train/%s.jpg'%k)\n img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n plt.subplot(1,10,i+1); plt.axis('off')\n plt.imshow(img)\nplt.show()\n\nprint('Images without Melanoma')\nimgs = train.loc[train.target==0].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n img = cv2.imread('../input/jpeg-melanoma-128x128/train/%s.jpg'%k)\n img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n plt.subplot(1,10,i+1); plt.axis('off')\n plt.imshow(img)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Parameters**\nIn order to be a proper cross validation with a meaningful overall CV score (aligned with LB score), **you need to choose the same** `IMG_SIZES`, `INC2019`, `INC2018`, and `EFF_NETS` **for each fold**. If your goal is to just run lots of experiments, then you can choose to have a different experiment in each fold. Then each fold is like a holdout validation experiment. When you find a configuration you like, you can use that configuration for all folds. \n\nFirst we choose a number of `EXPERIMENTS`. Then this notebook will perform that many experiments on the same KFold fold. Afterward we can compare validation scores. Or you can repeat the same experiment many times and assess validation score variability. If you have read my previous notebook [here][1], you are familar with most of these configuration variables. We will list the new parameters first:\n[1]: https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DEVICEis GPU or TPU.
SEEDa different seed produces a different triple stratified kfold split.
FOLDSnumber of folds. Best set to 3, 5, or 15 but can be any number between 2 and 15.
IMG_SIZESis a Python list of length FOLDS. These are the image sizes to use each fold.
INC2019This includes the new half of the 2019 competition data.
INC2018This includes the second half of the 2019 competition data.
BATCH_SIZESis a list of length FOLDS. These are batch sizes for each fold.
EPOCHSis a list of length FOLDS. These are maximum epochs.
EFF_NETSis a list of length FOLDS. These are the ConvNeXt to use each fold.
EFF_NETS_NUMis a list of length FOLDS. These are the index for each item of EFF_NETS.
WGTSthis should be `1/FOLDS` for each fold.
TTAtest time augmentation.
EXPERIMENTSnumber of experiments to perform
FNUMBERwhich of the KFolds to repeatedly perform experiments on.
M1is a list. Adds copies of malignant images from this year's comp data
M2is a list. Adds copies of malignant images from ISIC archive that are not in 2020, 2019, 2018, 2017 comp data
M3is a list. Adds copies of malignant images from 2019 comp data.
M4is a list. Adds copies of malignant images from 2017 and 2018 data.
DROP_FREQa list of floats between 0 and 1. Determines the proportion of train images to apply coarse dropout to
DROP_CTa list of ints. Determines how many squares to remove from train images when applying dropout
DROP_SIZEa list of floats between 0 and 1. Determines the size of square side equals IMG_SIZE * DROP_SIZE
INFER_TESTwhether to predict test images each experiment
\n","metadata":{}},{"cell_type":"markdown","source":"With this code I will perform 3 different experiments based on class-imbalance:\nCLASS_WEIGHT\nBINARYFOCALCROSSLOSS\nAUGMENTATIONS","metadata":{}},{"cell_type":"code","source":"DEVICE = \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\n# NUMBER OF FOLDS: 3,5,15 (1-15)\nFOLDS = 5\n\n# WHICH FOLD TO PERFORM EXPERIMENTS ON: (1-15)\n# NUMBER OF EXPERIMENTS FOR THE SELECTED FOLD\nFNUMBER = 1\nEXPERIMENTS = 1\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD: 128, 192, 256, 384, 512, 768 \nIMG_SIZES = [256]*EXPERIMENTS\n\n# INCLUDE OLD COMP DATA? YES=1 NO=0\nINC2019 = [1]*EXPERIMENTS\nINC2018 = [1]*EXPERIMENTS\n\n# UPSAMPLE MALIGNANT COUNT TIMES\nM1 = [0]*EXPERIMENTS #2020 malig\nM2 = [0]*EXPERIMENTS #ISIC malig\nM3 = [0]*EXPERIMENTS #2019 good malig\nM4 = [0]*EXPERIMENTS #2018 2017 malig\n\n# COARSE DROPOUT\nDROP_FREQ = [0.75,0.75,0.75] # between 0 and 1\nDROP_CT = [8,8,8] # may slow training if CT>16\nDROP_SIZE = [0.2,0.2,0.2] # between 0 and 1\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [16]*EXPERIMENTS\nEPOCHS = [25]*EXPERIMENTS\n\n# WHICH CONVNET NAME TO USE\nEFF_NETS = ['ResNet101']*FOLDS\n\nEFF_NETS_NUMS = [0,1,2,3,4]\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/EXPERIMENTS]*EXPERIMENTS\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 11\nINFER_TEST = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n print(\"connecting to TPU...\")\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Running on TPU ', tpu.master())\n except ValueError:\n print(\"Could not connect to TPU\")\n tpu = None\n\n if tpu:\n try:\n print(\"initializing TPU ...\")\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(\"TPU initialized\")\n except _:\n print(\"failed to initialize TPU\")\n else:\n DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n print(\"Using default strategy for CPU and single GPU\")\n strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n \n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 1: Preprocess Data**\nPreprocess has already been done and saved to TFRecords. Here we choose which size to load. We can use either 128x128, 192x192, 256x256, 384x384, 512x512, 768x768 by changing the `IMG_SIZES` variable in the preceeding code section. These TFRecords are discussed [here][1]. The advantage of using different input sizes is discussed [here][2]\n\n[1]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/155579\n[2]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/160147","metadata":{}},{"cell_type":"code","source":"GCS_PATH = [None]*FOLDS\nGCS_PATH2 = [None]*FOLDS\nGCS_PATH3 = [None]*FOLDS\n\nfor i,k in enumerate(IMG_SIZES[:FOLDS]):\n GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))\n GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\n GCS_PATH3[i] = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i'%(k,k))\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\nfiles_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Step 2: Data Augmentation**\nThis notebook uses rotation, sheer, zoom, shift augmentation first shown in this notebook [here][1] and successfully used in Melanoma comp by AgentAuers [here][2]. This notebook also uses horizontal flip, hue, saturation, contrast, brightness augmentation similar to last years winner and also similar to AgentAuers' notebook.\n\nAdditionally we can decide to use external data by changing the variables `INC2019` and `INC2018` in the preceeding code section. These variables respectively indicate whether to load last year 2019 data and/or year 2018 + 2017 data. These datasets are discussed [here][3]\n\nConsider experimenting with different augmenation and/or external data. The code to load TFRecords is taken from AgentAuers' notebook [here][2]. Thank you AgentAuers, this is great work.\n\n[1]: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\n[2]: https://www.kaggle.com/agentauers/incredible-tpus-finetune-effnetb0-b6-at-once\n[3]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910","metadata":{}},{"cell_type":"markdown","source":"### **Geometric Augmentations**","metadata":{}},{"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n # returns 3x3 transformmatrix which transforms indicies\n \n # CONVERT DEGREES TO RADIANS\n rotation = math.pi * rotation / 180.\n shear = math.pi * shear / 180.\n\n def get_3x3_mat(lst):\n return tf.reshape(tf.concat([lst],axis=0), [3,3])\n \n # ROTATION MATRIX\n c1 = tf.math.cos(rotation)\n s1 = tf.math.sin(rotation)\n one = tf.constant([1],dtype='float32')\n zero = tf.constant([0],dtype='float32')\n \n rotation_matrix = get_3x3_mat([c1, s1, zero, \n -s1, c1, zero, \n zero, zero, one]) \n # SHEAR MATRIX\n c2 = tf.math.cos(shear)\n s2 = tf.math.sin(shear) \n \n shear_matrix = get_3x3_mat([one, s2, zero, \n zero, c2, zero, \n zero, zero, one]) \n \n # ZOOM MATRIX\n zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero, \n zero, one/width_zoom, zero, \n zero, zero, one]) \n # SHIFT MATRIX\n shift_matrix = get_3x3_mat([one, zero, height_shift, \n zero, one, width_shift, \n zero, zero, one])\n \n return K.dot(K.dot(rotation_matrix, shear_matrix), \n K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, DIM=256): \n # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n # output - image randomly rotated, sheared, zoomed, and shifted\n XDIM = DIM%2 #fix for size 331\n \n rot = ROT_ * tf.random.normal([1], dtype='float32')\n shr = SHR_ * tf.random.normal([1], dtype='float32') \n h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n # GET TRANSFORMATION MATRIX\n m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n # LIST DESTINATION PIXEL INDICES\n x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n z = tf.ones([DIM*DIM], dtype='int32')\n idx = tf.stack( [x,y,z] )\n \n # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n idx2 = K.cast(idx2, dtype='int32')\n idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n \n # FIND ORIGIN PIXEL VALUES \n idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n d = tf.gather_nd(image, tf.transpose(idx3))\n \n return tf.reshape(d,[DIM, DIM,3])","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Dropout Augmentations**","metadata":{}},{"cell_type":"code","source":"RATE = 0.75; CT = 8; SIZE = 0.2\n\ndef dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):\n # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n # output - image with CT squares of side size SZ*DIM removed\n \n # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n P = tf.cast( tf.random.uniform([],0,1)