{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# install required dependencies\n\n# !conda install -c conda-forge pillow -y\n# !conda install -c conda-forge pydicom -y\n# !conda install -c conda-forge gdcm -y\n# !pip install pylibjpeg pylibjpeg-libjpeg\n\n# only required for tpu processing, which i don't use\n\n# !pip uninstall tensorflow -y\n# !pip uninstall tensorflow-io -y\n# !pip install -q tensorflow-gpu\n# !pip install -q tensorflow-io ","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:14.720130Z","iopub.execute_input":"2021-07-16T17:14:14.720537Z","iopub.status.idle":"2021-07-16T17:14:14.724408Z","shell.execute_reply.started":"2021-07-16T17:14:14.720434Z","shell.execute_reply":"2021-07-16T17:14:14.723491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport pickle\n\nfrom tqdm import tqdm\nimport glob\nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.pylab as pylab\nimport seaborn as sns\nimport pprint\nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport albumentations as A \nimport cv2\n\nfrom PIL import Image\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nos.listdir('../input')\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:14.729232Z","iopub.execute_input":"2021-07-16T17:14:14.729477Z","iopub.status.idle":"2021-07-16T17:14:17.009148Z","shell.execute_reply.started":"2021-07-16T17:14:14.729453Z","shell.execute_reply":"2021-07-16T17:14:17.008335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA and preprocessing\nPreparo il dataset ad essere utilizzato dal mio modello","metadata":{}},{"cell_type":"code","source":"basepath = '../input/siim-covid19-detection/'\n\ntrain_study_df = pd.read_csv(basepath + \"train_study_level.csv\")\ntrain_image_df = pd.read_csv(basepath + \"train_image_level.csv\")\n\ntrain_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:17.010634Z","iopub.execute_input":"2021-07-16T17:14:17.010988Z","iopub.status.idle":"2021-07-16T17:14:17.076883Z","shell.execute_reply.started":"2021-07-16T17:14:17.010953Z","shell.execute_reply":"2021-07-16T17:14:17.075957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_directory = basepath + \"train/\"\ntest_directory =  basepath + \"test/\"\n\n# merging study and image train dataframes\ntry:\n    train_study_df['StudyInstanceUID'] = train_study_df['id'].apply(lambda x: x.replace('_study', ''))\n    del train_study_df['id']\nexcept:\n    assert 'StudyInstanceUID' in train_study_df.columns, 'Something went wrong with the dataframe. Rerun previous cells'\n    print('train_study_df[id] already deleted')\ntrain_df = train_image_df.merge(train_study_df, on='StudyInstanceUID')\n\n# adding path to the df\ntraining_paths = []\nfor sid in tqdm(train_df['StudyInstanceUID']):\n    training_paths.append(glob.glob(os.path.join(train_directory, sid +\"/*/*\"))[0])\n\ntrain_df['path'] = training_paths\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:17.078667Z","iopub.execute_input":"2021-07-16T17:14:17.079029Z","iopub.status.idle":"2021-07-16T17:14:39.863023Z","shell.execute_reply.started":"2021-07-16T17:14:17.078994Z","shell.execute_reply":"2021-07-16T17:14:39.862295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:39.864499Z","iopub.execute_input":"2021-07-16T17:14:39.864823Z","iopub.status.idle":"2021-07-16T17:14:39.873109Z","shell.execute_reply.started":"2021-07-16T17:14:39.864788Z","shell.execute_reply":"2021-07-16T17:14:39.871995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A small preview of the distribution of data","metadata":{}},{"cell_type":"code","source":"params = {'legend.fontsize': 'x-large',\n          'figure.figsize': (20, 32),\n         'axes.labelsize': 'x-large',\n         'axes.titlesize':'x-large',\n         'xtick.labelsize':'x-large',\n         'ytick.labelsize':'x-large'}\npylab.rcParams.update(params)\n\nfig, ax = plt.subplots(2,2)\nsns.countplot(x = train_df[\"Negative for Pneumonia\"], ax=ax[0,0],color=\"#ffb4a2\")\nax[0,0].set_title(\"Negative for Pneumonia Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nsns.countplot(x = train_df[\"Typical Appearance\"], ax=ax[0,1],color=\"#e5989b\")\nax[0,1].set_title(\"Typical Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nsns.countplot(x = train_df[\"Indeterminate Appearance\"], ax=ax[1,0],color=\"#b5838d\")\nax[1,0].set_title(\"Indeterminate Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nsns.countplot(x = train_df[\"Atypical Appearance\"], ax=ax[1,1],color=\"#6d6875\")\nax[1,1].set_title(\"Atypical Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nfig.subplots_adjust(wspace=0.1, hspace=0.2, top=0.5)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-16T17:14:39.874516Z","iopub.execute_input":"2021-07-16T17:14:39.874883Z","iopub.status.idle":"2021-07-16T17:14:40.316415Z","shell.execute_reply.started":"2021-07-16T17:14:39.874832Z","shell.execute_reply":"2021-07-16T17:14:40.315578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, resample=Image.NEAREST):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    im.convert('1')\n    im = im.resize((size, size), resample)\n    \n    return np.asarray(im)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:40.317601Z","iopub.execute_input":"2021-07-16T17:14:40.318108Z","iopub.status.idle":"2021-07-16T17:14:40.323285Z","shell.execute_reply.started":"2021-07-16T17:14:40.318069Z","shell.execute_reply":"2021-07-16T17:14:40.322507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voi_lut=True\nfix_monochrome=True\nsize = 224\n\n# if you only want the pixel arary use this\ndef dicom_to_pixelarray(filename):\n    dicom_header = dicom.dcmread(filename) \n    data = apply_voi_lut(dicom_header.pixel_array, dicom_header)\n    data = resize(data, size)\n\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom_header.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    modified_image_data = (data * 255).astype(np.uint8)\n    return modified_image_data\n\ndef dicom_dataset_to_dict(filename,func):\n    \"\"\"Credit: https://github.com/pydicom/pydicom/issues/319\n               https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    \"\"\"\n    \n    dicom_header = dicom.dcmread(filename) \n    \n    #====== DICOM FILE DATA ======\n    dicom_dict = {}\n    repr(dicom_header)\n    for dicom_value in dicom_header.values():\n        if dicom_value.tag == (0x7fe0, 0x0010):\n            # discard pixel data\n            continue\n        if type(dicom_value.value) == dicom.dataset.Dataset:\n            dicom_dict[dicom_value.name] = dicom_dataset_to_dict(dicom_value.value)\n        else:\n            v = _convert_value(dicom_value.value)\n            dicom_dict[dicom_value.name] = v\n      \n    del dicom_dict['Pixel Representation']\n    \n    if func!='metadata_df':\n        #====== DICOM IMAGE DATA ======\n        # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n        if voi_lut:\n            data = apply_voi_lut(dicom_header.pixel_array, dicom_header)\n        else:\n            data = dicom_header.pixel_array\n        # depending on this value, X-ray may look inverted - fix that:\n        if fix_monochrome and dicom_header.PhotometricInterpretation == \"MONOCHROME1\":\n            data = np.amax(data) - data\n        data = data - np.min(data)\n        data = data / np.max(data)\n        modified_image_data = (data * 255).astype(np.uint8)\n    \n        return dicom_dict, modified_image_data\n    \n    else:\n        return dicom_dict\n\ndef _sanitise_unicode(s):\n    return s.replace(u\"\\u0000\", \"\").strip()\n\ndef _convert_value(v):\n    t = type(v)\n    if t in (list, int, float):\n        cv = v\n    elif t == str:\n        cv = _sanitise_unicode(v)\n    elif t == bytes:\n        s = v.decode('ascii', 'replace')\n        cv = _sanitise_unicode(s)\n    elif t == dicom.valuerep.DSfloat:\n        cv = float(v)\n    elif t == dicom.valuerep.IS:\n        cv = int(v)\n    else:\n        cv = repr(v)\n    return cv\n\nfilename = train_df.path[0]\n    \ndf, img_array = dicom_dataset_to_dict(filename, 'fetch_both_values')\n\nfig, ax = plt.subplots(1, 2, figsize=[15, 8])\nax[0].imshow(img_array, cmap=plt.cm.gray)\nax[1].imshow(img_array, cmap=plt.cm.plasma)    \nplt.show()\n\npprint.pprint(df)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:40.324572Z","iopub.execute_input":"2021-07-16T17:14:40.324912Z","iopub.status.idle":"2021-07-16T17:14:43.921128Z","shell.execute_reply.started":"2021-07-16T17:14:40.324875Z","shell.execute_reply":"2021-07-16T17:14:43.920377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !! This takes a few minutes to process all the dicom files !!\ndef load_pixel_array(df):    \n    pixel_arrays = []\n    y = []\n    for i, row in tqdm(df.iterrows(), total=df.shape[0]):\n        try:\n            pixel_arrays.append(dicom_to_pixelarray(row['path']))\n            y.append(np.array([row['Negative for Pneumonia'], \n                              row['Typical Appearance'], \n                              row['Indeterminate Appearance'], \n                              row['Atypical Appearance']]))\n        except RuntimeError:\n            continue\n\n    pixel_arrays = np.asarray(pixel_arrays)\n    x = np.repeat(pixel_arrays[..., np.newaxis], 3, -1)\n    y = np.asarray(y)\n    return x, y","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:43.923478Z","iopub.execute_input":"2021-07-16T17:14:43.923772Z","iopub.status.idle":"2021-07-16T17:14:43.931778Z","shell.execute_reply.started":"2021-07-16T17:14:43.923742Z","shell.execute_reply":"2021-07-16T17:14:43.930529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reload = False\nif reload:\n    x_train, y_train = load_pixel_array(train_df)\nelif os.path.isfile('../input/reti-siim-covid/x_train.npy'):\n    x_train = np.load('../input/reti-siim-covid/x_train.npy')\n    y_train = np.load('../input/reti-siim-covid/y_train.npy')\nelif os.path.isfile('/kaggle/working/x_train.npy'):\n    x_train = np.load('/kaggle/working/x_train.npy')\n    y_train = np.load('/kaggle/working/y_train.npy')\nelse:\n    x_train, y_train = load_pixel_array(train_df)\n\nprint(x_train.shape, y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:43.933778Z","iopub.execute_input":"2021-07-16T17:14:43.934144Z","iopub.status.idle":"2021-07-16T17:14:51.075922Z","shell.execute_reply.started":"2021-07-16T17:14:43.934108Z","shell.execute_reply":"2021-07-16T17:14:51.075046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# salvo gli output così non ci metto troppo temo a caricare nuovamente gli array una volta chiuso il notebook\nnp.save('/kaggle/working/x_train.npy', x_train)\nnp.save('/kaggle/working/y_train.npy', y_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:51.077264Z","iopub.execute_input":"2021-07-16T17:14:51.077629Z","iopub.status.idle":"2021-07-16T17:14:51.682755Z","shell.execute_reply.started":"2021-07-16T17:14:51.077593Z","shell.execute_reply":"2021-07-16T17:14:51.681819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data augmentation with TPU acceleration:\nUsing [this notebook](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu) I try to speed up the learning process \n\nThere is an error when I try to run it on the TPU which i didn't have time to fix, so i w won't use this code","metadata":{}},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nimport tensorflow.keras as K\nimport tensorflow as tf\ntry:\n    import tensorflow_io as tfio\nexcept:\n    print('tensorflow_io not installed')\nimport re","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:51.685888Z","iopub.execute_input":"2021-07-16T17:14:51.686301Z","iopub.status.idle":"2021-07-16T17:14:56.005149Z","shell.execute_reply.started":"2021-07-16T17:14:51.686259Z","shell.execute_reply":"2021-07-16T17:14:56.004051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ndef get_strategy():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return strategy","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:56.006721Z","iopub.execute_input":"2021-07-16T17:14:56.007306Z","iopub.status.idle":"2021-07-16T17:14:56.016471Z","shell.execute_reply.started":"2021-07-16T17:14:56.007259Z","shell.execute_reply":"2021-07-16T17:14:56.015399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nstrategy = get_strategy()\n\n# Configuration\nIMAGE_SIZE = [224, 224]\nEPOCHS = 25\nFOLDS = 3\nSEED = 777\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nFIRST_FOLD_ONLY = False\n\n\n# Enable mixed precision and XLA?\n# Kaggle TPUs and GPUs don't fully support these yet\nMIXED_PRECISION = False\nXLA_ACCELERATE = False\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:56.017902Z","iopub.execute_input":"2021-07-16T17:14:56.018293Z","iopub.status.idle":"2021-07-16T17:14:56.036999Z","shell.execute_reply.started":"2021-07-16T17:14:56.018253Z","shell.execute_reply":"2021-07-16T17:14:56.036147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data access\nGCS_PATH = KaggleDatasets().get_gcs_path('siim-covid19-detection')\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*/*/*.dcm')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*/*/*.dcm') # predictions on this dataset should be submitted for the competition\nlen(TRAINING_FILENAMES), len(TEST_FILENAMES)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:56.038355Z","iopub.execute_input":"2021-07-16T17:14:56.038925Z","iopub.status.idle":"2021-07-16T17:14:57.598623Z","shell.execute_reply.started":"2021-07-16T17:14:56.038891Z","shell.execute_reply":"2021-07-16T17:14:57.597725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_id = train_df.set_index('id')\n\ndef parse_image(filename):    \n    id = bytes.decode(filename.numpy()).split('/')[-1].split('.')[0] + '_image'\n    label = tf.constant(train_df_id.loc[id][['Negative for Pneumonia',\n                                               'Typical Appearance',\n                                               'Indeterminate Appearance',\n                                               'Atypical Appearance']],\n                        dtype = tf.uint8, shape=[4])\n    image_bytes = tf.io.read_file(filename)\n\n    # if bad decoding - raise an error\n    image = tfio.image.decode_dicom_image(image_bytes, on_error='strict',\n        dtype=tf.float32) \n    image = tf.image.resize(image, IMAGE_SIZE) # optional\n    \n    return image, label\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.Dataset.list_files(GCS_PATH + '/train/*/*/*.dcm') # automatically interleaves reads from multiple files    \n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(lambda x: tf.py_function(parse_image, [x], [tf.float32, tf.uint8]), num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    return len(TRAINING_FILENAMES)\n\nNUM_TRAINING_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (1./FOLDS) )\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:57.600186Z","iopub.execute_input":"2021-07-16T17:14:57.600585Z","iopub.status.idle":"2021-07-16T17:14:57.624526Z","shell.execute_reply.started":"2021-07-16T17:14:57.600545Z","shell.execute_reply":"2021-07-16T17:14:57.623640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uncomment to use TPU\n# d = load_dataset(TRAINING_FILENAMES)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:57.627149Z","iopub.execute_input":"2021-07-16T17:14:57.627421Z","iopub.status.idle":"2021-07-16T17:14:57.633296Z","shell.execute_reply.started":"2021-07-16T17:14:57.627397Z","shell.execute_reply":"2021-07-16T17:14:57.632461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show output of dataset\n# for element in d.take(1):\n#     plt.imshow(element[0].numpy()[0])\n#     print(element[0].numpy()[0].shape)\n#     print(element[1].numpy())","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:57.634512Z","iopub.execute_input":"2021-07-16T17:14:57.634882Z","iopub.status.idle":"2021-07-16T17:14:57.643123Z","shell.execute_reply.started":"2021-07-16T17:14:57.634832Z","shell.execute_reply":"2021-07-16T17:14:57.642159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:57.644552Z","iopub.execute_input":"2021-07-16T17:14:57.644970Z","iopub.status.idle":"2021-07-16T17:14:57.664579Z","shell.execute_reply.started":"2021-07-16T17:14:57.644932Z","shell.execute_reply":"2021-07-16T17:14:57.663839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# row = 3; col = 4;\n# all_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES[:3]),do_aug=False).unbatch()\n# one_element = tf.data.Dataset.from_tensors( next(iter(all_elements)) )\n# augmented_element = one_element.repeat().map(transform).batch(row*col)\n\n# for (img,label) in augmented_element:\n#     plt.figure(figsize=(15,int(15*row/col)))\n#     for j in range(row*col):\n#         plt.subplot(row,col,j+1)\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show()\n#     break","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:57.665421Z","iopub.execute_input":"2021-07-16T17:14:57.665665Z","iopub.status.idle":"2021-07-16T17:14:57.677259Z","shell.execute_reply.started":"2021-07-16T17:14:57.665634Z","shell.execute_reply":"2021-07-16T17:14:57.676318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model definition\nProvo a sfruttare transfer learning usando il modello ResNet50, pretrained in keras, facendo training solo sul layer FC finale","metadata":{}},{"cell_type":"code","source":"import tensorflow.keras as K\nimport tensorflow as tf\nimport datetime as dt","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:14:57.678394Z","iopub.execute_input":"2021-07-16T17:14:57.678675Z","iopub.status.idle":"2021-07-16T17:14:57.687058Z","shell.execute_reply.started":"2021-07-16T17:14:57.678650Z","shell.execute_reply":"2021-07-16T17:14:57.686071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# building the actual model and enabling GPU acceleration\n# TPU acceleration takes a while to set up and i dont have the time\n# need to use Google SDK to pipe data directly into tpu, otherwise the \n# TPU is bottlenecked by data transfer\n\ndef get_model(file=None):\n    if file is not None:\n        with strategy.scope():\n            model = K.models.load_model(file)\n            return model\n    with strategy.scope():\n        model = K.models.Sequential()\n\n        input_t = K.Input(shape=(224, 224, 3))\n        res_model = K.applications.ResNet50(include_top=False,\n                                           weights=\"imagenet\",\n                                           input_tensor=input_t)\n#         for layer in res_model.layers:\n#             layer.trainable = False\n\n        model.add(res_model)\n        model.add(tf.keras.layers.GlobalAveragePooling2D())\n        model.add(K.layers.Dense(4, activation='softmax'))\n\n        model.compile(loss='categorical_crossentropy',\n                      optimizer=K.optimizers.RMSprop(lr=2e-5),\n                      metrics=['accuracy', \n                               K.metrics.AUC(multi_label=True),\n                               K.metrics.AUC(name='prc', curve='PR'),])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:16:54.966460Z","iopub.execute_input":"2021-07-16T17:16:54.966798Z","iopub.status.idle":"2021-07-16T17:16:54.974973Z","shell.execute_reply.started":"2021-07-16T17:16:54.966768Z","shell.execute_reply":"2021-07-16T17:16:54.973843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VAL_SPLIT = 0.15\ndatagen = K.preprocessing.image.ImageDataGenerator(rotation_range=30,\n                                                   width_shift_range=0.1,\n                                                   height_shift_range=0.1,\n                                                   brightness_range=(0.8, 1.2),\n                                                   shear_range=15,\n                                                   horizontal_flip=True,\n                                                   vertical_flip=True,\n                                                   validation_split=VAL_SPLIT,\n                                                  )\n\n\ntrain_it = datagen.flow(x_train,y_train, batch_size=BATCH_SIZE,subset='training')\n\nvalidation_it = datagen.flow(x_train, y_train, batch_size=BATCH_SIZE, subset='validation')\n\n# generate samples and plot\nbatch = train_it.next()\nfor i in range(9):\n    # define subplot\n    plt.subplot(330 + 1 + i)\n    # generate batch of images\n    # convert to unsigned integers for viewing\n    image = batch[0][i].astype('uint8')\n    # plot raw pixel data\n    plt.imshow(image)\n# show the figure\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:16:55.118252Z","iopub.execute_input":"2021-07-16T17:16:55.118563Z","iopub.status.idle":"2021-07-16T17:16:58.039822Z","shell.execute_reply.started":"2021-07-16T17:16:55.118536Z","shell.execute_reply":"2021-07-16T17:16:58.038893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = '../input/resmodel-weights/resmodel_weights.h5'\nmodel = get_model(file=None)\nmodel.summary()\n#  628,404 = numero trainable parameters\nprint(f'numero data points: {len(x_train)}\\nnumero trainable parameters/numero data points: {628404/len(x_train)}')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:16:58.041383Z","iopub.execute_input":"2021-07-16T17:16:58.041705Z","iopub.status.idle":"2021-07-16T17:16:59.779067Z","shell.execute_reply.started":"2021-07-16T17:16:58.041672Z","shell.execute_reply":"2021-07-16T17:16:59.778278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model, to_file='model.png', show_shapes=True,show_layer_names=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:16:59.781959Z","iopub.execute_input":"2021-07-16T17:16:59.782211Z","iopub.status.idle":"2021-07-16T17:16:59.955720Z","shell.execute_reply.started":"2021-07-16T17:16:59.782186Z","shell.execute_reply":"2021-07-16T17:16:59.954850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_point = K.callbacks.ModelCheckpoint(filepath=\"/kaggle/working/resmodel_weights_ckp.h5\",\n                                          monitor=\"val_acc\",\n                                          mode=\"max\",\n                                          save_best_only=True,\n                                          )\n\nearly_stopping = K.callbacks.EarlyStopping(monitor='loss', patience = 5)\n\nhistory = model.fit(train_it, batch_size=BATCH_SIZE, epochs=100, \n                    verbose=1, validation_data=validation_it, callbacks=[check_point, early_stopping])\n\nnow = dt.datetime.now()\nname = 'resmodel_pooled'\nmodel.summary()\nmodel.save(f\"/kaggle/working/{name}_weights_{now.strftime('%d_%m_%Y__%H_%M')}.h5\")\nwith open(f\"/kaggle/working/{name}_history_{now.strftime('%d_%m_%Y__%H_%M')}.pkl\", 'wb') as file_pi:\n        pickle.dump(history.history, file_pi)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:16:59.957486Z","iopub.execute_input":"2021-07-16T17:16:59.957854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# auc history\nplt.plot(history.history['auc'])\nplt.plot(history.history['val_auc'])\nplt.title('model auc')\nplt.ylabel('auc')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:15:08.094376Z","iopub.status.idle":"2021-07-16T17:15:08.095074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef TPU_train_cross_validate(folds = 5):\n    histories = []\n    models = []\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        print(); print('#'*25)\n        print('### FOLD',f+1)\n        print('#'*25)\n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = [lr_callback],#, early_stopping],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=2\n        )\n        models.append(model)\n        histories.append(history)\n    return histories, models\n\ndef TPU_train_and_predict(folds = 5):\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Start training %i folds'%folds)\n    histories, models = train_cross_validate(folds = folds)\n    print('Computing predictions...')\n    # get the mean probability of the folds models\n    probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    predictions = np.argmax(probabilities, axis=-1)\n    print('Generating submission.csv file...')\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n    return histories, models\n    \n# run train and predict for TPU\n# histories, models = train_and_predict(folds = FOLDS)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T17:15:08.096444Z","iopub.status.idle":"2021-07-16T17:15:08.097141Z"},"trusted":true},"execution_count":null,"outputs":[]}]}