{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tqdm import tqdm\nimport seaborn as sns\nimport wandb\nfrom wandb.keras import WandbCallback\nimport keras\nfrom keras.models import Sequential\nimport pydicom\nimport matplotlib.pyplot as plt\nimport cv2\nimport pathlib\nfrom os import listdir\nfrom scipy.stats import gmean\n\nSQRT2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32)) #ouch\n\nfrom pfutils import (get_test_data, get_train_data, get_pseudo_test_data, get_exponential_decay_lr_callback, TTA_on_test,\n                     build_model, get_cosine_annealing_lr_callback, get_fold_indices, DataGenerator, make_lungmask)\n\nfrom pfutils import (absolute_delta_error, sigma_cost, delta_over_sigma, optimal_sigma_loss_function, \n                    Laplace_metric, Laplace_log_likelihood, experimental_loss_function)\n\nWANDB = False\nSUBMIT = True\nTRAIN_ON_BACKWARD_WEEKS = False\n\n#If TEST is False use this to simulate tractable testcases. Should be 0 if SUBMIT = True\nPSEUDO_TEST_PATIENTS = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if SUBMIT:\n    PSEUDO_TEST_PATIENTS = 0\n    WANDB = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if WANDB:    \n    # retrieve W&B key\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    wandb_key = user_secrets.get_secret(\"wandb_key\")\n    assert wandb_key, \"Please create a key.txt or Kaggle Secret with your W&B API key\"\n\n    !pip install wandb -qqq --upgrade\n    !wandb login $wandb_key","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Settings And network"},{"metadata":{"trusted":true},"cell_type":"code","source":"#State whether model should predict slope or single weeks\n#Predicting the slope is making the assumption that the decrease is linear\nPREDICT_SLOPE = False\n\n#Image Flags\nUSE_IMAGES = False\nAPPLY_LUNGMASK = False\nDIM = 224\nIMG_FEATURES = 22\nEFFNET = 0\nUSE_THREE_LAYERS = True\n\nOPTIMAL_SIGMA_LOSS = False\nCOSINE_CYCLES = 5\n\n#Dropout rate\nDROP_OUT_RATE = 0\nDROP_OUT_LAYERS = [] # [0,1,2] voor dropout in de eerste 3 lagen\n\n#L2-Regularization\nL2_REGULARIZATION = False\nREGULARIZATION_CONSTANT = 0.0001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of folds. A number between 1 and 176-PSEUDO_TEST_PATIENTS. 176 = 2^4 * 11\nFOLDS = 5\n\n#Batch size\nBATCH_SIZE = 128\n\n#Amount of features inputted in NN\nNUMBER_FEATURES = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TTA steps and TTA gaussian multiplier\nTTA_STEPS = 1\nTTA_MULTIPLIER = 0\n\n#Hidden layers\nHIDDEN_LAYERS = [32,32]\n\n#Gaussian Noise (the reported std error for FVC measurement devices is 70)\n#NOISE_SDS : [WeekInit, WeekTarget, WeekDiff, FVC, Percent, Age, Sex, CurrentlySmokes, Ex-smoker, Never Smoked]\nNOISE_SDS = [10,10,1] + [500, 0, 10] + [0.25] + 3*[0.25]\n#GAUSSIAN_NOISE_CORRELATED is a boolean indicating if the gaussians added to FVC on X and y are perfectly correlated or independent\nGAUSSIAN_NOISE_FVC_CORRELATED = True\nADD_NOISE_FVC_TO_PERCENT = True\n                           \n#Activation function to use ('swish', 'leakyrelu' or 'relu')\nACTIVATION_FUNCTION = 'swish'\n\n#Experimenting with loss\nLOSS_MODIFICATION = 1 #(sqrt2 * delta / 70) * LOSS_MODIFICATION is added to the loss function (a value of 1 gives roughly equal weight to delta and sigma)\n\n#Batch normalization\nBATCH_NORMALIZATION = False\nPRE_BATCH_NORMALIZATION = False\nBATCH_RENORMALIZATION = False\n\n#Train length\nEPOCHS = 500\n\n#Input and/or output normalization\nINPUT_NORMALIZATION = True\nOUTPUT_NORMALIZATION = True\nNEGATIVE_NORMALIZATION = False\n\n#Learning rate\nLEARNING_RATE_SCHEDULER = 'exp' #'exp', 'cos' or None\nMAX_LEARNING_RATE = 0.001\nEPOCHS_PER_OOM_DECAY = 300 #OoM : Order of Magnitude\n\nMODEL_NAME = \"TestNoPercentNoOverfit\"\n\nconfig = dict(NUMBER_FEATURES = NUMBER_FEATURES, L2_REGULARIZATION = L2_REGULARIZATION, INPUT_NORMALIZATION = INPUT_NORMALIZATION, BATCH_RENORMALIZATION = BATCH_RENORMALIZATION,\n              ACTIVATION_FUNCTION = ACTIVATION_FUNCTION, DROP_OUT_RATE = DROP_OUT_RATE, OUTPUT_NORMALIZATION = OUTPUT_NORMALIZATION, PRE_BATCH_NORMALIZATION = PRE_BATCH_NORMALIZATION,\n              EPOCHS = EPOCHS, MAX_LEARNING_RATE = MAX_LEARNING_RATE, LOSS_MODIFICATION = LOSS_MODIFICATION, NOISE_SDS = NOISE_SDS, OPTIMAL_SIGMA_LOSS = OPTIMAL_SIGMA_LOSS,\n              COSINE_CYCLES = COSINE_CYCLES, MODEL_NAME=MODEL_NAME, LEARNING_RATE_SCHEDULER = LEARNING_RATE_SCHEDULER, PREDICT_SLOPE = PREDICT_SLOPE,\n              HIDDEN_LAYERS = HIDDEN_LAYERS, REGULARIZATION_CONSTANT = REGULARIZATION_CONSTANT, EPOCHS_PER_OOM_DECAY = EPOCHS_PER_OOM_DECAY,\n              DROP_OUT_LAYERS = DROP_OUT_LAYERS, BATCH_SIZE = BATCH_SIZE, GAUSSIAN_NOISE_FVC_CORRELATED = GAUSSIAN_NOISE_FVC_CORRELATED, TTA_STEPS = TTA_STEPS,\n              ADD_NOISE_FVC_TO_PERCENT = ADD_NOISE_FVC_TO_PERCENT, NEGATIVE_NORMALIZATION = NEGATIVE_NORMALIZATION, BATCH_NORMALIZATION = BATCH_NORMALIZATION,\n              APPLY_LUNGMASK = APPLY_LUNGMASK, USE_IMAGES = USE_IMAGES, DIM = DIM, IMG_FEATURES = IMG_FEATURES, EFFNET = EFFNET, TTA_MULTIPLIER = TTA_MULTIPLIER)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if SUBMIT:\n    test_data, submission = get_test_data(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\n    \ntest_data[\"Percent\"] = 0\ntrain_data, train_images, train_labels = get_train_data('../input/osic-pulmonary-fibrosis-progression/train.csv', PSEUDO_TEST_PATIENTS, TRAIN_ON_BACKWARD_WEEKS, USE_IMAGES, APPLY_LUNGMASK, DIM)\ntrain_data[\"Percent\"] = 0\nnp.save(\"train_data.npy\", train_data.to_numpy())\nnp.save(\"train_images.npy\", np.array(train_images))\nnp.save(\"train_labels.npy\", train_labels.to_numpy())\n\nif PSEUDO_TEST_PATIENTS > 0:\n    test_data, test_check = get_pseudo_test_data('../input/osic-pulmonary-fibrosis-progression/train.csv', PSEUDO_TEST_PATIENTS, INPUT_NORMALIZATION)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"model = build_model(config)\n#tf.keras.utils.plot_model(model)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Folds and Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"fold_pos = get_fold_indices(FOLDS, train_data)\nprint(fold_pos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = []\n\nfor fold in range(FOLDS):\n    \n    train_ID = list(range(fold_pos[0],fold_pos[fold])) + list(range(fold_pos[fold+1],fold_pos[-1]))\n    val_ID = list(range(fold_pos[fold], fold_pos[fold+1]))\n    # Generators\n    training_generator = DataGenerator(train_ID, config)\n    validation_generator = DataGenerator(val_ID, config, validation = True)\n    \n    model = build_model(config)\n    \n    sv = tf.keras.callbacks.ModelCheckpoint(\n    'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n    save_weights_only=True, mode='min', save_freq='epoch')\n    callbacks = [sv]\n    if LEARNING_RATE_SCHEDULER == 'exp':\n        callbacks.append(get_exponential_decay_lr_callback(config))\n    if LEARNING_RATE_SCHEDULER == 'cos':\n        callbacks.append(get_cosine_annealing_lr_callback(config))\n\n    print(fold+1, \"of\", FOLDS)\n    if WANDB:\n        name = MODEL_NAME + '-F{}'.format(fold+1)\n        config.update({'fold': fold+1})\n        wandb.init(project=\"pulfib\", name = name, config=config)\n        wandb_cb = WandbCallback()\n        callbacks.append(wandb_cb)\n        \n    history = model.fit(training_generator, validation_data = validation_generator, epochs = EPOCHS,\n                            verbose = 0, callbacks = callbacks)\n\n    if SUBMIT or PSEUDO_TEST_PATIENTS > 0:\n        model.load_weights('fold-%i.h5'%fold)\n        TTA_test_data = TTA_on_test(test_data.to_numpy(), config)\n        for j in range(TTA_STEPS):\n            predictions.append(model.predict(TTA_test_data[:,:,j], batch_size = 256))\n            \n    if WANDB:\n        # finalize run\n        wandb.join()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if SUBMIT:\n    predictions = np.abs(predictions)\n    predictions[:,:,1] = np.power(predictions[:,:,1],2)\n    predictions = np.mean(predictions, axis = 0)\n    predictions[:,1] = np.power(predictions[:,1],0.5)\n    for i in range(1,len(test_data)+1):\n        submission.loc[i,\"FVC\"] = predictions[i-1,0]\n        submission.loc[i, \"Confidence\"] = predictions[i-1,1]\n    submission.to_csv(\"submission.csv\", index = False)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if PSEUDO_TEST_PATIENTS > 0:\n    quadraticmeans = []\n    for j in range(0,11):\n        result = []\n        for i in range(-20,20):\n            postprocess = np.abs(predictions[j])\n            if i == 0:\n                postprocess[:,:,1] = gmean(postprocess[:,:,1], axis = 0)\n                postprocess = np.mean(postprocess, axis = 0)\n            else:\n                postprocess[:,:,1] = np.power(postprocess[:,:,1],i)\n                postprocess = np.mean(postprocess, axis = 0)\n                postprocess[:,1] = np.power(postprocess[:,1],1/i)\n            FVC_true = test_check[\"TargetFVC\"].values\n            FVC_pred = postprocess[:,0]\n            sigma = postprocess[:,1]\n\n            sigma_clip = np.maximum(np.abs(sigma), 70)\n            delta = np.abs(FVC_true - FVC_pred)\n            delta = np.minimum(delta, 1000)\n\n            sq2 = np.sqrt(2)\n            loss = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip * sq2)\n            result.append(np.mean(loss))\n            if i == 2:\n                quadraticmeans.append(result[-1])\n        plt.plot(np.arange(-20,20),result)\n        plt.show()\nplt.plot(0.1*np.arange(0,11),quadraticmeans)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}