{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index\n!pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index\n\nimport os\nimport cv2\nimport pydicom\nimport pandas as pd\nimport numpy as np \nimport efficientnet.tfkeras\nimport tensorflow as tf \nimport matplotlib.pyplot as plt \nimport random\nfrom tqdm.notebook import tqdm \nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow_addons.optimizers import RectifiedAdam\nfrom tensorflow.keras import Model\nfrom keras.models import load_model, Model\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom tensorflow.keras.optimizers import Nadam\nimport seaborn as sns\nfrom PIL import Image\n\nconfig = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)\n\nfrom sklearn.cluster import KMeans\nfrom skimage import morphology\nfrom skimage import measure","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_to_hu(slices):\n    \n    images = np.stack([file.pixel_array for file in slices])\n    images = images.astype(np.int16)\n    \n    preffered_shape = [128, 128]\n    if images.shape[1:] !=preffered_shape:\n        images_new = np.zeros((images.shape[0],preffered_shape[0],preffered_shape[1]))\n        for i in range(images.shape[0]):\n                images_new[i]=cv2.resize(images[i], (preffered_shape[0], preffered_shape[1]))\n        images=np.copy(images_new)\n    \n    images[images <= -1000] = 0\n    \n    # convert to HU\n    for n in range(len(slices)):\n        \n        intercept = slices[n].RescaleIntercept\n        slope = slices[n].RescaleSlope\n        \n        if slope != 1:\n            images[n] = images[n].astype(np.float64)*slope\n            images[n] = images[n].astype(np.int16)\n            \n        images[n] += np.int16(intercept)\n    return np.array(images, dtype=np.int16)\n\ndef z_normalization(image):\n\n    preffered_z_size = 5\n    new_image = np.zeros((image.shape[1], image.shape[2], preffered_z_size ))\n\n    if image.shape[0] < preffered_z_size:\n        step = int(preffered_z_size/image.shape[0])\n        for i in range(image.shape[0]):\n            new_image[:,:,i*step] = image[i]\n    elif image.shape[0] > preffered_z_size:\n        step = int(image.shape[0]/preffered_z_size)\n        for i in range(preffered_z_size):\n            new_image[:,:,i] = image[i*step]\n    else:\n        new_image = image\n    \n    new_image = (new_image)/2**11\n    normalization_c = image.shape[0]/preffered_z_size\n    return new_image, normalization_c\n\ndef get_img(dcm_path):\n    \n    files = os.listdir(dcm_path)\n    file_nums = [np.int(file.split(\".\")[0]) for file in files]\n    sorted_file_nums = np.sort(file_nums)[::-1]\n    slices = [pydicom.dcmread(dcm_path + \"/\" + str(file_num) + \".dcm\" ) for file_num in sorted_file_nums]\n    \n    slices_hu = transform_to_hu(slices)\n    slices_norm, c_norm = z_normalization(slices_hu)\n    \n    return slices_norm, c_norm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tab(df , norm, num_weeks):\n    vector = [(df.Age.values[0]) / 100] \n    \n    if df.Sex.values[0] == 'male':\n       vector.append(0)\n    else:\n       vector.append(1)\n    \n    if df.SmokingStatus.values[0] == 'Never smoked':\n        vector.extend([1, 0, 0, 0])\n    elif df.SmokingStatus.values[0] == 'Ex-smoker':\n        vector.extend([0, 1, 0, 0])\n    elif df.SmokingStatus.values[0] == 'Currently smokes':\n        vector.extend([0, 0, 1, 0])\n    else:\n        vector.extend([0, 0, 0, 1])\n    \n    vector.append(norm/100)\n    vector.append(num_weeks/100)\n    return np.array(vector)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.50, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import (\n    Dense,Conv3D, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)\nimport efficientnet.tfkeras as efn\n\ndef get_efficientnet(model, shape):\n    models_dict = efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False)\n    return models_dict\n\ndef build_model(shape=(128,128,5), model_class='b5'):\n    inp1 = Input(shape=shape)\n    inp2 = Input(shape=(10))\n    \n    y = Conv2D(filters=3,kernel_size=3, padding='same', strides=(1, 1),data_format='channels_last')(inp1)\n   # y = Conv2D(filters=1,kernel_size=3, padding='same', strides=(1, 1),data_format='channels_last')(y)\n    \n    base = get_efficientnet(model_class, (128,128,3))\n    x1 = base(y)\n    base.trainable = False\n    x1 = BatchNormalization()(x1)\n    x1 = GlobalAveragePooling2D()(x1)\n    \n    x1 = Dropout(0.385)(x1) \n    x1= Dense(20,activation=\"relu\")(x1)\n    \n    #x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x2 = L.Dense(50, activation=\"relu\")(inp2)\n    x2 = L.Dense(30, activation=\"relu\")(x2)\n\n    y = Concatenate()([x1,x2])\n    \n    p1 = L.Dense(3, activation=\"linear\")(y)\n    p2 = L.Dense(3, activation=\"relu\")(y)\n    \n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1))([p1, p2])\n     \n    \n    model = Model([inp1, inp2] , preds)\n    #model.load_weights('../input/osic-model-weights/' + 'efficientnetb5-50epochs.h5',by_name=True, skip_mismatch=True)\n    \n    model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model\n\n\nmodel_class = 'b5' ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv') \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sun_model =build_model((128,128,5), model_class)\n# print('OK')\nsun_model.load_weights(\"../input/retro-nn/modelnn_sun.h5\")\n# print('OK')\nsun_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Root = '../input/osic-pulmonary-fibrosis-progression/'\nsub = pd.read_csv(f\"{Root}/sample_submission.csv\")\nnh=len(sub)\nnum_week=nh//len(test)\nfvc = []\nconfidence = []\nnames_w = []\n\nfor i, pat in enumerate((test.Patient.unique())):\n    print(pat)\n    \n    images_new = np.zeros((num_week, 128, 128, 5))\n    vector_new = np.zeros((num_week, 9))\n    \n    num_weeks = 1\n    \n    images, c_norm = get_img (Root + 'test/' + pat + '/')\n    images = images.astype('float32')\n    vector = get_tab (test.loc[test.Patient == pat, :], c_norm, num_weeks)\n    base_week = np.resize(np.array(np.arange(num_week)),(num_week,1))\n        \n    for j in range(num_week):\n        images_new[j] = images\n        vector_new[j,:-1] = vector\n        vector_new[j,-1] = test.Percent.loc[test.Patient == pat]/100\n        names_w.append(pat+'_'+str(base_week[j,0]-12))\n    \n    \n    vector_new = np.concatenate((vector_new, base_week), axis = 1)\n    vector_new = vector_new.astype('float32')\n    print(vector_new)\n    buff_predict = sun_model.predict([images_new, vector_new])\n    \n    confidence.extend(buff_predict[:,2] - buff_predict[:,0])\n    fvc.extend(buff_predict[:,1])\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(confidence)\n# print(len(fvc))\n# print(names_w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(PREDICTIONS.shape)\nd = {'Patient_Week': names_w, 'FVC': fvc,'Confidence':confidence}\nsubmis=pd.DataFrame(data=d)\n# print(submis.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(test.head())\n\nfor i in range(len(test)):\n    submis.loc[submis['Patient_Week']==test.Patient[i]+'_'+str(test.Weeks[i]), 'FVC'] = test.FVC[i]\n    submis.loc[submis['Patient_Week']==test.Patient[i]+'_'+str(test.Weeks[i]), 'Confidence'] = 0.1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = submis.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)\nprint(df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}