{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pydicom\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\n\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Viendo un poco los datos ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#levantamos solo el train set \ndf = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\")\ngiven_test = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv\")\nsub = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\ndf.shape, given_test.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"given_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# chequeamos si el test dato pertenece al train y vemos que efectivamente lo hace. \nset(given_test.Patient.values).issubset(set(df.Patient.values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# de qué forma deberíamos entregar el submision?--> \nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Analisamos un poco más del submision sample que es el archivo el cual tenemos que predecir? ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']].drop(columns = ['Patient_Week'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#los pacientes del sub son los mismos que los de test. \nset(sub.Patient.unique()) - set(given_test.Patient.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#en sub hay semanas que no tengo en train ? \n#tomo ejemplo de un solo paciente \nid_ = sub.Patient.unique()[4]\nshapesub = sub[sub.Patient == id_].shape\nshapedf = df[df.Patient == id_].shape\nprint('hay un total de muestras: {} para paciente x en submision'.format(shapesub[0]))\nprint('hay un total de muestras: {} para paciente x en train'.format(shapedf[0]))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Entonces acá es donde esta el verdadero quilombo \n- Hay solo 10 semanas de datos para predecir 130 aprox ? \n- es un problema temporal, dado cierto tiempo de datos predecir próximo tiempo. \n- Se evaluan la efectividad solamente en las últimas 3 semanas pero hay que predecir todas. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Algunas especificaciones","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#tipos de columna\ndf.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#algunos valores estadísticos \ndf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#chequeamos si alguna columna es nula (creo que con un solo valor nan ya salta True: chequear)\ndf.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#hay valores duplicados en el train? \ndf.duplicated().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cantidad_pacientes = df.Patient.nunique()\ncantidad_sex = df['Sex'].value_counts()\ncantidad_status = df['SmokingStatus'].value_counts()\nprint('Cantidad total de pacientes :' + str(cantidad_pacientes))\nprint(cantidad_sex)\nprint(cantidad_status)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preproceso de los datos: \nEl preproceso de los datos debe aplicarse tanto al train como al test set. Por ende se realiza un preproceso general luego se divide los sets de interes. \nEl submision set es lo que hay que predecir. Hay que chequear si se repiten semanas del submision en el train, no estaría bien predecir sobre datos de entrenamiento. \nLuego del procesado y las predicciones el submision hay que volverlo a acomodar para enviarlo.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#este merge me parece al pedo pero lo meto por las dudas ya que así esta donde lo robe \nsub = sub.merge(given_test.drop('Weeks', axis=1), on=\"Patient\")\ndf['WHERE'] = 'train'\ngiven_test['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = df.append([given_test, sub]) #juntamos test y sub ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['Female'] = data.Sex.apply(lambda x: 1 if x == 'Female' else 0)\ndata['Male'] = data.Sex.apply(lambda x: 1 if x == 'Male' else 0)\ndata['ExSmoker'] = data.SmokingStatus.apply(lambda x: 1 if x == 'Ex-smoker' else 0)\ndata['NeverSmoked'] = data.SmokingStatus.apply(lambda x: 1 if x == 'Never smoked' else 0)\ndata['CurrentlySmokes'] = data.SmokingStatus.apply(lambda x: 1 if x == 'Currently smokes' else 0)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['semana_min'] = data['Weeks']\ndata['semana_min'] = data.groupby('Patient')['semana_min'].transform('min')\nbase = data.loc[data.Weeks == data.semana_min]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\nbase.head(20)\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data.Weeks - data.semana_min\ndata = data.astype({\"base_week\": float})\n\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Normalización de los datos. Las columnas a usar en el modelo son (week, Percent, Age, Female,Male,ExSmoker,NeverSmoked,CurrentlySmokes, min_FVC)\n#Los nombres no son exactamente igual a los del flaco pero es lo mismo\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['Percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\ndata['Age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['min_FVC'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#El valor de week queda distinto al chabon no se porque","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.50, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model(nh):\n    z = L.Input((nh,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}