{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom\nimport os\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.ensemble import RandomForestRegressor\nfrom skimage import morphology\nfrom skimage import measure\nfrom skimage.transform import resize\nimport tensorflow as tf\nfrom sklearn.cluster import KMeans\nimport matplotlib.patches as patches","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# note : you are asked to predict every patient's FVC measurement for every possible week.****","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Substance\t HU\nAir         −1000\nLung\t    −500\nFat\t        −100 to −50\nWater\t    0\nBlood\t    +30 to +70\nMuscle\t    +10 to +40\nLiver\t    +40 to +60\nBone\t     +700 (cancellous bone) to +3000 (cortical bone)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**okay so i thought this out . we have all the history of fvc 's of the training set . so i can use mathematical methods like lagrange interpolation and some regression methods to make the predictions of fvc of any week . but the problem comes in the test set as we are provided with only one fvc measurement . but we are also provided with base ct scans of the patients so lets see...**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"trying different techniques to figure out something","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''unique_ids=train_csv['Patient'].unique()\nweek_x=[]\nfvc_y=[]\npercent=[]\nfor id in unique_ids:\n    week=np.array(train_csv[train_csv['Patient']==id]['Weeks'])\n    fvc=np.array(train_csv[train_csv['Patient']==id]['FVC'])\n    per=np.array(train_csv[train_csv['Patient']==id]['Percent'])\n    week_x.append(week)\n    fvc_y.append(fvc)\n    percent.append(per)\n    \nunique_train=pd.DataFrame(train_csv['Patient'].unique(),columns=['Patient'])\nunique_train['week_x']=week_x\nunique_train['fvc_y']=fvc_y\nunique_train['percent']=percent\nunique_train.head()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''X=unique_train['week_x'][0]\nY=unique_train['fvc_y'][0]'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def using_poly_reg(X,Y,degree=3):\n    poly_features=PolynomialFeatures(degree=degree,include_bias=False)\n    x_poly=poly_features.fit_transform(X[:,np.newaxis])\n\n    lin_reg=LinearRegression()\n    lin_reg.fit(x_poly,Y)\n\n    x_test=np.arange(-12,133)[:,np.newaxis]\n    x_test_poly=poly_features.fit_transform(x_test)\n    plt.plot(x_test,lin_reg.predict(x_test_poly))\n    plt.plot(X,Y)\n    #plt.ylim(0,6400)\n    #plt.xlim(-12,133)\n    plt.grid(True)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''using_poly_reg(X,Y,degree=3)  #here we can customize the degree of the polynomial so it is better\n                              #by the way if degree=len(X)-1 then it is same as interpolation'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"getting info about a healthy person fvc related to age sex and smoking status\n\n# the percent actually can be used to take out the healthy person fvc with same characteristics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''train_csv['healthy_person_FVC']=(train_csv['FVC']/(train_csv['Percent']/100)).round()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''train_csv'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''healthy_fvc_info=train_csv.groupby(['Age','Sex','SmokingStatus'])['healthy_person_FVC'].mean().round()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''plt.plot(healthy_fvc_info[:,'Male','Ex-smoker'],label='male ex smoker')\nplt.plot(healthy_fvc_info[:,'Male','Never smoked'],label='male never smoked')\nplt.plot(healthy_fvc_info[:,'Male','Currently smokes'],label='male currently smokes')\n\nplt.plot(healthy_fvc_info[:,'Female','Ex-smoker'],label='female ex smoker')\nplt.plot(healthy_fvc_info[:,'Female','Never smoked'],label='female never smoked')\nplt.plot(healthy_fvc_info[:,'Female','Currently smokes'],label='female currently smokes')\n\nplt.title('healthy fvc related to age,sex and smoking status')\nplt.legend()\nplt.grid(True)'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"so this is good i now have a  healthy persons fvc information related to sex age and smoking status.probably i can make use of this","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def RForestRegressor(x,y):\n    reg=RandomForestRegressor(n_estimators=50)\n    reg.fit(x[:,np.newaxis],y)\n    x_test=np.arange(0,100)\n    y_test=reg.predict(x_test[:,np.newaxis])\n    plt.plot(x_test,y_test,label='predicted')\n    plt.plot(x,y,label='real')\n    plt.grid(True)\n    plt.legend()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''#x=np.array(healthy_fvc_info[:,'Male','Ex-smoker'].index)\n#y=np.array(healthy_fvc_info[:,'Male','Ex-smoker'].values)\nx=np.array(healthy_fvc_info[:,'Male','Never smoked'].index)\ny=np.array(healthy_fvc_info[:,'Male','Never smoked'].values)\n\nRForestRegressor(x,y)'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''X=unique_train['week_x'][0]\nY=unique_train['fvc_y'][0]\nRForestRegressor(X,Y)'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"here we see if the age of the patient is recorded only once in the train set........","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''age=train_csv.groupby('Patient')['Age'].unique()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''for item in age:\n    if len(item)==1:\n        continue\n    else:\n        print(item.index)'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"nothing in output means age is only recorded once for each patient ........ nice :-) . that means we can add age column in unique_train dataset","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"now same thing for smoking status","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''SS=train_csv.groupby('Patient')['SmokingStatus'].unique()\nfor item in SS:\n    if len(item)==1:\n        continue\n    else:\n        print(item)'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"great that means Smoking is recorded once and kept same for each patient for all visits","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"so in the test set we are given only one fvc count and only one fvc is not enough for claculating fvc for every possible week.and also we are given only one ct scan for only one visit which alone is not enough but can be used as a singular feature. the only thing we can apply regression on is the weeks of visit . ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**week no**(changes per visit) , **age**(fixed for each visit) , **sex**(fixed for each visit) , **Smoking status**(fixed for each visit)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"i can add one more feature like intial lung capacity with help pf initial dicom. but what is the use?\nWHAT IS THE GAME PLAN::::---\n\n1. FIRSTLY I CAN PREDICT HEALTHY FVC FOR EVERY AGE OF A PERSON IF I FIX HIS SEX AND SMOKING STATUS. THERE WILL BE TOTAL 6 REGRESSORS\n\n2. I HAVE TO ADD ONE MORE FEATURE RELATED TO LUNG CAPACITY WHICH CAN BE GENERATED FROM DICOMS . THIS FEATURE WILL HELP DIFFERENTIATE TWO DIFFERENT PERSON WHO HAVE SAME AGE,SEX,SMOKING STATUS AND WEEK .\n\nCURRENTLY I HAVE 5 FEATURES : AGE,SEX,SMOKING STATUS,WEEK AND LUNG CAPACITY\n\nAGE IS FIXED FOR ONE PATIENT \nSEX IS FIXED FOR ONE PATIENT\nSMOKING STATUS IS FIXED FOR ONE PATIENT\nLUNG CAPACITY IS FIXED FOR ONE PERSON BUT UNIQUE FOR EVERY PATIENT\nWEEK IS VARIABLE FOR EACH PATIENT\n\n3. NOW I HAVE TO SOMEHOW PREDICT FVC FOR EACH WEEK USING ABOVE FEATURES.\n4. I DONT KNOW MUCH ABOUT CONFIDENCE BUT IT HAS TO BE IN ML SO WHAT I WILL DO IS I WILL PREDICT FVC FOR THE BASE WEEK GIVEN IN THE TEST SET AND THEN I WILL SUBTRACT THE TWO VALUES  ie:  MOD(predicted-exact value) . ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''sex=[]\nfor id in unique_train['Patient']:\n    sex.append(train_csv[train_csv['Patient']==id]['Sex'].unique()[0])\n    \nunique_train['sex']=sex'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''age=[]\nfor id in unique_train['Patient']:\n    age.append(train_csv[train_csv['Patient']==id]['Age'].unique()[0])\n    \nunique_train['age']=age'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''ss=[]\nfor id in unique_train['Patient']:\n    ss.append(train_csv[train_csv['Patient']==id]['SmokingStatus'].unique()[0])\n    \nunique_train['smoking-status']=ss'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''unique_train'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I STILL DONT have to do dth with dicoms.so ..........","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# KMEANS","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''from sklearn.cluster import KMeans\n\nlung=pydicom.dcmread('../input/osic-pulmonary-fibrosis-progression/train/ID00012637202177665765362/26.dcm')\nimage=lung.pixel_array\nX = image.reshape(-1,1)\n#X=image\n\n#good_init=np.array([[-2048],[-1000],[892],[-177],[190]])\n#kmeans = KMeans(n_clusters=8,init=good_init,n_init=1).fit(X)\n\nkmeans = KMeans(n_clusters=6).fit(X)\n\nsegmented_img = kmeans.cluster_centers_[kmeans.labels_]\nsegmented_img = segmented_img.reshape(image.shape)\nplt.imshow(segmented_img)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fitter(img):\n    \n    row_size= img.shape[0]\n    col_size = img.shape[1]\n    \n    mean = np.mean(img)\n    std = np.std(img)\n    img = img-mean\n    img = img/std\n    # Find the average pixel value near the lungs\n    # to renormalize washed out images\n    middle = img[int(col_size/4):int(col_size/4*3),int(row_size/4):int(row_size/4*3)] \n    mean = np.mean(middle)  \n    max = np.max(img)\n    min = np.min(img)\n    # To improve threshold finding, I'm moving the \n    # underflow and overflow on the pixel spectrum\n    img[img==max]=mean\n    img[img==min]=mean\n    #\n    # Using Kmeans to separate foreground (soft tissue / bone) and background (lung/air)\n    #\n    kmeans = KMeans(n_clusters=2).fit(np.reshape(middle,[np.prod(middle.shape),1]))\n    return kmeans\n\nlung=pydicom.dcmread('../input/osic-pulmonary-fibrosis-progression/train/ID00012637202177665765362/26.dcm')\nimage=lung.pixel_array*lung.RescaleSlope+lung.RescaleIntercept\nkmeans=fitter(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_lungmask(img,kmeans,display=False):\n    image=img\n    row_size= img.shape[0]\n    col_size = img.shape[1]\n    mean = np.mean(img)\n    std = np.std(img)\n    img = img-mean\n    img = img/std\n    # Find the average pixel value near the lungs\n    # to renormalize washed out images\n    middle = img[int(col_size/4):int(col_size/4*3),int(row_size/4):int(row_size/4*3)] \n    mean = np.mean(middle)  \n    max = np.max(img)\n    min = np.min(img)\n    # To improve threshold finding, I'm moving the \n    # underflow and overflow on the pixel spectrum\n    img[img==max]=mean\n    img[img==min]=mean\n    #\n    # Using Kmeans to separate foreground (soft tissue / bone) and background (lung/air)\n    #\n    centers = sorted(kmeans.cluster_centers_.flatten())\n    threshold = np.mean(centers)\n    thresh_img = np.where(img<threshold,1.0,0.0)  # threshold the image\n\n    # First erode away the finer elements, then dilate to include some of the pixels surrounding the lung.  \n    # We don't want to accidentally clip the lung.\n\n    eroded = morphology.erosion(thresh_img,np.ones([5,5]))\n    dilation = morphology.dilation(eroded,np.ones([8,8]))\n\n    labels = measure.label(dilation) # Different labels are displayed in different colors\n    label_vals = np.unique(labels)\n    regions = measure.regionprops(labels)\n    good_labels = []\n    #for prop in regions:\n    #    b = prop.bbox\n    #    if (abs((b[2]+b[0])/2-(row_size/2))<100) and ( (abs((b[3]+b[1])/2-(col_size/4))<110) or (abs((b[3]+b[1])/2-(col_size/4)*3)<110) ):\n    #        good_labels.append(prop.label)\n            \n    for prop in regions:\n        b = prop.bbox\n        lung_row=abs((b[2]+b[0])/2-(row_size/2))\n        left_lung_col=abs((b[3]+b[1])/2-(col_size/4))\n        right_lung_col=abs((b[3]+b[1])/2-(col_size/4)*3)\n        \n        if lung_row<100 and (left_lung_col<110 or right_lung_col<110):\n            good_labels.append(prop.label)\n            \n    mask = np.ndarray([row_size,col_size],dtype=np.int8)\n    mask[:] = 0\n\n    #\n    #  After just the lungs are left, we do another large dilation\n    #  in order to fill in and out the lung mask \n    #\n    for N in good_labels:\n        mask = mask + np.where(labels==N,1,0)\n    mask = morphology.dilation(mask,np.ones([8,8])) # one last dilation\n\n    if (display):\n        fig, ax = plt.subplots(3, 2, figsize=[12, 12])\n        ax[0, 0].set_title(\"Original\")\n        ax[0, 0].imshow(img, cmap='gray')\n        ax[0, 0].axis('off')\n        ax[0, 1].set_title(\"Threshold\")\n        ax[0, 1].imshow(thresh_img, cmap='gray')\n        ax[0, 1].axis('off')\n        ax[1, 0].set_title(\"After Erosion and Dilation\")\n        ax[1, 0].imshow(dilation, cmap='gray')\n        ax[1, 0].axis('off')\n        ax[1, 1].set_title(\"Color Labels\")\n        ax[1, 1].imshow(labels)\n        ax[1, 1].axis('off')\n        ax[2, 0].set_title(\"Final Mask\")\n        ax[2, 0].imshow(mask, cmap='gray')\n        ax[2, 0].axis('off')\n        ax[2, 1].set_title(\"Apply Mask on Original\")\n        ax[2, 1].imshow(mask*img, cmap='gray')\n        ax[2, 1].axis('off')\n        \n        plt.show()\n        \n    air=[]\n    for i in range(image.shape[0]):\n        for j in range(image.shape[1]):\n            if mask[i][j]==1:\n                air.append(image[i][j])\n    if len(air)==0 :\n        air_percent=0.0\n    else:\n        air_percent=abs((sum(air)/len(air))/10).round(4)\n    return mask,air_percent","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#id='ID00011637202177653955184'\n#path='../input/osic-pulmonary-fibrosis-progression/train/'+id+'/' \n#filenames=os.listdir(path) \n#fileno=int(len(filenames)/2)\n#img=pydicom.dcmread(path+str(fileno)+'.dcm')\n#make_lungmask(img,kmeans,display=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_plots=True\n\nif show_plots:\n    fig=plt.figure(figsize=(20,20)) \ncol=14\nrow=14 \ni=1 \nair_percent_dict={}\nfor id in train_csv['Patient'].unique(): \n    path='../input/osic-pulmonary-fibrosis-progression/train/'+id+'/' \n    filenames=os.listdir(path) \n    fileno=int(len(filenames)/2)\n    for item in filenames:\n        number=int(item.split('.')[0])\n        if number==fileno:\n            break\n        else:\n            continue\n    try:\n        lung=pydicom.dcmread(path+item) \n        image=lung.pixel_array*lung.RescaleSlope+lung.RescaleIntercept\n        mask,air_percent=make_lungmask(image,kmeans,display=False)\n        air_percent_dict[id]=air_percent\n        if show_plots:\n            fig.add_subplot(row,col,i) \n            plt.title(air_percent)\n            plt.imshow(mask,cmap='gray')\n            plt.grid(False)\n            plt.axis(False)\n    except: \n        air_percent_dict[id]=np.nan \n    i=i+1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#here i define a thresold for excluding values with clipped lungs :\nfor key in air_percent_dict:\n    if air_percent_dict[key]<35.0:\n        air_percent_dict[key]=np.nan\n        print(key)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=train_csv[['Patient', 'Weeks', 'FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus']]\nfrom sklearn.preprocessing import LabelEncoder\nlb=LabelEncoder()#sex\ntrain.iloc[:,5]=lb.fit_transform(train.iloc[:,5])\nlb2=LabelEncoder()#ss\ntrain.iloc[:,6]=lb2.fit_transform(train.iloc[:,6])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lung_percent=[]\nfor id in train['Patient']:\n    lung_percent.append(float(air_percent_dict[id]))\ntrain['lung percent']=lung_percent\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=train.dropna()\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# now preparing the test images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\ntest.iloc[:,5]=lb.transform(test.iloc[:,5])\ntest.iloc[:,6]=lb2.transform(test.iloc[:,6])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"air_percent_dict={}\nfor id in test['Patient'].unique(): \n    path='../input/osic-pulmonary-fibrosis-progression/test/'+id+'/' \n    filenames=os.listdir(path) \n    fileno=int(len(filenames)/2)\n    for item in filenames:\n        number=int(item.split('.')[0])\n        if number==fileno:\n            break\n        else:\n            continue\n    try:\n        lung=pydicom.dcmread(path+item) \n        image=lung.pixel_array*lung.RescaleSlope+lung.RescaleIntercept\n        mask,air_percent=make_lungmask(image,kmeans,display=False)\n        air_percent_dict[id]=air_percent\n    except: \n        print(id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"air_percent_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lung_percent_test=[]\nfor id in test['Patient']:\n    lung_percent_test.append(float(air_percent_dict[id]))\ntest['lung percent']=lung_percent_test\ntest","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# healthy fvc predictor","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def healthy_fvc_predictor(age,sex,smoking_status):\n    x=np.array(healthy_fvc_info[:,sex,smoking_status].index)\n    y=np.array(healthy_fvc_info[:,sex,smoking_status].values)\n    reg=RandomForestRegressor(n_estimators=50)\n    reg.fit(x[:,np.newaxis],y)\n    return reg.predict([[age]])'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# submission prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric(actual_fvc, predicted_fvc, confidence, return_values = False):\n    \"\"\"\n    Calculates the modified Laplace Log Likelihood score for this competition.\n    \"\"\"\n    sd_clipped = np.maximum(confidence, 70)\n    delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)\n    metric = - np.sqrt(2) * delta / sd_clipped - np.log(np.sqrt(2) * sd_clipped)\n\n    if return_values:\n        return metric\n    else:\n        return np.mean(metric)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def RForestRegressor(x,y):\n    reg=RandomForestRegressor(n_estimators=400)\n    reg.fit(np.array(x),np.array(y))\n    return reg\n\nx=train[['Weeks','Percent','lung percent','SmokingStatus','Sex','Age']]\n#x=train[['Percent','lung percent','Weeks','Sex']]\ny=train['FVC']\n\n\nfrom sklearn.model_selection import train_test_split\nxtrain,xvalid,ytrain,yvalid=train_test_split(x,y,test_size=0.2)\n\npercent_reg=RForestRegressor(xtrain,ytrain)\n\npreds=percent_reg.predict(np.array(xvalid))\nconfidence=abs(preds-np.array(yvalid))\nprint(metric(np.array(yvalid),preds,confidence))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def neural(x):\n    model=tf.keras.models.Sequential()\n    model.add(tf.keras.layers.Dense(100,activation='relu',input_shape=x.shape[1:]))\n    model.add(tf.keras.layers.Dense(100,activation='relu'))\n    model.add(tf.keras.layers.Dense(100,activation='relu'))\n    model.add(tf.keras.layers.Dense(1))\n    model.compile(loss='mse',optimizer='adam')\n    return model\n\nx=train[['Weeks','Percent','lung percent','SmokingStatus','Sex','Age']]\n#x=train[['Percent','lung percent','Weeks','Sex']]\ny=train['FVC']\n\n\nfrom sklearn.model_selection import train_test_split\nxtrain,xvalid,ytrain,yvalid=train_test_split(x,y,test_size=0.2)\n\nmodel=neural(x)\n\nclass metric_callback(tf.keras.callbacks.Callback):\n    def __init__(self,metrics,xvalid,yvalid):\n        self.metrics=metrics\n        self.xvalid=xvalid\n        self.yvalid=yvalid\n    def on_epoch_end(self,epoch,logs={}):\n        preds=self.model.predict(np.array(self.xvalid))\n        confidence=abs(preds-np.array(self.yvalid))\n        metric=self.metrics(np.array(self.yvalid),preds,confidence)\n        print('\\r val metrics score :',metric)\n\nhistory=model.fit(np.array(xtrain),np.array(ytrain),epochs=10,callbacks=[metric_callback(metric,xvalid,yvalid)])'''\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_fi(forest,X):\n    importances = forest.feature_importances_\n    std = np.std([tree.feature_importances_ for tree in forest.estimators_],axis=0)\n    indices = np.argsort(importances)[::-1]\n\n    # Print the feature ranking\n    print(\"Feature ranking:\")\n\n    for f in range(X.shape[1]):\n        print(\"%d. feature : %s (%f)\" % (f + 1, np.array(X.columns)[indices[f]], importances[indices[f]]))\n\n    # Plot the impurity-based feature importances of the forest\n    plt.figure()\n    plt.title(\"Feature importances\")\n    plt.bar(range(X.shape[1]), importances[indices],color=\"g\", yerr=std[indices])\n    plt.xticks(range(X.shape[1]),np.array(X.columns)[indices])\n    plt.xlim([-1, X.shape[1]])\n    plt.show()\n    \nplot_fi(percent_reg,x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv=test[['Patient', 'Weeks', 'FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus','lung percent']]\nweeks=np.arange(-12,134)\nresult={}\nfor id in test_csv['Patient'].unique():\n    percent=np.array(test_csv[test_csv['Patient']==id]['Percent'])\n    sex=np.array(test_csv[test_csv['Patient']==id]['Sex'])\n    age=np.array(test_csv[test_csv['Patient']==id]['Age'])\n    ss=np.array(test_csv[test_csv['Patient']==id]['SmokingStatus'])\n    lp=np.array(test_csv[test_csv['Patient']==id]['lung percent'])\n    percent=np.repeat(percent,len(weeks))\n    sex=np.repeat(sex,len(weeks))\n    age=np.repeat(age,len(weeks))\n    ss=np.repeat(ss,len(weeks))\n    lp=np.repeat(lp,len(weeks))\n    x=np.concatenate([weeks[:,np.newaxis],percent[:,np.newaxis],lp[:,np.newaxis],ss[:,np.newaxis],sex[:,np.newaxis],age[:,np.newaxis]],axis=1)\n    outcome=percent_reg.predict(x)\n    result[id]=outcome","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ans_df_list=[]\nfor id in result:\n    ID=np.repeat(id,len(weeks))\n    ans=np.concatenate([ID[:,np.newaxis],weeks[:,np.newaxis],result[id][:,np.newaxis]],axis=1)\n    ans=pd.DataFrame(ans)\n    ans_df_list.append(ans)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit=pd.concat(ans_df_list,ignore_index=True)\nsubmit.columns=['Patient','Weeks','FVC']\n\nsubmit['FVC']=submit['FVC'].astype(float)\nsubmit['Weeks']=submit['Weeks'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''healthy_fvc_dict={}\nfor i in range(len(test_csv)):\n    hfvc=healthy_fvc_predictor(test_csv.iloc[i,4],test_csv.iloc[i,5],test_csv.iloc[i,6])\n    healthy_fvc_dict[test_csv.iloc[i,0]]=hfvc.ravel()[0]'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''hfvc_list=[]\nfor i in range(len(submit)):\n    hfvc_list.append(healthy_fvc_dict[submit.iloc[i,0]])\n    \nsubmit['healthy_fvc']=hfvc_list\nsubmit['Percent']=submit['Percent'].astype(float)\nsubmit['FVC']=(submit['healthy_fvc']*submit['Percent'])/100'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# for confidence","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"confidence_dict={}\nfor id in submit['Patient'].unique():\n    real=float(test_csv[test_csv['Patient']==id]['FVC'])\n    \n    week=int(test_csv[test_csv['Patient']==id]['Weeks'])\n    \n    predicted=float(submit[(submit['Patient']==id) & (submit['Weeks']==week) ]['FVC'])\n    \n    confidence_dict[id]=abs(real-predicted)\n    \n    #confidence_dict[id]=np.std(np.array(submit[submit['Patient']==id]['FVC']).astype(float))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confidence_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confidence=[]\nfor i in range(len(submit)):\n    confidence.append(confidence_dict[submit.iloc[i,0]])\nsubmit['Confidence']=confidence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['Patient']=submit['Patient']+'_'+(submit['Weeks'].astype(str))\nsubmit.drop(['Weeks'],axis=1,inplace=True)\nsubmit.columns=['Patient_Week','FVC','Confidence']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}