{"cells":[{"metadata":{"trusted":true,"_uuid":"5ad0cdfe464278e47565e0c4c9f04a9cdde31c1a"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89e938253ad351ca3fc9319b32b6652d4f5d0ec9"},"cell_type":"code","source":"traindata = pd.read_csv('../input/train.csv')\ntestdata = pd.read_csv('../input/test.csv')\nmergeData = [traindata, testdata]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"433aa5ebc2ad0958af3c1bc2871b38b8989fcff4"},"cell_type":"markdown","source":"# 1. FEATURE ENGINEERING"},{"metadata":{"_uuid":"d3672719d0213f7161ed396c96fd33ded6672e66"},"cell_type":"markdown","source":"## 1.1. Get Titles: based on name"},{"metadata":{"trusted":true,"_uuid":"76c48227554865e6576731d6f2bfc064f0e4b436"},"cell_type":"code","source":"import re\n\ndef title(Name):\n    getTitle = re.search('([A-Za-z]+)\\.',Name)\n    \n    if getTitle:\n        return getTitle.group(1)\n    return \"\"\n\nfor data in mergeData:\n    data['Titles'] = data['Name'].apply(title)\n\ncrosstabTrain = pd.crosstab(traindata['Titles'], traindata['Sex'])\ncrosstabTest = pd.crosstab(testdata['Titles'], testdata['Sex'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e57563baccafd5c883d462dd015505c17866f67d"},"cell_type":"code","source":"### check if there two sex in one title\ncrosstabTrain[(crosstabTrain.female != 0) & (crosstabTrain.male !=0)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"56eec58849a49b1a2830bff5009580d4f912c545"},"cell_type":"code","source":"# Dr are female & male\ncrosstabTest[(crosstabTest.female != 0) & (crosstabTest.male !=0)]\n# Just in Train data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54dc55b1b69a6b814351b7b148d682490ac55bca"},"cell_type":"markdown","source":"## MALE"},{"metadata":{"trusted":false,"_uuid":"d41c59223e523b91326a5fadd7881a5a08b7eaa3"},"cell_type":"code","source":"crosstabTest.loc[crosstabTest.male != 0, 'male'].sort_values(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"460fba1e21a9e46ef8f6c5738e0de1932fe2c62b"},"cell_type":"code","source":"#titles in data train have represent titles in data test\nmale = crosstabTrain.loc[crosstabTrain.male != 0, 'male'].sort_values(ascending=False)\nmale","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"11390d0d1d12b9e8c013d168685e9aaeb0e4dd75"},"cell_type":"code","source":"# Age distibution for titles: Mr. and Master\nManOld = traindata[['Titles','Age']][traindata.Titles.isin(['Mr','Master'])]\n\nf,axes = plt.subplots(1,2, figsize=(12,6))\nvisManOld = sns.violinplot(data=ManOld, x='Titles', y='Age',ax=axes[0], palette=\"husl\")\n\n\nplt.rcParams['figure.figsize'] = 10,8\nvisMO1 = sns.distplot(ManOld[ManOld.Titles=='Mr'].Age, hist=False, kde_kws={\"shade\": True}, color=\"r\", label ='Mr')\nvisMO1 = sns.distplot(ManOld[ManOld.Titles=='Master'].Age, hist=False, kde_kws={\"shade\": True}, color=\"g\", label='Master')\nvisMO1.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"791ac10c342460e7ac7f0f11deee9a32a2d0d7ac"},"cell_type":"code","source":"mr = ['Rev','Dr','Major', 'Col', 'Sir', 'Jonkheer', 'Don', 'Capt']\ntraindata[(traindata.Titles.isin(mr)) & (traindata.Age<=20)][['Titles','Age']]\n#all of them greater than 20 y.o\n# testdata[(testdata.Titles.isin(maleL)) & (testdata.Age<=20) ][['Titles','Age']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"eea5c233faf534ef5785419ac25788cc6c5f2431"},"cell_type":"code","source":"for data in mergeData:\n    data.loc[data.Titles.isin(mr) & (data.Sex == 'male'),'Titles'] = 'Mr'","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"60d566129fd72deca511717dffda038af8acf84c"},"cell_type":"code","source":"#female Dr\ntraindata[traindata.Titles == 'Dr']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a311b469aa027751b6be624bd0a87d2459d56dbe"},"cell_type":"markdown","source":"## FEMALE"},{"metadata":{"trusted":false,"_uuid":"9666517a79ed00bafd206acb6f5abb58e48cceca"},"cell_type":"code","source":"crosstabTest.loc[crosstabTest.female != 0, 'female'].sort_values(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"969b037e4755b6008646e378e4f7135172b86460"},"cell_type":"code","source":"crosstabTrain.loc[crosstabTrain.female != 0, 'female'].sort_values(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"06877cbbd0601f3e04e429f82bf60aa6947c2db1"},"cell_type":"code","source":"woman = traindata[['Titles','Age']][traindata.Titles.isin(['Miss','Mrs'])]\n\nf,axes = plt.subplots(1,2, figsize=(12,6))\nviswoman = sns.violinplot(data=woman, x='Titles', y='Age',ax=axes[0], palette=\"husl\")\n\n\nplt.rcParams['figure.figsize'] = 10,8\nvisMO2 = sns.distplot(woman[woman.Titles=='Miss'].Age, hist=False, kde_kws={\"shade\": True}, color=\"r\", label ='Miss')\nvisMO2 = sns.distplot(woman[woman.Titles=='Mrs'].Age, hist=False, kde_kws={\"shade\": True}, color=\"g\", label='Mrs')\nvisMO2.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"17a4b2de2a70da8d1380da71887219c4715d61ff"},"cell_type":"code","source":"mrs = ['Mlle','Ms','Mme','Lady','Dr','Countess', 'Dona']\ntraindata[traindata.Titles.isin(mrs) & (traindata.Age <=20)][['Titles','Age']]\n#All of them greater than 20 y.o\n#testdata[testdata.Titles.isin(mrs) & (testdata.Age <=20)][['Titles','Age']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bf412e9e452420e41268c85442b452fa1ba770e2"},"cell_type":"code","source":"for data in mergeData:\n    data.loc[(data.Titles.isin(mrs))&(data.Sex == 'female'),'Titles'] = 'Mrs'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e881400c5a1a73f917f616e9b25f6393ed432cd"},"cell_type":"markdown","source":"## 1.2. fillna Age: based on Titles"},{"metadata":{"trusted":false,"_uuid":"a4fd5c5cbeb9401eae8233fcadf0a875afd26f28"},"cell_type":"code","source":"titles = ['Mr','Master','Mrs','Miss']\nfor title in titles:\n    for data in mergeData:\n        dt = data[data.Titles==title]\n\n        ageAvg = dt.Age.mean()\n        ageStd = dt.Age.std()\n        ageMin = ageAvg-ageStd\n        ageMax = ageAvg+ageStd\n        numNull = data.Age.isnull().sum()\n\n        ageRand = np.random.randint(ageMin,ageMax,size=numNull)\n        data['Age'][np.isnan(data['Age'])] = ageRand\n        #data['Age'] = data['Age'].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f206a23dd3d073483a36dab430e25c95494d02bb"},"cell_type":"markdown","source":"## 1.3. Alone or not alone"},{"metadata":{"trusted":false,"_uuid":"caa44cfda7ff1777e868c17238ef2ab221731e2f"},"cell_type":"code","source":"for data in mergeData:\n    data.loc[(data.SibSp == 0) & (data.Parch == 0), 'Alone'] = 1\n    data.loc[data.Alone.isnull(), 'Alone'] = 0 \n    data.Alone = data.Alone.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d97c9607a4bb35c01b6184d38a99054c39cd8985"},"cell_type":"markdown","source":"## 1.4. fillna fare : based on Pclass\nthere is correlation between Fare & Pclass"},{"metadata":{"trusted":false,"_uuid":"2d706d3b99a50f961b9d81306caba759e4c5782b"},"cell_type":"code","source":"sns.heatmap(traindata.corr(),annot=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3045a3ee7af92431c5b3bcefe5a21ffd559c4147"},"cell_type":"code","source":"Pclass = list(traindata.Pclass.unique())\nfor pc in Pclass:\n    for data in mergeData:\n        df = data[data.Pclass == pc]\n        \n        fareAvg = df.Fare.mean()\n        fareStd = df.Fare.std()\n        fareMin = fareAvg-fareStd\n        fareMax = fareAvg+fareStd\n        numNull = data.Fare.isnull().sum()\n        \n        fareRand = np.random.randint(fareMin,fareMax,size=numNull)\n        data['Fare'][np.isnan(data['Fare'])] = fareRand","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b31f42b73a3eeeeaca8f3f28ddce5bc2287b0c6"},"cell_type":"markdown","source":"## 1.5. fillna Embarked: based on mode    "},{"metadata":{"trusted":false,"_uuid":"937a9bda5ac1a4c90dd4fb4ebce9355b51b92172"},"cell_type":"code","source":"for data in mergeData:\n    data.loc[data.Embarked.isnull(),'Embarked'] = 'S'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ef54b7e5ab378168d144a841b7a665c0cda3687"},"cell_type":"markdown","source":"# remove columns"},{"metadata":{"trusted":false,"_uuid":"b28df32564066c08b4fbc06e88507ad5df1a4569"},"cell_type":"code","source":"dropColumns = ['PassengerId','Name','SibSp','Parch','Ticket','Cabin']\nfor data in mergeData:\n    data.drop(dropColumns, axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"251298de9debb1c09ca202596500cdd554f5267b"},"cell_type":"markdown","source":"# 2. CATEGORICAL"},{"metadata":{"trusted":false,"_uuid":"87beeca3e185542c5dd6dcae66f234e6082c1585"},"cell_type":"code","source":"# Age Interval\nsplitAge = traindata.copy()\nsplitAge['Age'] = pd.cut(traindata['Age'], 5)\nsplitAge.groupby('Age').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6c142dfa84124f45a252e9c24e853302649949a7"},"cell_type":"code","source":"# Fare Interval\nsplitFare = traindata.copy()\nsplitFare['Fare'] = pd.qcut(traindata['Fare'], 6)\nsplitFare.groupby('Fare').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"dd173b9f94d92f4720a5fc102712a1a43ef42a1e"},"cell_type":"code","source":"for data in mergeData:\n    #AGE\n    data.loc[data.Age <= 16.3, 'Age'] = 0\n    data.loc[(data.Age >16.3) & (data.Age <= 32.3), 'Age'] = 1\n    data.loc[(data.Age >32.3) & (data.Age <= 48.2), 'Age'] = 2\n    data.loc[(data.Age >48.2) & (data.Age <= 64.1), 'Age'] = 3\n    data.loc[data.Age > 64.1, 'Age'] = 4   \n    \n    #FARE\n    data.loc[data.Fare <= 7.8, 'Fare'] = 0\n    data.loc[(data.Fare > 7.8)&(data.Fare <= 8.7), 'Fare'] = 1\n    data.loc[(data.Fare > 8.7)&(data.Fare <= 14.5), 'Fare'] = 2\n    data.loc[(data.Fare > 14.5)&(data.Fare <= 26.0), 'Fare'] = 3\n    data.loc[(data.Fare > 26.0)&(data.Fare <= 52.4), 'Fare'] = 4\n    data.loc[(data.Fare > 52.4), 'Fare'] = 5\n    \n    data['Sex'] = data['Sex'].map({'female':0,'male':1}).astype('int')\n    data['Titles'] = data['Titles'].map({'Mr':0, 'Mrs':1, 'Miss':2, 'Master':3})\n    data['Embarked'] = data['Embarked'].map({'Q':0, 'C':1, 'S':2})\n    \n    data['Age'] = data['Age'].astype(int)\n    data['Fare'] = data['Fare'].astype(int)\n    data['Embarked'] = data['Embarked'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f72bfc263e3c79afd1d46a52a1699511ea285d74"},"cell_type":"code","source":"trainData = traindata[['Embarked','Titles','Pclass','Sex','Age','Fare','Alone','Survived']]\ntestData= testdata[['Embarked','Titles','Pclass','Sex','Age','Fare','Alone']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e0eecfabcf465b293eddf1f2937564da53a86857"},"cell_type":"code","source":"X = trainData.iloc[:,:-1].values\ny = trainData.iloc[:,7:8].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"795697ba6d7a4bb89fe254d3fe73e6a871be1d2d"},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"33f546d0bc60a8c8fdb47fddb3f7e47fdedfed11"},"cell_type":"markdown","source":"# 3. ONEHOTENCODER \nEmbarked & Titles"},{"metadata":{"trusted":false,"_uuid":"19178526e7bb3b5856445a232b8fcd5e0dc2cb62"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n\n# EMBARKED\nonehotencoderE = OneHotEncoder(categorical_features = [0])\nX = onehotencoderE.fit_transform(X).toarray()\n#avoiding dummy var trap\nX = X[:,1:]\n\n#TITLES\nonehotencoderT = OneHotEncoder(categorical_features = [2])\nX = onehotencoderT.fit_transform(X).toarray()\n#avoiding dummy var trap\nX = X[:,1:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9126cbf2c24e7f17eafbdad4cbdd5a623549e451"},"cell_type":"markdown","source":"# 4. BACKWARD ELIMINATION\nthe optimum model, without Fare & Alone"},{"metadata":{"trusted":false,"_uuid":"816d661b6ec6077b596fb292ae9abb511beaf1d0"},"cell_type":"code","source":"import statsmodels.formula.api as sm\n# ADD BIAS FIRST\nX = np.append(arr = np.ones((891,1)).astype(int), values=X, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"223816bcb70dbc60f554e941ff2dc14f3c953e48"},"cell_type":"code","source":"X_opt = X[:,[0,1,2,3,5,6,7,8,10]] #remove index that have Pvalues > Significant Level\ncekOLS = sm.OLS(endog = y, exog=X_opt).fit()\ncekOLS.summary()\n\n#X -- bias=0 #Embarked=1 #Titles=2 #Pclass=3 #Sex=4 #Age=5 ##Fare=6 ##Alone=7\n#constant has index 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a9e46e0869a83b98d4c34124ff981b235497046"},"cell_type":"markdown","source":"# 5. NEURAL NETWORK"},{"metadata":{"trusted":false,"_uuid":"7216352f56dc8d7821671a6c6be10f744fb984a2"},"cell_type":"code","source":"X = X[:,[0,1,2,3,5,6,7,8,10]]\ny = y","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1e10637bcd42e4b826a1321cd858744919f2b2e7"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nXtrain,Xval,ytrain,yval = train_test_split(X,y,stratify=y, test_size=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"053c97d73095acc28bade9ca7fc7988aaafc3219"},"cell_type":"code","source":"def sigmoid(z):\n    return(1 / (1 + np.exp(-z)))\n\ndef sigmoidGradient(z):\n    return(sigmoid(z)*(1-sigmoid(z)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3f4db0c25cd7c40167246572857a9811e0a2689c"},"cell_type":"code","source":"def getOptTheta(nn_params, input_layer_size, hidden_layer_size, num_labels, features, classes, reg,alpha,num_iter):\n    \n    theta1 = nn_params[0:(hidden_layer_size*(input_layer_size+1))].reshape(hidden_layer_size,(input_layer_size+1))\n    theta2 = nn_params[(hidden_layer_size*(input_layer_size+1)):].reshape(num_labels,(hidden_layer_size+1))\n\n    m = features.shape[0]\n    if num_labels == 1:\n        y_matrix = classes\n    else:\n        y_matrix = pd.get_dummies(classes.ravel()).as_matrix() \n        \n    for i in np.arange(num_iter):\n        \n        # Cost\n        a1 = features \n\n        z2 = theta1.dot(a1.T) \n        a2 = np.c_[np.ones((features.shape[0],1)),sigmoid(z2.T)] \n\n        z3 = theta2.dot(a2.T) \n        a3 = sigmoid(z3)\n\n        # Gradients\n        d3 = a3.T - y_matrix \n        d2 = theta2[:,1:].T.dot(d3.T)*sigmoidGradient(z2) \n\n        delta1 = d2.dot(a1) \n        delta2 = d3.T.dot(a2)\n\n        theta1_ = np.c_[np.ones((theta1.shape[0],1)),theta1[:,1:]]\n        theta2_ = np.c_[np.ones((theta2.shape[0],1)),theta2[:,1:]]\n\n        theta1_grad = delta1/m + (theta1_*reg)/m\n        theta2_grad = delta2/m + (theta2_*reg)/m\n\n        theta1 = theta1 - (alpha*theta1_grad)\n        theta2 = theta2 - (alpha*theta2_grad)\n        \n    return(theta1,theta2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"879bbd98514713ee1b4e64e39c119ef13327694c"},"cell_type":"code","source":"def prediction(X,thetaOpt1,thetaOpt2):\n    [m,n] = X.shape\n    \n    a1 = X \n    a2 = sigmoid(X.dot(thetaOpt1.T) ) \n    hypo = sigmoid(np.c_[np.ones((a2.shape[0],1)),a2].dot(thetaOpt2.T)) \n    \n    if thetaOpt2.shape[0] == 1:\n        for i in range(hypo.shape[0]):\n            if hypo[i] >= 0.5:\n                hypo[i] = 1\n            else:\n                hypo[i] = 0\n        opt = hypo\n        \n    else:\n        opt = (np.argmax(hypo, axis=1))\n        opt = opt.reshape(-1,1)\n    \n    return(opt)  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8662ffdf8cd891eb148168b3276e19bf30f28127"},"cell_type":"markdown","source":"# 6. MODEL SELECTION"},{"metadata":{"_uuid":"1f4608883370382994309f999637e7909b246d16"},"cell_type":"markdown","source":"## 6.1. Model Selection: Hidden Layer Size"},{"metadata":{"trusted":false,"_uuid":"ad308e90421245af2931b7cd6982c9b6982b9b80"},"cell_type":"code","source":"list_hidden_layer_size = [6,7,8,9,10,12,16,21,31]\n\nreg=0.01\nalpha= 0.1\nnum_iter=10000\n\ninput_layer_size = 8\nnum_labels = 1\n#--------------------------------\n\nAThl = []\nAVhl = []\n\nfor hidden_layer_size in list_hidden_layer_size:\n    \n    #create initial random params\n    eps = 0.12\n    initialTheta1 = np.random.rand(hidden_layer_size,input_layer_size+1)*2*eps-eps\n    initialTheta2 = np.random.rand(num_labels,hidden_layer_size+1)*2*eps-eps\n    initial_nn_params = np.r_[initialTheta1.ravel(), initialTheta2.ravel()]\n    \n    [tOpt1,tOpt2] = getOptTheta(initial_nn_params, input_layer_size, hidden_layer_size, num_labels, Xtrain, ytrain, reg,alpha,num_iter)\n        \n     \n    predTrain = prediction(Xtrain,tOpt1,tOpt2)\n    accTrain = np.mean(predTrain==ytrain)*100\n    \n    predVal = prediction(Xval,tOpt1,tOpt2)\n    accVal = np.mean(predVal==yval)*100\n    \n    AThl.append(accTrain)\n    AVhl.append(accVal)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ede982d492b6489dc9bd90482b36444f94dd7b43"},"cell_type":"code","source":"plt.plot(list_hidden_layer_size,AThl, label = 'Accuracy Train' )\nplt.plot(list_hidden_layer_size,AVhl, label='Accuracy Val')\nplt.title('accurate for each hidden layer size')\nplt.xlabel('number of hidden layer size')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"48c624dd200f38fd335046cbe37a1cceda7069ce"},"cell_type":"code","source":"gethl = pd.DataFrame({'hidden_layer_size':list_hidden_layer_size,'Train Accuracy':AThl,'Val Accuracy':AVhl})\ngethl.sort_values('Val Accuracy', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87979b81bd003470631436e2e34e2903b55eb6f3"},"cell_type":"markdown","source":"## 6.2. Model Selection: Lambda (regularization)"},{"metadata":{"trusted":false,"_uuid":"c4e6124c11d819f7b172e96ccdd40939caf93076"},"cell_type":"code","source":"list_reg=[0, 0.01, 0.1, 1, 10, 100]\n\nalpha= 0.1\nnum_iter=10000\n\nhidden_layer_size = 7\ninput_layer_size = 8\nnum_labels = 1\n#--------------------------------\n\nATlr = []\nAVlr = []\n\nfor reg in list_reg:\n    \n    #CREATE INITIAL RANDOM PARAMS\n    eps = 0.12\n    initialTheta1 = np.random.rand(hidden_layer_size,input_layer_size+1)*2*eps-eps\n    initialTheta2 = np.random.rand(num_labels,hidden_layer_size+1)*2*eps-eps\n    initial_nn_params = np.r_[initialTheta1.ravel(), initialTheta2.ravel()]\n    \n    [tOpt1,tOpt2] = getOptTheta(initial_nn_params, input_layer_size, hidden_layer_size, num_labels, Xtrain, ytrain, reg,alpha,num_iter)\n        \n     \n    predTrain = prediction(Xtrain,tOpt1,tOpt2)\n    accTrain = np.mean(predTrain==ytrain)*100\n    \n    predVal = prediction(Xval,tOpt1,tOpt2)\n    accVal = np.mean(predVal==yval)*100\n    \n    ATlr.append(accTrain)\n    AVlr.append(accVal)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f19084006aa868c5beeba38bdb7e484ba72ae042"},"cell_type":"code","source":"plt.plot(list_reg,ATlr, label = 'Accuracy Train' )\nplt.plot(list_reg,AVlr, label='Accuracy Val')\nplt.title('Model selection : lambda')\nplt.xlabel('lambda')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2226f01c1a984e940fc18260df7e179f635dd2c4"},"cell_type":"code","source":"getlr = pd.DataFrame({'Lambda - reg':list_reg,'Train Accuracy':ATlr,'Val Accuracy':AVlr})\ngetlr.sort_values('Val Accuracy', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ba00398c1405ced9a91f885574966b08835ab20"},"cell_type":"markdown","source":"## 6.3. Model Selection: NumIters for GradientDescent"},{"metadata":{"trusted":false,"_uuid":"65f0301c476d61f7dd6710c418a3764cd639ebde"},"cell_type":"code","source":"numIters=[1000,2000,3000,4000,5000,10000,15000,20000,25000,30000]\n\nreg=0.01\nalpha= 0.1\n\nhidden_layer_size = 7\ninput_layer_size = 8\nnum_labels = 1\n#--------------------------------\n\nATni = []\nAVni = []\n\nfor num_iter in numIters:\n    \n    #CREATE INITIAL RANDOM PARAMS\n    eps = 0.12\n    initialTheta1 = np.random.rand(hidden_layer_size,input_layer_size+1)*2*eps-eps\n    initialTheta2 = np.random.rand(num_labels,hidden_layer_size+1)*2*eps-eps\n    initial_nn_params = np.r_[initialTheta1.ravel(), initialTheta2.ravel()]\n    \n    [tOpt1,tOpt2] = getOptTheta(initial_nn_params, input_layer_size, hidden_layer_size, num_labels, Xtrain, ytrain, reg,alpha,num_iter)\n        \n     \n    predTrain = prediction(Xtrain,tOpt1,tOpt2)\n    accTrain = np.mean(predTrain==ytrain)*100\n    \n    predVal = prediction(Xval,tOpt1,tOpt2)\n    accVal = np.mean(predVal==yval)*100\n    \n    ATni.append(accTrain)\n    AVni.append(accVal)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"22e2bfa8eab1f77698a633d4e78d9037f7da982e"},"cell_type":"code","source":"plt.plot(numIters,ATni, label = 'Accuracy Train' )\nplt.plot(numIters,AVni, label='Accuracy Val')\nplt.title('Model selection : Num Iter for GradientDescent')\nplt.xlabel('Num iter')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1762240480ce79244cb2de5bae7c94bae06b1693"},"cell_type":"code","source":"getni = pd.DataFrame({'Num Iters':numIters,'Train Accuracy':ATni,'Val Accuracy':AVni})\ngetni.sort_values('Val Accuracy', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4adc6266d8e7eb5697a2e58a199f40f0cb18f720"},"cell_type":"markdown","source":"## 6.4.Model Selection: Alpha"},{"metadata":{"trusted":false,"_uuid":"5cec1105bc06b0e303288e99665f18327bb4c696"},"cell_type":"code","source":"alphaL= [0.01,0.03,0.1,0.3,1,3]\n\nreg=0.01\nnum_iter=20000\n\nhidden_layer_size = 7\ninput_layer_size = 8\nnum_labels = 1\n#--------------------------------\n\nATa = []\nAVa = []\n\nfor alpha in alphaL:\n    \n    #CREATE INITIAL RANDOM PARAMS\n    eps = 0.12\n    initialTheta1 = np.random.rand(hidden_layer_size,input_layer_size+1)*2*eps-eps\n    initialTheta2 = np.random.rand(num_labels,hidden_layer_size+1)*2*eps-eps\n    initial_nn_params = np.r_[initialTheta1.ravel(), initialTheta2.ravel()]\n    \n    [tOpt1,tOpt2] = getOptTheta(initial_nn_params, input_layer_size, hidden_layer_size, num_labels, Xtrain, ytrain, reg,alpha,num_iter)\n        \n     \n    predTrain = prediction(Xtrain,tOpt1,tOpt2)\n    accTrain = np.mean(predTrain==ytrain)*100\n    \n    predVal = prediction(Xval,tOpt1,tOpt2)\n    accVal = np.mean(predVal==yval)*100\n    \n    ATa.append(accTrain)\n    AVa.append(accVal)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4c8d2855b9fdac05f13b0c1c3b2232ba42eab10a"},"cell_type":"code","source":"plt.plot(alphaL,ATa, label = 'Accuracy Train' )\nplt.plot(alphaL,AVa, label='Accuracy Val')\nplt.title('Model selection : Num Iter for GradientDescent')\nplt.xlabel('Num iter')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"42008823f013034420661d566d7f31c48a026324"},"cell_type":"code","source":"geta = pd.DataFrame({'Alpha - learning rate':alphaL,'Train Accuracy':ATa,'Val Accuracy':AVa})\ngeta.sort_values('Val Accuracy', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab7b3a8ca5c64d3197ddcf6036ce0cefdcc71f84"},"cell_type":"markdown","source":"## 🌹get different initial_nn_params and different accuracy 🌹"},{"metadata":{"trusted":false,"_uuid":"c65b7988da7fabef9f10fdf4f65e7753b7cbaf40"},"cell_type":"code","source":"accuracyTrainLr = []\naccuracyValLr = []\ntOptl1 = []\ntOptl2 = []\n\ntrying = np.arange(1,11)\nfor iters in trying:\n\n    reg=0.01\n    alpha= 0.1\n    num_iter=20000\n\n    hidden_layer_size = 7\n    input_layer_size = 8\n    num_labels = 1\n    #--------------------------------\n\n    #CREATE INITIAL RANDOM PARAMS\n    eps = 0.12\n    initialTheta1 = np.random.rand(hidden_layer_size,input_layer_size+1)*2*eps-eps\n    initialTheta2 = np.random.rand(num_labels,hidden_layer_size+1)*2*eps-eps\n    initial_nn_params = np.r_[initialTheta1.ravel(), initialTheta2.ravel()]\n\n    [tOpt1,tOpt2] = getOptTheta(initial_nn_params, input_layer_size, hidden_layer_size, num_labels, Xtrain, ytrain, reg,alpha,num_iter)\n\n\n    predTrain = prediction(Xtrain,tOpt1,tOpt2)\n    accTrain = np.mean(predTrain==ytrain)*100\n\n    predVal = prediction(Xval,tOpt1,tOpt2)\n    accVal = np.mean(predVal==yval)*100\n\n    accuracyTrainLr.append(accTrain)\n    accuracyValLr.append(accVal)\n    tOptl1.append(tOpt1)\n    tOptl2.append(tOpt2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"01b1cc761eee8f64b779b058269d36c20f49dc0d"},"cell_type":"code","source":"plt.plot(trying,accuracyTrainLr, label = 'Train Accuracy' )\nplt.plot(trying,accuracyValLr, label='Val Accuracy')\nplt.title('run n times for different param and accuracy')\nplt.xlabel('running n times')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"035519699078d928bb8199c1fa77e0c5a868098f"},"cell_type":"code","source":"tri = accuracyTrainLr.index(np.max(accuracyTrainLr))\ntrv = np.max(accuracyTrainLr)\ntvi = accuracyValLr.index(np.max(accuracyValLr))\ntvv = np.max(accuracyValLr)\nprint('the highest accuracy in data train in index:',tri,'with accuracy =',trv)\nprint('the highest accuracy in data CROSSVAL in index:',tvi,'with accuracy =',tvv)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4be695f8034e03e2a3a88fa1eb9975f82fcc473"},"cell_type":"markdown","source":"## get the optimum theta"},{"metadata":{"trusted":false,"_uuid":"50d134ec111ac07627c6ed4bb6ce996db23a2525"},"cell_type":"code","source":"imax = accuracyValLr.index(np.max(accuracyValLr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"acd6d8ff68ac9dd74c4a9fd9f38fa993bb5e7048"},"cell_type":"code","source":"Theta1Optimum = tOptl1[imax]\nTheta2Optimum = tOptl2[imax]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"87f665df5b2f664fd8dfaec1af958196e6960d18"},"cell_type":"code","source":"predTrain = prediction(Xtrain,Theta1Optimum,Theta2Optimum)\naccTrain = np.mean(predTrain==ytrain)*100\naccTrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c264a66592e2f8c2f3da216f167162da062ef0e8"},"cell_type":"code","source":"predVal = prediction(Xval,Theta1Optimum,Theta2Optimum)\naccVal = np.mean(predVal==yval)*100\naccVal","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"277ad40cf99c35fd9b1387132c97875747f0e51e"},"cell_type":"markdown","source":"# 7. EVALUATION"},{"metadata":{"trusted":false,"_uuid":"c0011ccfa4c4848b912e6cc8a39edb1a8997f919"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, precision_recall_curve, auc, roc_auc_score, roc_curve, recall_score, classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c5f1682e783a318398e9ea4cba7f2ad5b6f26eef"},"cell_type":"code","source":"import itertools\n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    sns.set_style('white')\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=0)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        #print(\"Normalized confusion matrix\")\n    else:\n        1#print('Confusion matrix, without normalization')\n\n    #print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce29f6566b3e8b16fa5d40b2f7b1e38d06e63cd8"},"cell_type":"markdown","source":"## 7.1. Confusion Matrix"},{"metadata":{"trusted":false,"_uuid":"d143aa1500ecf404096589f0e0b2b8dbec136528"},"cell_type":"code","source":"cm = confusion_matrix(yval,predVal)\n\n#bukan make index untuk akses confusion matrix. \nprint(\"Recall =\", round(cm[1,1]/(cm[1,0]+cm[1,1]),2))\nprint(\"Precision =\",round(cm[1,1]/(cm[0,1]+cm[1,1]),2))\nprint(\"Accuracy=\",(cm[1,1]+cm[0,0])/(cm[1,1]+cm[0,0]+cm[1,0]+cm[0,1]) )\n# Plot non-normalized confusion matrix\nclass_names = [0,1]\nplt.figure()\nplot_confusion_matrix(cm\n                      , classes=class_names\n                      , title='Confusion matrix')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1e9bd23e7756788e266e9b6fcfbf4054419c8780"},"cell_type":"code","source":"print(classification_report(yval,predVal))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5da13bc1b8795632ce24917bbfde4c6017dd888f"},"cell_type":"markdown","source":"## 7.2. ROC Curve"},{"metadata":{"trusted":false,"_uuid":"e958d9bac77b19360c838a33cd6dd35402fa8e50"},"cell_type":"code","source":"fpr, tpr, thresholds = roc_curve(yval,predVal)\nroc_auc = auc(fpr,tpr)\n\nprint('ROC AUC=',round((roc_auc*100),2),'%')\n# Plot ROC\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, 'b',label='AUC = %0.2f'% roc_auc)\nplt.legend(loc='lower right')\nplt.plot([0,1],[0,1],'r--')\nplt.xlim([-0.1,1.0])\nplt.ylim([-0.1,1.01])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"884845adfdfc22847d0d85a76ebdb75713531970"},"cell_type":"markdown","source":"# 8. PREDICTING TEST DATA"},{"metadata":{"trusted":false,"_uuid":"4fb63e791abf1b36b9c35f7878547570bf38bdd5"},"cell_type":"code","source":"testData.iloc[:,:-2].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6f38f88aa4460ea1bf7c5074462923962e505002"},"cell_type":"code","source":"testData.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"72e80cb026fdfed5d3a3fd04ae825f1dadcea975"},"cell_type":"code","source":"Xtest = testData.iloc[:,:].values\nXtest.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2557082a8cba0fbe6a3e17cfa8d56576a1a98147"},"cell_type":"markdown","source":"## OneHotEncoder Embarked & Titles"},{"metadata":{"trusted":false,"_uuid":"1ba22371bd28e1e0c6552c08bdfd531951692919"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n\n#EMBARKED\nonehotencoderXtestE = OneHotEncoder(categorical_features = [0])\nXtest = onehotencoderXtestE.fit_transform(Xtest).toarray()\n#avoiding dummy variable trap\nXtest= Xtest[:,1:]\n\n#TITLES\nonehotencoderXtestT = OneHotEncoder(categorical_features = [2])\nXtest = onehotencoderXtestT.fit_transform(Xtest).toarray()\n#avoiding dummy variable trap\nXtest= Xtest[:,1:]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bee00c0c9fc1aec30cd915c249da716d4151143e"},"cell_type":"code","source":"Xtest = np.c_[np.ones((Xtest.shape[0])),Xtest]\nXtest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"36ec66c804be6927ee626a7788321e15b349548f"},"cell_type":"code","source":"Xtest = Xtest[:,[0,1,2,3,5,6,7,8,10]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cccbbf89d191583320d29c681a2447623b64ce91"},"cell_type":"code","source":"Xtest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"def2cb527e1f0a103a153244334ec45a5b1490ce"},"cell_type":"code","source":"predTest = prediction(Xtest,Theta1Optimum,Theta2Optimum)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}