{"cells":[{"metadata":{"_uuid":"f58d9dc12fae65dfae0d415a92b9b99b3c78cb94"},"cell_type":"markdown","source":"## Titanic:  MLP (Keras) vs LightGBM vs CatBoost vs XGBoost with same features"},{"metadata":{"trusted":true,"_uuid":"6702c34bd58b243538a43f91d8874d641969fe15"},"cell_type":"code","source":"from math import *\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport warnings\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.preprocessing import Imputer, StandardScaler\nimport matplotlib.pyplot as plt\nfrom matplotlib import gridspec\n\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, BatchNormalization, Activation\nfrom keras.optimizers import Adam\nfrom keras import backend as K\nfrom tensorflow.python.client import device_lib\n\nimport lightgbm as lgb\nimport catboost as cb\nimport xgboost as xgb\n\nimport seaborn as sns\nsns.set_style(\"whitegrid\")\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 200)\nwarnings.filterwarnings('ignore')\n\nprint(device_lib.list_local_devices())\n\nconfig = tf.ConfigProto(device_count={\"CPU\": 1, \"GPU\" : 1})\nsession = tf.Session(config=config)\nK.set_session(session)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c3e021c9ab646ee505b7008edb0e18b8984b3c1a"},"cell_type":"code","source":"#test = pd.read_csv(r\"c:\\work\\dataset\\titanic\\test.csv\", \",\")\n#train = pd.read_csv(r\"c:\\work\\dataset\\titanic\\train.csv\", \",\")\ntest = pd.read_csv(\"../input/test.csv\", \",\")\ntrain = pd.read_csv(\"../input/train.csv\", \",\")\ntest[\"is_test\"] = True\ntrain[\"is_test\"] = False\ncommon = pd.concat([test, train],axis=0).loc[:,[\"PassengerId\", \"Survived\", \"is_test\", \n                                                \"Age\", \"Cabin\", \"Embarked\", \n                                                \"Fare\", \"Name\", \"Parch\", \"Pclass\", \n                                                \"Sex\", \"SibSp\", \"Ticket\"]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95a936edc05eda270f63617eecce8b1b85c68b78"},"cell_type":"markdown","source":"### Feature engineering\nTicket based features:"},{"metadata":{"trusted":false,"_uuid":"20b127264e4ae18c781f4cc220a654106821dc6d"},"cell_type":"code","source":"common[\"Ticket\"].count() - len(common[\"Ticket\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5246346e0f9a8076de803e990249ee905d304b28"},"cell_type":"markdown","source":"Ticket has repeated values, so use grouping. We can get number of passengers per ticket and number/percent of females per single ticket. Also ticket can be pure digital or with some text info."},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"acde5e0aef9818075f1b21a020b28222d0d42934"},"cell_type":"code","source":"t = train.groupby(by=\"Ticket\", as_index=False).agg({\"PassengerId\" : 'count', \"Sex\" : lambda x : x[x==\"female\"].count()})\nt.columns = [\"Ticket\", \"SameTicket\", \"FemalesOnTicket\"]\ncommon = pd.merge(common, t, how=\"left\", on=\"Ticket\")\n\ncommon[\"TicketDigits\"] = pd.to_numeric(common[\"Ticket\"].str.split(\" \").str[-1], errors=\"coerce\").astype(np.str).str.len()\ncommon[\"TicketIsNumber\"] = ~common[\"Ticket\"].str.contains(\"[A-Za-z]\", regex = True)\ncommon[\"FemalesPerTicketPart\"] = common[\"FemalesOnTicket\"]/common[\"SameTicket\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5bfdaad8b8a82112af08e2d43b7ed68bbe001c21"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20, 5))\nsns.barplot(x=\"TicketDigits\", y=\"Survived\", data=common, ax=ax[0])\nsns.barplot(x=\"TicketIsNumber\", y=\"Survived\", data=common, ax = ax[1])\nsns.regplot(x=\"FemalesPerTicketPart\", y=\"Survived\", data=common, ax = ax[2])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abc36d49effaeaeb7f0150a0c0fc24ff7b44eb5a"},"cell_type":"markdown","source":"TicketIsNumber seems to be not valued\n\nNow process Name field. Some records have two passengers in single field name. "},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"1099af781b1ebc1bf5342c9d83b26978b76f8ea3"},"cell_type":"code","source":"common[\"DoubleName\"] = common[\"Name\"].str.contains(\"\\(\")\ncommon[\"NameLen\"] = common[\"Name\"].str.len()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fb6bd424cce38128498ca5a8210a93e2cd1a060c"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(20, 5))\nsns.barplot(x=\"DoubleName\", y=\"Survived\", data=common, ax=ax[0])\nsns.regplot(x=\"NameLen\", y=\"Survived\", data=common, ax = ax[1])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad93ee27bc63df965e4355d129df17e14b583a2b"},"cell_type":"markdown","source":"Get title from Name, and average survivability per title"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"86edfd2d293ff7f1cc4f74a695bb705444b3eb23"},"cell_type":"code","source":"common[\"Title\"] = common[\"Name\"].str.split(\", \").str[1].str.split(\" \").str[0]\ncommon.loc[common[\"Title\"].str[-1]!=\".\", \"Title\"]=\"Bad\"\nrare_title = common[\"Title\"].value_counts()[common[\"Title\"].value_counts() < 5].index\ncommon[\"Title\"] = common[\"Title\"].apply(lambda x: 'Rare' if x in rare_title else x)\n\ntitletarget = common.groupby(by=\"Title\", as_index=False).agg({\"Survived\" : 'mean'})\ntitletarget.columns = [\"Title\", \"TargetByTitle\"]\ncommon = pd.merge(common, titletarget, how=\"left\", on=\"Title\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"034d2f51253903a518f94b51cb057178f3405440"},"cell_type":"code","source":"sns.barplot(x=\"Title\", y=\"TargetByTitle\", data=common)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"769ac3db217ef7f786808cd88eb648bc2233db71"},"cell_type":"markdown","source":"Make features related to family size"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"53fdcd0d85491900748285b7e51030d22129e9f9"},"cell_type":"code","source":"common[\"Family\"] = common[\"Parch\"] + common[\"SibSp\"] + 1\ncommon[\"Alone\"] = common[\"Family\"] == 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"015ae4b77ce5b159284eac5cfc9a2c300b143440"},"cell_type":"markdown","source":"Process cabin number, get deck (vertical coordinate in ship), cabin digit number as distance from ship's nose, and check is number odd/even as side of ship axis. Also we can get average survivability per deck."},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"02d91325db8936c4838cdca9ba04b836a69817ec"},"cell_type":"code","source":"cap = train.groupby(by=\"Cabin\", as_index=False).agg({\"PassengerId\" : 'count'})\ncap.columns = [\"Cabin\", \"SameCabin\"]\ncommon = pd.merge(common, cap, how=\"left\", on=\"Cabin\")\ncommon[\"CabinNumber\"] = pd.to_numeric(common[\"Cabin\"].str[1:], errors = \"coerce\")\ncommon[\"CabinEven\"] = common[\"CabinNumber\"] %2\ncommon[\"CabinsPerMan\"] = common[\"Cabin\"].str.split(\" \").str.len()\ncommon[\"Deck\"] = common[\"Cabin\"].str[0].rank().fillna(-1)\n\ndecktarget = common.groupby(by=\"Deck\", as_index=False).agg({\"Survived\" : 'mean'})\ndecktarget.columns = [\"Deck\", \"TargetByDeck\"]\ncommon = pd.merge(common, decktarget, how=\"left\", on=\"Deck\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f4a595feb5f945f6dce67a14bb041972b54ae6e5"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20, 5))\nsns.barplot(x=\"Deck\", y=\"TargetByDeck\", data=common.sort_values(\"Deck\"), ax=ax[0])\nsns.barplot(x=\"CabinEven\", y=\"Survived\", data=common, ax=ax[1])\nsns.regplot(x=\"CabinNumber\", y=\"Survived\", data=common, ax=ax[2])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0dbd0387b7016ad804eb7a2e9db0f985d274cbfd"},"cell_type":"markdown","source":"Now process age field. We can split it on ranges and get average survivality per one. Also calculate average age of family."},{"metadata":{"trusted":false,"_uuid":"1f303880d48c9ab6219d2ce877b312cf480a32b2"},"cell_type":"code","source":"sns.swarmplot(x=\"Survived\", y=\"Age\", hue=\"Sex\", palette=sns.color_palette([\"#20AFCF\",\"#cf4040\"]), data=common)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"60c39afe3db1f6555d0b8c038d97fda966b0e512"},"cell_type":"code","source":"common['AgeGroup'] = pd.qcut(common['Age'].fillna(common['Age'].mean()).astype(int), 6)\nagetarget = common.groupby(by=\"AgeGroup\", as_index=False).agg({\"Survived\" : 'mean'})\nagetarget.columns = [\"AgeGroup\", \"TargetByAgeGroup\"]\ncommon = pd.merge(common, agetarget, how=\"left\", on=\"AgeGroup\")\n\ncommon[\"IsTinyChild\"] = common[\"Age\"]<1\ncommon[\"IsChild\"] = common[\"Age\"]<10\ncommon[\"AverageAge\"] = common[\"Age\"] / common[\"Family\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d006d5e012df2f78046e54605a3daabcce81dd27"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20, 5))\nsns.barplot(x=\"AgeGroup\", y=\"TargetByAgeGroup\", data=common, ax = ax[0])\nsns.barplot(x=\"IsChild\", y=\"Survived\", data=common, ax = ax[1])\nsns.regplot(x=\"AverageAge\", y=\"Survived\", data=common, ax = ax[2])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3749ce09a9599b5c84a9b09c83ef3fc08c726678"},"cell_type":"markdown","source":"Fare processing. We also can split it on ranges and get survivability per fare range. "},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"22743b63a5e8bb627c25ea80caa52b84420834be"},"cell_type":"code","source":"common['FareGroup'] = pd.qcut(common['Fare'].fillna(common['Fare'].mean()).astype(int), 6)\nfaretarget = common.groupby(by=\"FareGroup\", as_index=False).agg({\"Survived\" : 'mean'})\nfaretarget.columns = [\"FareGroup\", \"TargetByFareGroup\"]\ncommon = pd.merge(common, faretarget, how=\"left\", on=\"FareGroup\")\ncommon[\"AverageFareByFamily\"] = common[\"Fare\"] / common[\"Family\"]\ncommon[\"AverageFareByTicket\"] = common[\"Fare\"] / common[\"SameTicket\"]\ncommon[\"FareLog\"] = np.log(common[\"Fare\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ffb1a76fd92e43e6dab63d2d079fdd9ae63d9a92"},"cell_type":"code","source":"sns.barplot(x=\"FareGroup\", y=\"TargetByFareGroup\", data=common)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"091c1f9a4238d764763d54225647441c0ed3d42b"},"cell_type":"markdown","source":"Now calculate average survivability for each left categorical fields"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"70dec814080a4fe1f88098cafa253b4f80825173"},"cell_type":"code","source":"pclasstarget = common.groupby(by=\"Pclass\", as_index=False).agg({\"Survived\" : 'mean'})\npclasstarget.columns = [\"Pclass\", \"TargetByPclass\"]\ncommon = pd.merge(common, pclasstarget, how=\"left\", on=\"Pclass\")\n\nEmbarkedtarget = common.groupby(by=\"Embarked\", as_index=False).agg({\"Survived\" : 'mean'})\nEmbarkedtarget.columns = [\"Embarked\", \"TargetByEmbarked\"]\ncommon = pd.merge(common, Embarkedtarget, how=\"left\", on=\"Embarked\")\n\nSextarget = common.groupby(by=\"Sex\", as_index=False).agg({\"Survived\" : 'mean'})\nSextarget.columns = [\"Sex\", \"TargetBySex\"]\ncommon = pd.merge(common, Sextarget, how=\"left\", on=\"Sex\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fc430a37fcc73bd0553abb180f3b1d0149607a70"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20, 5))\nsns.barplot(x=\"Pclass\", y=\"TargetByPclass\", data=common, ax=ax[0])\nsns.barplot(x=\"Embarked\", y=\"TargetByEmbarked\", data=common, ax = ax[1])\nsns.barplot(x=\"Sex\", y=\"TargetBySex\", data=common, ax = ax[2])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57697c441213d8a2e4b27fecc60f090746b4eb23"},"cell_type":"markdown","source":"### Feature selection and preprocessing"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"7b0fa7672dde2005eedd1016947685e18a1cfcf8"},"cell_type":"code","source":"allfeatures = [\n    \"PassengerId\", \n    \"is_test\", \n    \"Survived\", \n    \"Age\", \n    \"Fare\", \n    \"Parch\", \n    \"Pclass\",\n    \"SibSp\", \n    \"Sex\", \n    \"Embarked\", \n    \"SameTicket\", \n    \"FemalesOnTicket\", \n    \"SameCabin\", \n    \"Deck\", \n    \"TargetByDeck\", \n    \"TargetByTitle\", \n    \"TargetByAgeGroup\", \n    \"TargetByFareGroup\",\n    \"TargetByPclass\",\n    \"TargetByEmbarked\",\n    \"TargetBySex\",\n    \"Title\", \n    \"CabinNumber\", \n    \"CabinEven\", \n    \"CabinsPerMan\", \n    \"DoubleName\", \n    \"NameLen\", \n    \"TicketDigits\",\n    \"TicketIsNumber\",\n    \"IsTinyChild\", \n    \"IsChild\", \n    \"Alone\", \n    \"Family\", \n    \"AverageAge\",\n    \"AverageFareByFamily\",\n    \"AverageFareByTicket\",\n    \"FemalesPerTicketPart\"\n]\n\nc = common.loc[:, allfeatures]\nc = pd.get_dummies(c, columns=[\n    \"Title\", \n    \"Embarked\", \n    \"Pclass\",\n    \"Sex\"\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"daf1a295a861328586012b3d03052c9df9c7f397"},"cell_type":"code","source":"c.describe().T.sort_values(\"count\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2182b90e1424ed48e97b737e33359e1a4ce16ac"},"cell_type":"markdown","source":"Several features requires imputation and normalization"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c0a32f787d02e8dcca372e2e8c42309ec0306b68"},"cell_type":"code","source":"c.iloc[:,3:] = Imputer(strategy=\"most_frequent\").fit_transform(c.iloc[:,3:])\n\ndep = c[c[\"is_test\"] == False].loc[:, [\"Survived\"]]\nindep = c[c[\"is_test\"] == False].iloc[:, 3:]\nres = c[c[\"is_test\"] == True].iloc[:, 3:]\nres_index = c[c[\"is_test\"] == True].loc[:, \"PassengerId\"]\n\nindep.iloc[:,3:] = StandardScaler().fit_transform(indep.iloc[:,3:])\nres.iloc[:,3:] = StandardScaler().fit_transform(res.iloc[:,3:])\n\nindep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=47)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a399631bd4670bc0d987012c928db85bb09f10c9"},"cell_type":"markdown","source":"### Model 1. Keras MLP\nMake simple sequental MLP NN. (Parameters selected manually)"},{"metadata":{"trusted":false,"_uuid":"4641b533277f57fa61bc26826a2be976e9c75f4e"},"cell_type":"code","source":"with tf.device('/device:CPU:0'):\n    gs1 = Sequential()\n    gs1.add(Dense(45 ,activation='linear', input_dim=45))\n    gs1.add(BatchNormalization())\n\n    gs1.add(Dense(9,activation='linear'))\n    gs1.add(BatchNormalization())\n    gs1.add(Dropout(0.4))\n\n    gs1.add(Dense(5,activation='linear'))\n    gs1.add(BatchNormalization())\n    gs1.add(Dropout(0.2))\n\n    gs1.add(Dense(1,activation='relu', ))\n    gs1.compile(optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0), loss='binary_crossentropy', metrics=['accuracy'])\n\n    gs1.fit(indep_train, dep_train, epochs=500, batch_size=30, validation_data=(indep_test,dep_test), verbose=False)\n    g=gs1.predict_classes(res)[:,0]\n    print(accuracy_score(dep_test, gs1.predict_classes(indep_test)), accuracy_score(dep_train, gs1.predict_classes(indep_train)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82ff8dc2b5fe2ce6856bdce0b53dcbbd333d8ab2"},"cell_type":"markdown","source":"Lets see how accuracy changes during epochs"},{"metadata":{"trusted":false,"_uuid":"2a51483c791b381e3cb45b06037abc01d0de2dd9"},"cell_type":"code","source":"fig, ax = plt.subplots(2, 1, sharex='col', figsize=(20, 10))\nax[0].set_title('Model accuracy history')\nax[0].plot(gs1.history.history['acc'])\nax[0].plot(gs1.history.history['val_acc'])\nax[0].set_ylabel('Accuracy')\nax[0].legend(['train', 'test'], loc='right')\nax[0].grid()\n\nax[1].set_title('Model loss history')\nax[1].plot(gs1.history.history['loss'])\nax[1].plot(gs1.history.history['val_loss'])\nax[1].set_ylabel('Loss')\nax[1].legend(['train', 'test'], loc='right')\nax[1].grid()\nplt.xlabel('Epoch')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b63b3d32f35c55270e62a092cb6e2831764a12fd"},"cell_type":"markdown","source":"Get same NN on 5 folds and average result"},{"metadata":{"trusted":false,"_uuid":"5985581f508ebc79e8be053e5916405b0b458cc5"},"cell_type":"code","source":"cvscores = []\ndata = pd.DataFrame()\ni=1\nwith tf.device('/device:CPU:0'):\n    for train, test in StratifiedKFold(n_splits=5, shuffle=True, random_state=1).split(indep, dep.iloc[:,0]):\n        X = indep.reindex().iloc[train,:]\n        Y = dep.reindex().iloc[train,0]\n        Xv = indep.reindex().iloc[test,:]\n        Yv = dep.reindex().iloc[test,0]\n        \n        gs1 = Sequential()\n        gs1.add(Dense(45 ,activation='linear', input_dim=45))\n        gs1.add(BatchNormalization())\n\n        gs1.add(Dense(9,activation='linear'))\n        gs1.add(BatchNormalization())\n        gs1.add(Dropout(0.4))\n\n        gs1.add(Dense(5,activation='linear'))\n        gs1.add(BatchNormalization())\n        gs1.add(Dropout(0.2))\n\n        gs1.add(Dense(1,activation='relu', ))\n        gs1.compile(optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0), loss='binary_crossentropy', metrics=['accuracy'])\n\n        gs1.fit(X, Y, epochs=500, batch_size=30, validation_data=(Xv, Yv), verbose=False)\n        data[i] = gs1.predict_classes(res)[:,0]\n        scores = gs1.evaluate(Xv, Yv, verbose=0)\n        print(gs1.metrics_names[1], scores[1])\n        cvscores.append(scores[1])\n        i+=1\nmlp_mean = np.mean(cvscores)\nmlp_stdev = np.std(cvscores)\nprint(mlp_mean, mlp_stdev)\n\ng = np.round(data.mean(axis=1))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"11735bcfca56307198bafce4b722feb43f782dca"},"cell_type":"code","source":"#result = pd.DataFrame(res_index.astype(np.int), columns=[\"PassengerId\"])\n#result[\"Survived\"] = g.astype(np.int)\n#result.to_csv(r\"c:\\work\\dataset\\titanic\\mlp.csv\", \",\", index=None)\n#result.to_csv(\"mlp.csv\", \",\", index=None)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"32cb747cb0ef0114958ae0b2c4874522fb73df53"},"cell_type":"markdown","source":"Unfortunatelly, accuracy on LB too low (0.75119 - 0.79425 with different seeds)"},{"metadata":{"_uuid":"06832dd3c3c5846c45b1021959ec0df08bff87c3"},"cell_type":"markdown","source":"### Model 2. LightGBM\nAll parameters selected by brute force, sequentally by 1-2 parameters. No HyperOpt nor GridSearchCV geven such accuracy for me."},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"53f401b5f62f692e2851b5c6001e6bf2303af27e"},"cell_type":"code","source":"# Example of manual parameter tuning\n\"\"\"\nfor i in range(1,10):\n    params = {}\n    params[\"max_depth\"] = i\n    params[\"learning_rate\"] = 0.45\n    params[\"lambda_l1\"] = 0.1\n    params[\"lambda_l2\"] = 0.01\n    params[\"n_estimators\"] = 5000\n    params[\"n_jobs\"]=5 \n    params[\"objective\"] = \"binary\"\n    \n    params[\"boosting\"] = \"dart\"\n    params[\"colsample_bytree\"] = 0.9\n    params[\"subsample\"] =0.9\n\n    train_data = lgb.Dataset(data=indep_train, label=dep_train, free_raw_data=False, feature_name = list(indep_train))\n    cv_result = lgb.cv(params, train_data, nfold=5, stratified=False, metrics=['binary_error'], early_stopping_rounds=50)\n    print(i, 1-np.mean(cv_result[\"binary_error-mean\"]))\n    \"\"\";","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a5e8d997547d5077f9e088925a8697c74b841a61"},"cell_type":"code","source":"indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=47)\ngs1 = lgb.LGBMClassifier(max_depth = 7,\n                         lambda_l1 = 0.1,\n                         lambda_l2 = 0.01,\n                         learning_rate = 0.01, \n                         n_estimators = 500, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,\n                         n_jobs = 5)\ngs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], eval_metric='accuracy', verbose=False, early_stopping_rounds=50);\n\n\n\ng = gs1.predict(res)\na = accuracy_score(dep_test, gs1.predict(indep_test))\nb = accuracy_score(dep_train, gs1.predict(indep_train))\nprint(a, b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fae732b284c1dd1ee4299643ba5d59a39dfb9729"},"cell_type":"code","source":"attr2 = {k: v for k, v in zip(indep.columns, gs1.feature_importances_) if v>0}\nattr2 = sorted(attr2.items(), key=lambda x: x[1], reverse = False)\nx1,y1 = zip(*attr2)\ni1=range(len(x1))\nplt.figure(num=None, figsize=(9, 7), dpi=300, facecolor='w', edgecolor='k')\nplt.barh(i1, y1)\nplt.title(\"LGBM\")\nplt.yticks(i1, x1)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b95e5012fa59f67710be6eaad6f640ca53a4c6b2"},"cell_type":"markdown","source":"Now blend results of models with different seeds (i experimentally got optimal number of models - 90)"},{"metadata":{"trusted":false,"_uuid":"bfad6cdca0a178e1f863f904ad641cd37035dc19"},"cell_type":"code","source":"model = []\ncvscores = []\n\nfor i in range(0,90):\n    indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=i)\n    gs1 = lgb.LGBMClassifier(max_depth = 7,\n                             lambda_l1 = 0.1,\n                             lambda_l2 = 0.01,\n                             learning_rate =  0.01, num_iterations=20000,\n                             n_estimators = 5000, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,\n                             n_jobs = 5, boosting='dart' )\n    gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], eval_metric='accuracy', verbose=False, early_stopping_rounds=50);\n    model.append(gs1)\n    cvscores.append(accuracy_score(dep_test, gs1.predict(indep_test)))\n\ndata = pd.DataFrame()\nte = pd.DataFrame()\nfor i in range(0,90):\n    data[i] = model[i].predict(res)\n    te[i] = model[i].predict(indep_test)\n\ng = np.round(data.mean(axis=1))\nt = np.round(te.mean(axis=1))\n\nlgb_mean = np.mean(cvscores)\nlgb_stdev = np.std(cvscores)\nprint(lgb_mean, lgb_stdev)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6b01154a9f65468801238cdd29e1329c51bf1e2"},"cell_type":"markdown","source":"This got <b>0.82296</b> on LB"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f5786f407f7d304bc48f67a764e66f91d58fe371"},"cell_type":"code","source":"result = pd.DataFrame(res_index.astype(np.int), columns=[\"PassengerId\"])\nresult[\"Survived\"] = g.astype(np.int)\nresult.to_csv(\"lgbm.csv\", \",\", index=None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1007033acca9cf812f3470c13497a0af916e67b"},"cell_type":"markdown","source":"### Model 3. CatBoost\nSame approach"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"1bc7beb4d8622176d9d4ba17ec4806fe52a697e2"},"cell_type":"code","source":"gs1 = cb.CatBoostClassifier(depth = 9, reg_lambda=0.1,\n                         learning_rate = 0.09, \n                         iterations = 500)\ngs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)],  verbose=False, early_stopping_rounds=50);\n\ng = gs1.predict(res)\na = accuracy_score(dep_test, gs1.predict(indep_test))\nb = accuracy_score(dep_train, gs1.predict(indep_train))\n#cv = cross_val_score(gs1, indep_train, dep_train, cv=5)\nprint(a, b)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"0f52238764c8a92fa69481347a57281758769a9f"},"cell_type":"code","source":"attr2 = {k: v for k, v in zip(indep.columns, gs1.feature_importances_) if v>0}\nattr2 = sorted(attr2.items(), key=lambda x: x[1], reverse = False)\nx1,y1 = zip(*attr2)\ni1=range(len(x1))\nplt.figure(num=None, figsize=(9, 8), dpi=300, facecolor='w', edgecolor='k')\nplt.barh(i1, y1)\nplt.title(\"CatBoost\")\nplt.yticks(i1, x1)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"8a59341f06de16871a37f392a5fde6687925ce25"},"cell_type":"code","source":"model = []\ncvscores = []\nfor i in range(0, 90):\n    indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=i)\n    gs1 = cb.CatBoostClassifier(depth = 9, reg_lambda=0.1,\n                     learning_rate = 0.09, \n                     iterations = 500)\n    gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)],  verbose=False, early_stopping_rounds=50);\n    model.append(gs1)\n    cvscores.append(accuracy_score(dep_test, gs1.predict(indep_test)))\n    \ndata = pd.DataFrame()\nte = pd.DataFrame()\n\nfor i in range(0, 90):\n    data[i] = model[i].predict(res)\n    te[i] = model[i].predict(indep_test)\n\ng = np.round(data.mean(axis=1))\nt = np.round(te.mean(axis=1))\n\ncb_mean = np.mean(cvscores)\ncb_stdev = np.std(cvscores)\nprint(cb_mean, cb_stdev)\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"526c957ab10769569f9731c3e108682e4ae5425b"},"cell_type":"code","source":"#result = pd.DataFrame(res_index.astype(np.int), columns=[\"PassengerId\"])\n#result[\"Survived\"] = g.astype(np.int)\n#result.to_csv(r\"c:\\work\\dataset\\titanic\\catboost.csv\", \",\", index=None)\n#result.to_csv(\"catboost.csv\", \",\", index=None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a06e73ce5fe9f60e3b37f31e6fc26edf5b5ebf2a"},"cell_type":"markdown","source":"### Model 4. XGBoost"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"1519f3d341a99bc8645efc2c9ca847a8eda0e70b"},"cell_type":"code","source":"gs1 = xgb.XGBClassifier(max_depth = 9,\n                         learning_rate = 0.01, \n                         n_estimators = 500, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,\n                         n_jobs = 5)\ngs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)],  verbose=False, early_stopping_rounds=50);\n\ng = gs1.predict(res)\na = accuracy_score(dep_test, gs1.predict(indep_test))\nb = accuracy_score(dep_train, gs1.predict(indep_train))\nprint(a, b)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"320663fd968c73161b62b722eceb6c51b7452ade"},"cell_type":"code","source":"attr2 = {k: v for k, v in zip(indep.columns, gs1.feature_importances_) if v>0}\nattr2 = sorted(attr2.items(), key=lambda x: x[1], reverse = False)\nx1,y1 = zip(*attr2)\ni1=range(len(x1))\nplt.figure(num=None, figsize=(9, 7), dpi=300, facecolor='w', edgecolor='k')\nplt.barh(i1, y1)\nplt.title(\"XGBoost\")\nplt.yticks(i1, x1)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"939d8b561aacc5839e897d23091f3aa7227f6fa9"},"cell_type":"code","source":"model = []\ncvscores = []\nfor i in range(0, 90):\n    indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=i)\n    gs1 = xgb.XGBClassifier(max_depth = 7, reg_lambda = 0.02,\n                         learning_rate = 0.01, \n                         n_estimators = 5000, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,\n                         n_jobs = 5)\n    gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)],  verbose=False, early_stopping_rounds=50);\n    model.append(gs1)\n    cvscores.append(accuracy_score(dep_test, gs1.predict(indep_test)))\n    \ndata = pd.DataFrame()\nte = pd.DataFrame()\n\nfor i in range(0, 90):\n    data[i] = model[i].predict(res)\n    te[i] = model[i].predict(indep_test)\n\ng = np.round(data.mean(axis=1))\nt = np.round(te.mean(axis=1))\n\nxgb_mean = np.mean(cvscores)\nxgb_stdev = np.std(cvscores)\nprint(xgb_mean, xgb_stdev)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"14fc3744552ba3f79f85bd5bfdee906781fe28d2"},"cell_type":"code","source":"#result = pd.DataFrame(res_index.astype(np.int), columns=[\"PassengerId\"])\n#result[\"Survived\"] = g.astype(np.int)\n#result.to_csv(r\"c:\\work\\dataset\\titanic\\xgb.csv\", \",\", index=None)\n#result.to_csv(\"lgbm.csv\", \",\", index=None)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"7356a8436fd005ada3ece9b16347fd5f8f964b45"},"cell_type":"code","source":"d = {'Model':[\"Keras MLP\", \"LightGBM\", \"CatBoost\", \"XGBoost\"], \n     'Mean accuracy': [mlp_mean, lgb_mean, cb_mean, xgb_mean], \n     'Std. Dev.': [mlp_stdev, lgb_stdev, cb_stdev, xgb_stdev],\n    'Leaderboard': [0.77033, 0.82296, 0.78947, 0.77511]}\npd.DataFrame(data=d, columns=[\"Model\", \"Mean accuracy\", \"Std. Dev.\", \"Leaderboard\"]).sort_values(\"Mean accuracy\", ascending=False).head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.1"}},"nbformat":4,"nbformat_minor":1}