{"cells":[{"metadata":{"_cell_guid":"6125ae30-9bc8-4f18-87c8-bd27ee880709","_uuid":"913fb7a20d26955600d3b7b7151ed8e7623e3017"},"cell_type":"markdown","source":"**Machine Learning from Disaster**\n\nThis is my first submission in Kaggle. I have been teaching myself machine learning over the past few months and decided to apply what I have learned into something more practical. I took some bits and pieces of inspiration from other Kernels on the site. \n\nThe model is a simple MLP built with Keras."},{"metadata":{"_cell_guid":"8a4e9016-933f-4783-a488-8e34397b9e76","_uuid":"33f1eb162588113e3948ec91ad6a29f715ab78a8","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport xgboost as xgb\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a75d3c2c-5caa-4ef5-876d-b0bde302210a","_uuid":"cdf85e635061be2ef4c6af3cb33f9ceda9550f7d","scrolled":true,"trusted":false},"cell_type":"code","source":"#read in database and inspect the headings\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b8442ea8-0173-4cb1-997a-04b8f0554e73","_uuid":"90aa5a94233012e99fe6070836279995dd40de8f","trusted":false},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6a81785d-19df-43cd-844e-5513ac2afc85","_uuid":"0372a1e0129dcb45170386fa5f43d8894808011f","trusted":false},"cell_type":"code","source":"#we can jump right in to modify categories so they can be understood by the classifier\n#we also fill missing values \nfrom sklearn import preprocessing\nle = preprocessing.LabelEncoder()\ntrain[\"Sex\"] = le.fit_transform(train[\"Sex\"]) \ntrain[\"Embarked\"] = train[\"Embarked\"].fillna(\"S\") #fill with most common value\ntrain[\"Embarked\"] = le.fit_transform(train[\"Embarked\"])\ntrain[\"Age\"] = train[\"Age\"].fillna(train[\"Age\"].median()) #fill missing values with median age\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cd563739-1981-4e7b-8c79-d3e69b5be211","_uuid":"69a1102d25803abbec2d99df8ae5a9908c64f4c7","scrolled":true,"trusted":false},"cell_type":"code","source":"#select dependent, independent variables for analysis. \n#A priori expectations -> PClass, age and sex likely  significant\nX_train = train[[\"Pclass\",\"Sex\",\"Age\",\"SibSp\",\"Parch\",\"Fare\",\"Embarked\"]]\n#X_train = train[[\"Pclass\",\"Sex\",\"Age\"]]\nX_train = X_train.values\ny_train = train[[\"Survived\"]].values\n\n#standardize data for MLP\n#from sklearn.preprocessing import MinMaxScaler\nfrom sklearn import preprocessing\nX_train = preprocessing.scale(X_train)\nprint(X_train)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"33a25ee8-3f64-44f6-8463-03ef674da012","_uuid":"85f8d19186304e3c57f1b805acf0fa7a9f1adaac","trusted":false},"cell_type":"code","source":"print(X_train.shape)\nprint(y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8b07e58e-ce6f-4cd3-90b1-f5049c9fc3df","_uuid":"2781023e2b8533286e4f0973fe3a876038c14c92","scrolled":true,"trusted":false},"cell_type":"code","source":"#build, compile and fit the model\n#deep networks seem to require a lot of tinkering so we can simplify things with a shallower...\n#and wider network. Since database is small we set batch_size = 1 to update weights more frequently\n#although there might be a better choice for this\nfrom keras.layers import *\nfrom keras.models import Sequential\nfrom keras import optimizers, regularizers\nfrom keras.callbacks import EarlyStopping\nimport keras \n\n\nmodel = Sequential()\nmodel.add(Dense(32,\n                kernel_initializer = \"he_uniform\",\n                bias_initializer = \"zeros\",\n                #kernel_regularizer=regularizers.l2(0.01),\n                activation = \"relu\",\n                input_dim = X_train.shape[1] ))\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(32,\n                kernel_initializer = \"he_uniform\",\n                bias_initializer = \"zeros\",\n                #kernel_regularizer=regularizers.l2(0.01),\n                activation = \"relu\",\n                ))\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(32,\n                kernel_initializer = \"he_uniform\",\n                bias_initializer = \"zeros\",\n                #kernel_regularizer=regularizers.l2(0.01),\n                activation = \"relu\",\n                ))\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(1, activation = \"sigmoid\"))\n\n# opt = optimizers.Adam(lr = 0.0001,decay = 0, beta_1=0.9, beta_2=0.999, \n#                       epsilon = None, amsgrad = True)\nlr = 0.01\nepochs = 10\ndecay = lr / epochs\nmomentum = 0.8\nopt = optimizers.SGD(lr=lr, decay=decay, momentum=momentum, nesterov=False)\n#opt = optimizers.SGD(lr=0.001)\n\nmodel.compile(loss = \"binary_crossentropy\", optimizer = opt, \n              metrics = [\"accuracy\"])\n\n\n#track model performance\nstop = EarlyStopping(monitor = 'val_loss', patience = epochs, verbose = 1)\nhistory = model.fit(X_train, y_train, epochs = epochs, validation_split = 0.33, shuffle = True, \n        verbose = 1, batch_size = 8, callbacks = [stop])\n\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a2e0a172-9080-4ea5-864d-7c58e2ad981e","_uuid":"174f7b36ea3ec2e8adea4a8d4a050c2199474479","collapsed":true,"scrolled":true,"trusted":false},"cell_type":"code","source":"#prepare test data for predictions\ntest = pd.read_csv(\"../input/test.csv\")\ntest[\"Sex\"] = le.fit_transform(test[\"Sex\"])\ntest[\"Embarked\"] = test[\"Embarked\"].fillna(\"S\")\ntest[\"Embarked\"] = le.fit_transform(test[\"Embarked\"])\ntest[\"Name\"] = le.fit_transform(test[\"Name\"])\n#test[\"Age\"] = test[\"Age\"].fillna(test[\"Age\"].median()) \nX_test = test[[\"Pclass\",\"Sex\",\"Age\",\"SibSp\",\"Parch\",\"Fare\",\"Embarked\"]]\nX_test[\"Age\"] = X_test[\"Age\"].fillna(X_test[\"Age\"].median())\nX_test[\"Fare\"] = X_test[\"Fare\"].fillna(X_test[\"Fare\"].mean())\nX_test = X_test.values\nX_test = preprocessing.scale(X_test)\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b4967a90-19e0-42dc-9d4c-693db878eaf3","_uuid":"994ca59cd5f26e200624a1c54f903fbcb303244a","collapsed":true,"trusted":false},"cell_type":"code","source":"#sigmoid activation gives probabilities of 1 (Surviving). \n#binarizer turns this into appropiate output for competition guidelines\npredictions = model.predict(X_test, batch_size=8)\nfrom sklearn.preprocessing import Binarizer\nbinarizer = Binarizer(threshold = 0.5)\npredictions = binarizer.fit_transform(predictions)\npredictions = predictions.astype(np.int32)                       \n\n#create submission file\nPassengerId = test[\"PassengerId\"]\nevaluation = PassengerId.to_frame()\nevaluation[\"Survived\"] = predictions\nevaluation.to_csv(\"evaluation_submission.csv\", index = False)\n\"evalutation_submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"855cbb33db2b3746a2137ecc9f29c58ca289c86e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}