{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport plotly.express as px\n\nplt.style.use('seaborn')\n\n#Sklearn imports\nfrom sklearn.preprocessing import OrdinalEncoder, StandardScaler\nfrom sklearn.compose import make_column_transformer\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.model_selection import cross_val_score, train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\n#Hiperparameter tunning\nfrom skopt import gp_minimize\nfrom skopt.plots import plot_convergence\n\nfrom xgboost import XGBClassifier\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n#Machine learning","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:01.446882Z","iopub.execute_input":"2022-07-27T11:27:01.447418Z","iopub.status.idle":"2022-07-27T11:27:11.160140Z","shell.execute_reply.started":"2022-07-27T11:27:01.447318Z","shell.execute_reply":"2022-07-27T11:27:11.159311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDataset = '../input/spaceship-titanic/train.csv'\ntestDataset = '../input/spaceship-titanic/test.csv'\n\ndfRaw = pd.read_csv(trainDataset)\ndfRaw.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.161661Z","iopub.execute_input":"2022-07-27T11:27:11.162561Z","iopub.status.idle":"2022-07-27T11:27:11.240096Z","shell.execute_reply.started":"2022-07-27T11:27:11.162525Z","shell.execute_reply":"2022-07-27T11:27:11.239399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfRaw.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.241150Z","iopub.execute_input":"2022-07-27T11:27:11.241622Z","iopub.status.idle":"2022-07-27T11:27:11.253537Z","shell.execute_reply.started":"2022-07-27T11:27:11.241594Z","shell.execute_reply":"2022-07-27T11:27:11.252595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfRaw.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.255844Z","iopub.execute_input":"2022-07-27T11:27:11.256605Z","iopub.status.idle":"2022-07-27T11:27:11.263943Z","shell.execute_reply.started":"2022-07-27T11:27:11.256567Z","shell.execute_reply":"2022-07-27T11:27:11.262900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfWork = dfRaw.copy()\ndef splitCabin(cabin):\n    var1 = var2 = var3 = 0\n    if (type(cabin) == str) & (cabin != np.nan):\n        var1 = cabin.split('/')[0]\n        var2 = cabin.split('/')[1]\n        var3 = cabin.split('/')[2]\n    else:\n        var1 = var2 = var3 = np.nan\n    return var1, var2, var3\n\ndef splitPassengerId(pId):\n    group = pId.split('_')[0]\n    idInGroup = pId.split('_')[1]\n    return group, idInGroup\n\ndef prepareForAnalysis(df, train = True):\n    #Change the Cryosleep column\n    df['CryoSleep'] = df['CryoSleep'].map({False:'No',True:'Yes'})\n    #Same for Vip column\n    df['VIP'] = df['VIP'].map({False:'No',True:'Yes'})\n    #Applying a function in order to split the cabin column in 3\n    df['deck'], df['num'], df['side'] = zip(*df['Cabin'].apply(splitCabin))\n    dfWork['num'] = dfWork['num'].astype(int, errors = 'ignore')\n    #Trying to treat the passenger Id variable\n    df['PassengerGroup'], df['PassengerGroupId'] = zip(*df['PassengerId'].apply(splitPassengerId))\n    dfWork['PassengerGroup'] = dfWork['PassengerGroup'].astype(int)\n    dfWork['PassengerGroupId'] = dfWork['PassengerGroupId'].astype(int)\n\n    df.drop(['Name','Cabin','PassengerId'], axis = 1, inplace = True)\n\n    if train:\n        df['Transported'] = df['Transported'].map({False:'No',True:'Yes'})\n    return df\n\n\ndfWork = prepareForAnalysis(dfWork,train=True)\ndfWork\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.264986Z","iopub.execute_input":"2022-07-27T11:27:11.265896Z","iopub.status.idle":"2022-07-27T11:27:11.354804Z","shell.execute_reply.started":"2022-07-27T11:27:11.265862Z","shell.execute_reply":"2022-07-27T11:27:11.353670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"objectCols = dfWork.select_dtypes(include='object').columns\nnumCols = dfWork.select_dtypes(exclude='object').columns\nobjectCols, numCols","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.356609Z","iopub.execute_input":"2022-07-27T11:27:11.357474Z","iopub.status.idle":"2022-07-27T11:27:11.373193Z","shell.execute_reply.started":"2022-07-27T11:27:11.357428Z","shell.execute_reply":"2022-07-27T11:27:11.372017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfWork.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.375143Z","iopub.execute_input":"2022-07-27T11:27:11.376234Z","iopub.status.idle":"2022-07-27T11:27:11.384708Z","shell.execute_reply.started":"2022-07-27T11:27:11.376189Z","shell.execute_reply":"2022-07-27T11:27:11.383828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndfWork[objectCols].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.385831Z","iopub.execute_input":"2022-07-27T11:27:11.386829Z","iopub.status.idle":"2022-07-27T11:27:11.408007Z","shell.execute_reply.started":"2022-07-27T11:27:11.386786Z","shell.execute_reply":"2022-07-27T11:27:11.406869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfWork[numCols].dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.409283Z","iopub.execute_input":"2022-07-27T11:27:11.409619Z","iopub.status.idle":"2022-07-27T11:27:11.418421Z","shell.execute_reply.started":"2022-07-27T11:27:11.409588Z","shell.execute_reply":"2022-07-27T11:27:11.417545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfWork[numCols].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.422476Z","iopub.execute_input":"2022-07-27T11:27:11.422886Z","iopub.status.idle":"2022-07-27T11:27:11.465558Z","shell.execute_reply.started":"2022-07-27T11:27:11.422854Z","shell.execute_reply":"2022-07-27T11:27:11.464750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 500)\ndfWork.groupby('Transported')[numCols].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.466829Z","iopub.execute_input":"2022-07-27T11:27:11.467393Z","iopub.status.idle":"2022-07-27T11:27:11.531270Z","shell.execute_reply.started":"2022-07-27T11:27:11.467361Z","shell.execute_reply":"2022-07-27T11:27:11.530209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfWork['Transported'].value_counts().plot.bar();","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.532715Z","iopub.execute_input":"2022-07-27T11:27:11.533046Z","iopub.status.idle":"2022-07-27T11:27:11.669512Z","shell.execute_reply.started":"2022-07-27T11:27:11.533015Z","shell.execute_reply":"2022-07-27T11:27:11.668561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2,ncols = 3, figsize = (18,12))\n\n\nax[0,0].set_title('Distribution of Age')\nsns.boxplot(y = 'Age', data = dfWork, color = 'c', ax = ax[0,0])\n\nax[0,1].set_title('Distribution of RoomService')\nsns.boxplot(y = 'RoomService', data = dfWork, color = 'g', ax = ax[0,1])\n\nax[0,2].set_title('Distribution of FoodCourt')\nsns.boxplot(y = 'FoodCourt', data = dfWork, color = 'b', ax = ax[0,2])\n\nax[1,0].set_title('Distribution of ShoppingMall')\nsns.boxplot(y = 'ShoppingMall', data = dfWork, color = 'm', ax = ax[1,0])\n\nax[1,1].set_title('Distribution of Spa')\nsns.boxplot(y = 'Spa', data = dfWork, color = 'y', ax = ax[1,1])\n\nax[1,2].set_title('Distribution of VRDeck')\nsns.boxplot(y = 'VRDeck', data = dfWork, color = 'y', ax = ax[1,2])\n\nplt.tight_layout(pad = 2)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:11.671294Z","iopub.execute_input":"2022-07-27T11:27:11.672442Z","iopub.status.idle":"2022-07-27T11:27:12.320265Z","shell.execute_reply.started":"2022-07-27T11:27:11.672397Z","shell.execute_reply":"2022-07-27T11:27:12.319353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.figure(figsize=(16,10))\ndfWork[['Age','RoomService','FoodCourt', 'ShoppingMall','Spa','VRDeck']].hist(bins = 60, figsize=(16,10));\nplt.tight_layout(pad = 2)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:12.321145Z","iopub.execute_input":"2022-07-27T11:27:12.321476Z","iopub.status.idle":"2022-07-27T11:27:13.781761Z","shell.execute_reply.started":"2022-07-27T11:27:12.321446Z","shell.execute_reply":"2022-07-27T11:27:13.780703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the plot above we can easily see that appart from the variable Age all others have really a lot of outliers","metadata":{}},{"cell_type":"markdown","source":"The above plots show a lot of usefull information from the dataset, we can actually see:\n1. If the home planet is Earth it is much more likely that the person will not be transported than those comming from Europa or Mars\n2. CryoSleep seems to increase a lot in the chances that someone will transported\n3. Destination = 55 Cancri e increases the chance of being transported\n4. The number of VIP is actually quiet low but it decreases the chance of being transported","metadata":{}},{"cell_type":"code","source":"plt.hist(x =dfWork[dfWork['Transported'] == 'No']['Age'], bins = 45,color = 'b',alpha = 0.5, label = 'Not transported' );\nplt.hist(x =dfWork[dfWork['Transported'] == 'Yes']['Age'], bins = 45,color = 'r',alpha = 0.5, label = 'Transported' );","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:13.783067Z","iopub.execute_input":"2022-07-27T11:27:13.783426Z","iopub.status.idle":"2022-07-27T11:27:14.053275Z","shell.execute_reply.started":"2022-07-27T11:27:13.783396Z","shell.execute_reply":"2022-07-27T11:27:14.052026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Calculating the correlation of the numerical variables\ncorr = dfWork[numCols].corr()\nplt.figure(figsize=(10,6))\nsns.heatmap(corr,annot=True )\nplt.yticks(rotation = 45)\nplt.xticks(rotation = 45)\nplt.title(\"Correlation of the numerical variables\");","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:14.054506Z","iopub.execute_input":"2022-07-27T11:27:14.054825Z","iopub.status.idle":"2022-07-27T11:27:14.515798Z","shell.execute_reply.started":"2022-07-27T11:27:14.054778Z","shell.execute_reply":"2022-07-27T11:27:14.514712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_matrix(dfWork,dimensions=['Age','RoomService','FoodCourt', 'ShoppingMall','Spa','VRDeck'], color = 'Transported', width = 1200,height=800)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:14.517144Z","iopub.execute_input":"2022-07-27T11:27:14.517497Z","iopub.status.idle":"2022-07-27T11:27:15.627876Z","shell.execute_reply.started":"2022-07-27T11:27:14.517467Z","shell.execute_reply":"2022-07-27T11:27:15.626524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### analyzing the Categorical variables","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2,ncols = 2, figsize = (12,8))\n\nax[0,0].set_title('Home planet vs Transported')\nsns.countplot(x = 'HomePlanet', data = dfWork, hue = 'Transported', palette = 'Blues', ax = ax[0,0])\nax[0,1].set_title('CryoSleep vs Transported')\nsns.countplot(x = 'CryoSleep', data = dfWork, hue = 'Transported', palette = 'RdPu', ax = ax[0,1])\nax[1,0].set_title('Destination vs Transported')\nsns.countplot(x = 'Destination', data = dfWork, hue = 'Transported', palette = 'BuPu', ax = ax[1,0])\nax[1,1].set_title('VIP vs Transported')\nsns.countplot(x = 'VIP', data = dfWork, hue = 'Transported', palette = 'BuGn', ax = ax[1,1])\n\nplt.tight_layout(pad = 2)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:15.629515Z","iopub.execute_input":"2022-07-27T11:27:15.630239Z","iopub.status.idle":"2022-07-27T11:27:16.210760Z","shell.execute_reply.started":"2022-07-27T11:27:15.630198Z","shell.execute_reply":"2022-07-27T11:27:16.209595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (12,6))\nax[0].set_title('Relating passenger groupID and Transported')\nsns.countplot(x = 'PassengerGroupId', data = dfWork, hue = 'Transported', ax = ax[0]);\nax[1].set_title('Relating passenger group and Transported')\nsns.histplot(data = dfWork, x = 'PassengerGroup', hue = 'Transported', ax = ax[1])\nplt.tight_layout(pad = 3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:16.212670Z","iopub.execute_input":"2022-07-27T11:27:16.213132Z","iopub.status.idle":"2022-07-27T11:27:16.760210Z","shell.execute_reply.started":"2022-07-27T11:27:16.213087Z","shell.execute_reply":"2022-07-27T11:27:16.759158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Passengers traveling alone have a slightly better chance to avoid beying transported","metadata":{}},{"cell_type":"code","source":"dfWork.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:16.761608Z","iopub.execute_input":"2022-07-27T11:27:16.762436Z","iopub.status.idle":"2022-07-27T11:27:16.782677Z","shell.execute_reply.started":"2022-07-27T11:27:16.762401Z","shell.execute_reply":"2022-07-27T11:27:16.781567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (12,6))\nax[0].set_title('Relating Cabin deck and Transported')\nsns.countplot(x = 'deck', data = dfWork, hue = 'Transported', ax = ax[0]);\nax[1].set_title('Relating Cabin side and Transported')\nsns.countplot(x = 'side', data = dfWork, hue = 'Transported', ax = ax[1]);\n\n\nplt.tight_layout(pad = 3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:16.784570Z","iopub.execute_input":"2022-07-27T11:27:16.785404Z","iopub.status.idle":"2022-07-27T11:27:17.200679Z","shell.execute_reply.started":"2022-07-27T11:27:16.785358Z","shell.execute_reply":"2022-07-27T11:27:17.199804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preparing the data for machine learning","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(trainDataset)\ntest = pd.read_csv(testDataset)\ntrain = prepareForAnalysis(train, train = True)\ntest = prepareForAnalysis(test, train=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.202094Z","iopub.execute_input":"2022-07-27T11:27:17.202561Z","iopub.status.idle":"2022-07-27T11:27:17.588925Z","shell.execute_reply.started":"2022-07-27T11:27:17.202527Z","shell.execute_reply":"2022-07-27T11:27:17.587400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.590652Z","iopub.execute_input":"2022-07-27T11:27:17.591101Z","iopub.status.idle":"2022-07-27T11:27:17.626356Z","shell.execute_reply.started":"2022-07-27T11:27:17.591045Z","shell.execute_reply":"2022-07-27T11:27:17.625473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.628705Z","iopub.execute_input":"2022-07-27T11:27:17.630242Z","iopub.status.idle":"2022-07-27T11:27:17.662725Z","shell.execute_reply.started":"2022-07-27T11:27:17.630185Z","shell.execute_reply":"2022-07-27T11:27:17.661078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.664174Z","iopub.execute_input":"2022-07-27T11:27:17.664773Z","iopub.status.idle":"2022-07-27T11:27:17.693008Z","shell.execute_reply.started":"2022-07-27T11:27:17.664730Z","shell.execute_reply":"2022-07-27T11:27:17.691679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(trainDF, testDF):\n    catColsToImpute = ['HomePlanet','CryoSleep','Destination','VIP','deck','side']\n    catTransformer = make_column_transformer((SimpleImputer(strategy = 'most_frequent'),catColsToImpute))\n    catTransformer.fit(trainDF)\n    trainDF[catColsToImpute] = catTransformer.transform(trainDF[catColsToImpute])\n    testDF[catColsToImpute] = catTransformer.transform(testDF[catColsToImpute])\n    catColsToEncode = ['HomePlanet','CryoSleep','Destination','deck','side']\n    catEncoder = make_column_transformer((OrdinalEncoder(),catColsToEncode))\n    catEncoder.fit(trainDF)\n    trainDF[catColsToEncode] = catEncoder.transform(trainDF[catColsToEncode] )\n    testDF[catColsToEncode] = catEncoder.transform(testDF[catColsToEncode] )\n\n\n    numericColsToImpute = ['Age','RoomService','FoodCourt','ShoppingMall','Spa','VRDeck']\n\n    numTransformer = make_column_transformer((SimpleImputer(strategy='mean'),numericColsToImpute))\n    numTransformer.fit(trainDF)\n    trainDF[numericColsToImpute] = numTransformer.transform(trainDF)\n    testDF[numericColsToImpute] = numTransformer.transform(testDF)\n\n    trainDF['num'] = trainDF['num'].fillna(method = 'ffill')\n    testDF['num'] = testDF['num'].fillna(method = 'ffill')\n\n    trainDF['Transported'] = trainDF['Transported'].map({'No':0,'Yes':1})\n    trainDF['VIP'] = trainDF['VIP'].map({'No':0,'Yes':1})\n    testDF['VIP'] = testDF['VIP'].map({'No':0,'Yes':1})\n\n    trainDF['PassengerGroup'] = trainDF['PassengerGroup'].astype(int)\n    trainDF['PassengerGroupId'] = trainDF['PassengerGroupId'].astype(int)\n    trainDF['num'] = trainDF['num'].astype(int)\n    testDF['PassengerGroup'] = testDF['PassengerGroup'].astype(int)\n    testDF['PassengerGroupId'] = testDF['PassengerGroupId'].astype(int)\n    testDF['num'] = testDF['num'].astype(int)\n\n    #columnsToScale = ['Age','RoomService','ShoppingMall','Spa','VRDeck','deck','num','PassengerGroup']\n    #scaleTransformer = make_column_transformer((StandardScaler(),columnsToScale))\n    #scaleTransformer.fit(trainDF)\n    #trainDF[columnsToScale] = scaleTransformer.transform(trainDF[columnsToScale])\n    #testDF[columnsToScale] = scaleTransformer.transform(testDF[columnsToScale])\n    \n    return trainDF, testDF\ntrain, test = preprocess(train, test)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.694100Z","iopub.execute_input":"2022-07-27T11:27:17.694698Z","iopub.status.idle":"2022-07-27T11:27:17.800673Z","shell.execute_reply.started":"2022-07-27T11:27:17.694653Z","shell.execute_reply":"2022-07-27T11:27:17.799473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.802571Z","iopub.execute_input":"2022-07-27T11:27:17.803131Z","iopub.status.idle":"2022-07-27T11:27:17.812439Z","shell.execute_reply.started":"2022-07-27T11:27:17.803097Z","shell.execute_reply":"2022-07-27T11:27:17.811319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.818616Z","iopub.execute_input":"2022-07-27T11:27:17.819444Z","iopub.status.idle":"2022-07-27T11:27:17.840985Z","shell.execute_reply.started":"2022-07-27T11:27:17.819408Z","shell.execute_reply":"2022-07-27T11:27:17.839750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.842227Z","iopub.execute_input":"2022-07-27T11:27:17.842573Z","iopub.status.idle":"2022-07-27T11:27:17.901889Z","shell.execute_reply.started":"2022-07-27T11:27:17.842545Z","shell.execute_reply":"2022-07-27T11:27:17.900901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier(n_estimators=200, n_jobs=-1)\n\nX = train.drop('Transported', axis = 1)\ny = train['Transported']\n\ncvRF = cross_val_score(model,X = X, y = y, cv = 5, scoring = 'accuracy')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:17.903311Z","iopub.execute_input":"2022-07-27T11:27:17.903667Z","iopub.status.idle":"2022-07-27T11:27:24.059032Z","shell.execute_reply.started":"2022-07-27T11:27:17.903638Z","shell.execute_reply":"2022-07-27T11:27:24.057591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cvRF.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:24.061116Z","iopub.execute_input":"2022-07-27T11:27:24.061634Z","iopub.status.idle":"2022-07-27T11:27:24.070489Z","shell.execute_reply.started":"2022-07-27T11:27:24.061581Z","shell.execute_reply":"2022-07-27T11:27:24.069172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(n_jobs = -1)\n\nX = train.drop('Transported', axis = 1)\ny = train['Transported']\n\ncross_val_score(model,X = X, y = y, cv = 5, scoring = 'accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:24.072350Z","iopub.execute_input":"2022-07-27T11:27:24.072870Z","iopub.status.idle":"2022-07-27T11:27:28.760153Z","shell.execute_reply.started":"2022-07-27T11:27:24.072825Z","shell.execute_reply":"2022-07-27T11:27:28.759117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scoreModelRF(params):\n    model = RandomForestClassifier(n_jobs=-1, n_estimators=params[0], max_depth=params[1],min_samples_leaf=params[2],min_samples_split=params[3])\n\n    cv = cross_val_score(model, X, y, cv = 4, scoring = 'accuracy')\n    return -cv.mean()\n\nparams = [\n    (100,2000),\n    (2,100),\n    (1,100),\n    (2,100)\n]\nresult = gp_minimize(func=scoreModelRF, dimensions=params, n_calls = 20, n_initial_points=10, random_state=1)\nresult","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:27:28.761760Z","iopub.execute_input":"2022-07-27T11:27:28.762811Z","iopub.status.idle":"2022-07-27T11:31:40.024814Z","shell.execute_reply.started":"2022-07-27T11:27:28.762770Z","shell.execute_reply":"2022-07-27T11:31:40.023362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(result.x)\nplot_convergence(result);","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:31:40.028710Z","iopub.execute_input":"2022-07-27T11:31:40.029173Z","iopub.status.idle":"2022-07-27T11:31:40.450372Z","shell.execute_reply.started":"2022-07-27T11:31:40.029131Z","shell.execute_reply":"2022-07-27T11:31:40.449263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier(n_jobs=-1, n_estimators=320, max_depth=28,min_samples_leaf=49,min_samples_split=7)\ncross_val_score(model, X, y, cv=5, scoring='accuracy')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:31:40.451791Z","iopub.execute_input":"2022-07-27T11:31:40.452934Z","iopub.status.idle":"2022-07-27T11:31:46.399391Z","shell.execute_reply.started":"2022-07-27T11:31:40.452887Z","shell.execute_reply":"2022-07-27T11:31:46.397955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop('Transported', axis = 1)\ny = train['Transported']\n\ndef scoreXGB(params):\n    model = XGBClassifier(n_jobs = -1, n_estimators = 1000,learning_rate = 0.1, max_depth = params[0], min_child_weight = params[1], gamma = params[2],\n    subsample = params[3], colsample_bytree = params[4], objective = 'binary:logistic' )\n    return -cross_val_score(model, X, y, cv = 5, scoring = 'accuracy').mean()\n\n\n\nparams = [\n    (3,10),     #max_depth\n    (1,10),     #min_child_weight\n    (0,0.2),    #gamma\n    (0.5,0.9),  #subsample\n    (0.5,0.9)   #colsample_by_tree\n]\nresult = gp_minimize(func=scoreXGB, dimensions=params, n_calls = 20, n_initial_points=10, random_state=1)\nprint(result.x)\nplot_convergence(result)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:31:46.400881Z","iopub.execute_input":"2022-07-27T11:31:46.401217Z","iopub.status.idle":"2022-07-27T11:44:08.013139Z","shell.execute_reply.started":"2022-07-27T11:31:46.401185Z","shell.execute_reply":"2022-07-27T11:44:08.012281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(n_jobs = -1, max_depth = 4, min_child_weight = 10, gamma = 0.04,subsample = 0.5, colsample_bytree = 0.9 , n_estimators = 1000 , learning_rate = 0.1);\ncross_val_score(model, X, y, cv = 5, scoring = 'accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:44:08.014510Z","iopub.execute_input":"2022-07-27T11:44:08.015233Z","iopub.status.idle":"2022-07-27T11:44:36.135667Z","shell.execute_reply.started":"2022-07-27T11:44:08.015190Z","shell.execute_reply":"2022-07-27T11:44:36.134533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop('Transported', axis = 1)\ny = train['Transported']\n\nX_train, X_valid, y_train, y_valid = train_test_split(X,y, test_size=0.3)\n\ninput_shape = [X.shape[1]]\ndropoutRatio = 0.5\n\nmodelNN = keras.Sequential([\n    layers.BatchNormalization(input_shape = input_shape),\n    layers.Dense(512, activation='relu'),\n    layers.BatchNormalization(),   \n    layers.Dropout(dropoutRatio),\n    layers.Dense(512, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(dropoutRatio),\n    layers.Dense(1,activation='sigmoid')\n\n])\n\nmodelNN.compile(optimizer='adam',\n    loss = 'binary_crossentropy',\n    metrics = ['binary_accuracy'])\n\n\nearly_stopping = keras.callbacks.EarlyStopping(\n    patience=5,\n    min_delta=0.001,\n    restore_best_weights=True,\n)\nhistory = modelNN.fit(\n    X_train, y_train,\n    validation_data=(X_valid, y_valid),\n    batch_size=512,\n    epochs=200,\n    callbacks=[early_stopping],)\n\n\nhistory_df = pd.DataFrame(history.history)\nhistory_df.loc[:, ['loss', 'val_loss']].plot(title=\"Cross-entropy\")\nhistory_df.loc[:, ['binary_accuracy', 'val_binary_accuracy']].plot(title=\"Accuracy\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:44:36.137381Z","iopub.execute_input":"2022-07-27T11:44:36.137796Z","iopub.status.idle":"2022-07-27T11:44:58.577363Z","shell.execute_reply.started":"2022-07-27T11:44:36.137753Z","shell.execute_reply":"2022-07-27T11:44:58.575991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = modelNN.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:44:58.579003Z","iopub.execute_input":"2022-07-27T11:44:58.579379Z","iopub.status.idle":"2022-07-27T11:44:58.953635Z","shell.execute_reply.started":"2022-07-27T11:44:58.579347Z","shell.execute_reply":"2022-07-27T11:44:58.952478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfTestRaw = pd.read_csv(testDataset)\ndfSubmission = pd.DataFrame(dfTestRaw['PassengerId'])\ndfSubmission['Transported'] = pred\ndfSubmission['Transported'] = dfSubmission['Transported'].map(lambda x: True if x >=0.5 else False)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:48:27.500930Z","iopub.execute_input":"2022-07-27T11:48:27.501354Z","iopub.status.idle":"2022-07-27T11:48:27.525621Z","shell.execute_reply.started":"2022-07-27T11:48:27.501318Z","shell.execute_reply":"2022-07-27T11:48:27.524363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfSubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:48:30.593473Z","iopub.execute_input":"2022-07-27T11:48:30.594505Z","iopub.status.idle":"2022-07-27T11:48:30.606139Z","shell.execute_reply.started":"2022-07-27T11:48:30.594456Z","shell.execute_reply":"2022-07-27T11:48:30.604813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfSubmission.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T11:48:42.175816Z","iopub.execute_input":"2022-07-27T11:48:42.177215Z","iopub.status.idle":"2022-07-27T11:48:42.190433Z","shell.execute_reply.started":"2022-07-27T11:48:42.177160Z","shell.execute_reply":"2022-07-27T11:48:42.188796Z"},"trusted":true},"execution_count":null,"outputs":[]}]}