{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-12T06:38:11.546657Z","iopub.execute_input":"2022-08-12T06:38:11.547131Z","iopub.status.idle":"2022-08-12T06:38:11.606694Z","shell.execute_reply.started":"2022-08-12T06:38:11.547093Z","shell.execute_reply":"2022-08-12T06:38:11.604834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basic libraries of python for numeric and dataframe computations\n\nfrom numpy import asarray\nfrom numpy import *\nfrom numba import njit\n\n# Basic library for data visualization\nimport matplotlib.pyplot as plt     \n\n# Slightly advanced library for data visualization            \nimport seaborn as sns                          \n\n# Featauretools for feature engineering\nimport featuretools as ft\n\n\n\nimport statsmodels.formula.api as smf\n\n#Importing our ML toolkit\nfrom sklearn.metrics import r2_score\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_validate, StratifiedKFold, RepeatedStratifiedKFold, RandomizedSearchCV, GridSearchCV\nfrom sklearn.model_selection import cross_val_score, learning_curve\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.impute import KNNImputer\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.preprocessing import StandardScaler\n\n\n\n# Importing primitives\nfrom featuretools.primitives import (Minute, Hour, Day, Month,\n                                     Weekday, IsWeekend, Count, Sum, Mean, Median, Std, Min, Max)\n\n# Used to ignore the warning given as output of the code\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:11.715875Z","iopub.execute_input":"2022-08-12T06:38:11.716839Z","iopub.status.idle":"2022-08-12T06:38:11.790362Z","shell.execute_reply.started":"2022-08-12T06:38:11.716774Z","shell.execute_reply":"2022-08-12T06:38:11.788997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating a function for comprehensive DEA, generic to any dataset\n#______________________________________________________________________\n#the input contains the dataset in data frame, and an optional column to drop, e.g. the target column\n#i.e. the input parameter 'target' is optional\n\ndef discover(df, target=''):\n    \n    #Creating Seris with Feature Types, removing the target feature\n    if target != '':\n        df1 = df.drop(target, axis = 1, inplace = False)\n    else:\n        df1 = df\n\n    #listing feature type\n    feature_type = []\n    for j in range(len(df1.columns)):       \n        if df1[df1.columns[j]].nunique()==2:\n            feature_type.append('Binary')            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()>10:\n            feature_type.append('Numerical')            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Ordinal')                    \n        elif df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Categorical')        \n        else:\n            feature_type.append('Identifier')\n    \n    #listing sample feature content\n    features_content = []\n    for i in range(len(df1.columns)):\n        if df1[df1.columns[i]].nunique()<10: \n            features_content.append(df1[df1.columns[i]].unique())\n        else:\n            features_content.append('n = ' + str(df1[df1.columns[i]].nunique()))\n            \n    #listing empty cells per feature\n    num_of_empty = []\n    for k in df1.columns:\n        num_of_empty.append(df1[k].isnull().sum())\n\n    #listing empty cells ratio feature type\n    ratio_of_empty = []\n    for k in df1.columns:\n        ratio_of_empty.append(round(100*df1[k].isnull().sum()/df1[k].isnull().count(),2))\n        \n\n    discovered = pd.DataFrame({'Features': df1.columns, \n                               'Features Content': features_content, \n                               'Feature Type': feature_type, \n                               'Empty Cells': num_of_empty,\n                               '% Empty': ratio_of_empty\n                             })\n    discovered = discovered.sort_values(['Feature Type','Empty Cells'],ascending=False)\n    \n    return discovered\n#_______________________________________________________________________________________________________________\n#function to visualize the dataset, works best with classification datasets\n#to modify later to take also numerical target data\n#the function takes: df: the dataframe containing the dataset, and \n#target: a string containing the column name of the target column (i.e. the one containing the classes), and \n#cols: the number of graphs shown side by side in a row\n#_______________________________________________________________________________________________________________\n\ndef graph(df, target, cols = 5):\n    \n        \n    if target != '':\n        df1 = df.drop(target, axis = 1, inplace = False)\n    else:\n        df1 = df\n\n#listing feature types\n    feature_type = []\n    for j in range(len(df1.columns)):       \n        if df1[df1.columns[j]].nunique()==2:\n            feature_type.append('Binary')            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()>10:\n            feature_type.append('Numerical')            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Ordinal')                    \n        elif df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Categorical')        \n        else:\n            feature_type.append('Identifier')\n            \n#listing feature content\n    features_content = []\n    for i in range(len(df1.columns)):\n        if df1[df1.columns[i]].nunique()<10: \n            features_content.append(df1[df1.columns[i]].unique())\n        else:\n            features_content.append('n = ' + str(df1[df1.columns[i]].nunique()))\n\n#listing empty cells feature type\n    num_of_empty = []\n    for k in df1.columns:\n        num_of_empty.append(df1[k].isnull().sum())\n\n    discovered = pd.DataFrame({'Features': df1.columns, \n                                   'Features Content': features_content, \n                                   'Feature Type': feature_type, \n                                   'Empty Cells': num_of_empty\n                                 })\n\n#_________________________________________________________________________________________________    \n# Plotting:\n#_________________________________________________________________________________________________    \n\n#plotting the target column\n    plt.figure(figsize=(10,4))\n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n    ax = df[target].value_counts().plot.barh()\n    plt.title(f'{target} Count Plot')\n\n#plotting Categorical & binary values:\n    \n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n    plots = 0\n    legends = df[target].unique()\n    columns = int(min(cols,len(discovered['Features'][discovered['Feature Type'] .isin(['Categorical', 'Ordinal', 'Binary'])])))\n    rows = int(np.ceil(len(legends)*len(discovered['Features'][discovered['Feature Type'] .isin(['Categorical', 'Ordinal', 'Binary'])])/cols))\n    plt.figure(figsize=(20,5*rows))\n    \n    for i in discovered['Features'][discovered['Feature Type'] .isin(['Categorical', 'Ordinal', 'Binary'])]:\n        plots = plots + 1\n        plt.subplot(rows,columns,plots)\n        sns.countplot(x=i, hue=target, data=df, palette=\"Set1\")\n\n\n#plotting Numerical values:\n\n\n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n    plots = 0\n    columns = int(min(cols,len(discovered['Features'][discovered['Feature Type'] =='Numerical'])))\n    rows = int(np.ceil(len(discovered['Features'][discovered['Feature Type'] =='Numerical'])/cols))\n    plt.figure(figsize=(20,5*rows))\n    for i in discovered['Features'][discovered['Feature Type'] =='Numerical']:\n        plots = plots + 1\n        plt.subplot(rows,columns,plots)\n        sns.histplot(data = df, x = df[i], hue = target, hue_order = legends, kde = True, element = 'step')\n        #plt.yscale('symlog')\n        if df[i].skew(axis = 0, skipna = True)>2:\n            plt.xscale('symlog')\n            plt.xlim(0,max(df[i]))\n    plt.show()\n    \n#Plotting Overall Correlation Heatmap\n    \n    columns = int(min(cols,len(discovered['Features'][discovered['Feature Type'] =='Numerical'])))\n    rows    = int(np.ceil(len(discovered['Features'][discovered['Feature Type'] =='Numerical'])/cols))\n    plt.figure(figsize=(5*columns,5*columns))\n    cmap = sns.diverging_palette(230, 20, as_cmap = True)\n    sns.heatmap(df.corr().abs(), annot = True, fmt = '.2f', cmap = cmap, square=True )\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:11.942694Z","iopub.execute_input":"2022-08-12T06:38:11.943465Z","iopub.status.idle":"2022-08-12T06:38:12.031390Z","shell.execute_reply.started":"2022-08-12T06:38:11.943411Z","shell.execute_reply":"2022-08-12T06:38:12.030377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to calculate mutual information score of the data:\n\ndef make_mi_scores(X, y):\n    X = X.copy()\n    for colname in X.select_dtypes([\"object\", \"category\"]):\n        X[colname], _ = X[colname].factorize()\n    # All discrete features should now have integer dtypes\n    discrete_features = [pd.api.types.is_integer_dtype(t) for t in X.dtypes]\n    mi_scores = mutual_info_regression(X, y, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    df_mi_scores = pd.DataFrame({'Mutual Information Scores': mi_scores})\n    return df_mi_scores\n\n\ndef plot_mi_scores(scores):\n    scores = scores.sort_values(ascending=True)\n    width = np.arange(len(scores))\n    ticks = list(scores.index)\n    plt.barh(width, scores)\n    plt.yticks(width, ticks)\n    plt.title(\"Mutual Information Scores\")\n\n\n#to calculate the VIF of the data:\n\ndef get_vif(exogs, data):\n\n    # initialize dictionaries\n    vif_dict, tolerance_dict = {}, {}\n\n    # form input data for each exogenous variable\n    for exog in exogs:\n        not_exog = [i for i in exogs if i != exog]\n        X, y = data[not_exog], data[exog]\n\n        # extract r-squared from the fit\n        r_squared = LinearRegression().fit(X, y).score(X, y)\n\n        # calculate VIF\n        vif = 1/(1 - r_squared)\n        vif_dict[exog] = vif\n\n        # calculate tolerance\n        tolerance = 1 - r_squared\n        tolerance_dict[exog] = tolerance\n\n    # return VIF DataFrame\n    df_vif = pd.DataFrame({'VIF': vif_dict, 'Tolerance': tolerance_dict})\n    df_vif = df_vif.sort_values(['VIF'])\n    return df_vif\n\n#combine the above two functions in a comprehensive table for feature selection\ndef feature_significance (data, target):\n    #target = the target variable or the classicifation lables column\n    ymiscore = data[target]\n    xmiscore = data.drop(labels = [target],axis = 1)\n    mi_scores = make_mi_scores(xmiscore, ymiscore)\n\n    xvif = data.drop(labels = [target],axis = 1)\n    Feature_Strength = get_vif(xvif,target).join(mi_scores)\n    Feature_Strength.sort_values(['Mutual Information Scores'], ascending = False).style.background_gradient(cmap='coolwarm',axis=0, vmax = 5).set_precision(3)\n\n#function to score the models:\ndef score(X, y, model, cv):\n    scoring = [\"roc_auc\"]\n    scores = cross_validate(\n        model, X, y, scoring=scoring, cv=cv, return_train_score=True,\n    )\n    scores = pd.DataFrame(scores).T\n    return scores.assign(\n        mean = lambda x: x.mean(axis=1),\n        std = lambda x: x.std(axis=1),\n    )\n\n#to perform PCA and plot a scatterplot for the results vs. a target value:\n\ndef graph_PCA(data,target, n='' ):\n    if n != '':\n        n\n    else:\n        n= int((data.shape[1])*0.5)\n    #Finding principal components for the data\n    pca1 = PCA(n_components=n, random_state=1)\n    data_pca = pd.DataFrame(PCA(n_components=n, random_state=1).fit_transform(data))\n    #data_scaled = pd.DataFrame(data_scaled, columns=data.columns)\n    \n    dataandpca = pd.concat([data_pca,data], axis = 1)\n    dataandpca[target] = df_train[target]\n    plt.figure(figsize=(20,8))\n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n    plt.title(f'{target} in PCA')\n    sns.scatterplot(dataandpca[0],dataandpca[1],data=dataandpca, hue= target)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:12.207082Z","iopub.execute_input":"2022-08-12T06:38:12.207514Z","iopub.status.idle":"2022-08-12T06:38:12.275414Z","shell.execute_reply.started":"2022-08-12T06:38:12.207474Z","shell.execute_reply":"2022-08-12T06:38:12.274079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/spaceship-titanic/train.csv',encoding='utf-8')\ndf_test =  pd.read_csv  ('../input/spaceship-titanic/test.csv', encoding='utf-8')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:12.483212Z","iopub.execute_input":"2022-08-12T06:38:12.483980Z","iopub.status.idle":"2022-08-12T06:38:12.574829Z","shell.execute_reply.started":"2022-08-12T06:38:12.483926Z","shell.execute_reply":"2022-08-12T06:38:12.573823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discover(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:12.754479Z","iopub.execute_input":"2022-08-12T06:38:12.755084Z","iopub.status.idle":"2022-08-12T06:38:12.845981Z","shell.execute_reply.started":"2022-08-12T06:38:12.755043Z","shell.execute_reply":"2022-08-12T06:38:12.844786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph(df_train, 'HomePlanet')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:12.910673Z","iopub.execute_input":"2022-08-12T06:38:12.911066Z","iopub.status.idle":"2022-08-12T06:38:16.273581Z","shell.execute_reply.started":"2022-08-12T06:38:12.911032Z","shell.execute_reply":"2022-08-12T06:38:16.272212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:16.276170Z","iopub.execute_input":"2022-08-12T06:38:16.276688Z","iopub.status.idle":"2022-08-12T06:38:16.349051Z","shell.execute_reply.started":"2022-08-12T06:38:16.276640Z","shell.execute_reply":"2022-08-12T06:38:16.347657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph(df_train, 'Transported')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:16.350859Z","iopub.execute_input":"2022-08-12T06:38:16.351182Z","iopub.status.idle":"2022-08-12T06:38:19.452332Z","shell.execute_reply.started":"2022-08-12T06:38:16.351152Z","shell.execute_reply":"2022-08-12T06:38:19.450987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:19.456643Z","iopub.execute_input":"2022-08-12T06:38:19.457117Z","iopub.status.idle":"2022-08-12T06:38:19.529903Z","shell.execute_reply.started":"2022-08-12T06:38:19.457078Z","shell.execute_reply":"2022-08-12T06:38:19.528813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cabin:\n#creating this early as it will help with imputing relevnat data\n\ncombine = [df_train, df_test]\nfor dataset in combine:\n    dataset[['Deck', 'Num', 'Side']]  = dataset['Cabin'].str.split(\"/\", expand = True)\n   ","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:19.531555Z","iopub.execute_input":"2022-08-12T06:38:19.531918Z","iopub.status.idle":"2022-08-12T06:38:19.611021Z","shell.execute_reply.started":"2022-08-12T06:38:19.531876Z","shell.execute_reply":"2022-08-12T06:38:19.609645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph(df_train,'Deck' )","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:19.612675Z","iopub.execute_input":"2022-08-12T06:38:19.613346Z","iopub.status.idle":"2022-08-12T06:38:24.913045Z","shell.execute_reply.started":"2022-08-12T06:38:19.613308Z","shell.execute_reply":"2022-08-12T06:38:24.912039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discover(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:24.914223Z","iopub.execute_input":"2022-08-12T06:38:24.914856Z","iopub.status.idle":"2022-08-12T06:38:25.042148Z","shell.execute_reply.started":"2022-08-12T06:38:24.914817Z","shell.execute_reply":"2022-08-12T06:38:25.040888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer  # noqa\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n\ncombine = [df_train, df_test]\ncatempty = ['HomePlanet', 'Deck', 'Destination','CryoSleep', 'VIP', 'Side']\nfor dataset in combine:\n    dataset[catempty] = dataset[catempty].apply(lambda series: pd.Series(\n        LabelEncoder().fit_transform(series[series.notnull()]),\n        index=series[series.notnull()].index\n    ))\n    imp_cat = IterativeImputer(estimator=RandomForestClassifier(),                                initial_strategy='most_frequent',                               max_iter=10, random_state=0)\n\n    dataset[catempty] = imp_cat.fit_transform(dataset[catempty])","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:25.043635Z","iopub.execute_input":"2022-08-12T06:38:25.043989Z","iopub.status.idle":"2022-08-12T06:38:53.443606Z","shell.execute_reply.started":"2022-08-12T06:38:25.043947Z","shell.execute_reply":"2022-08-12T06:38:53.442307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph(df_train,'Transported' )","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:53.445715Z","iopub.execute_input":"2022-08-12T06:38:53.446209Z","iopub.status.idle":"2022-08-12T06:38:57.191196Z","shell.execute_reply.started":"2022-08-12T06:38:53.446163Z","shell.execute_reply":"2022-08-12T06:38:57.189769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Imputing:\nimputer = KNNImputer(n_neighbors=5)\ncombine = [df_train, df_test]\nfor dataset in combine:\n    dataset[['Age','VIP', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck','Num']]=imputer.fit_transform(dataset[['Age','VIP', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck','Num']])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:57.194368Z","iopub.execute_input":"2022-08-12T06:38:57.194723Z","iopub.status.idle":"2022-08-12T06:38:58.319548Z","shell.execute_reply.started":"2022-08-12T06:38:57.194691Z","shell.execute_reply":"2022-08-12T06:38:58.318513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating new features:\n\n\n#PassengerId:\nfor dataset in combine:\n    dataset[['Group', 'NumInGroup']]  = dataset['PassengerId'].str.split(\"_\", expand = True)\n\n    \n\n#Below 18\nfor dataset in combine:\n    dataset['Below_18']                   = 0\n    dataset['Below_18'][dataset['Age']<18]= 1\n\n#IsAlone\nfor dataset in combine:\n    dataset['IsAlone']                             = 0\n    dataset['IsAlone'][dataset['NumInGroup']=='01']= 1\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:58.321072Z","iopub.execute_input":"2022-08-12T06:38:58.321433Z","iopub.status.idle":"2022-08-12T06:38:58.824758Z","shell.execute_reply.started":"2022-08-12T06:38:58.321400Z","shell.execute_reply":"2022-08-12T06:38:58.823075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lable Encoding:\n\nle = LabelEncoder()\ndf_train.Transported = le.fit_transform(df_train.Transported)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:58.826219Z","iopub.execute_input":"2022-08-12T06:38:58.826685Z","iopub.status.idle":"2022-08-12T06:38:58.874027Z","shell.execute_reply.started":"2022-08-12T06:38:58.826648Z","shell.execute_reply":"2022-08-12T06:38:58.873188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.Transported","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:58.875533Z","iopub.execute_input":"2022-08-12T06:38:58.876706Z","iopub.status.idle":"2022-08-12T06:38:58.929733Z","shell.execute_reply.started":"2022-08-12T06:38:58.876586Z","shell.execute_reply":"2022-08-12T06:38:58.928343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#evaluating feature significance:\ndf_light = df_train.drop(['Name','Cabin','PassengerId'],axis = 1)\nymiscore = df_light['Transported']\nxmiscore = df_light.drop(labels = ['Transported'],axis = 1)\nmi_scores = make_mi_scores(xmiscore, ymiscore)\n\nxvif = df_light.drop(labels = ['Transported'],axis = 1)\nFeature_Strength = get_vif(xvif,df_light).join(mi_scores)\nFeature_Strength.sort_values(['Mutual Information Scores'], ascending = False).style.background_gradient(cmap='coolwarm',axis=0, vmax = 5).set_precision(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:38:58.934078Z","iopub.execute_input":"2022-08-12T06:38:58.934462Z","iopub.status.idle":"2022-08-12T06:39:01.714174Z","shell.execute_reply.started":"2022-08-12T06:38:58.934425Z","shell.execute_reply":"2022-08-12T06:39:01.712311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:39:01.716172Z","iopub.execute_input":"2022-08-12T06:39:01.718207Z","iopub.status.idle":"2022-08-12T06:39:01.792693Z","shell.execute_reply.started":"2022-08-12T06:39:01.718157Z","shell.execute_reply":"2022-08-12T06:39:01.791373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combine = [df_train, df_test]\nfor dataset in combine:\n    dataset = dataset.drop(['PassengerId','Cabin','Name'], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:39:01.794350Z","iopub.execute_input":"2022-08-12T06:39:01.794876Z","iopub.status.idle":"2022-08-12T06:39:01.849173Z","shell.execute_reply.started":"2022-08-12T06:39:01.794839Z","shell.execute_reply":"2022-08-12T06:39:01.848176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"#Creating different compositions of dataset to iterate through, using series of columns 'dropfeature' \n#to choose which features to discard at each iteration\n\ndropfeature = []\ndropfeature.append([])\ndropfeature.append('Below_18')\ndropfeature.append('Age')\ndropfeature.append(['Side']+['IsAlone']+ ['Below_18'])\ndropfeature.append(['NumInGroup']+['Side']+['IsAlone']+ ['Below_18'])\ndropfeature.append(['NumInGroup']+['Side']+['IsAlone']+ ['Below_18'] + ['VIP'])","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:44:17.924059Z","iopub.execute_input":"2022-08-12T06:44:17.925187Z","iopub.status.idle":"2022-08-12T06:44:17.980139Z","shell.execute_reply.started":"2022-08-12T06:44:17.925136Z","shell.execute_reply":"2022-08-12T06:44:17.978723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fitting logistic regressor through the different compositions\n#to choose the best one\n\nfor dropcol in dropfeature:\n    columns = df_train.drop(dropcol, axis = 1).columns\n    X = df_train.drop('Transported', axis = 1).drop(dropcol, axis = 1)\n    Y = df_train['Transported']\n    Test = df_test[X.columns]\n\n\n    std_scaler = StandardScaler()\n    X = std_scaler.fit_transform(X)\n    Test = std_scaler.transform(Test)\n\n    skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n    model_lr = LogisticRegression(max_iter = 500, C=0.0001, penalty='l2', solver='newton-cg')\n    scores = score(X, Y, model_lr, cv=skf)\n    print('Columns Dropped:\\n',dropcol)\n    display(scores)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:44:28.353683Z","iopub.execute_input":"2022-08-12T06:44:28.354209Z","iopub.status.idle":"2022-08-12T06:44:33.213678Z","shell.execute_reply.started":"2022-08-12T06:44:28.354160Z","shell.execute_reply":"2022-08-12T06:44:33.212380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#winning configuration is:\ndropcol = (['NumInGroup', 'Side', 'IsAlone', 'Below_18'])\n\n#Iterating through different fine-tuned classifiers:\n\nkfold = StratifiedKFold(n_splits=10)\n\ncolumns = df_train.drop(dropcol, axis = 1).columns\nX = df_train.drop('Transported', axis = 1).drop(dropcol, axis = 1)\nY = df_train['Transported']\nTest = df_test[X.columns]\n\nstd_scaler = StandardScaler()\nX = std_scaler.fit_transform(X)\nTest = std_scaler.transform(Test)\n\n\n# Gradient boosting tunning\nGBC = GradientBoostingClassifier()\ngb_param_grid = {'loss' : [\"deviance\"],\n              'n_estimators' : [100,200,300],\n              'learning_rate': [0.1, 0.05, 0.01],\n              'max_depth': [4, 8],\n              'min_samples_leaf': [100,150],\n              'max_features': [0.3, 0.1] \n              }\ngsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring=\"roc_auc\", n_jobs= -1, verbose = 1)\ngsGBC.fit(X,Y)\nGBC_best = gsGBC.best_estimator_\ngsGBC.best_score_\n\n#ExtraTrees \nExtC = ExtraTreesClassifier()\nex_param_grid = {\"max_depth\": [None],\n              \"max_features\": [1, 3, 10],\n              \"min_samples_split\": [2, 3, 10],\n              \"min_samples_leaf\": [1, 3, 10],\n              \"bootstrap\": [False],\n              \"n_estimators\" :[100,300],\n              \"criterion\": [\"gini\"]}\ngsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring=\"roc_auc\", n_jobs= 4, verbose = 1)\ngsExtC.fit(X,Y)\nExtC_best = gsExtC.best_estimator_\ngsExtC.best_score_\n\n# RFC Parameters tunning \nRFC = RandomForestClassifier()\nrf_param_grid = {\"max_depth\": [None],\n              \"max_features\": [1, 3, 10],\n              \"min_samples_split\": [2, 3, 10],\n              \"min_samples_leaf\": [1, 3, 10],\n              \"bootstrap\": [False],\n              \"n_estimators\" :[100,300],\n              \"criterion\": [\"gini\"]}\ngsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring=\"roc_auc\", n_jobs= 4, verbose = 1)\ngsRFC.fit(X,Y)\nRFC_best = gsRFC.best_estimator_\ngsRFC.best_score_\n\n# SVC classifier\nSVMC = SVC(probability=True)\nsvc_param_grid = {'kernel': ['rbf'], \n                  'gamma': [ 0.001, 0.01, 0.1, 1],\n                  'C': [1, 10, 50, 100,200,300, 1000]}\ngsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring=\"roc_auc\", n_jobs= 4, verbose = 1)\ngsSVMC.fit(X,Y)\nSVMC_best = gsSVMC.best_estimator_\ngsSVMC.best_score_\n\n# Adaboost\nDTC = DecisionTreeClassifier()\nadaDTC = AdaBoostClassifier(DTC, random_state=7)\nada_param_grid = {\"base_estimator__criterion\" : [\"gini\", \"entropy\"],\n              \"base_estimator__splitter\" :   [\"best\", \"random\"],\n              \"algorithm\" : [\"SAMME\",\"SAMME.R\"],\n              \"n_estimators\" :[1,2],\n              \"learning_rate\":  [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}\ngsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring=\"roc_auc\", n_jobs= 4, verbose = 1)\ngsadaDTC.fit(X,Y)\nada_best = gsadaDTC.best_estimator_\ngsadaDTC.best_score_\n\n\n# Concatenate all classifier results\ntest_Transported_RFC = pd.Series(gsRFC.best_score_, name=\"RFC\")\ntest_Transported_ExtC = pd.Series(gsExtC.best_score_, name=\"ExtC\")\ntest_Transported_SVMC = pd.Series(gsSVMC.best_score_, name=\"SVC\")\ntest_Transported_AdaC = pd.Series(gsadaDTC.best_score_, name=\"Ada\")\ntest_Transported_GBC = pd.Series(gsGBC.best_score_, name=\"GBC\")\n\nensemble_results = pd.concat([test_Transported_RFC,test_Transported_ExtC,test_Transported_AdaC,test_Transported_GBC, test_Transported_SVMC],axis=1)\n\nplt.ylim(ensemble_results.min().mean(), ensemble_results.max().mean())\ng= sns.barplot(data = ensemble_results)\nfor bar in g.patches: \n    g.annotate(format(bar.get_height(), '.4f'),\n                   (bar.get_x() + bar.get_width() / 2,\n                    bar.get_height()), ha='center', va='center',\n                   size=9, xytext=(0, 8),\n                   textcoords='offset points')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T06:47:59.322761Z","iopub.execute_input":"2022-08-12T06:47:59.323809Z","iopub.status.idle":"2022-08-12T08:19:11.529009Z","shell.execute_reply.started":"2022-08-12T06:47:59.323761Z","shell.execute_reply":"2022-08-12T08:19:11.527488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = np.array(pd.read_csv('../input/spaceship-titanic/test.csv', encoding='utf-8').PassengerId)\nb = gsSVMC.predict(Test)\nIbrahim_Seyam_Submission = pd.DataFrame([a,b], index =['PassengerId','Transported']).T\nIbrahim_Seyam_Submission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:33:55.130333Z","iopub.execute_input":"2022-08-12T08:33:55.133394Z","iopub.status.idle":"2022-08-12T08:33:56.602138Z","shell.execute_reply.started":"2022-08-12T08:33:55.133293Z","shell.execute_reply":"2022-08-12T08:33:56.600815Z"},"trusted":true},"execution_count":null,"outputs":[]}]}