{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport sklearn\nimport re\n%matplotlib inline\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport seaborn as sns # used for plot interactive graph.\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.compose import make_column_transformer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.preprocessing import StandardScaler\n\npd.options.mode.chained_assignment = None\npd.set_option('display.max_rows', 25)\npd.set_option('display.max_columns', 25)\npd.set_option('display.width', 1000)\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-08-09T07:30:31.582624Z","iopub.execute_input":"2022-08-09T07:30:31.583777Z","iopub.status.idle":"2022-08-09T07:30:31.608199Z","shell.execute_reply.started":"2022-08-09T07:30:31.583727Z","shell.execute_reply":"2022-08-09T07:30:31.606946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/spaceship-titanic/train.csv')\ntest = pd.read_csv('/kaggle/input/spaceship-titanic/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.610904Z","iopub.execute_input":"2022-08-09T07:30:31.611767Z","iopub.status.idle":"2022-08-09T07:30:31.668171Z","shell.execute_reply.started":"2022-08-09T07:30:31.611720Z","shell.execute_reply":"2022-08-09T07:30:31.667315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['dataset']= 'train'\ntest['dataset'] = 'test'","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.669917Z","iopub.execute_input":"2022-08-09T07:30:31.670429Z","iopub.status.idle":"2022-08-09T07:30:31.676703Z","shell.execute_reply.started":"2022-08-09T07:30:31.670396Z","shell.execute_reply":"2022-08-09T07:30:31.675709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([train, test], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.678572Z","iopub.execute_input":"2022-08-09T07:30:31.679263Z","iopub.status.idle":"2022-08-09T07:30:31.692655Z","shell.execute_reply.started":"2022-08-09T07:30:31.679226Z","shell.execute_reply":"2022-08-09T07:30:31.691602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.694985Z","iopub.execute_input":"2022-08-09T07:30:31.695521Z","iopub.status.idle":"2022-08-09T07:30:31.701719Z","shell.execute_reply.started":"2022-08-09T07:30:31.695487Z","shell.execute_reply":"2022-08-09T07:30:31.700816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['dataset'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.703153Z","iopub.execute_input":"2022-08-09T07:30:31.703723Z","iopub.status.idle":"2022-08-09T07:30:31.715881Z","shell.execute_reply.started":"2022-08-09T07:30:31.703690Z","shell.execute_reply":"2022-08-09T07:30:31.714758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We will use only the 'Train' dataset for explorations and processing to ensure that we dont do any data leakage","metadata":{}},{"cell_type":"code","source":"def data_desc(df):\n    print ()\n    print(\"Overall Data Description\")\n    print (\"Total number of records\", df.shape[0])\n    print (\"Total number of columns/features\", df.shape[1])\n    print (\"\")\n    cols=df.columns\n    data_type =[]\n    for col in df.columns:\n        data_type.append(df[col].dtype)\n    n_uni = df.nunique()\n    n_miss = df.isna().sum()\n    names = list(zip(cols, data_type,n_uni,n_miss))\n    variable_desc = pd.DataFrame(names,columns=[\"Name\",\"Type\",\"Unique levels\",\"Missing\"])\n    print (variable_desc)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.718404Z","iopub.execute_input":"2022-08-09T07:30:31.719398Z","iopub.status.idle":"2022-08-09T07:30:31.727697Z","shell.execute_reply.started":"2022-08-09T07:30:31.719351Z","shell.execute_reply":"2022-08-09T07:30:31.726829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_desc(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.735433Z","iopub.execute_input":"2022-08-09T07:30:31.735887Z","iopub.status.idle":"2022-08-09T07:30:31.765623Z","shell.execute_reply.started":"2022-08-09T07:30:31.735850Z","shell.execute_reply":"2022-08-09T07:30:31.764363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering ","metadata":{}},{"cell_type":"markdown","source":"* ## We have an object variable [Cabin] with 6560 levels. Need to check it out and see if any processing required\n* ## Passenger Id should be removed from any analysis\n* ## Passenger Name .. may not have any significance. But still we can see if there are any weird reasons if there is any link between them\n* ## Need to create bins for spend variables","metadata":{}},{"cell_type":"code","source":"train['Cabin']","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.767457Z","iopub.execute_input":"2022-08-09T07:30:31.767897Z","iopub.status.idle":"2022-08-09T07:30:31.776446Z","shell.execute_reply.started":"2022-08-09T07:30:31.767865Z","shell.execute_reply":"2022-08-09T07:30:31.775194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cabin = train['Cabin'].str.split(\"/\",expand=True)\ntrain = pd.concat([train,cabin], axis=1)\ntrain.rename(columns={0:'Deck', 1:'Num', 2:'Port'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.783839Z","iopub.execute_input":"2022-08-09T07:30:31.784399Z","iopub.status.idle":"2022-08-09T07:30:31.812170Z","shell.execute_reply.started":"2022-08-09T07:30:31.784365Z","shell.execute_reply":"2022-08-09T07:30:31.810852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cabin = test['Cabin'].str.split(\"/\",expand=True)\ntest = pd.concat([test,cabin], axis=1)\ntest.rename(columns={0:'Deck', 1:'Num', 2:'Port'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.814508Z","iopub.execute_input":"2022-08-09T07:30:31.816147Z","iopub.status.idle":"2022-08-09T07:30:31.836314Z","shell.execute_reply.started":"2022-08-09T07:30:31.816092Z","shell.execute_reply":"2022-08-09T07:30:31.835061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.837960Z","iopub.execute_input":"2022-08-09T07:30:31.838858Z","iopub.status.idle":"2022-08-09T07:30:31.845390Z","shell.execute_reply.started":"2022-08-09T07:30:31.838820Z","shell.execute_reply":"2022-08-09T07:30:31.844330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.853057Z","iopub.execute_input":"2022-08-09T07:30:31.853824Z","iopub.status.idle":"2022-08-09T07:30:31.862247Z","shell.execute_reply.started":"2022-08-09T07:30:31.853758Z","shell.execute_reply":"2022-08-09T07:30:31.860716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_value = train.isnull().sum().sort_values(ascending = False)\nmissing_perc = (train.isnull().sum()*100/train.shape[0]).sort_values(ascending = False)\nvalue = pd.concat([missing_value,missing_perc],axis=1,keys=['Count','%'])\ndisplay(value.head(20).style.background_gradient(cmap = 'Reds', axis = 0))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.878172Z","iopub.execute_input":"2022-08-09T07:30:31.879140Z","iopub.status.idle":"2022-08-09T07:30:31.915476Z","shell.execute_reply.started":"2022-08-09T07:30:31.879096Z","shell.execute_reply":"2022-08-09T07:30:31.914254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nsns.displot(\n    data=train.isna().melt(value_name=\"missing\"),\n    y=\"variable\",\n    hue=\"missing\",\n    multiple=\"fill\",\n    aspect=2\n)\nplt.title(\"Missing Value Proportion Each Feature\");","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:31.917400Z","iopub.execute_input":"2022-08-09T07:30:31.918305Z","iopub.status.idle":"2022-08-09T07:30:32.800246Z","shell.execute_reply.started":"2022-08-09T07:30:31.918265Z","shell.execute_reply":"2022-08-09T07:30:32.798883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final = train.copy()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.802968Z","iopub.execute_input":"2022-08-09T07:30:32.803792Z","iopub.status.idle":"2022-08-09T07:30:32.813314Z","shell.execute_reply.started":"2022-08-09T07:30:32.803745Z","shell.execute_reply":"2022-08-09T07:30:32.812092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_desc(train_final)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.814665Z","iopub.execute_input":"2022-08-09T07:30:32.815142Z","iopub.status.idle":"2022-08-09T07:30:32.848356Z","shell.execute_reply.started":"2022-08-09T07:30:32.815097Z","shell.execute_reply":"2022-08-09T07:30:32.847103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Num'] = pd.to_numeric(train_final['Num'], errors='coerce')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.851791Z","iopub.execute_input":"2022-08-09T07:30:32.852324Z","iopub.status.idle":"2022-08-09T07:30:32.865479Z","shell.execute_reply.started":"2022-08-09T07:30:32.852274Z","shell.execute_reply":"2022-08-09T07:30:32.864193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['Num'] = pd.to_numeric(test['Num'], errors='coerce')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.866742Z","iopub.execute_input":"2022-08-09T07:30:32.867604Z","iopub.status.idle":"2022-08-09T07:30:32.881366Z","shell.execute_reply.started":"2022-08-09T07:30:32.867566Z","shell.execute_reply":"2022-08-09T07:30:32.880290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['VRDeck'].value_counts(dropna=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.883015Z","iopub.execute_input":"2022-08-09T07:30:32.884140Z","iopub.status.idle":"2022-08-09T07:30:32.894998Z","shell.execute_reply.started":"2022-08-09T07:30:32.884100Z","shell.execute_reply":"2022-08-09T07:30:32.893972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nnum_imputer = SimpleImputer(missing_values=np.nan,strategy = 'median')\ncat_imputer = SimpleImputer(missing_values=np.nan, strategy = 'most_frequent')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.897259Z","iopub.execute_input":"2022-08-09T07:30:32.897731Z","iopub.status.idle":"2022-08-09T07:30:32.903981Z","shell.execute_reply.started":"2022-08-09T07:30:32.897685Z","shell.execute_reply":"2022-08-09T07:30:32.902718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spend variables \n","metadata":{"execution":{"iopub.execute_input":"2022-07-23T09:02:41.585127Z","iopub.status.busy":"2022-07-23T09:02:41.584745Z","iopub.status.idle":"2022-07-23T09:02:41.595464Z","shell.execute_reply":"2022-07-23T09:02:41.594452Z","shell.execute_reply.started":"2022-07-23T09:02:41.585084Z"}}},{"cell_type":"code","source":"train_final['inhouse_spend'] = train_final['RoomService'] + train_final['Spa'] +  train_final['VRDeck']\ntrain_final['out_spend'] = train_final['FoodCourt'] + train_final['ShoppingMall'] \n\ntest['inhouse_spend'] = test['RoomService'] + test['Spa'] +  test['VRDeck']\ntest['out_spend'] = test['FoodCourt'] + test['ShoppingMall'] \n\n\ntrain_final['spend_ratio'] = train_final['out_spend']/train_final['inhouse_spend']\ntest['spend_ratio'] = test['out_spend']/test['inhouse_spend']\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.905839Z","iopub.execute_input":"2022-08-09T07:30:32.906588Z","iopub.status.idle":"2022-08-09T07:30:32.924015Z","shell.execute_reply.started":"2022-08-09T07:30:32.906538Z","shell.execute_reply":"2022-08-09T07:30:32.922503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = train_final.loc[train_final['spend_ratio'] != np.inf, 'spend_ratio'].max()\ntrain_final['spend_ratio'].replace(np.inf,m,inplace=True)\n\nn = test.loc[test['spend_ratio'] != np.inf, 'spend_ratio'].max()\ntest['spend_ratio'].replace(np.inf,n,inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.925994Z","iopub.execute_input":"2022-08-09T07:30:32.926780Z","iopub.status.idle":"2022-08-09T07:30:32.938145Z","shell.execute_reply.started":"2022-08-09T07:30:32.926730Z","shell.execute_reply":"2022-08-09T07:30:32.936882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['spend_ratio'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.943963Z","iopub.execute_input":"2022-08-09T07:30:32.946077Z","iopub.status.idle":"2022-08-09T07:30:32.959418Z","shell.execute_reply.started":"2022-08-09T07:30:32.946031Z","shell.execute_reply":"2022-08-09T07:30:32.957848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Food_Court_sp']= train_final['FoodCourt'].apply(lambda x:1 if x < 300  else (2 if x > 300 else None ))\ntest['Food_Court_sp']= test['FoodCourt'].apply(lambda x:1 if x < 300  else (2 if x > 300 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.961132Z","iopub.execute_input":"2022-08-09T07:30:32.961894Z","iopub.status.idle":"2022-08-09T07:30:32.976083Z","shell.execute_reply.started":"2022-08-09T07:30:32.961849Z","shell.execute_reply":"2022-08-09T07:30:32.975230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Food_Court_sp'] = train_final['Food_Court_sp'].astype('category')\ntest['Food_Court_sp'] = test['Food_Court_sp'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.978035Z","iopub.execute_input":"2022-08-09T07:30:32.978771Z","iopub.status.idle":"2022-08-09T07:30:32.988100Z","shell.execute_reply.started":"2022-08-09T07:30:32.978726Z","shell.execute_reply":"2022-08-09T07:30:32.986871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['in_spenders']= train_final['inhouse_spend'].apply(lambda x:1 if x > 0 else (0 if x == 0 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:32.990403Z","iopub.execute_input":"2022-08-09T07:30:32.990766Z","iopub.status.idle":"2022-08-09T07:30:33.001915Z","shell.execute_reply.started":"2022-08-09T07:30:32.990735Z","shell.execute_reply":"2022-08-09T07:30:33.000971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['in_spenders']= test['inhouse_spend'].apply(lambda x:1 if x > 0 else (0 if x == 0 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.003382Z","iopub.execute_input":"2022-08-09T07:30:33.004109Z","iopub.status.idle":"2022-08-09T07:30:33.013669Z","shell.execute_reply.started":"2022-08-09T07:30:33.004064Z","shell.execute_reply":"2022-08-09T07:30:33.012834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Age_brac']= train_final['Age'].apply(lambda x:1 if x < 1 else (2 if x > 1 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.015088Z","iopub.execute_input":"2022-08-09T07:30:33.016153Z","iopub.status.idle":"2022-08-09T07:30:33.029511Z","shell.execute_reply.started":"2022-08-09T07:30:33.016108Z","shell.execute_reply":"2022-08-09T07:30:33.028362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['Age_brac']= test['Age'].apply(lambda x:1 if x < 5 else (2 if x > 5 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.031037Z","iopub.execute_input":"2022-08-09T07:30:33.031513Z","iopub.status.idle":"2022-08-09T07:30:33.044500Z","shell.execute_reply.started":"2022-08-09T07:30:33.031467Z","shell.execute_reply":"2022-08-09T07:30:33.043408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Age_brac'].value_counts(dropna=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.046136Z","iopub.execute_input":"2022-08-09T07:30:33.047223Z","iopub.status.idle":"2022-08-09T07:30:33.057703Z","shell.execute_reply.started":"2022-08-09T07:30:33.047162Z","shell.execute_reply":"2022-08-09T07:30:33.056316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['spend_brac'] = train_final['inhouse_spend'].apply(lambda x:1 if x < 185 else (2 if x > 185 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.059747Z","iopub.execute_input":"2022-08-09T07:30:33.060844Z","iopub.status.idle":"2022-08-09T07:30:33.072202Z","shell.execute_reply.started":"2022-08-09T07:30:33.060773Z","shell.execute_reply":"2022-08-09T07:30:33.070973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['spend_brac'] = test['inhouse_spend'].apply(lambda x:1 if x < 185 else (2 if x > 185 else None ))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.075764Z","iopub.execute_input":"2022-08-09T07:30:33.076951Z","iopub.status.idle":"2022-08-09T07:30:33.086905Z","shell.execute_reply.started":"2022-08-09T07:30:33.076911Z","shell.execute_reply":"2022-08-09T07:30:33.085838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create bins for spend variables based on the analysis by the dependent variable","metadata":{}},{"cell_type":"code","source":"agg_func_describe = {'inhouse_spend': ['describe']}\ntrain_final.groupby(['Transported']).agg(agg_func_describe).round(2)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.088364Z","iopub.execute_input":"2022-08-09T07:30:33.089299Z","iopub.status.idle":"2022-08-09T07:30:33.125155Z","shell.execute_reply.started":"2022-08-09T07:30:33.089264Z","shell.execute_reply":"2022-08-09T07:30:33.124337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def q95(x):\n    return x.quantile(0.95)\n\n# 90th Percentile\ndef q90(x):\n    return x.quantile(0.9)\n\ntrain_final.groupby(['Transported']).agg({'RoomService': ['describe', q90, q95, 'max']})\nprint(\"\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.126238Z","iopub.execute_input":"2022-08-09T07:30:33.126972Z","iopub.status.idle":"2022-08-09T07:30:33.150759Z","shell.execute_reply.started":"2022-08-09T07:30:33.126937Z","shell.execute_reply":"2022-08-09T07:30:33.149439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final.groupby(['Transported']).agg({'Spa': ['describe', q90, q95, 'max']})\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.152377Z","iopub.execute_input":"2022-08-09T07:30:33.152744Z","iopub.status.idle":"2022-08-09T07:30:33.192557Z","shell.execute_reply.started":"2022-08-09T07:30:33.152712Z","shell.execute_reply":"2022-08-09T07:30:33.191240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final.groupby(['Transported']).agg({'VRDeck': ['describe', q90, q95, 'max']})\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.193954Z","iopub.execute_input":"2022-08-09T07:30:33.194308Z","iopub.status.idle":"2022-08-09T07:30:33.234261Z","shell.execute_reply.started":"2022-08-09T07:30:33.194276Z","shell.execute_reply":"2022-08-09T07:30:33.233367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final.groupby(['Transported']).agg({'ShoppingMall': ['describe', q90, q95, 'max']})\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.235889Z","iopub.execute_input":"2022-08-09T07:30:33.236597Z","iopub.status.idle":"2022-08-09T07:30:33.275770Z","shell.execute_reply.started":"2022-08-09T07:30:33.236552Z","shell.execute_reply":"2022-08-09T07:30:33.274555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final.groupby(['Transported']).agg({'FoodCourt': ['describe', q90, q95, 'max']})\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.277232Z","iopub.execute_input":"2022-08-09T07:30:33.277780Z","iopub.status.idle":"2022-08-09T07:30:33.317255Z","shell.execute_reply.started":"2022-08-09T07:30:33.277744Z","shell.execute_reply":"2022-08-09T07:30:33.316199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final.groupby(['Transported']).agg({'inhouse_spend': ['describe', q90, q95, 'max']})\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.318661Z","iopub.execute_input":"2022-08-09T07:30:33.319761Z","iopub.status.idle":"2022-08-09T07:30:33.360947Z","shell.execute_reply.started":"2022-08-09T07:30:33.319723Z","shell.execute_reply":"2022-08-09T07:30:33.359902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final.groupby(['Transported']).agg({'out_spend': ['describe', q90, q95, 'max']})","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.362843Z","iopub.execute_input":"2022-08-09T07:30:33.363299Z","iopub.status.idle":"2022-08-09T07:30:33.402632Z","shell.execute_reply.started":"2022-08-09T07:30:33.363256Z","shell.execute_reply":"2022-08-09T07:30:33.401841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['RoomService_rev'] = pd.cut(train_final['RoomService'] , [0, 470, 4000, 15000])\ntrain_final['Spa_rev'] = pd.cut(train_final['Spa'] , [0, 450, 4000, 25000])\ntrain_final['VRDeck_rev'] = pd.cut(train_final['VRDeck'] , [0, 400, 5000, 25000])\ntrain_final['FoodCourt_rev'] = pd.cut(train_final['FoodCourt'] , [0, 180, 25000, 30000])\ntrain_final['Shop_rev'] = pd.cut(train_final['ShoppingMall'] , [0, 100, 620, 25000])\ntrain_final['inspend_rev'] = pd.cut(train_final['inhouse_spend'] , [0, 750, 6000, 29000])\ntrain_final['outspend_rev'] = pd.cut(train_final['out_spend'] , [0, 60, 1300, 30000])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.409065Z","iopub.execute_input":"2022-08-09T07:30:33.409680Z","iopub.status.idle":"2022-08-09T07:30:33.437252Z","shell.execute_reply.started":"2022-08-09T07:30:33.409638Z","shell.execute_reply":"2022-08-09T07:30:33.436287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['RoomService_rev'] = pd.cut(test['RoomService'] , [0, 470, 4000, 15000])\ntest['Spa_rev'] = pd.cut(test['Spa'] , [0, 450, 4000, 25000])\ntest['VRDeck_rev'] = pd.cut(test['VRDeck'] , [0, 400, 5000, 25000])\ntest['FoodCourt_rev'] = pd.cut(test['FoodCourt'] , [0, 180, 25000, 30000])\ntest['Shop_rev'] = pd.cut(test['ShoppingMall'] , [0, 100, 620, 25000])\ntest['inspend_rev'] = pd.cut(test['inhouse_spend'] , [0, 750, 6000, 29000])\ntest['outspend_rev'] = pd.cut(test['out_spend'] , [0, 60, 1300, 30000])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.438447Z","iopub.execute_input":"2022-08-09T07:30:33.439118Z","iopub.status.idle":"2022-08-09T07:30:33.465733Z","shell.execute_reply.started":"2022-08-09T07:30:33.439081Z","shell.execute_reply":"2022-08-09T07:30:33.464501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.crosstab(train_final['VRDeck_rev'],train_final['Transported'],normalize='columns')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.469086Z","iopub.execute_input":"2022-08-09T07:30:33.469525Z","iopub.status.idle":"2022-08-09T07:30:33.495541Z","shell.execute_reply.started":"2022-08-09T07:30:33.469489Z","shell.execute_reply":"2022-08-09T07:30:33.494322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.crosstab(train_final['RoomService_rev'],train_final['Transported'],normalize='columns')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.496837Z","iopub.execute_input":"2022-08-09T07:30:33.497782Z","iopub.status.idle":"2022-08-09T07:30:33.527350Z","shell.execute_reply.started":"2022-08-09T07:30:33.497744Z","shell.execute_reply":"2022-08-09T07:30:33.526274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.crosstab(train_final['spend_brac'],train_final['Transported'])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.528744Z","iopub.execute_input":"2022-08-09T07:30:33.529103Z","iopub.status.idle":"2022-08-09T07:30:33.553196Z","shell.execute_reply.started":"2022-08-09T07:30:33.529071Z","shell.execute_reply":"2022-08-09T07:30:33.552012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = train_final['Name'].str.split(\" \",expand=True)\ntrain_final = pd.concat([train_final,name], axis=1)\ntrain_final.rename(columns={0:'Firstname', 1:'Lastname'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.554886Z","iopub.execute_input":"2022-08-09T07:30:33.555334Z","iopub.status.idle":"2022-08-09T07:30:33.584754Z","shell.execute_reply.started":"2022-08-09T07:30:33.555287Z","shell.execute_reply":"2022-08-09T07:30:33.583899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Firstname'] = train_final['Firstname'].astype('|S')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.586197Z","iopub.execute_input":"2022-08-09T07:30:33.586748Z","iopub.status.idle":"2022-08-09T07:30:33.594094Z","shell.execute_reply.started":"2022-08-09T07:30:33.586715Z","shell.execute_reply":"2022-08-09T07:30:33.592952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Firstname'] = train_final['Firstname'].apply(lambda x:\"None\" if  x == None else x)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.595494Z","iopub.execute_input":"2022-08-09T07:30:33.595850Z","iopub.status.idle":"2022-08-09T07:30:33.607522Z","shell.execute_reply.started":"2022-08-09T07:30:33.595792Z","shell.execute_reply":"2022-08-09T07:30:33.606667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Lastname']  = train_final['Lastname'].str.replace('[^a-zA-Z]', '')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.608574Z","iopub.execute_input":"2022-08-09T07:30:33.609270Z","iopub.status.idle":"2022-08-09T07:30:33.626491Z","shell.execute_reply.started":"2022-08-09T07:30:33.609238Z","shell.execute_reply":"2022-08-09T07:30:33.625301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Lastname'] = train_final['Lastname'].astype('|S')\ntrain_final['Lastname'] = train_final['Lastname'].apply(lambda x:\"None\" if  x == None else x)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.628266Z","iopub.execute_input":"2022-08-09T07:30:33.628617Z","iopub.status.idle":"2022-08-09T07:30:33.639541Z","shell.execute_reply.started":"2022-08-09T07:30:33.628584Z","shell.execute_reply":"2022-08-09T07:30:33.638222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Lastname'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.640720Z","iopub.execute_input":"2022-08-09T07:30:33.641078Z","iopub.status.idle":"2022-08-09T07:30:33.650512Z","shell.execute_reply.started":"2022-08-09T07:30:33.641046Z","shell.execute_reply":"2022-08-09T07:30:33.649433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = test['Name'].str.split(\" \",expand=True)\ntest = pd.concat([test,name], axis=1)\ntest.rename(columns={0:'Firstname', 1:'Lastname'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.652485Z","iopub.execute_input":"2022-08-09T07:30:33.653129Z","iopub.status.idle":"2022-08-09T07:30:33.673184Z","shell.execute_reply.started":"2022-08-09T07:30:33.653093Z","shell.execute_reply":"2022-08-09T07:30:33.671795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['Firstname'] = test['Firstname'].astype('|S')\ntest['Firstname'] = test['Firstname'].apply(lambda x:\"None\" if  x == None else x)\ntest['Lastname']  = test['Lastname'].str.replace('[^a-zA-Z]', '')\ntest['Lastname'] = test['Lastname'].astype('|S')\ntest['Lastname'] = test['Lastname'].apply(lambda x:\"None\" if  x == None else x)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.675047Z","iopub.execute_input":"2022-08-09T07:30:33.675926Z","iopub.status.idle":"2022-08-09T07:30:33.697619Z","shell.execute_reply.started":"2022-08-09T07:30:33.675880Z","shell.execute_reply":"2022-08-09T07:30:33.696711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['in_spenders'] = train_final['in_spenders'].astype('category')\ntest['in_spenders'] = test['in_spenders'].astype('category')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.699021Z","iopub.execute_input":"2022-08-09T07:30:33.700027Z","iopub.status.idle":"2022-08-09T07:30:33.708166Z","shell.execute_reply.started":"2022-08-09T07:30:33.699991Z","shell.execute_reply":"2022-08-09T07:30:33.707014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Age_brac'] = train_final['Age_brac'].astype('category')\ntest['Age_brac'] = test['Age_brac'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.709924Z","iopub.execute_input":"2022-08-09T07:30:33.710543Z","iopub.status.idle":"2022-08-09T07:30:33.719763Z","shell.execute_reply.started":"2022-08-09T07:30:33.710511Z","shell.execute_reply":"2022-08-09T07:30:33.718680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['spend_brac'] = train_final['spend_brac'].astype('category')\ntest['spend_brac'] = test['spend_brac'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.721531Z","iopub.execute_input":"2022-08-09T07:30:33.723104Z","iopub.status.idle":"2022-08-09T07:30:33.731960Z","shell.execute_reply.started":"2022-08-09T07:30:33.723055Z","shell.execute_reply":"2022-08-09T07:30:33.730860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cont_var = []\ncat_var=[]\nfor col in train_final.columns:\n    if (train_final[col].dtype == \"object\") | (train_final[col].dtype == \"category\"):\n        cat_var.append(col)\n    elif ((train_final[col].dtype == \"float64\") | (train_final[col].dtype == \"int\")):\n        cont_var.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.736163Z","iopub.execute_input":"2022-08-09T07:30:33.736857Z","iopub.status.idle":"2022-08-09T07:30:33.747136Z","shell.execute_reply.started":"2022-08-09T07:30:33.736792Z","shell.execute_reply":"2022-08-09T07:30:33.745772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final[cont_var] = num_imputer.fit_transform(train_final[cont_var])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.748891Z","iopub.execute_input":"2022-08-09T07:30:33.749589Z","iopub.status.idle":"2022-08-09T07:30:33.778413Z","shell.execute_reply.started":"2022-08-09T07:30:33.749546Z","shell.execute_reply":"2022-08-09T07:30:33.777317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final[cat_var] = cat_imputer.fit_transform(train_final[cat_var])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:33.779976Z","iopub.execute_input":"2022-08-09T07:30:33.780743Z","iopub.status.idle":"2022-08-09T07:30:34.254327Z","shell.execute_reply.started":"2022-08-09T07:30:33.780696Z","shell.execute_reply":"2022-08-09T07:30:34.253205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[cont_var] = num_imputer.fit_transform(test[cont_var])\ntest[cat_var] = cat_imputer.fit_transform(test[cat_var])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:34.255680Z","iopub.execute_input":"2022-08-09T07:30:34.256057Z","iopub.status.idle":"2022-08-09T07:30:34.605497Z","shell.execute_reply.started":"2022-08-09T07:30:34.256024Z","shell.execute_reply":"2022-08-09T07:30:34.604333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Univariate analysis - Distributions","metadata":{}},{"cell_type":"code","source":"train_final.hist(bins=12,figsize=(20,15))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:34.607063Z","iopub.execute_input":"2022-08-09T07:30:34.607402Z","iopub.status.idle":"2022-08-09T07:30:36.346335Z","shell.execute_reply.started":"2022-08-09T07:30:34.607372Z","shell.execute_reply":"2022-08-09T07:30:36.345180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nto_remove = ['PassengerId','Name','Cabin','dataset']\n\n# create new list using list comprehension\ncat_var = [i for i in cat_var if i not in to_remove]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:36.347991Z","iopub.execute_input":"2022-08-09T07:30:36.348628Z","iopub.status.idle":"2022-08-09T07:30:36.354847Z","shell.execute_reply.started":"2022-08-09T07:30:36.348582Z","shell.execute_reply":"2022-08-09T07:30:36.353600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Transported'].value_counts(dropna=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:36.356709Z","iopub.execute_input":"2022-08-09T07:30:36.357156Z","iopub.status.idle":"2022-08-09T07:30:36.369539Z","shell.execute_reply.started":"2022-08-09T07:30:36.357115Z","shell.execute_reply":"2022-08-09T07:30:36.368257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final['Transported']=train_final['Transported'].astype('int')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:36.371785Z","iopub.execute_input":"2022-08-09T07:30:36.372201Z","iopub.status.idle":"2022-08-09T07:30:36.380228Z","shell.execute_reply.started":"2022-08-09T07:30:36.372148Z","shell.execute_reply":"2022-08-09T07:30:36.379301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Relationship of continuous variables with DV","metadata":{}},{"cell_type":"code","source":"sns.set_theme(style=\"white\", rc={\"axes.facecolor\": (0, 0, 0, 0)})\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:36.381174Z","iopub.execute_input":"2022-08-09T07:30:36.381479Z","iopub.status.idle":"2022-08-09T07:30:36.391126Z","shell.execute_reply.started":"2022-08-09T07:30:36.381451Z","shell.execute_reply":"2022-08-09T07:30:36.390098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for var in cont_var:\n    sns.barplot(x='Transported',y=var,data=train_final)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:36.393948Z","iopub.execute_input":"2022-08-09T07:30:36.394859Z","iopub.status.idle":"2022-08-09T07:30:39.794724Z","shell.execute_reply.started":"2022-08-09T07:30:36.394795Z","shell.execute_reply":"2022-08-09T07:30:39.792599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Relationship of categorical values with DV","metadata":{}},{"cell_type":"markdown","source":"for var in cat_var:\n    sns.barplot(x='Transported',y=var,data=train_final)\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2022-07-23T09:02:44.778621Z","iopub.status.busy":"2022-07-23T09:02:44.777905Z","iopub.status.idle":"2022-07-23T09:02:48.845457Z","shell.execute_reply":"2022-07-23T09:02:48.844276Z","shell.execute_reply.started":"2022-07-23T09:02:44.778573Z"}}},{"cell_type":"code","source":"train_final.corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.796153Z","iopub.execute_input":"2022-08-09T07:30:39.796463Z","iopub.status.idle":"2022-08-09T07:30:39.819194Z","shell.execute_reply.started":"2022-08-09T07:30:39.796435Z","shell.execute_reply":"2022-08-09T07:30:39.818197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set up column transformers & pipeline","metadata":{}},{"cell_type":"code","source":"X = [y for x in [cont_var, cat_var] for y in x]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.820637Z","iopub.execute_input":"2022-08-09T07:30:39.820975Z","iopub.status.idle":"2022-08-09T07:30:39.826309Z","shell.execute_reply.started":"2022-08-09T07:30:39.820944Z","shell.execute_reply":"2022-08-09T07:30:39.825269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.827686Z","iopub.execute_input":"2022-08-09T07:30:39.828296Z","iopub.status.idle":"2022-08-09T07:30:39.839321Z","shell.execute_reply.started":"2022-08-09T07:30:39.828262Z","shell.execute_reply":"2022-08-09T07:30:39.838566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IV  = train_final[X]\ny = train_final['Transported']","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.840604Z","iopub.execute_input":"2022-08-09T07:30:39.841119Z","iopub.status.idle":"2022-08-09T07:30:39.853847Z","shell.execute_reply.started":"2022-08-09T07:30:39.841087Z","shell.execute_reply":"2022-08-09T07:30:39.852609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idCol=test.PassengerId.to_numpy()\ntest.set_index('PassengerId', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.855438Z","iopub.execute_input":"2022-08-09T07:30:39.856234Z","iopub.status.idle":"2022-08-09T07:30:39.866467Z","shell.execute_reply.started":"2022-08-09T07:30:39.856195Z","shell.execute_reply":"2022-08-09T07:30:39.865307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nohe = OneHotEncoder(handle_unknown='ignore',categories='auto')\nlabel = LabelEncoder()\noe = OrdinalEncoder()\nscaler = StandardScaler()\nlogreg = LogisticRegression(solver='liblinear',multi_class='auto', random_state=1,max_iter=1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.868065Z","iopub.execute_input":"2022-08-09T07:30:39.869090Z","iopub.status.idle":"2022-08-09T07:30:39.875602Z","shell.execute_reply.started":"2022-08-09T07:30:39.869056Z","shell.execute_reply":"2022-08-09T07:30:39.874863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\n\ncv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.877049Z","iopub.execute_input":"2022-08-09T07:30:39.877849Z","iopub.status.idle":"2022-08-09T07:30:39.887854Z","shell.execute_reply.started":"2022-08-09T07:30:39.877782Z","shell.execute_reply":"2022-08-09T07:30:39.886869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ct = make_column_transformer(\n   #(num_imputer,['Age','RoomService','FoodCourt','ShoppingMall','Spa','VRDeck']),\n   #(CountVectorizer(), 'Firstname','Lastname'),\n   #(scaler, [ 'spend_ratio' ,'CryoSleep']),\n\n   (scaler, [ 'spend_ratio' ,'CryoSleep','Age','FoodCourt','ShoppingMall','inhouse_spend','Num','RoomService','Spa','VRDeck',\"inhouse_spend\",\"out_spend\"]),\n  #  (scaler, ['CryoSleep','inhouse_spend']),\n   #(cat_imputer,['Deck','Port','HomePlanet','Destination']),\n   (ohe, ['VIP','in_spenders','Deck','Port','HomePlanet','Destination','Age_brac','FoodCourt_rev','RoomService_rev','Spa_rev','VRDeck_rev',\"Shop_rev\",\"inspend_rev\",\"outspend_rev\"]),\n  \n    remainder='drop')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.889317Z","iopub.execute_input":"2022-08-09T07:30:39.889948Z","iopub.status.idle":"2022-08-09T07:30:39.899213Z","shell.execute_reply.started":"2022-08-09T07:30:39.889913Z","shell.execute_reply":"2022-08-09T07:30:39.898374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.900665Z","iopub.execute_input":"2022-08-09T07:30:39.901257Z","iopub.status.idle":"2022-08-09T07:30:39.913472Z","shell.execute_reply.started":"2022-08-09T07:30:39.901223Z","shell.execute_reply":"2022-08-09T07:30:39.912651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe_LR = make_pipeline(ct, logreg)\npipe_LR.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:39.914733Z","iopub.execute_input":"2022-08-09T07:30:39.915098Z","iopub.status.idle":"2022-08-09T07:30:40.598214Z","shell.execute_reply.started":"2022-08-09T07:30:39.915067Z","shell.execute_reply":"2022-08-09T07:30:40.596848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ncv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n\ncross_val_score(pipe_LR, IV, y, cv=cv, scoring='accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:40.600359Z","iopub.execute_input":"2022-08-09T07:30:40.600868Z","iopub.status.idle":"2022-08-09T07:30:59.777368Z","shell.execute_reply.started":"2022-08-09T07:30:40.600822Z","shell.execute_reply":"2022-08-09T07:30:59.775731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {}\nparams['logisticregression__penalty'] = ['l1', 'l2']\nparams['logisticregression__C'] = [0.1, 1, 10]\nparams","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:59.796456Z","iopub.execute_input":"2022-08-09T07:30:59.797352Z","iopub.status.idle":"2022-08-09T07:30:59.806695Z","shell.execute_reply.started":"2022-08-09T07:30:59.797301Z","shell.execute_reply":"2022-08-09T07:30:59.805398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\ngrid = GridSearchCV(pipe_LR, params, cv=cv, scoring='accuracy')\ngrid.fit(IV, y);","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:30:59.808608Z","iopub.execute_input":"2022-08-09T07:30:59.809411Z","iopub.status.idle":"2022-08-09T07:34:43.504305Z","shell.execute_reply.started":"2022-08-09T07:30:59.809364Z","shell.execute_reply":"2022-08-09T07:34:43.503079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.DataFrame(grid.cv_results_)\nresults.sort_values('rank_test_score')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:34:43.506221Z","iopub.execute_input":"2022-08-09T07:34:43.507025Z","iopub.status.idle":"2022-08-09T07:34:43.545943Z","shell.execute_reply.started":"2022-08-09T07:34:43.506978Z","shell.execute_reply":"2022-08-09T07:34:43.544699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid.best_score_\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:34:43.547332Z","iopub.execute_input":"2022-08-09T07:34:43.547671Z","iopub.status.idle":"2022-08-09T07:34:43.555252Z","shell.execute_reply.started":"2022-08-09T07:34:43.547640Z","shell.execute_reply":"2022-08-09T07:34:43.554047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:34:43.557226Z","iopub.execute_input":"2022-08-09T07:34:43.558269Z","iopub.status.idle":"2022-08-09T07:34:43.565166Z","shell.execute_reply.started":"2022-08-09T07:34:43.558223Z","shell.execute_reply":"2022-08-09T07:34:43.564299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nRF = RandomForestClassifier()\npipe_RF = make_pipeline(ct, RF)\npipe_RF.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:34:43.566421Z","iopub.execute_input":"2022-08-09T07:34:43.570516Z","iopub.status.idle":"2022-08-09T07:34:45.517026Z","shell.execute_reply.started":"2022-08-09T07:34:43.570476Z","shell.execute_reply":"2022-08-09T07:34:45.515836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_val_score(pipe_RF, IV, y, cv=cv, scoring='accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:34:45.518343Z","iopub.execute_input":"2022-08-09T07:34:45.518724Z","iopub.status.idle":"2022-08-09T07:35:36.393929Z","shell.execute_reply.started":"2022-08-09T07:34:45.518692Z","shell.execute_reply":"2022-08-09T07:35:36.392645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:35:36.395778Z","iopub.execute_input":"2022-08-09T07:35:36.396205Z","iopub.status.idle":"2022-08-09T07:35:36.401513Z","shell.execute_reply.started":"2022-08-09T07:35:36.396169Z","shell.execute_reply":"2022-08-09T07:35:36.400314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nXGB = XGBClassifier()\npipe_XGB = make_pipeline(ct, XGB)\npipe_XGB.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:35:36.402982Z","iopub.execute_input":"2022-08-09T07:35:36.403352Z","iopub.status.idle":"2022-08-09T07:35:38.524093Z","shell.execute_reply.started":"2022-08-09T07:35:36.403305Z","shell.execute_reply":"2022-08-09T07:35:38.522877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_val_score(pipe_XGB, IV, y, cv=cv, scoring='accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:35:38.525531Z","iopub.execute_input":"2022-08-09T07:35:38.525867Z","iopub.status.idle":"2022-08-09T07:36:39.034586Z","shell.execute_reply.started":"2022-08-09T07:35:38.525837Z","shell.execute_reply":"2022-08-09T07:36:39.033723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier\nmodel=CatBoostClassifier(iterations=300,\n                         eval_metric='Accuracy',\n                        verbose=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:36:39.035735Z","iopub.execute_input":"2022-08-09T07:36:39.036702Z","iopub.status.idle":"2022-08-09T07:36:39.041216Z","shell.execute_reply.started":"2022-08-09T07:36:39.036665Z","shell.execute_reply":"2022-08-09T07:36:39.040249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npipe_CAT = make_pipeline(ct, model)\npipe_CAT.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:36:39.042367Z","iopub.execute_input":"2022-08-09T07:36:39.043133Z","iopub.status.idle":"2022-08-09T07:36:41.538664Z","shell.execute_reply.started":"2022-08-09T07:36:39.043085Z","shell.execute_reply":"2022-08-09T07:36:41.537467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_val_score(pipe_CAT, IV, y, cv=cv, scoring='accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:36:41.540014Z","iopub.execute_input":"2022-08-09T07:36:41.540351Z","iopub.status.idle":"2022-08-09T07:37:54.964184Z","shell.execute_reply.started":"2022-08-09T07:36:41.540321Z","shell.execute_reply":"2022-08-09T07:37:54.962984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RF with Hyperparameter Optimization","metadata":{}},{"cell_type":"code","source":"!pip install parameter-sherpa","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:37:54.966579Z","iopub.execute_input":"2022-08-09T07:37:54.966933Z","iopub.status.idle":"2022-08-09T07:39:06.491756Z","shell.execute_reply.started":"2022-08-09T07:37:54.966903Z","shell.execute_reply":"2022-08-09T07:39:06.489994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sherpa\nimport sherpa.algorithms.bayesian_optimization as bayesian_optimization","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:39:06.494152Z","iopub.execute_input":"2022-08-09T07:39:06.494672Z","iopub.status.idle":"2022-08-09T07:39:07.220450Z","shell.execute_reply.started":"2022-08-09T07:39:06.494625Z","shell.execute_reply":"2022-08-09T07:39:07.219106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters = [sherpa.Discrete('n_estimators', [100,200,250, 300,400,500]),\n              sherpa.Choice('criterion', ['gini', 'entropy']),\n              sherpa.Discrete(\"max_depth\", [1, 2, 5, 7, 11, 15]), \n              sherpa.Continuous('max_features', [0.1, 0.9])]\n\nalgorithm = bayesian_optimization.GPyOpt(max_concurrent=1,model_type='GP_MCMC',acquisition_type='EI_MCMC',max_num_trials=10)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:39:07.222094Z","iopub.execute_input":"2022-08-09T07:39:07.222665Z","iopub.status.idle":"2022-08-09T07:39:07.230030Z","shell.execute_reply.started":"2022-08-09T07:39:07.222628Z","shell.execute_reply":"2022-08-09T07:39:07.228899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = sherpa.Study(parameters=parameters,\n                     algorithm=algorithm,\n                     lower_is_better=False)\n\nfor trial in study:\n    print(\"Trial \", trial.id, \" with parameters \", trial.parameters)\n    Rf = RandomForestClassifier(criterion=trial.parameters['criterion'],\n                                 max_features=trial.parameters['max_features'],\n                                 n_estimators=trial.parameters['n_estimators'],\n                                 random_state=0)\n    pipe_RFnew = make_pipeline(ct, Rf)\n    scores = cross_val_score(pipe_RFnew, IV, y, cv=5)\n    print(\"Score: \", scores.mean())\n    study.add_observation(trial, iteration=1, objective=scores.mean())\n    study.finalize(trial)\nprint(study.get_best_result())","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:39:07.231506Z","iopub.execute_input":"2022-08-09T07:39:07.232161Z","iopub.status.idle":"2022-08-09T07:45:18.233238Z","shell.execute_reply.started":"2022-08-09T07:39:07.232125Z","shell.execute_reply":"2022-08-09T07:45:18.231776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RF2 = RandomForestClassifier(criterion = 'entropy', max_depth = 1, max_features = 0.11995440348098675, n_estimators = 138)\npipe_RF = make_pipeline(ct, RF2)\npipe_RF.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:45:18.235249Z","iopub.execute_input":"2022-08-09T07:45:18.235897Z","iopub.status.idle":"2022-08-09T07:45:19.157129Z","shell.execute_reply.started":"2022-08-09T07:45:18.235846Z","shell.execute_reply":"2022-08-09T07:45:19.155945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_val_score(pipe_RF, IV, y, cv=cv, scoring='accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:45:19.158638Z","iopub.execute_input":"2022-08-09T07:45:19.159686Z","iopub.status.idle":"2022-08-09T07:45:44.898132Z","shell.execute_reply.started":"2022-08-09T07:45:19.159648Z","shell.execute_reply":"2022-08-09T07:45:44.896918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stacking Classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import StackingClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:45:44.899789Z","iopub.execute_input":"2022-08-09T07:45:44.900298Z","iopub.status.idle":"2022-08-09T07:45:44.906350Z","shell.execute_reply.started":"2022-08-09T07:45:44.900260Z","shell.execute_reply":"2022-08-09T07:45:44.904823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import GaussianNB\nfrom matplotlib import pyplot\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier, ExtraTreesClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:45:44.908467Z","iopub.execute_input":"2022-08-09T07:45:44.908930Z","iopub.status.idle":"2022-08-09T07:45:44.922944Z","shell.execute_reply.started":"2022-08-09T07:45:44.908892Z","shell.execute_reply":"2022-08-09T07:45:44.921879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_stacking():\n# define the base models\n    level0 = list()\n    level0.append(('lr', LogisticRegression(solver='liblinear',multi_class='auto', random_state=1,max_iter=1000)))\n    level0.append(('knn', KNeighborsClassifier()))\n    level0.append(('cart', DecisionTreeClassifier()))\n    level0.append(('XGB', XGBClassifier()))\n    level0.append(('bayes', GaussianNB()))\n    level0.append(('RF', RF2))\n    level0.append(('CAT',CatBoostClassifier(iterations=300,\n                         eval_metric='Accuracy',\n                        verbose=0)))\n\n    # define meta learner model\n    level1 = LogisticRegression()\n    # define the stacking ensemble\n    model = StackingClassifier(estimators=level0, final_estimator=level1, cv=5)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:46:57.583088Z","iopub.execute_input":"2022-08-09T07:46:57.583866Z","iopub.status.idle":"2022-08-09T07:46:57.593273Z","shell.execute_reply.started":"2022-08-09T07:46:57.583796Z","shell.execute_reply":"2022-08-09T07:46:57.592034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe_ensem = make_pipeline(ct, get_stacking())\npipe_ensem.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:46:59.184778Z","iopub.execute_input":"2022-08-09T07:46:59.185538Z","iopub.status.idle":"2022-08-09T07:47:25.668034Z","shell.execute_reply.started":"2022-08-09T07:46:59.185495Z","shell.execute_reply":"2022-08-09T07:47:25.666739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_model(model, X, y):\n\tcv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n\tscores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')\n\treturn scores","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:48:21.432061Z","iopub.execute_input":"2022-08-09T07:48:21.432581Z","iopub.status.idle":"2022-08-09T07:48:21.439610Z","shell.execute_reply.started":"2022-08-09T07:48:21.432542Z","shell.execute_reply":"2022-08-09T07:48:21.438411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(pipe_ensem,IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:48:24.175736Z","iopub.execute_input":"2022-08-09T07:48:24.176192Z","iopub.status.idle":"2022-08-09T07:58:55.935376Z","shell.execute_reply.started":"2022-08-09T07:48:24.176153Z","shell.execute_reply":"2022-08-09T07:58:55.933882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Voting Classifier ","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:58:55.938559Z","iopub.execute_input":"2022-08-09T07:58:55.939606Z","iopub.status.idle":"2022-08-09T07:58:55.944722Z","shell.execute_reply.started":"2022-08-09T07:58:55.939558Z","shell.execute_reply":"2022-08-09T07:58:55.943991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voting_clf = VotingClassifier(estimators=[('CAT',CatBoostClassifier(iterations=300,\n                         eval_metric='Accuracy',\n                        verbose=0)), ('RF',RF2),('XGB',XGB),('bayes', GaussianNB()),('KNN', KNeighborsClassifier()), ('DTree',DecisionTreeClassifier()), ('LogReg', LogisticRegression(solver='liblinear',multi_class='auto', random_state=1,max_iter=1000))], voting='hard')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:58:55.945754Z","iopub.execute_input":"2022-08-09T07:58:55.946628Z","iopub.status.idle":"2022-08-09T07:58:55.956181Z","shell.execute_reply.started":"2022-08-09T07:58:55.946596Z","shell.execute_reply":"2022-08-09T07:58:55.955406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe_vote = make_pipeline(ct, voting_clf)\npipe_vote.fit(IV,y)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:58:55.958299Z","iopub.execute_input":"2022-08-09T07:58:55.958772Z","iopub.status.idle":"2022-08-09T07:59:00.998520Z","shell.execute_reply.started":"2022-08-09T07:58:55.958741Z","shell.execute_reply":"2022-08-09T07:59:00.997402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_val_score(pipe_vote, IV, y, cv=5, scoring='accuracy').mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:59:00.999822Z","iopub.execute_input":"2022-08-09T07:59:01.000131Z","iopub.status.idle":"2022-08-09T07:59:24.658684Z","shell.execute_reply.started":"2022-08-09T07:59:01.000102Z","shell.execute_reply":"2022-08-09T07:59:24.656973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vote_pred = pipe_vote.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:59:24.660746Z","iopub.execute_input":"2022-08-09T07:59:24.661542Z","iopub.status.idle":"2022-08-09T07:59:25.985449Z","shell.execute_reply.started":"2022-08-09T07:59:24.661493Z","shell.execute_reply":"2022-08-09T07:59:25.983614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensem_Pred = pipe_CAT.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T07:59:25.987749Z","iopub.execute_input":"2022-08-09T07:59:25.989614Z","iopub.status.idle":"2022-08-09T07:59:26.363528Z","shell.execute_reply.started":"2022-08-09T07:59:25.989550Z","shell.execute_reply":"2022-08-09T07:59:26.362311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(columns=[\"PassengerId\",\"Transported\"])\nsubmission[\"PassengerId\"] = idCol\nsubmission.set_index('PassengerId')\nsubmission[\"Transported\"] = pipe_ensem.predict(test).astype(bool)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-08-09T08:34:32.076894Z","iopub.execute_input":"2022-08-09T08:34:32.078702Z","iopub.status.idle":"2022-08-09T08:34:33.305829Z","shell.execute_reply.started":"2022-08-09T08:34:32.078629Z","shell.execute_reply":"2022-08-09T08:34:33.304418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\nprint('Submission succesful!')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T08:34:38.260528Z","iopub.execute_input":"2022-08-09T08:34:38.260983Z","iopub.status.idle":"2022-08-09T08:34:38.276264Z","shell.execute_reply.started":"2022-08-09T08:34:38.260946Z","shell.execute_reply":"2022-08-09T08:34:38.275220Z"},"trusted":true},"execution_count":null,"outputs":[]}]}