{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nimport itertools\nfrom sklearn.preprocessing import StandardScaler, OrdinalEncoder, MinMaxScaler\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\nfrom imblearn.over_sampling import SMOTE","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.317392Z","iopub.execute_input":"2022-08-09T09:36:01.317874Z","iopub.status.idle":"2022-08-09T09:36:01.326760Z","shell.execute_reply.started":"2022-08-09T09:36:01.317840Z","shell.execute_reply":"2022-08-09T09:36:01.325728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Import data\n# ====================================================\n\n\ndf_train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntarget  = df_train['failure']\ndf_test = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsample_sub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\n\ncdata = pd.concat([df_train ,df_test], ignore_index = True)\n\ncdata.drop(['id','failure'], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.348453Z","iopub.execute_input":"2022-08-09T09:36:01.348877Z","iopub.status.idle":"2022-08-09T09:36:01.536491Z","shell.execute_reply.started":"2022-08-09T09:36:01.348839Z","shell.execute_reply":"2022-08-09T09:36:01.534751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Dropping Categorical variables\n# ====================================================\n\ncdata2 = cdata.copy()\ncdata2.drop(['product_code','attribute_0','attribute_1'],axis = 1, inplace = True)\n\n\nnumeric_features  = cdata2.select_dtypes(np.number)\nna_numeric_features = [feat for feat in numeric_features if feat in cdata2.loc[:,cdata2.isna().sum()>0].columns ]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.538804Z","iopub.execute_input":"2022-08-09T09:36:01.539207Z","iopub.status.idle":"2022-08-09T09:36:01.645102Z","shell.execute_reply.started":"2022-08-09T09:36:01.539158Z","shell.execute_reply":"2022-08-09T09:36:01.643833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdata2","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.646698Z","iopub.execute_input":"2022-08-09T09:36:01.647069Z","iopub.status.idle":"2022-08-09T09:36:01.688225Z","shell.execute_reply.started":"2022-08-09T09:36:01.647036Z","shell.execute_reply":"2022-08-09T09:36:01.686919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Feature engg\n# ====================================================\n\n# cdata2['m_mean'] = cdata2.iloc[:,6:-1].mean(axis=1)\n# cdata2['m_std']= cdata2.iloc[:,6:-1].std(axis=1)\n\n#ref - https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342319\ncdata2['m_3_missing'] = cdata2.measurement_3.isna()\ncdata2['m_5_missing'] =cdata2.measurement_5.isna()\n\ncdata2['attribute_2*3'] = cdata2['attribute_2'] * cdata2['attribute_3']\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.690724Z","iopub.execute_input":"2022-08-09T09:36:01.691044Z","iopub.status.idle":"2022-08-09T09:36:01.701424Z","shell.execute_reply.started":"2022-08-09T09:36:01.691014Z","shell.execute_reply":"2022-08-09T09:36:01.700255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Imputing median \n# ====================================================\n\n\nfor feat in na_numeric_features:\n     cdata2[feat].fillna(cdata2[feat].median(),inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.702542Z","iopub.execute_input":"2022-08-09T09:36:01.702865Z","iopub.status.idle":"2022-08-09T09:36:01.744869Z","shell.execute_reply.started":"2022-08-09T09:36:01.702836Z","shell.execute_reply":"2022-08-09T09:36:01.743289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdata2 = pd.get_dummies(cdata2, drop_first = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.747274Z","iopub.execute_input":"2022-08-09T09:36:01.747716Z","iopub.status.idle":"2022-08-09T09:36:01.765507Z","shell.execute_reply.started":"2022-08-09T09:36:01.747679Z","shell.execute_reply":"2022-08-09T09:36:01.764092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Scaling \n# ====================================================\n\ncdata3 = cdata2.copy()\n\nscaler = StandardScaler()\nscaler.fit(cdata3)\ncdata3 = pd.DataFrame(scaler.transform(cdata3), index = cdata3.index, columns = cdata3.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:01.767151Z","iopub.execute_input":"2022-08-09T09:36:01.767598Z","iopub.status.idle":"2022-08-09T09:36:02.064490Z","shell.execute_reply.started":"2022-08-09T09:36:01.767546Z","shell.execute_reply":"2022-08-09T09:36:02.062858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# ====================================================\n# Balancing the dataset\n# ====================================================\n\n\nx_train  = cdata3.iloc[:df_train.shape[0],:]\nx_test  = cdata3.iloc[df_train.shape[0]:,:]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:02.066211Z","iopub.execute_input":"2022-08-09T09:36:02.066620Z","iopub.status.idle":"2022-08-09T09:36:02.075212Z","shell.execute_reply.started":"2022-08-09T09:36:02.066582Z","shell.execute_reply":"2022-08-09T09:36:02.073586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier, ExtraTreesClassifier, RandomForestClassifier\nfrom sklearn.linear_model import RidgeClassifier, LogisticRegression\nfrom lightgbm import LGBMClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import CategoricalNB\nfrom xgboost import XGBClassifier\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:02.076478Z","iopub.execute_input":"2022-08-09T09:36:02.076856Z","iopub.status.idle":"2022-08-09T09:36:02.086548Z","shell.execute_reply.started":"2022-08-09T09:36:02.076822Z","shell.execute_reply":"2022-08-09T09:36:02.085173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {'xgb':XGBClassifier(random_state=42,\n        max_depth=2,\n        min_child_weight=14,\n        gamma=13,\n        n_estimators=100,),\n          \n           'gbc':GradientBoostingClassifier(n_estimators = 200, loss = 'deviance',learning_rate = 0.01),\n           'lr':LogisticRegression(C= 0.01, penalty = 'l1', solver = 'liblinear', max_iter = 200),\n          'lda': LinearDiscriminantAnalysis(),\n#           'knn': KNeighborsClassifier(algorithm = 'brute', n_jobs=-1)\n         }","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:02.090283Z","iopub.execute_input":"2022-08-09T09:36:02.090736Z","iopub.status.idle":"2022-08-09T09:36:02.099947Z","shell.execute_reply.started":"2022-08-09T09:36:02.090698Z","shell.execute_reply":"2022-08-09T09:36:02.098588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, cross_val_score\ndef kf_cross_val(model,X,y):\n    \n    scores,feature_imp, features = [],[], []\n    \n    kf = KFold(n_splits=3,shuffle = True, random_state=42)\n    \n    for fold, (train_index, test_index) in enumerate(kf.split(X, y)):\n        \n        x_train = X.iloc[train_index]\n        y_train = y.loc[train_index]\n        x_test = X.loc[test_index]\n        y_test = y.loc[test_index]\n        \n        model.fit(x_train,y_train)\n        \n        y_pred = model.predict_proba(x_test)[:,1]     # edit \n        scores.append(roc_auc_score(y_test,y_pred))\n        \n        try:\n            feature_imp.append(model.feature_importances_)\n            features.append(model.feature_names_)\n        except AttributeError: # if model does not have .feature_importances_ attribute\n            pass\n        \n    return feature_imp, scores, features","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:02.101862Z","iopub.execute_input":"2022-08-09T09:36:02.102376Z","iopub.status.idle":"2022-08-09T09:36:02.115792Z","shell.execute_reply.started":"2022-08-09T09:36:02.102334Z","shell.execute_reply":"2022-08-09T09:36:02.114366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Cross Validation results / comparing models\n# ====================================================\n\nresults  = {}\n\n\nfor name,model in models.items():\n    \n    feature_imp,result,features = kf_cross_val(model, x_train, target)\n    results[name] = result\n\nfor name, result in results.items():\n    print(\"----------\\n\" + name)\n    print(np.mean(result))\n    print(np.std(result))\n    print(feature_imp)\n    \n   ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:36:02.117620Z","iopub.execute_input":"2022-08-09T09:36:02.118023Z","iopub.status.idle":"2022-08-09T09:37:05.556081Z","shell.execute_reply.started":"2022-08-09T09:36:02.117989Z","shell.execute_reply":"2022-08-09T09:37:05.554416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ----------\n# xgb\n# 0.5864963608770714\n# 0.005108446593833111\n# []\n# ----------\n# gbc\n# 0.5868586510174651\n# 0.005620482965321298\n# []\n# ----------\n# lr\n# 0.5892217131558347\n# 0.0059300239055273506\n# []\n# ----------\n# lda\n# 0.5880375690386336\n# 0.007091633302138498","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:37:05.558461Z","iopub.execute_input":"2022-08-09T09:37:05.560573Z","iopub.status.idle":"2022-08-09T09:37:05.568365Z","shell.execute_reply.started":"2022-08-09T09:37:05.560515Z","shell.execute_reply":"2022-08-09T09:37:05.566723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\noversample = SMOTE()\nx_train, target = oversample.fit_resample(x_train, target)\nprint(x_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:37:05.570563Z","iopub.execute_input":"2022-08-09T09:37:05.573523Z","iopub.status.idle":"2022-08-09T09:37:06.536765Z","shell.execute_reply.started":"2022-08-09T09:37:05.573450Z","shell.execute_reply":"2022-08-09T09:37:06.535342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Ensemble weights \n# ====================================================\n\nweights = {'xgb':0.0,\n           'gbc':0.1,\n           'lr':0.5,\n          'lda':0.4}","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:37:06.538038Z","iopub.execute_input":"2022-08-09T09:37:06.538390Z","iopub.status.idle":"2022-08-09T09:37:06.544632Z","shell.execute_reply.started":"2022-08-09T09:37:06.538358Z","shell.execute_reply":"2022-08-09T09:37:06.543072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name,model in models.items():\n    model.fit(x_train, target)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:37:06.546379Z","iopub.execute_input":"2022-08-09T09:37:06.546745Z","iopub.status.idle":"2022-08-09T09:38:06.761023Z","shell.execute_reply.started":"2022-08-09T09:37:06.546712Z","shell.execute_reply":"2022-08-09T09:38:06.759097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds  = {}\n\nfor name,model in models.items():\n    \n    pred = pd.DataFrame(model.predict_proba(x_test)).iloc[:,1]  # second column\n    preds[name] = pred","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:38:06.763716Z","iopub.execute_input":"2022-08-09T09:38:06.764983Z","iopub.status.idle":"2022-08-09T09:38:06.913645Z","shell.execute_reply.started":"2022-08-09T09:38:06.764914Z","shell.execute_reply":"2022-08-09T09:38:06.912492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred  = np.zeros(x_test.shape[0])\nfor name,pred in  preds.items():\n    y_pred = y_pred + weights[name] * pred\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:38:06.915392Z","iopub.execute_input":"2022-08-09T09:38:06.920951Z","iopub.status.idle":"2022-08-09T09:38:06.936402Z","shell.execute_reply.started":"2022-08-09T09:38:06.920858Z","shell.execute_reply":"2022-08-09T09:38:06.934630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.Series(y_pred, name='failure')\nsample_sub['failure'] = y_pred\nsubmission = sample_sub.copy()\nsubmission.to_csv('submission.csv',index= False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:38:06.945048Z","iopub.execute_input":"2022-08-09T09:38:06.949692Z","iopub.status.idle":"2022-08-09T09:38:07.055996Z","shell.execute_reply.started":"2022-08-09T09:38:06.949600Z","shell.execute_reply":"2022-08-09T09:38:07.055050Z"},"trusted":true},"execution_count":null,"outputs":[]}]}