{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import scipy as sp\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\ndf_train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntarget  = df_train['failure']\ndf_test = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsample_sub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\ncdata = pd.concat([df_train ,df_test], ignore_index = True)\ncdata['failure'].value_counts(normalize = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:55.210544Z","iopub.execute_input":"2022-08-14T01:33:55.211057Z","iopub.status.idle":"2022-08-14T01:33:55.404810Z","shell.execute_reply.started":"2022-08-14T01:33:55.211012Z","shell.execute_reply":"2022-08-14T01:33:55.403617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Podemos ver que o dataset em geral possui muito mais ocorrências sem falha do que com, esse desequilibrio pode influenciar negativamente no resultado, uma maneira de lidar com isso é fazendo oversampling.","metadata":{}},{"cell_type":"code","source":"##Correlações\ncorr = cdata.corr(method='spearman')\nmask = np.zeros_like(corr)\nmask[np.triu_indices_from(mask)] = True\n\nfig, ax = plt.subplots()\nax.figure.set_size_inches(15, 15)\nsns.heatmap(corr, ax=ax, annot=True, mask=mask, fmt='.2f', cbar=False, square=True, cmap=plt.cm.Blues, center=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:55.406618Z","iopub.execute_input":"2022-08-14T01:33:55.407045Z","iopub.status.idle":"2022-08-14T01:33:59.527842Z","shell.execute_reply.started":"2022-08-14T01:33:55.407014Z","shell.execute_reply":"2022-08-14T01:33:59.526619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Em geral vemos que a variável alvo tem muita pouca correlação com a maioria das variáveis, a mais considerável seria a variável loading mas ainda com uma correlação pequena, principalmente ao comparar a variável measurement_17 com outras colunas do dataset.","metadata":{}},{"cell_type":"code","source":"##Removendo os ids e a variável alvo\ncdata.drop(['id','failure'], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.529668Z","iopub.execute_input":"2022-08-14T01:33:59.530386Z","iopub.status.idle":"2022-08-14T01:33:59.550296Z","shell.execute_reply.started":"2022-08-14T01:33:59.530343Z","shell.execute_reply":"2022-08-14T01:33:59.549105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Vendo o começo do dataset já sem a variável alvo e os ids\ncdata","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.552936Z","iopub.execute_input":"2022-08-14T01:33:59.553345Z","iopub.status.idle":"2022-08-14T01:33:59.591135Z","shell.execute_reply.started":"2022-08-14T01:33:59.553308Z","shell.execute_reply":"2022-08-14T01:33:59.590168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Podemos ver que existem algumas variáveis categóricas, nos experimentos para o resultado optei por removê-las","metadata":{}},{"cell_type":"code","source":"##Removendo variável categórica do dataset(attribute_0 e attribute_1)\ncdata2 = cdata.copy()\ncdata2.drop(['product_code','attribute_0','attribute_1'],axis = 1, inplace = True)\n\n\nnumeric_features  = cdata2.select_dtypes(np.number)\nna_numeric_features = [feat for feat in numeric_features if feat in cdata2.loc[:,cdata2.isna().sum()>0].columns ]","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.592196Z","iopub.execute_input":"2022-08-14T01:33:59.592827Z","iopub.status.idle":"2022-08-14T01:33:59.686628Z","shell.execute_reply.started":"2022-08-14T01:33:59.592794Z","shell.execute_reply":"2022-08-14T01:33:59.685638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Valores ausentes para o dataset\ncdata2.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.688140Z","iopub.execute_input":"2022-08-14T01:33:59.688848Z","iopub.status.idle":"2022-08-14T01:33:59.700385Z","shell.execute_reply.started":"2022-08-14T01:33:59.688801Z","shell.execute_reply":"2022-08-14T01:33:59.699254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Para o dataset podemos ver que existem valores faltantes, uma forma de lidar com isso é fazer o preenchimento de valores ausentes com a mediana","metadata":{}},{"cell_type":"code","source":"#insight da discussão - https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342319\ncdata2['m_3_missing'] = cdata2.measurement_3.isna()\ncdata2['m_5_missing'] =cdata2.measurement_5.isna()\ncdata2['attribute_2*3'] = cdata2['attribute_2'] * cdata2['attribute_3']","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.701595Z","iopub.execute_input":"2022-08-14T01:33:59.702005Z","iopub.status.idle":"2022-08-14T01:33:59.710870Z","shell.execute_reply.started":"2022-08-14T01:33:59.701972Z","shell.execute_reply":"2022-08-14T01:33:59.709626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Preenchimento dos ausentes com mediana\nfor feat in na_numeric_features:\n     cdata2[feat].fillna(cdata2[feat].median(),inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.712444Z","iopub.execute_input":"2022-08-14T01:33:59.713087Z","iopub.status.idle":"2022-08-14T01:33:59.746446Z","shell.execute_reply.started":"2022-08-14T01:33:59.713043Z","shell.execute_reply":"2022-08-14T01:33:59.745173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdata2 = pd.get_dummies(cdata2, drop_first = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.749302Z","iopub.execute_input":"2022-08-14T01:33:59.749787Z","iopub.status.idle":"2022-08-14T01:33:59.765789Z","shell.execute_reply.started":"2022-08-14T01:33:59.749741Z","shell.execute_reply":"2022-08-14T01:33:59.764708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Scaling de cdata2\nfrom sklearn.preprocessing import StandardScaler\ncdata3 = cdata2.copy()\nscaler = StandardScaler()\nscaler.fit(cdata3)\ncdata3 = pd.DataFrame(scaler.transform(cdata3), index = cdata3.index, columns = cdata3.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:33:59.769930Z","iopub.execute_input":"2022-08-14T01:33:59.771161Z","iopub.status.idle":"2022-08-14T01:34:00.066083Z","shell.execute_reply.started":"2022-08-14T01:33:59.771110Z","shell.execute_reply":"2022-08-14T01:34:00.064907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Divisão em xtrain e xtest\nx_train  = cdata3.iloc[:df_train.shape[0],:]\nx_test  = cdata3.iloc[df_train.shape[0]:,:]","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:34:00.067606Z","iopub.execute_input":"2022-08-14T01:34:00.068057Z","iopub.status.idle":"2022-08-14T01:34:00.074658Z","shell.execute_reply.started":"2022-08-14T01:34:00.068013Z","shell.execute_reply":"2022-08-14T01:34:00.073404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.linear_model import RidgeClassifier, LogisticRegression\nfrom lightgbm import LGBMClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import CategoricalNB\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:34:00.076148Z","iopub.execute_input":"2022-08-14T01:34:00.076605Z","iopub.status.idle":"2022-08-14T01:34:00.086079Z","shell.execute_reply.started":"2022-08-14T01:34:00.076562Z","shell.execute_reply":"2022-08-14T01:34:00.085149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Modelos a serem usados nos testes utilizando fine tuning de parametros\nmodels = {'xgb':XGBClassifier(random_state=42,\n        max_depth=2,\n        min_child_weight=14,\n        gamma=13,\n        n_estimators=100,),\n          \n           'gbc':GradientBoostingClassifier(n_estimators = 100, loss = 'deviance',learning_rate = 0.01),\n           'lr':LogisticRegression(C= 0.01, penalty = 'l1', solver = 'liblinear', max_iter = 100),\n          'knn': KNeighborsClassifier(algorithm = 'brute', n_jobs=-1)\n         }","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:49:08.199346Z","iopub.execute_input":"2022-08-14T01:49:08.199742Z","iopub.status.idle":"2022-08-14T01:49:08.207112Z","shell.execute_reply.started":"2022-08-14T01:49:08.199709Z","shell.execute_reply":"2022-08-14T01:49:08.206163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Kfold e resultados\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\ndef kf_cross_val(model,X,y):\n    \n    scores,feature_imp, features = [],[], []\n    kf = KFold(n_splits=3,shuffle = True, random_state=42)\n    \n    for fold, (train_index, test_index) in enumerate(kf.split(X, y)):\n        \n        x_train = X.iloc[train_index]\n        y_train = y.loc[train_index]\n        x_test = X.loc[test_index]\n        y_test = y.loc[test_index]\n        \n        model.fit(x_train,y_train)\n        \n        y_pred = model.predict_proba(x_test)[:,1]     # edit \n        scores.append(roc_auc_score(y_test,y_pred))\n        \n        try:\n            feature_imp.append(model.feature_importances_)\n            features.append(model.feature_names_)\n        except AttributeError: # if model does not have .feature_importances_ attribute\n            pass\n        \n    return feature_imp, scores, features","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:49:17.474437Z","iopub.execute_input":"2022-08-14T01:49:17.475058Z","iopub.status.idle":"2022-08-14T01:49:17.484327Z","shell.execute_reply.started":"2022-08-14T01:49:17.475021Z","shell.execute_reply":"2022-08-14T01:49:17.483280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results  = {}\n\n\nfor name,model in models.items():\n    \n    feature_imp,result,features = kf_cross_val(model, x_train, target)\n    results[name] = result\n\nfor name, result in results.items():\n    print(\"----------\\n\" + name)\n    print(np.mean(result))\n    print(np.std(result))\n    print(feature_imp)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:49:25.327092Z","iopub.execute_input":"2022-08-14T01:49:25.327527Z","iopub.status.idle":"2022-08-14T01:50:11.932467Z","shell.execute_reply.started":"2022-08-14T01:49:25.327492Z","shell.execute_reply":"2022-08-14T01:50:11.931186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Oversampling\nfrom imblearn.over_sampling import SMOTE\nprint(x_train.shape)\noversample = SMOTE()\nx_train, target = oversample.fit_resample(x_train, target)\nprint(x_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:50:39.146577Z","iopub.execute_input":"2022-08-14T01:50:39.147017Z","iopub.status.idle":"2022-08-14T01:50:39.986836Z","shell.execute_reply.started":"2022-08-14T01:50:39.146977Z","shell.execute_reply":"2022-08-14T01:50:39.985397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = {'xgb':0.1,\n           'gbc':0.2,\n           'lr':0.4,\n          'knn':0.3}","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:52:28.382654Z","iopub.execute_input":"2022-08-14T01:52:28.383157Z","iopub.status.idle":"2022-08-14T01:52:28.388609Z","shell.execute_reply.started":"2022-08-14T01:52:28.383117Z","shell.execute_reply":"2022-08-14T01:52:28.387602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name,model in models.items():\n    model.fit(x_train, target)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:53:29.219046Z","iopub.execute_input":"2022-08-14T01:53:29.219542Z","iopub.status.idle":"2022-08-14T01:53:59.815690Z","shell.execute_reply.started":"2022-08-14T01:53:29.219502Z","shell.execute_reply":"2022-08-14T01:53:59.814782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds  = {}\n\nfor name,model in models.items():\n    \n    pred = pd.DataFrame(model.predict_proba(x_test)).iloc[:,1]  # second column\n    preds[name] = pred","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:54:59.114566Z","iopub.execute_input":"2022-08-14T01:54:59.115043Z","iopub.status.idle":"2022-08-14T01:55:22.643785Z","shell.execute_reply.started":"2022-08-14T01:54:59.115004Z","shell.execute_reply":"2022-08-14T01:55:22.642593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred  = np.zeros(x_test.shape[0])\nfor name,pred in  preds.items():\n    y_pred = y_pred + weights[name] * pred","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:55:31.191758Z","iopub.execute_input":"2022-08-14T01:55:31.192191Z","iopub.status.idle":"2022-08-14T01:55:31.201023Z","shell.execute_reply.started":"2022-08-14T01:55:31.192152Z","shell.execute_reply":"2022-08-14T01:55:31.200181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.Series(y_pred, name='failure')\nsample_sub['failure'] = y_pred\nsubmission = sample_sub.copy()\nsubmission.to_csv('submission.csv',index= False)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T01:55:54.595679Z","iopub.execute_input":"2022-08-14T01:55:54.596110Z","iopub.status.idle":"2022-08-14T01:55:54.654761Z","shell.execute_reply.started":"2022-08-14T01:55:54.596075Z","shell.execute_reply":"2022-08-14T01:55:54.653823Z"},"trusted":true},"execution_count":null,"outputs":[]}]}