{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

Ponto de partida (EDA)

\n
\n\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/26480/logos/header.png?t=2021-04-09-00-57-05)","metadata":{"id":"dyZU8TQF3EC_","papermill":{"duration":0.108455,"end_time":"2022-05-04T04:22:03.356410","exception":false,"start_time":"2022-05-04T04:22:03.247955","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"A edição agosto do Tabular Playgroud Series de 2022 é uma oportunidade para ajudar a empresa fictícia Keep It Dry a melhorar seu principal produto Super Soaker, o produto é usado em fábricas para absorver \n\nA empresa acaba de concluir um grande estudo de teste para diferentes protótipos de produtos, podemos usar esses dados para construir um modelo que prevê falhas do produto? Esse é o grande desafio. ","metadata":{"id":"_63OYyaQFle4","papermill":{"duration":0.102426,"end_time":"2022-05-04T04:22:03.561482","exception":false,"start_time":"2022-05-04T04:22:03.459056","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n

OBJETIVO

\n
","metadata":{"id":"MhoTTWqXTluy","papermill":{"duration":0.10217,"end_time":"2022-05-04T04:22:03.766987","exception":false,"start_time":"2022-05-04T04:22:03.664817","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Neste notebook vamos fazer uma análise (EDA) para conhencer os dados e estabelecer uma linha de base, mostrarei as etapas iniciais de uma competição do Kaggle - desde a compreensão do conjunto de dados até a preparação dos dados para serem usados em um modelo machine learning. Vamos passar pelas seguintes tarefas:\n\n- Leitura no conjunto de dados\n- Calculando estatísticas sobre o conjunto de dados\n- Visualização univariada\n- Visualizando multivariada\n- Pré-processamento \n\n\n","metadata":{"id":"Q0e0UrygTkr4","papermill":{"duration":0.103047,"end_time":"2022-05-04T04:22:03.977664","exception":false,"start_time":"2022-05-04T04:22:03.874617","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"---","metadata":{"id":"BVBGntvGT4r4","papermill":{"duration":0.102474,"end_time":"2022-05-04T04:22:04.183305","exception":false,"start_time":"2022-05-04T04:22:04.080831","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n

1. IMPORTAÇÕES

\n
","metadata":{"id":"Cjx0CaVr3EDF","papermill":{"duration":0.102396,"end_time":"2022-05-04T04:22:04.389491","exception":false,"start_time":"2022-05-04T04:22:04.287095","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1.1. Instalações","metadata":{"id":"HkllrkTTeqs5","papermill":{"duration":0.10291,"end_time":"2022-05-04T04:22:04.594960","exception":false,"start_time":"2022-05-04T04:22:04.492050","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# https://pub.towardsai.net/use-google-colab-like-a-pro-39a97184358d\nCOLAB = 'google.colab' in str(get_ipython()) \n\nif COLAB: \n !pip install --q scikit-plot\n !pip install --q category_encoders\n !pip install --q shap\n !pip install --q inflection \n !pip install --q optbinning\n !pip install --q catboost\n !pip install --q pandas-profiling\n !pip install --q pycaret\n \n from google.colab import drive\n drive.mount('/content/drive')","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:35:55.175765Z","start_time":"2022-08-01T21:35:55.146728Z"},"executionInfo":{"elapsed":8633,"status":"ok","timestamp":1651533974650,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"_O4QLmfE3VfS","outputId":"d0a40c48-15d4-478d-c092-b8f227cfbeb2","papermill":{"duration":0.147758,"end_time":"2022-05-04T04:22:04.844921","exception":false,"start_time":"2022-05-04T04:22:04.697163","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:49.090706Z","iopub.execute_input":"2022-08-02T17:15:49.091098Z","iopub.status.idle":"2022-08-02T17:15:49.109156Z","shell.execute_reply.started":"2022-08-02T17:15:49.091068Z","shell.execute_reply":"2022-08-02T17:15:49.108190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2. Bibliotecas ","metadata":{"id":"2TUVX7H53EDG","papermill":{"duration":0.172636,"end_time":"2022-05-04T04:22:05.203774","exception":false,"start_time":"2022-05-04T04:22:05.031138","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import warnings\nimport random\nimport os\nimport gc\nimport torch\nimport sklearn.exceptions\nimport datetime\n#import shap","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:35:58.274654Z","start_time":"2022-08-01T21:35:57.095442Z"},"executionInfo":{"elapsed":5375,"status":"ok","timestamp":1651517147077,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"7CkqtXYc3EDG","papermill":{"duration":3.621538,"end_time":"2022-05-04T04:22:08.989725","exception":false,"start_time":"2022-05-04T04:22:05.368187","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:49.787778Z","iopub.execute_input":"2022-08-02T17:15:49.788692Z","iopub.status.idle":"2022-08-02T17:15:51.993716Z","shell.execute_reply.started":"2022-08-02T17:15:49.788633Z","shell.execute_reply":"2022-08-02T17:15:51.992402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt \nimport seaborn as sns\nimport joblib as jb\nimport scipy.stats as stats\nimport plotly.express as px\nimport scikitplot as skplt\nimport plotly.graph_objs as go","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:35:58.909654Z","start_time":"2022-08-01T21:35:58.390652Z"},"executionInfo":{"elapsed":635,"status":"ok","timestamp":1651517147709,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"wlxzri_03EDH","outputId":"ee0bb97d-14c9-4425-dcb7-e2f19d82478c","papermill":{"duration":0.231612,"end_time":"2022-05-04T04:22:09.326572","exception":false,"start_time":"2022-05-04T04:22:09.094960","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:51.995681Z","iopub.execute_input":"2022-08-02T17:15:51.996227Z","iopub.status.idle":"2022-08-02T17:15:53.580304Z","shell.execute_reply.started":"2022-08-02T17:15:51.996190Z","shell.execute_reply":"2022-08-02T17:15:53.579036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, KFold, StratifiedKFold, GroupKFold\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, FunctionTransformer, MaxAbsScaler\nfrom sklearn.preprocessing import PowerTransformer, RobustScaler, Normalizer, PolynomialFeatures\nfrom sklearn.preprocessing import QuantileTransformer, LabelEncoder, OneHotEncoder\nfrom sklearn.metrics import roc_auc_score","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:15:11.166145Z","start_time":"2022-08-02T02:15:11.158123Z"},"executionInfo":{"elapsed":261,"status":"ok","timestamp":1651522243969,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"EQ1kIGrLFle-","papermill":{"duration":0.110081,"end_time":"2022-05-04T04:22:09.540566","exception":false,"start_time":"2022-05-04T04:22:09.430485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:53.581680Z","iopub.execute_input":"2022-08-02T17:15:53.582039Z","iopub.status.idle":"2022-08-02T17:15:53.588297Z","shell.execute_reply.started":"2022-08-02T17:15:53.582000Z","shell.execute_reply":"2022-08-02T17:15:53.587113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.compose import ColumnTransformer, make_column_transformer\nfrom sklearn.pipeline import Pipeline, make_pipeline, FeatureUnion\nfrom sklearn.feature_selection import VarianceThreshold\nfrom sklearn.impute import SimpleImputer","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:55:14.167456Z","start_time":"2022-08-02T01:55:14.085417Z"},"execution":{"iopub.status.busy":"2022-08-02T17:15:53.590885Z","iopub.execute_input":"2022-08-02T17:15:53.591243Z","iopub.status.idle":"2022-08-02T17:15:53.682339Z","shell.execute_reply.started":"2022-08-02T17:15:53.591211Z","shell.execute_reply":"2022-08-02T17:15:53.680879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\nfrom sklearn.linear_model import LogisticRegressionCV, LogisticRegression","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:24:31.150769Z","start_time":"2022-08-02T02:24:31.143777Z"},"execution":{"iopub.status.busy":"2022-08-02T17:40:07.649588Z","iopub.execute_input":"2022-08-02T17:40:07.650004Z","iopub.status.idle":"2022-08-02T17:40:07.655016Z","shell.execute_reply.started":"2022-08-02T17:40:07.649970Z","shell.execute_reply":"2022-08-02T17:40:07.653778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{"id":"H4ae2YzqFle_","papermill":{"duration":0.104138,"end_time":"2022-05-04T04:22:09.962615","exception":false,"start_time":"2022-05-04T04:22:09.858477","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1.3. Funções\nAqui centralizamos todas as funções desenvolvidas durante o projeto para melhor organização do código.","metadata":{"id":"e0gRHehq3EDJ","papermill":{"duration":0.102595,"end_time":"2022-05-04T04:22:10.168779","exception":false,"start_time":"2022-05-04T04:22:10.066184","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Ultil():\n \n import sklearn.exceptions\n \n def __init__(self):\n pass\n \n def jupyter_setting():\n \n %matplotlib inline\n\n #os.environ[\"WANDB_SILENT\"] = \"true\" \n #plt.style.use('bmh') \n #plt.rcParams['figure.figsize'] = [20,15]\n #plt.rcParams['font.size'] = 13\n\n pd.options.display.max_columns = None\n #pd.set_option('display.expand_frame_repr', False)\n\n warnings.filterwarnings(action='ignore')\n warnings.simplefilter('ignore')\n warnings.filterwarnings('ignore')\n warnings.filterwarnings('ignore', category=DeprecationWarning)\n warnings.filterwarnings('ignore', category=FutureWarning)\n warnings.filterwarnings('ignore', category=RuntimeWarning)\n warnings.filterwarnings('ignore', category=UserWarning)\n warnings.filterwarnings(\"ignore\", category=sklearn.exceptions.UndefinedMetricWarning)\n warnings.filterwarnings(\"ignore\", category= sklearn.exceptions.UndefinedMetricWarning)\n\n pd.set_option('display.max_rows', 200)\n pd.set_option('display.max_columns', 500)\n pd.set_option('display.max_colwidth', None)\n\n icecream = [\"#00008b\", \"#960018\",\"#008b00\", \"#00468b\", \"#8b4500\", \"#582c00\"]\n #sns.palplot(sns.color_palette(icecream))\n\n colors = [\"lightcoral\", \"sandybrown\", \"darkorange\", \"mediumseagreen\",\n \"lightseagreen\", \"cornflowerblue\", \"mediumpurple\", \"palevioletred\",\n \"lightskyblue\", \"sandybrown\", \"yellowgreen\", \"indianred\",\n \"lightsteelblue\", \"mediumorchid\", \"deepskyblue\"]\n\n # Colors\n dark_red = \"#b20710\"\n black = \"#221f1f\"\n green = \"#009473\"\n myred = '#CD5C5C'\n myblue = '#6495ED'\n mygreen = '#90EE90' \n color_cols = [myred, myblue,mygreen]\n\n return icecream, colors, color_cols\n\n def missing_zero_values_table(df):\n mis_val = df.isnull().sum()\n mis_val_percent = round(df.isnull().mean().mul(100), 2)\n mz_table = pd.concat([mis_val, mis_val_percent], axis=1)\n mz_table = mz_table.rename(columns = {df.index.name:'col_name', \n 0 : 'Valores ausentes', \n 1 : '% de valores totais'})\n \n mz_table['Tipo de dados'] = df.dtypes\n mz_table = mz_table[mz_table.iloc[:,1] != 0 ]. \\\n sort_values('% de valores totais', ascending=False)\n \n msg = \"Seu dataframe selecionado tem {} colunas e {} \" + \\\n \"linhas. \\nExistem {} colunas com valores ausentes.\"\n \n print (msg.format(df.shape[1], df.shape[0], mz_table.shape[0]))\n \n return mz_table.reset_index()\n \n def reduce_memory_usage(df, verbose=True):\n \n numerics = [\"int8\", \"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\"]\n start_mem = df.memory_usage().sum() / 1024 ** 2\n\n for col in df.columns:\n\n col_type = df[col].dtypes\n\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n\n if str(col_type)[:3] == \"int\":\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n else:\n if (\n c_min > np.finfo(np.float16).min\n and c_max < np.finfo(np.float16).max\n ):\n df[col] = df[col].astype(np.float16)\n elif (\n c_min > np.finfo(np.float32).min\n and c_max < np.finfo(np.float32).max\n ):\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n end_mem = df.memory_usage().sum() / 1024 ** 2\n if verbose:\n print(\n \"Mem. usage decreased to {:.2f} Mb ({:.1f}% reduction)\".format(\n end_mem, 100 * (start_mem - end_mem) / start_mem\n )\n )\n\n return df\n \n def graf_label(ax, total):\n\n for i in ax.patches:\n # get_width pulls left or right; get_y pushes up or down\n width, height = i.get_width() -.2 , i.get_height()\n\n x, y = i.get_xy() \n color = 'white'\n alt = .5\n soma = 0 \n\n if height < 70:\n color = 'black'\n alt = 1\n soma = 10\n\n ax.annotate(str(round((i.get_height() * 100.0 / total), 1) )+'%', \n (i.get_x()+.3*width, \n i.get_y()+soma + alt*height),\n color = color,\n weight = 'bold',\n size = 14)\n \n def graf_bar(df, col, title, xlabel, ylabel, tol = 0):\n \n #ax = df.groupby(['churn_cat'])['churn_cat'].count()\n ax = df \n colors = col\n\n if tol == 0: \n total = sum(ax)\n ax = (ax).plot(kind ='bar',\n stacked = True,\n width = .5,\n rot = 0,\n color = colors, \n grid = False)\n else:\n total = tol \n\n ax = (ax).plot(kind ='bar',\n stacked = True,\n width = .5,\n rot = 0,\n figsize = (10,6),\n color = colors,\n grid = False)\n\n #ax.yaxis.set_major_formatter(mtick.PercentFormatter())\n\n #y_fmt = tick.FormatStrFormatter('%.0f') \n #ax.yaxis.set_major_formatter(y_fmt)\n\n title = title #+ ' \\n'\n xlabel = '\\n ' + xlabel \n ylabel = ylabel + ' \\n'\n\n ax.set_title(title , fontsize=22)\n ax.set_xlabel(xlabel, fontsize=12)\n ax.set_ylabel(ylabel, fontsize=12) \n\n min = [0,23000000]\n #ax.set_ylim(min)\n\n Ultil.graf_label(ax, total)\n\n def correlation(df_, threshold_):\n col_corr = set() \n corr_matrix = df_.corr()\n \n for i in range(len(corr_matrix.columns)):\n for j in range(i):\n if abs(corr_matrix.iloc[i, j]) > threshold_: \n colname = corr_matrix.columns[i] \n col_corr.add(colname)\n \n return col_corr\n\n def graf_fature_corr(df_, annot_=False, threshold_=.8, print_var_=False, \n print_graf_=True, mask_=True, title_=''):\n\n df = df_.copy().corr(method ='pearson').round(5)\n\n if print_graf_: \n # Máscara para ocultar a parte superior direita do gráfico, pois é uma duplicata\n mask = np.zeros_like(df)\n mask[np.triu_indices_from(mask)] = mask_\n\n # Making a plot\n ax = sns.heatmap(df, annot=annot_, \n mask=mask, \n cmap=\"RdBu\", \n annot_kws={\"weight\": \"bold\", \"fontsize\":13}\n )\n\n ax.set_title(\"\\n Correlação das variável {} \\n\".format(title_), fontsize=17)\n\n plt.setp(ax.get_xticklabels(), \n rotation = 90, \n ha = \"right\",\n rotation_mode = \"anchor\", \n weight = \"normal\")\n\n plt.setp(ax.get_yticklabels(), \n weight = \"normal\",\n rotation_mode = \"anchor\", \n rotation = 0, \n ha = \"right\")\n\n plt.show();\n\n if print_var_: \n df_corr = df[abs(df)>threshold_][df!=1.0].unstack().dropna().reset_index()\n if len(df_corr)>0: \n print('Variáveis autocorrelacionadas threshold={:2.2f}'.format(threshold_))\n df_corr.columns = ['var_1', 'var_2', 'corr']\n display(df_corr)\n else: \n print('Não tem variáveis autocorrelacionadas threshold={:2.2f}'.format(threshold_))\n \n def describe(df):\n var = df.columns\n\n # Medidas de tendência central, média e mediana \n ct1 = pd.DataFrame(df[var].apply(np.mean)).T\n ct2 = pd.DataFrame(df[var].apply(np.median)).T\n\n # Dispensão - str, min , max range skew, kurtosis\n d1 = pd.DataFrame(df[var].apply(np.std)).T\n d2 = pd.DataFrame(df[var].apply(min)).T\n d3 = pd.DataFrame(df[var].apply(max)).T\n d4 = pd.DataFrame(df[var].apply(lambda x: x.max() - x.min())).T\n d5 = pd.DataFrame(df[var].apply(lambda x: x.skew())).T\n d6 = pd.DataFrame(df[var].apply(lambda x: x.kurtosis())).T\n d7 = pd.DataFrame(df[var].apply(lambda x: (3 *( np.mean(x) - np.median(x)) / np.std(x) ))).T\n\n # concatenete \n m = pd.concat([d2, d3, d4, ct1, ct2, d1, d5, d6, d7]).T.reset_index()\n m.columns = ['attrobutes', 'min', 'max', 'range', 'mean', 'median', 'std','skew', 'kurtosis','coef_as']\n\n return m\n\n def graf_outlier(df, feature):\n col = [(0,4), (5,9)]\n\n df_plot = ((df[feature] - df[feature].min())/\n (df[feature].max() - df[feature].min()))\n\n fig, ax = plt.subplots(len(col), 1, figsize=(15,7))\n\n for i, (x) in enumerate(col): \n sns.boxplot(data = df_plot.iloc[:, x[0]:x[1] ], ax = ax[i]); \n\n def diff(t_a, t_b):\n from dateutil.relativedelta import relativedelta\n t_diff = relativedelta(t_b, t_a) # later/end time comes first!\n return '{h}h {m}m {s}s'.format(h=t_diff.hours, m=t_diff.minutes, s=t_diff.seconds)\n\n def free_gpu_cache():\n\n # https://www.kaggle.com/getting-started/140636\n #print(\"Initial GPU Usage\")\n #gpu_usage() \n\n #cuda.select_device(0)\n #cuda.close()\n #cuda.select_device(0) \n\n gc.collect()\n torch.cuda.empty_cache()\n\n def graf_eval():\n\n results = model.evals_result()\n ntree_limit = model.best_ntree_limit\n\n plt.figure(figsize=(20,7))\n\n for i, error in enumerate(['mlogloss', 'merror']):#\n\n plt.subplot(1,2,i+1)\n plt.plot(results[\"validation_0\"][error], label=\"Treinamento\")\n plt.plot(results[\"validation_1\"][error], label=\"Validação\")\n\n plt.axvline(ntree_limit, \n color=\"gray\", \n label=\"N. de árvore ideal {}\".format(ntree_limit))\n\n\n title_name ='\\n' + error.upper() + ' PLOT \\n'\n plt.title(title_name)\n plt.xlabel(\"Número de árvores\")\n plt.ylabel(error)\n plt.legend();\n\n def linear_fit_slope(y):\n \"\"\"Return the slope of a linear fit to a series.\"\"\"\n y_pure = y.dropna()\n length = len(y_pure)\n x = np.arange(0, length)\n slope, intercept = np.polyfit(x, y_pure.values, deg=1)\n return slope\n\n def linear_fit_intercept(y):\n \"\"\"Return the intercept of a linear fit to a series.\"\"\"\n y_pure = y.dropna()\n length = len(y_pure)\n x = np.arange(0, length)\n slope, intercept = np.polyfit(x, y_pure.values, deg=1)\n return intercept\n\n def cromer_v(x, y):\n cm = pd.crosstab(x, y).to_numpy() \n n = cm.sum()\n r, k = cm.shape\n chi2 = stats.chi2_contingency(cm)[0]\n chi2corr = max(0, chi2 - (k-1) * (r-1) /(n-1))\n kcorr = k - (k-1) **2/(n-1)\n rcorr = r - (r-1) **2/(n-1) \n v = np.sqrt((chi2corr/n) / (min(kcorr-1, rcorr-1))) \n return v \n\n def generate_category_table(data):\n\n cols = data.select_dtypes(include='object').columns\n dataset = pd.DataFrame()\n\n for i in cols:\n corr = []\n for x in cols: \n corr.append(Ultil.cromer_v(data[i],data[x]))\n\n aux = pd.DataFrame({i:corr})\n dataset = pd.concat([dataset, aux], axis=1) \n\n return dataset.set_index(dataset.columns)\n \n def graf_feature_corr(df_, annot_=False, threshold_=.8, print_var_=False, \n print_graf_=True, mask_=True, title_='', method_='pearson'):\n\n df = df_.corr(method=method_).round(5)\n\n if print_graf_: \n # Máscara para ocultar a parte superior direita do gráfico, pois é uma duplicata\n mask = np.zeros_like(df)\n mask[np.triu_indices_from(mask)] = mask_\n\n # Making a plot\n ax = sns.heatmap(df, annot = annot_, \n mask = mask, \n cmap = \"RdBu\", \n fmt = \".2f\",\n annot_kws = {\"weight\": \"bold\", \"fontsize\":10}\n )\n\n \n ax.set_title(\"\\n Correlação das variável {} \\n\".format(title_), fontsize=17)\n\n plt.setp(ax.get_xticklabels(), \n rotation = 90, \n ha = \"right\",\n rotation_mode = \"anchor\", \n weight = \"normal\")\n\n plt.setp(ax.get_yticklabels(), \n weight = \"normal\",\n rotation_mode = \"anchor\", \n rotation = 0, \n ha = \"right\")\n\n plt.show();\n\n if print_var_: \n df_corr = df[abs(df)>threshold_][df!=1.0].unstack().dropna().reset_index()\n if len(df_corr)>0: \n print('Variáveis autocorrelacionadas threshold={:2.2f}'.format(threshold_))\n df_corr.columns = ['var_1', 'var_2', 'corr']\n display(df_corr)\n else: \n print('Não tem variáveis autocorrelacionadas threshold={:2.2f}'.format(threshold_))\n \n return Ultil.correlation(df_, threshold_)\n\n def plot_roc_curve(fpr, tpr, label=None):\n fig, ax = plt.subplots()\n ax.plot(fpr, tpr, \"r-\", label=label)\n ax.plot([0, 1], [0, 1], transform=ax.transAxes, ls=\"--\", c=\".3\")\n plt.xlim([0.0, 1.0])\n plt.ylim([0.0, 1.0])\n plt.rcParams['font.size'] = 12\n plt.title('ROC curve for FLAI 08')\n plt.xlabel('False Positive Rate (1 - Specificity)')\n plt.ylabel('True Positive Rate (Sensitivity)')\n plt.legend(loc=\"lower right\")\n plt.grid(True)\n\n def feature_engineering(df_):\n\n var_f27 = ''\n for col in df_['f_27']: \n var_f27 +=col\n\n var_f27 = list(set(var_f27))\n var_f27.sort()\n\n df_[\"fe_f_27_unique\"] = df_[\"f_27\"].apply(lambda x: len(set(x)))\n\n for letra in var_f27: \n df_['fe_' + letra.lower() + '_count'] = df2_train[\"f_27\"].str.count(letra)\n\n return df_ \n\n def identifies_outliers(df):\n\n cols_num = df.select_dtypes(np.number).columns\n\n for col in cols_num: \n if col != 'unnamed': \n Q1 = df[col].quantile(0.25)\n Q3 = df[col].quantile(0.75)\n IQR = Q3-Q1\n lowqe_bound=Q1 - 1.5 * IQR\n upper_bound=Q3 + 1.5 * IQR\n\n df['outliers_'+ col] = 0\n df['outliers_'+ col][(df[col]<=lowqe_bound)|(df[col]>=upper_bound)] = 1 \n\n df[col] = np.where(df[col] > df[col].quantile(0.95),\n df[col].median(),\n df[col])\n\n def evaluation(y_, predictions_):\n mae = metrics.mean_absolute_error(y_, predictions_)\n mse = metrics.mean_squared_error(y_, predictions_)\n rmse = metrics.mean_squared_error(y_, predictions_, squared=False) \n mape = metrics.mean_absolute_percentage_error(y_, predictions_)\n r2 = metrics.r2_score(y_, predictions_) \n return rmse, mae, mse, mape, r2\n \n def feature_statistic(df, feature_float, feature_cat=None):\n df['fe_mean'] = df[feature_float].mean(axis=1) \n df['fe_std'] = df[feature_float].std(axis=1) \n df['fe_median'] = df[feature_float].median(axis=1) \n df['fe_var'] = df[feature_float].var(axis=1) \n df['fe_min'] = df[feature_float].min(axis=1) \n df['fe_max'] = df[feature_float].max(axis=1) \n df['fe_skew'] = df[feature_float].skew(axis=1) \n df['fe_quantile_25'] = df[feature_float].quantile(q=.25, axis=1)\n df['fe_quantile_50'] = df[feature_float].quantile(q=.5, axis=1)\n df['fe_quantile_75'] = df[feature_float].quantile(q=.75, axis=1)\n \n if feature_cat is not None:\n df['fe_dammy_count'] = df[feature_cat].sum(axis=1) \n \n return df","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:09:36.727755Z","start_time":"2022-08-02T00:09:36.608719Z"},"code_folding":[0,7,52,71,116,139,182,194,199,227,236,258,269,274,287,311,319,327,338,353,399,412,428,447,455],"executionInfo":{"elapsed":2,"status":"ok","timestamp":1651517170887,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"BTy3Pk6NFlfG","papermill":{"duration":0.109388,"end_time":"2022-05-04T04:22:15.146487","exception":false,"start_time":"2022-05-04T04:22:15.037099","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:53.975607Z","iopub.execute_input":"2022-08-02T17:15:53.975990Z","iopub.status.idle":"2022-08-02T17:15:54.108910Z","shell.execute_reply.started":"2022-08-02T17:15:53.975958Z","shell.execute_reply":"2022-08-02T17:15:54.107898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"icecream, colors, color_cols = Ultil.jupyter_setting()","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:36:36.583821Z","start_time":"2022-08-01T21:36:36.575844Z"},"execution":{"iopub.status.busy":"2022-08-02T17:15:54.110944Z","iopub.execute_input":"2022-08-02T17:15:54.111400Z","iopub.status.idle":"2022-08-02T17:15:54.125285Z","shell.execute_reply.started":"2022-08-02T17:15:54.111356Z","shell.execute_reply":"2022-08-02T17:15:54.124419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{"id":"zB4XYQAXFlfG","papermill":{"duration":0.102787,"end_time":"2022-05-04T04:22:15.352132","exception":false,"start_time":"2022-05-04T04:22:15.249345","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1.4. Dataset","metadata":{"id":"MocWXt1J3EDP","papermill":{"duration":0.102473,"end_time":"2022-05-04T04:22:15.556924","exception":false,"start_time":"2022-05-04T04:22:15.454451","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 1.4.1. Descrição de dados\n\nOs para essa competição representam os resultados de um grande estudo de teste de produtos. Para cada product_code recebemo um número de produtos attributes(fixo para o código), bem como uma série de measurement valores para cada produto individual, representando vários métodos de teste de laboratório. Cada produto é usado em um experimento simulado de ambiente do mundo real e absorve uma certa quantidade de fluido (loading) para ver se falha ou não.\n\nNossa tarefa é usar os dados para prever falhas de novos códigos de produtos individuais com seus resultados de testes de laboratório individuais. \n\n\n### 1.4.2. Arquivos \n- train.csv - os dados de treinamento, que incluem a variável alvo(failure);\n- test.csv - o conjunto de teste; a tarefa é prever a probabilidade de uma falha;\n- sample_submission.csv - um arquivo de envio de amostra no formato correto","metadata":{"id":"tye_0Gh23EDQ","papermill":{"duration":0.102246,"end_time":"2022-05-04T04:22:15.761286","exception":false,"start_time":"2022-05-04T04:22:15.659040","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 1.4.3. Estrutura de pasta\nA finalidade é criar um estrutura de pasta para armazenar os artefatos criados no processo de análise e modelagem.","metadata":{"id":"ZJiYmMHZFlfG","papermill":{"duration":0.175883,"end_time":"2022-05-04T04:22:16.105234","exception":false,"start_time":"2022-05-04T04:22:15.929351","status":"completed"},"tags":[]}},{"cell_type":"code","source":"path = '../input/tabular-playground-series-aug-2022/' \npath_data = '' \npath_automl = 'automl/'\ntarget = 'failure'","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:21:28.624980Z","start_time":"2022-08-02T00:21:28.609960Z"},"executionInfo":{"elapsed":277,"status":"ok","timestamp":1651517176821,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"vaTHvoMk3EDQ","papermill":{"duration":0.156132,"end_time":"2022-05-04T04:22:16.430124","exception":false,"start_time":"2022-05-04T04:22:16.273992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:54.127248Z","iopub.execute_input":"2022-08-02T17:15:54.128050Z","iopub.status.idle":"2022-08-02T17:15:54.140230Z","shell.execute_reply.started":"2022-08-02T17:15:54.128004Z","shell.execute_reply":"2022-08-02T17:15:54.139013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = ['img', 'Data', 'Data/pkl', 'Data/submission', 'Data/tunning', \n 'model', 'model/preds', 'model/optuna','model/preds/test', 'Data/shap',\n 'model/preds/test/n1', 'model/preds/test/n2', 'model/preds/test/n3', \n 'model/preds/train', 'model/preds/train/n1', 'model/preds/train/n2', \n 'model/preds/train/n3', 'model/preds/param', 'model/mdl', 'model/preds/folds' ]\n\nfor p in paths:\n try:\n os.mkdir(p) \n except:\n pass ","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:47:23.824825Z","start_time":"2022-08-01T21:47:23.808830Z"},"execution":{"iopub.status.busy":"2022-08-02T17:15:54.142116Z","iopub.execute_input":"2022-08-02T17:15:54.142843Z","iopub.status.idle":"2022-08-02T17:15:54.152767Z","shell.execute_reply.started":"2022-08-02T17:15:54.142797Z","shell.execute_reply":"2022-08-02T17:15:54.151884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.4.4. Carrega dados","metadata":{"id":"5EmEWgvYFlfH","papermill":{"duration":0.101695,"end_time":"2022-05-04T04:22:16.634117","exception":false,"start_time":"2022-05-04T04:22:16.532422","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df1_train = pd.read_csv(path + path_data + 'train.csv')\ndf1_test = pd.read_csv(path + path_data + 'test.csv')\ndf_submission = pd.read_csv(path + path_data + 'sample_submission.csv')\n\ndf1_train.shape, df_submission.shape","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:47:25.758425Z","start_time":"2022-08-01T21:47:25.562393Z"},"executionInfo":{"elapsed":12975,"status":"ok","timestamp":1651517346987,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"BZO_GBDb3EDQ","outputId":"0583f64e-3968-45fa-8523-2e0ca256f79e","papermill":{"duration":12.648692,"end_time":"2022-05-04T04:22:29.596769","exception":false,"start_time":"2022-05-04T04:22:16.948077","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:54.156110Z","iopub.execute_input":"2022-08-02T17:15:54.157022Z","iopub.status.idle":"2022-08-02T17:15:54.445463Z","shell.execute_reply.started":"2022-08-02T17:15:54.156987Z","shell.execute_reply":"2022-08-02T17:15:54.444223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_train.head()","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:47:26.487049Z","start_time":"2022-08-01T21:47:26.465049Z"},"executionInfo":{"elapsed":261,"status":"ok","timestamp":1651517351546,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"15l3Wg3O3EDQ","outputId":"29a5a6fa-7d8b-4e6c-b12d-def887699427","papermill":{"duration":0.133399,"end_time":"2022-05-04T04:22:29.835330","exception":false,"start_time":"2022-05-04T04:22:29.701931","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:54.447183Z","iopub.execute_input":"2022-08-02T17:15:54.447802Z","iopub.status.idle":"2022-08-02T17:15:54.484792Z","shell.execute_reply.started":"2022-08-02T17:15:54.447767Z","shell.execute_reply":"2022-08-02T17:15:54.483731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{"id":"qi1l4r05FlfI","papermill":{"duration":0.1041,"end_time":"2022-05-04T04:22:30.282043","exception":false,"start_time":"2022-05-04T04:22:30.177943","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n

2. Análise Exploratória de Dados (EDA)

\n
\n\nAntes de trabalhar com qualquer tipo de dados é importante entendê-los. Um passo crucial para esse objetivo é a Análise Exploratória de Dados (EDA): uma combinação de visualizações e análise estatística (uni, bi e multivariada) que nos ajude a entender melhor os dados com os quais estamos trabalhando e obter insights sobre seus relacionamentos.\n","metadata":{"id":"WoOaHaA53EDS","papermill":{"duration":0.110636,"end_time":"2022-05-04T04:22:32.124310","exception":false,"start_time":"2022-05-04T04:22:32.013674","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df2_train = df1_train.copy()\ndf2_test = df1_test.copy()","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:27:59.791311Z","start_time":"2022-08-02T00:27:59.768276Z"},"executionInfo":{"elapsed":260,"status":"ok","timestamp":1651517362329,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"LhBEAJI4FlfJ","papermill":{"duration":0.170728,"end_time":"2022-05-04T04:22:32.400711","exception":false,"start_time":"2022-05-04T04:22:32.229983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:54.742598Z","iopub.execute_input":"2022-08-02T17:15:54.743067Z","iopub.status.idle":"2022-08-02T17:15:54.750314Z","shell.execute_reply.started":"2022-08-02T17:15:54.743029Z","shell.execute_reply":"2022-08-02T17:15:54.749069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.1. Dimensão do DataSet","metadata":{"id":"pFqhznWa3EDS","papermill":{"duration":0.105357,"end_time":"2022-05-04T04:22:32.611999","exception":false,"start_time":"2022-05-04T04:22:32.506642","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('TREINO')\nprint('Numero de linhas : {}'.format(df2_train.shape[0]))\nprint('Numero de colunas: {}'.format(df2_train.shape[1]), end='\\n\\n')","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:48:47.310450Z","start_time":"2022-08-01T21:48:47.300452Z"},"executionInfo":{"elapsed":260,"status":"ok","timestamp":1651517364891,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"Xh7IbAi93EDS","outputId":"7e40557f-000e-4e90-dca5-e09b27254985","papermill":{"duration":0.124815,"end_time":"2022-05-04T04:22:32.842121","exception":false,"start_time":"2022-05-04T04:22:32.717306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:57.867402Z","iopub.execute_input":"2022-08-02T17:15:57.867813Z","iopub.status.idle":"2022-08-02T17:15:57.873815Z","shell.execute_reply.started":"2022-08-02T17:15:57.867775Z","shell.execute_reply":"2022-08-02T17:15:57.872727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{"id":"chdM7_XIFlfJ","papermill":{"duration":0.105833,"end_time":"2022-05-04T04:22:33.052616","exception":false,"start_time":"2022-05-04T04:22:32.946783","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 2.2. Tipo de dados","metadata":{"id":"IbVYbhUO3EDS","papermill":{"duration":0.105077,"end_time":"2022-05-04T04:22:33.265969","exception":false,"start_time":"2022-05-04T04:22:33.160892","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df2_train.info()","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:48:54.891957Z","start_time":"2022-08-01T21:48:54.704960Z"},"executionInfo":{"elapsed":249,"status":"ok","timestamp":1651517368083,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"qWJZ4EgA3EDS","outputId":"f77e577f-0509-42f7-943a-9ff68b1454b7","papermill":{"duration":0.29537,"end_time":"2022-05-04T04:22:33.665805","exception":false,"start_time":"2022-05-04T04:22:33.370435","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:15:59.718293Z","iopub.execute_input":"2022-08-02T17:15:59.718731Z","iopub.status.idle":"2022-08-02T17:15:59.751974Z","shell.execute_reply.started":"2022-08-02T17:15:59.718689Z","shell.execute_reply":"2022-08-02T17:15:59.751064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'{3*\"=\"} For Pandas {10*\"=\"}\\n{(df2_train.dtypes).value_counts()}')","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:49:06.881618Z","start_time":"2022-08-01T21:49:06.873653Z"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1651517371986,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"WasdxyY83EDT","outputId":"0d9e3a12-0439-47de-9b7b-76ecc07ac7d7","papermill":{"duration":0.116422,"end_time":"2022-05-04T04:22:34.247132","exception":false,"start_time":"2022-05-04T04:22:34.130710","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:16:00.020144Z","iopub.execute_input":"2022-08-02T17:16:00.020556Z","iopub.status.idle":"2022-08-02T17:16:00.028383Z","shell.execute_reply.started":"2022-08-02T17:16:00.020521Z","shell.execute_reply":"2022-08-02T17:16:00.027227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_float = df2_train.select_dtypes(np.float64).columns.to_list() \nfeature_int = df2_train.select_dtypes(np.int64).columns.to_list() \nfeature_int.remove('id')","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:49:26.286258Z","start_time":"2022-08-01T21:49:25.929392Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:00.396411Z","iopub.execute_input":"2022-08-02T17:16:00.398369Z","iopub.status.idle":"2022-08-02T17:16:00.411512Z","shell.execute_reply.started":"2022-08-02T17:16:00.398313Z","shell.execute_reply":"2022-08-02T17:16:00.410414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Vamos dar uma olhada nas variáveis do tipo int64. ","metadata":{"ExecuteTime":{"end_time":"2022-05-02T00:10:59.293786Z","start_time":"2022-05-02T00:10:59.273751Z"},"id":"PdczFQmPFlfK","papermill":{"duration":0.106533,"end_time":"2022-05-04T04:22:34.459261","exception":false,"start_time":"2022-05-04T04:22:34.352728","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for col in df2_train[feature_int]: \n num = df2_train[col].unique().tolist()\n num.sort()\n print('-'* 70)\n print('{} unique: {}'.format(col, num)) \n print('-'* 70)\n print()","metadata":{"ExecuteTime":{"end_time":"2022-08-01T21:49:34.662178Z","start_time":"2022-08-01T21:49:34.641140Z"},"executionInfo":{"elapsed":277,"status":"ok","timestamp":1651517376183,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"tuK-oyKkFlfK","outputId":"06c2a9f8-e9da-4cd2-d22f-4ceaac6dd165","papermill":{"duration":0.204705,"end_time":"2022-05-04T04:22:34.769865","exception":false,"start_time":"2022-05-04T04:22:34.565160","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:16:00.946794Z","iopub.execute_input":"2022-08-02T17:16:00.947726Z","iopub.status.idle":"2022-08-02T17:16:00.956622Z","shell.execute_reply.started":"2022-08-02T17:16:00.947688Z","shell.execute_reply":"2022-08-02T17:16:00.955568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nPodemos observar que as variáveis: measurement_0, measurement_1 e measurement_2 seguem um ordem em relação as outras measurements, pode ser um indicativo que essas variáveis foram transformadas em ordinal no processo de geração dos datasets, sendo assim, temos que utilzar normalizador que não altere essa ordem, isso pode ser um diferencial para a modelagem.


\n \n

","metadata":{"id":"8UhDTTV5FlfK","papermill":{"duration":0.10939,"end_time":"2022-05-04T04:22:34.987352","exception":false,"start_time":"2022-05-04T04:22:34.877962","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"---","metadata":{"id":"QmmQYqi2FlfL","papermill":{"duration":0.108584,"end_time":"2022-05-04T04:22:38.930627","exception":false,"start_time":"2022-05-04T04:22:38.822043","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 2.3. Identificar NA","metadata":{"heading_collapsed":true,"id":"v9eqrfLE3EDT","papermill":{"duration":0.107266,"end_time":"2022-05-04T04:22:39.145057","exception":false,"start_time":"2022-05-04T04:22:39.037791","status":"completed"},"tags":[]}},{"cell_type":"code","source":"missing = Ultil.missing_zero_values_table(df2_train)\nmissing[:].style.background_gradient(cmap='Reds')","metadata":{"ExecuteTime":{"end_time":"2022-08-01T22:00:44.806140Z","start_time":"2022-08-01T22:00:44.493439Z"},"executionInfo":{"elapsed":1393,"status":"ok","timestamp":1651517401163,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"hidden":true,"id":"x2fit_153EDT","outputId":"32800401-4fcd-42ae-81b0-4e6015c5bf60","papermill":{"duration":2.730514,"end_time":"2022-05-04T04:22:41.982375","exception":false,"start_time":"2022-05-04T04:22:39.251861","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:16:02.275602Z","iopub.execute_input":"2022-08-02T17:16:02.276347Z","iopub.status.idle":"2022-08-02T17:16:02.370587Z","shell.execute_reply.started":"2022-08-02T17:16:02.276300Z","shell.execute_reply":"2022-08-02T17:16:02.369469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nComo podemos observar acima, o percentual de dados faltantes das variáveis é baixo, neste casso no primeiro momento vamos fazer a inputação da mediana para o modelo de linha de base.


\n \n

","metadata":{"hidden":true,"id":"qB6Pwjd_3EDW","papermill":{"duration":0.10922,"end_time":"2022-05-04T04:22:44.439110","exception":false,"start_time":"2022-05-04T04:22:44.329890","status":"completed"},"tags":[]}},{"cell_type":"code","source":"duplicates = df2_train['id'].duplicated().sum()\nprint('Duplicates in data: {0}'.format(duplicates))","metadata":{"ExecuteTime":{"end_time":"2022-08-01T22:21:03.055899Z","start_time":"2022-08-01T22:21:03.035861Z"},"hidden":true,"execution":{"iopub.status.busy":"2022-08-02T17:16:03.383922Z","iopub.execute_input":"2022-08-02T17:16:03.384328Z","iopub.status.idle":"2022-08-02T17:16:03.391350Z","shell.execute_reply.started":"2022-08-02T17:16:03.384293Z","shell.execute_reply":"2022-08-02T17:16:03.390113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.4 Estatística Descritiva\nAbaixo estão as estatísticas básicas para cada variável que contém informações sobre contagem, média, desvio padrão, mínimo, 1º quartil, mediana, 3º quartil e máximo.","metadata":{"id":"5rLwDaIx3EDW","papermill":{"duration":0.109184,"end_time":"2022-05-04T04:22:44.657432","exception":false,"start_time":"2022-05-04T04:22:44.548248","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.4.1. Descritivo","metadata":{}},{"cell_type":"code","source":"df2_train.drop('id', axis=1).describe().T.style.background_gradient(cmap='YlOrRd')","metadata":{"ExecuteTime":{"end_time":"2022-08-01T22:21:06.828655Z","start_time":"2022-08-01T22:21:06.725641Z"},"executionInfo":{"elapsed":2893,"status":"ok","timestamp":1651517413954,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"6NTHpUNo3EDX","outputId":"27c68a8f-d460-4422-95c8-0b017eac5040","papermill":{"duration":2.408593,"end_time":"2022-05-04T04:22:48.002923","exception":false,"start_time":"2022-05-04T04:22:45.594330","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:16:04.521822Z","iopub.execute_input":"2022-08-02T17:16:04.522596Z","iopub.status.idle":"2022-08-02T17:16:04.637587Z","shell.execute_reply.started":"2022-08-02T17:16:04.522559Z","shell.execute_reply":"2022-08-02T17:16:04.636340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nAs variáveis que mais me chamaram a atenção são measurement_17 e loading com média e desvio padrão muito alto em relação as outras variáveis.


\n \n

","metadata":{"id":"_MPplFrXFlfN","papermill":{"duration":0.110521,"end_time":"2022-05-04T04:22:48.238707","exception":false,"start_time":"2022-05-04T04:22:48.128186","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.4.2. Gráfico de Variância","metadata":{"ExecuteTime":{"end_time":"2022-07-21T02:18:35.178987Z","start_time":"2022-07-21T02:18:35.167027Z"}}},{"cell_type":"code","source":"threshold = 2.2\nselector = VarianceThreshold(threshold=threshold)\ndf_aux = df2_train.select_dtypes(np.number).drop(['id','failure'], axis=1)\nselector.fit_transform(df_aux)\n\nplt.figure(figsize=(20,7))\nsns.barplot(y=selector.variances_, x=df_aux.columns, orient='v')\nplt.title('Seleção de variáveis com VarianceThreshold',size=15);\nplt.ylabel('Variância');\nplt.axhline(y=threshold, color='r', linestyle='--', label='Threshold')\nplt.xticks(rotation=70)\nplt.legend();\n\ndel df_aux","metadata":{"ExecuteTime":{"end_time":"2022-08-01T23:15:19.136626Z","start_time":"2022-08-01T23:15:18.811600Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:05.288795Z","iopub.execute_input":"2022-08-02T17:16:05.289894Z","iopub.status.idle":"2022-08-02T17:16:05.777921Z","shell.execute_reply.started":"2022-08-02T17:16:05.289850Z","shell.execute_reply":"2022-08-02T17:16:05.776729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nAcima observamos duas variáveis se destacando com alta variância, vamos verificar a variância por tipo de variáveis:


\n \n

","metadata":{}},{"cell_type":"code","source":"threshold = 2.2\nselector = VarianceThreshold(threshold=threshold)\ndf_aux = df2_train.select_dtypes(np.int).drop(['id','failure'], axis=1)\nselector.fit_transform(df_aux)\n\nplt.figure(figsize=(20,5))\nsns.barplot(y=selector.variances_, x=df_aux.columns, orient='v')\nplt.title('Seleção de variáveis com VarianceThreshold',size=15);\n\nplt.ylabel('Variância');\nplt.axhline(y=threshold, color='r', linestyle='--', label='Threshold')\nplt.legend();\n\ndel df_aux","metadata":{"ExecuteTime":{"end_time":"2022-08-01T23:45:48.604945Z","start_time":"2022-08-01T23:45:48.417892Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:05.782401Z","iopub.execute_input":"2022-08-02T17:16:05.782920Z","iopub.status.idle":"2022-08-02T17:16:06.030923Z","shell.execute_reply.started":"2022-08-02T17:16:05.782885Z","shell.execute_reply":"2022-08-02T17:16:06.029585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nPor tipo de variável, neste caso int, emos uma melhor visualização e entendimento da variância, temos três variáveis com alta variância, sendo que duas se destacam: measurement_0 e measurement_1 e duas com baixa variância: attribute_2 e attribute_3. \n \n
","metadata":{}},{"cell_type":"code","source":"threshold = 2.5\nselector = VarianceThreshold(threshold=threshold)\ndf_aux = df2_train.select_dtypes(np.float64)\nselector.fit_transform(df_aux)\n\nplt.figure(figsize=(20,5))\nsns.barplot(y=selector.variances_, x=df_aux.columns, orient='v')\n\nplt.xticks(rotation=70)\nplt.title('Seleção de variáveis com VarianceThreshold',size=15);\nplt.ylabel('Variância')\nplt.axhline(y=threshold, color='r', linestyle='--', label='Threshold')\nplt.legend()\n\ndel df_aux;","metadata":{"ExecuteTime":{"end_time":"2022-08-01T23:45:56.693768Z","start_time":"2022-08-01T23:45:56.381737Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:06.241901Z","iopub.execute_input":"2022-08-02T17:16:06.242290Z","iopub.status.idle":"2022-08-02T17:16:06.606964Z","shell.execute_reply.started":"2022-08-02T17:16:06.242258Z","shell.execute_reply":"2022-08-02T17:16:06.605808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nAs variáveis loading e measurement_1 se destacam em relação as outras variáveis, vamos excluí-las do gráfico para termos um melhor entendimento das outras variáveis. \n \n
","metadata":{"ExecuteTime":{"end_time":"2022-08-01T23:27:00.994469Z","start_time":"2022-08-01T23:27:00.980472Z"}}},{"cell_type":"code","source":"\nthreshold = 1\nselector = VarianceThreshold(threshold=threshold)\ndf_aux = df2_train.select_dtypes(np.float64).drop(['measurement_17', 'loading'], axis=1)\nselector.fit_transform(df_aux)\n\nplt.figure(figsize=(20,5))\nsns.barplot(y=selector.variances_, x=df_aux.columns, orient='v')\n\nplt.xticks(rotation=70)\nplt.title('Seleção de variáveis com VarianceThreshold',size=15);\nplt.ylabel('Variância')\nplt.axhline(y=threshold, color='r', linestyle='--', label='Threshold')\nplt.legend()\n\ndel df_aux;","metadata":{"ExecuteTime":{"end_time":"2022-08-01T23:46:03.543331Z","start_time":"2022-08-01T23:46:03.253324Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:06.644954Z","iopub.execute_input":"2022-08-02T17:16:06.645720Z","iopub.status.idle":"2022-08-02T17:16:06.985618Z","shell.execute_reply.started":"2022-08-02T17:16:06.645674Z","shell.execute_reply":"2022-08-02T17:16:06.984775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nAgora temos um melhor entendimento da variância com os gráficos acima, podemos destacar: \n- Entre as variávies do inteiro temos duas com alta variância e duas com baixa variância, a terceira ( measurement_2) pode ser uma variável segnificativa na modelagem;

\n \n- A loading e measurement_1 são as variáveis que tem maior variância entre as variáveis continuas;

\n \n- As variáveis entre measurement_1 até a measurement_9 tem baixa variância entre as variáveis continuas, provavelmente não são relevantes para modelagem. \n \n \n


\n \n \n

","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{"id":"Qw-DiwyOFlfN","papermill":{"duration":0.110067,"end_time":"2022-05-04T04:22:48.459283","exception":false,"start_time":"2022-05-04T04:22:48.349216","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 2.5. Distribuição","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 5))\n\npie = ax.pie([len(df1_train), len(df1_test)],\n labels = [\"Train dataset\", \"Test dataset\"],\n colors = [\"salmon\", \"teal\"],\n textprops= {\"fontsize\": 15},\n autopct = '%1.1f%%')\n\nax.axis(\"equal\")\nax.set_title(\"Comparação de comprimento do conjunto de dados \\n\", fontsize=18)\nfig.set_facecolor('white')\nplt.show();","metadata":{"ExecuteTime":{"end_time":"2022-08-01T23:58:34.917893Z","start_time":"2022-08-01T23:58:34.828871Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:08.385038Z","iopub.execute_input":"2022-08-02T17:16:08.386044Z","iopub.status.idle":"2022-08-02T17:16:08.511672Z","shell.execute_reply.started":"2022-08-02T17:16:08.386007Z","shell.execute_reply":"2022-08-02T17:16:08.510356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5.1. Distribuição Train x Test","metadata":{}},{"cell_type":"code","source":"feature_float = df2_test.drop('id', axis=1).select_dtypes(np.number).columns.to_list()\n\n\nlines = int(len(feature_float)/2)+1\nfig, ax = plt.subplots(lines,2 ,figsize=(20,35))\n\nfor i,feature in enumerate(feature_float):\n plt.subplot(lines,2,i+1)\n sns.histplot(data=df2_train, x=df2_train[feature],color='blue', alpha=0.5, label='train', bins=100)\n sns.histplot(data=df2_test , x=df2_test[feature] ,color='teal', alpha=0.5, label='test' , bins=100) \n plt.xlabel(feature, fontsize=12)\n plt.legend()\n \nplt.suptitle('DistPlot: train & test data', fontsize=20)\nplt.tight_layout(pad=3.0);","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:34:38.008779Z","start_time":"2022-08-02T00:34:26.054220Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:09.944422Z","iopub.execute_input":"2022-08-02T17:16:09.944847Z","iopub.status.idle":"2022-08-02T17:16:23.992388Z","shell.execute_reply.started":"2022-08-02T17:16:09.944811Z","shell.execute_reply":"2022-08-02T17:16:23.990964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nAmbos os datasets seguem a mesma distribuição.\n \n
","metadata":{}},{"cell_type":"markdown","source":"### 2.5.2. Proporção de variáveis","metadata":{"run_control":{"marked":false}}},{"cell_type":"code","source":"feature_cat = df2_train.drop(['id', 'failure', 'attribute_1', 'attribute_2'], axis=1) \\\n .select_dtypes(int).columns.to_list()\nfeature_float = df2_train.select_dtypes(np.float64).columns.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-08-02T17:16:23.994447Z","iopub.execute_input":"2022-08-02T17:16:23.994837Z","iopub.status.idle":"2022-08-02T17:16:24.008651Z","shell.execute_reply.started":"2022-08-02T17:16:23.994794Z","shell.execute_reply":"2022-08-02T17:16:24.007771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 5))\n\nplt.pie([ len(feature_cat), len(feature_float)], \n labels=['Categorical', 'Continuos' ],\n textprops={'fontsize': 13},\n autopct='%1.1f%%')\n\n#ax.axis(\"equal\")\nax.set_title(\"Comparação variáveis continuas/categóricas \\n Dataset Treino/Teste\", fontsize=18)\nfig.set_facecolor('white')\nplt.show()","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:05:51.463151Z","start_time":"2022-08-02T00:05:51.367159Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:24.009950Z","iopub.execute_input":"2022-08-02T17:16:24.011108Z","iopub.status.idle":"2022-08-02T17:16:24.135939Z","shell.execute_reply.started":"2022-08-02T17:16:24.011065Z","shell.execute_reply":"2022-08-02T17:16:24.134633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5.3. Variável predidora","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,6)) \nUltil.graf_bar(df2_train.groupby(['failure'])['failure'].count(), \n icecream, 'Distribuição da variável alvo', 'failure', \n 'Quantidade de produto com falha');","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:12:08.055524Z","start_time":"2022-08-02T00:12:07.869508Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:24.139455Z","iopub.execute_input":"2022-08-02T17:16:24.140216Z","iopub.status.idle":"2022-08-02T17:16:24.324027Z","shell.execute_reply.started":"2022-08-02T17:16:24.140165Z","shell.execute_reply":"2022-08-02T17:16:24.322940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nObservamos no gráfico acima o desbalanceamento da variável alvo, teste de balanceamento podem ser uma solução para ajuda na modelagem.


\n \n \n

","metadata":{}},{"cell_type":"markdown","source":"## 2.6. Dados Númericos","metadata":{"ExecuteTime":{"end_time":"2022-05-02T02:41:58.832609Z","start_time":"2022-05-02T02:41:58.823610Z"},"id":"QsdWcChoFlfP","papermill":{"duration":0.122441,"end_time":"2022-05-04T04:24:23.584315","exception":false,"start_time":"2022-05-04T04:24:23.461874","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.6.1. Correlação","metadata":{"ExecuteTime":{"end_time":"2022-05-02T02:42:21.060547Z","start_time":"2022-05-02T02:42:21.054549Z"},"id":"Hd0_u8dRFlfP","papermill":{"duration":0.121916,"end_time":"2022-05-04T04:24:23.828033","exception":false,"start_time":"2022-05-04T04:24:23.706117","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sns.set(style=\"whitegrid\", palette=\"pastel\")\nplt.figure(figsize=(20,15))\n\nfeature_corr = \\\n Ultil.graf_feature_corr(df_ = df2_train.drop('id', axis=1).copy(), \n annot_ = True, \n threshold_ = .1, \n print_var_ = False, \n print_graf_ = True, \n mask_ = True);","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:16:40.754450Z","start_time":"2022-08-02T00:16:39.125381Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:24.325999Z","iopub.execute_input":"2022-08-02T17:16:24.326470Z","iopub.status.idle":"2022-08-02T17:16:25.905481Z","shell.execute_reply.started":"2022-08-02T17:16:24.326425Z","shell.execute_reply":"2022-08-02T17:16:25.904447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n\nNão temos autocorrelação entre as variáveis, principalmete com a variável alvo que é muito baixa, a variável com maior correlação é loading com 0.13. Nesta competição quem tive aptidão de criar novas variáveis com relevância será o diferêncial.


\n \n \n

","metadata":{"id":"nxFtCqAqFlfQ","papermill":{"duration":0.123237,"end_time":"2022-05-04T04:24:25.777454","exception":false,"start_time":"2022-05-04T04:24:25.654217","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.5.2. Histograma\n\nVamos plotar apenas as variáveis float. ","metadata":{"ExecuteTime":{"end_time":"2022-05-02T02:51:23.699986Z","start_time":"2022-05-02T02:51:23.685019Z"},"id":"MSiARrovFlfQ","papermill":{"duration":0.124984,"end_time":"2022-05-04T04:24:26.025993","exception":false,"start_time":"2022-05-04T04:24:25.901009","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sns.set(style=\"darkgrid\")","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:16:48.636115Z","start_time":"2022-08-02T00:16:48.622117Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:25.906949Z","iopub.execute_input":"2022-08-02T17:16:25.907303Z","iopub.status.idle":"2022-08-02T17:16:25.913279Z","shell.execute_reply.started":"2022-08-02T17:16:25.907271Z","shell.execute_reply":"2022-08-02T17:16:25.912179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20, 20))\n\nfor i, col in enumerate(feature_float): \n plt.subplot(int(len(feature_float)/3 +1),3,i+1)\n sns.kdeplot(data=df2_train, x=col, hue=target, legend=True, shade=True, multiple='stack');\n \nplt.tight_layout(pad=3.0)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:22:33.151667Z","start_time":"2022-08-02T00:22:28.670385Z"},"executionInfo":{"elapsed":78388,"status":"ok","timestamp":1651518090777,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"Et365dgPFlfQ","outputId":"112ffa89-95fe-4d7a-f378-38486734b049","papermill":{"duration":82.980144,"end_time":"2022-05-04T04:25:49.129212","exception":false,"start_time":"2022-05-04T04:24:26.149068","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:16:25.915053Z","iopub.execute_input":"2022-08-02T17:16:25.915430Z","iopub.status.idle":"2022-08-02T17:16:32.568038Z","shell.execute_reply.started":"2022-08-02T17:16:25.915397Z","shell.execute_reply":"2022-08-02T17:16:32.566847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n \n


\n \n

","metadata":{"id":"pXS0c3pkFlfQ","papermill":{"duration":0.14334,"end_time":"2022-05-04T04:25:49.416179","exception":false,"start_time":"2022-05-04T04:25:49.272839","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.5.3. Outliers","metadata":{"id":"HCNjqWhcFlfQ","papermill":{"duration":0.148186,"end_time":"2022-05-04T04:25:49.996007","exception":false,"start_time":"2022-05-04T04:25:49.847821","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def iqr_outliers(df,ft):\n \n q1 = df[ft].quantile(0.25)\n q3 = df[ft].quantile(0.75)\n iqr = q3-q1\n c = 0 \n \n Lower_tail = q1 - 1.5 * iqr\n Upper_tail = q3 + 1.5 * iqr\n \n for i in range(len(df[ft])):\n if df[ft][i] > Upper_tail or df[ft][i] < Lower_tail:\n c+=1\n return c","metadata":{"ExecuteTime":{"end_time":"2022-08-02T00:57:18.699002Z","start_time":"2022-08-02T00:57:18.686005Z"},"code_folding":[0],"execution":{"iopub.status.busy":"2022-08-02T17:16:32.569434Z","iopub.execute_input":"2022-08-02T17:16:32.569885Z","iopub.status.idle":"2022-08-02T17:16:32.580272Z","shell.execute_reply.started":"2022-08-02T17:16:32.569851Z","shell.execute_reply":"2022-08-02T17:16:32.578861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(20, 30))\nfeature = df2_train.select_dtypes(np.number).columns.drop(['id', target]).to_list() \nrow = int(len(feature)/3 +1)\n\nfor i, col in enumerate(feature): \n plt.subplot(row, 3, i+1)\n sns.boxplot(data=df2_train, y=col, x=target) \n plt.xlabel('')\n \nplt.tight_layout(pad=3.0)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:01:14.652574Z","start_time":"2022-08-02T01:01:11.390914Z"},"executionInfo":{"elapsed":5609,"status":"ok","timestamp":1651518288040,"user":{"displayName":"Rogério Delfim","userId":"04235763959036945343"},"user_tz":180},"id":"_Yu0BtkpFlfR","outputId":"e0383791-53dd-4007-bc20-7c3a8414dbf1","papermill":{"duration":5.486634,"end_time":"2022-05-04T04:25:55.627343","exception":false,"start_time":"2022-05-04T04:25:50.140709","status":"completed"},"run_control":{"marked":false},"tags":[],"execution":{"iopub.status.busy":"2022-08-02T17:16:32.581900Z","iopub.execute_input":"2022-08-02T17:16:32.582953Z","iopub.status.idle":"2022-08-02T17:16:35.811687Z","shell.execute_reply.started":"2022-08-02T17:16:32.582906Z","shell.execute_reply":"2022-08-02T17:16:35.810830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out = []\nod = {f:iqr_outliers(df2_train,f) for f in feature}\n\n# Plotting Outliers\nplt.figure(figsize=(16,5))\nplt.bar(x=od.keys(),height=od.values())\nplt.xlabel(\"Features\")\nplt.ylabel(\"Outliers\")\nplt.title('Outliers')\nplt.xticks(rotation=70)\nplt.show()","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:02:48.971604Z","start_time":"2022-08-02T01:02:42.808581Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:35.815052Z","iopub.execute_input":"2022-08-02T17:16:35.815555Z","iopub.status.idle":"2022-08-02T17:16:44.125206Z","shell.execute_reply.started":"2022-08-02T17:16:35.815523Z","shell.execute_reply":"2022-08-02T17:16:44.123780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n \n- Todas as variáveis tem outliers, talvez a criação de uma variável que indique que aquele dados (amostra) é um outlier pode ajudar na modelagem, podemos utilizar o intervalo interquartil para identificar os outliers e criar a nova variável;

\n \n- A variável loading se destaca com maior número de outliers, seguidas das variáveis measurements_0, measurements_10 e measurements_13..


\n \n \n

","metadata":{"id":"rZe2nX__FlfR","papermill":{"duration":0.2323,"end_time":"2022-05-04T04:25:55.990732","exception":false,"start_time":"2022-05-04T04:25:55.758432","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.8.4. Variáveis discretas","metadata":{"ExecuteTime":{"end_time":"2022-07-20T20:35:22.600355Z","start_time":"2022-07-20T20:35:22.576353Z"}}},{"cell_type":"code","source":"feature = feature_int.copy()\nfeature.append('attribute_1')\nfeature.append('product_code')\nfeature.remove(target)\nfeature.sort()\n\nfeature.append(target)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:43:08.945323Z","start_time":"2022-08-02T01:43:08.927329Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:44.126976Z","iopub.execute_input":"2022-08-02T17:16:44.127334Z","iopub.status.idle":"2022-08-02T17:16:44.133155Z","shell.execute_reply.started":"2022-08-02T17:16:44.127300Z","shell.execute_reply":"2022-08-02T17:16:44.132304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,20))\n\nrow = int(len(feature)/2)+1\n\nfor i, col in enumerate(feature):\n plt.subplot(row, 2, i+1)\n ax = sns.countplot(data=df2_train, x=col, hue=target) \n \nplt.tight_layout(pad=3.0)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:43:11.993930Z","start_time":"2022-08-02T01:43:10.095395Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:44.134524Z","iopub.execute_input":"2022-08-02T17:16:44.135031Z","iopub.status.idle":"2022-08-02T17:16:46.613073Z","shell.execute_reply.started":"2022-08-02T17:16:44.134998Z","shell.execute_reply":"2022-08-02T17:16:46.611857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n \nPodemos destacar os sequintes ponto: \n- A variável attribute_2 na categoria 6 tanto para classe positiva e negativa parece ter uma relevância maior que as outras categoria, transforma essa variável em one hot encoding ou outra transformação pode gerar bons resultado na modelagem, o mesmo pode ser adotado com as variáveis: attribute_1, attribute_3;

\n \n- As variáveis measuremente_0, measuremente_1 e measuremente_2, podemos criar uma classe que agrupem os valores com pouca frequência, o que pode reduzir os outliers.


\n \n

","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\n\ndf = df2_train[feature].astype(object)\ndf[target] = df2_train[target].astype(object)\n\ncorr = Ultil.generate_category_table(df)\nmask = np.triu(np.ones_like(corr, dtype=bool))\nsns.heatmap(corr, annot=True, mask= mask). \\\n set_title('Mapa de calor de correlação das variável categóricas', fontsize=17);\n\ndel df","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:43:19.180540Z","start_time":"2022-08-02T01:43:18.050562Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:46.614545Z","iopub.execute_input":"2022-08-02T17:16:46.615583Z","iopub.status.idle":"2022-08-02T17:16:48.138280Z","shell.execute_reply.started":"2022-08-02T17:16:46.615537Z","shell.execute_reply":"2022-08-02T17:16:48.137119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

NOTA:

\n \nObservamos que ao fazemos a transformação das variáveis do tipo interio para object melhoramos a correlação com a variável alvo. \n \n
","metadata":{}},{"cell_type":"markdown","source":"
\n

3. Modelagem (baseline)

\n
\n","metadata":{"id":"wXr6ZlxXFlfR","papermill":{"duration":0.202469,"end_time":"2022-05-04T04:25:56.440655","exception":false,"start_time":"2022-05-04T04:25:56.238186","status":"completed"},"tags":[]}},{"cell_type":"code","source":"seed = 12359\ndf3_train = df2_train.copy().drop('id', axis=1)\ndf3_test = df2_test.copy().drop('id', axis=1)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:47:56.853284Z","start_time":"2022-08-02T01:47:56.834288Z"},"execution":{"iopub.status.busy":"2022-08-02T17:16:48.139771Z","iopub.execute_input":"2022-08-02T17:16:48.140799Z","iopub.status.idle":"2022-08-02T17:16:48.154262Z","shell.execute_reply.started":"2022-08-02T17:16:48.140757Z","shell.execute_reply":"2022-08-02T17:16:48.152735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cat = [col for col in df3_test.columns if df3_test[col].dtype in [\"object\", \"int64\"]]\nfeature_num = [col for col in df3_test.columns if col not in feature_cat]","metadata":{"ExecuteTime":{"end_time":"2022-08-02T01:57:14.246564Z","start_time":"2022-08-02T01:57:14.239559Z"},"execution":{"iopub.status.busy":"2022-08-02T17:32:18.471774Z","iopub.execute_input":"2022-08-02T17:32:18.472157Z","iopub.status.idle":"2022-08-02T17:32:18.478443Z","shell.execute_reply.started":"2022-08-02T17:32:18.472126Z","shell.execute_reply":"2022-08-02T17:32:18.477485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_measurement = [i for i in df3_train.columns if \"measurement\" in i]\nfeature_measurement_int = [i for i in df3_train if df3_train[i].dtype == int]\nfeature_measurement_float = [i for i in df3_train.columns if df3_train[i].dtype == float]\nfeature_attribute = [i for i in df3_train.columns if \"attribute\" in i]\nfloat_cols = [i for i in df3_train.columns if df3_train[i].dtype == float]","metadata":{"execution":{"iopub.status.busy":"2022-08-02T17:31:52.330128Z","iopub.execute_input":"2022-08-02T17:31:52.330912Z","iopub.status.idle":"2022-08-02T17:31:52.339165Z","shell.execute_reply.started":"2022-08-02T17:31:52.330874Z","shell.execute_reply":"2022-08-02T17:31:52.338100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.1. Processamento","metadata":{"id":"h4EpRC8aFlfS","papermill":{"duration":0.144185,"end_time":"2022-05-04T04:26:02.896706","exception":false,"start_time":"2022-05-04T04:26:02.752521","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 3.1.1. Pipeline","metadata":{}},{"cell_type":"code","source":"params = {'seed': seed,\n 'n_jobs': -1,\n #'lambda_l2': 2,\n 'num_leaves': 100,\n #'boosting': 'dart',\n #'bagging_freq': 10, \n 'learning_rate': 0.01,\n 'objective': 'binary', \n #'min_data_in_leaf': 40,\n #'num_boost_round': 1000,\n #'feature_fraction': 0.20,\n #'bagging_fraction': 0.50, \n 'metric': \"binary_logloss\"}\n\nmodel_lgbm = LGBMClassifier(**params)\nmodel_lr_cv= LogisticRegressionCV(penalty = 'elasticnet', \n l1_ratios = np.arange(0, 1, 11),\n solver = 'saga', \n max_iter = 10000,\n random_state = seed)\n\nmodel_lr = LogisticRegression(penalty = 'elasticnet', \n l1_ratio = 0.8, \n C = 0.007, \n tol = 1e-2, \n solver = 'saga', \n max_iter = 1000, \n random_state = seed)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:29:19.312896Z","start_time":"2022-08-02T02:29:19.292912Z"},"execution":{"iopub.status.busy":"2022-08-02T18:33:23.449919Z","iopub.execute_input":"2022-08-02T18:33:23.450564Z","iopub.status.idle":"2022-08-02T18:33:23.459427Z","shell.execute_reply.started":"2022-08-02T18:33:23.450525Z","shell.execute_reply":"2022-08-02T18:33:23.458191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pré-processamento de dados categóricos\nnumerical_transformer = Pipeline(steps=[\n ('imputer', SimpleImputer(strategy='median')),\n ('scaler', PowerTransformer())\n])\n\n# Pré-processamento de dados categóricos\ncategorical_transformer = Pipeline(steps=[\n ('imputer', SimpleImputer(strategy='most_frequent')),\n ('onehot', OneHotEncoder(handle_unknown='ignore'))\n])\n\n# Pré-processamento de pacote para dados numéricos e categóricos\npreprocessor = ColumnTransformer(\n transformers=[\n ('num', numerical_transformer, feature_num),\n ('cat', categorical_transformer, feature_cat)\n ])\n\npipeline_lgbm = Pipeline(steps=[\n ('preprocessor', preprocessor),\n ('model', model_lgbm)\n])\n\npipeline_lr_cv = Pipeline(steps=[\n ('preprocessor', preprocessor),\n ('model', model_lr_cv)\n])\n\npipeline_lr = Pipeline(steps=[\n ('preprocessor', preprocessor),\n ('model', model_lr)\n])","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:29:40.743364Z","start_time":"2022-08-02T02:29:40.735331Z"},"run_control":{"marked":false},"execution":{"iopub.status.busy":"2022-08-02T18:30:48.854177Z","iopub.execute_input":"2022-08-02T18:30:48.854587Z","iopub.status.idle":"2022-08-02T18:30:48.863251Z","shell.execute_reply.started":"2022-08-02T18:30:48.854555Z","shell.execute_reply":"2022-08-02T18:30:48.862285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.4. Validação Cruzada","metadata":{"id":"DoT5P1-rFlfT","papermill":{"duration":0.228414,"end_time":"2022-05-04T04:27:02.274304","exception":false,"start_time":"2022-05-04T04:27:02.045890","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X = df3_train.drop(target, axis=1)\ny = df3_train[target]\nX_test = df3_test.copy() \ngroups = df3_train['product_code']","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:29:48.053409Z","start_time":"2022-08-02T02:29:48.029375Z"},"execution":{"iopub.status.busy":"2022-08-02T18:33:28.277728Z","iopub.execute_input":"2022-08-02T18:33:28.278821Z","iopub.status.idle":"2022-08-02T18:33:28.289197Z","shell.execute_reply.started":"2022-08-02T18:33:28.278774Z","shell.execute_reply":"2022-08-02T18:33:28.288218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_train (model, X_, y_, X_ts_, groups_): \n\n preds_oof = np.zeros(len(X_))\n preds_test = np.zeros(len(X_ts_))\n\n folds = GroupKFold(n_splits=5)\n\n for fold_idx, (train_idx, val_idx) in enumerate(folds.split(X_, y_, groups_)):\n\n x_train, x_val = X_.iloc[train_idx], X_.iloc[val_idx]\n y_train, y_val = y_.iloc[train_idx], y_.iloc[val_idx]\n\n model.fit(x_train, y_train)\n\n val_preds = model.predict_proba(x_val)[:, 1]\n\n\n print(\"FOLD: {} - ROC-AUC: {:2.5f}\".format(fold_idx+1, roc_auc_score(y_val, val_preds)))\n\n preds_test += model.predict_proba(X_ts_)[:, 1] / 5\n preds_oof[val_idx] = val_preds \n\n print()\n print(\"OOF ROC-AUC: {:2.5f}\".format(roc_auc_score(y, preds_oof)))\n \n return preds_test ","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:30:37.041140Z","start_time":"2022-08-02T02:30:32.629634Z"},"execution":{"iopub.status.busy":"2022-08-02T18:33:29.458220Z","iopub.execute_input":"2022-08-02T18:33:29.458979Z","iopub.status.idle":"2022-08-02T18:33:29.469113Z","shell.execute_reply.started":"2022-08-02T18:33:29.458941Z","shell.execute_reply":"2022-08-02T18:33:29.467977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test = model_train(pipeline_lgbm, X, y, X_test, groups)\n# OOF ROC-AUC: 0.56871","metadata":{"execution":{"iopub.status.busy":"2022-08-02T18:33:46.662576Z","iopub.execute_input":"2022-08-02T18:33:46.662987Z","iopub.status.idle":"2022-08-02T18:33:56.853074Z","shell.execute_reply.started":"2022-08-02T18:33:46.662953Z","shell.execute_reply":"2022-08-02T18:33:56.851304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test = model_train(pipeline_lr, X, y, X_test, groups)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T18:29:03.153205Z","iopub.execute_input":"2022-08-02T18:29:03.153941Z","iopub.status.idle":"2022-08-02T18:29:07.310241Z","shell.execute_reply.started":"2022-08-02T18:29:03.153892Z","shell.execute_reply":"2022-08-02T18:29:07.309026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\npipeline_lr_cv = Pipeline(steps=[\n ('preprocessor', preprocessor),\n ('model', model_lr_cv)\n])\n\npipeline_lr_cv.fit(X, y)\npreds_trai_lin = pipeline_lr_cv.predict_proba(X)[:, 1]\npreds_test_lin = pipeline_lr_cv.predict_proba(X_test)[:, 1]\n\nprint(\"OOF ROC-AUC: {:2.5f}\".format(roc_auc_score(y, preds_trai_lin)))\n# 0.59015","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:42:44.806934Z","start_time":"2022-08-02T02:39:29.195289Z"},"execution":{"iopub.status.busy":"2022-08-02T18:36:17.918972Z","iopub.execute_input":"2022-08-02T18:36:17.919381Z","iopub.status.idle":"2022-08-02T18:41:35.570279Z","shell.execute_reply.started":"2022-08-02T18:36:17.919345Z","shell.execute_reply":"2022-08-02T18:41:35.569085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gerar submission","metadata":{}},{"cell_type":"code","source":"df_submission[target] = preds_test_lin # ( (preds_test * 0.15) + (preds_test_lin * 0.85))\ndf_submission.to_csv('submission_0001.csv', index=False)","metadata":{"ExecuteTime":{"end_time":"2022-08-02T02:45:19.338506Z","start_time":"2022-08-02T02:45:19.228472Z"},"execution":{"iopub.status.busy":"2022-08-02T18:52:53.986733Z","iopub.execute_input":"2022-08-02T18:52:53.987141Z","iopub.status.idle":"2022-08-02T18:52:54.042139Z","shell.execute_reply.started":"2022-08-02T18:52:53.987106Z","shell.execute_reply":"2022-08-02T18:52:54.041209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}