{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom sklearn.cluster import KMeans\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.ensemble import RandomForestRegressor\nfrom scipy.stats import pointbiserialr\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import make_scorer, accuracy_score\nfrom sklearn.impute import KNNImputer\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.linear_model import Lasso\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import  Pipeline\nfrom sklearn.feature_selection import VarianceThreshold\nfrom imblearn.over_sampling import RandomOverSampler,SMOTE,ADASYN\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score, classification_report\nfrom imblearn.under_sampling import RandomUnderSampler, CondensedNearestNeighbour,TomekLinks,OneSidedSelection,EditedNearestNeighbours\n\nfrom sklearn.ensemble import (\n    RandomForestClassifier,\n    BaggingClassifier,\n    AdaBoostClassifier\n)\n\nfrom imblearn.ensemble import (\n    BalancedBaggingClassifier,\n    BalancedRandomForestClassifier,\n    RUSBoostClassifier,\n    EasyEnsembleClassifier\n)\n\nfrom sklearn.metrics import balanced_accuracy_score\nfrom scipy.stats import mode\nimport numpy as np\nfrom scipy import stats\nfrom sklearn.preprocessing import StandardScaler","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_classificador_novo_categoria(X_train, y_train, X_test, y_test):\n    # Pre-processamento para garantir que y_test não tenha valores nulos\n\n    \n    # Criar e treinar o modelo Random Forest\n    model = RandomForestClassifier(n_estimators=200, random_state=42, n_jobs=-1)\n    \n    # Treinando o modelo\n    model.fit(X_train, y_train)\n    \n    # Fazer previsões no conjunto de teste\n    y_pred = model.predict(X_test)\n    \n    # Avaliar a precisão do modelo\n    accuracy = accuracy_score(y_test, y_pred)\n    print(f'Accuracy: {accuracy}')\n    \n    return model,y_pred","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validar_amostra(df,colunas,quant_amostras):\n\n    X = df.drop(['sii'], axis = 1)\n    y = df['sii']\n    X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.1,random_state = 42)\n\n    \n    #Obter valores de intervalo das colunas que irão ser selecionados\n    ####df_selecionados = df[colunas]\n    df_selecionados = pd.DataFrame(X_train)[colunas]\n\n\n\n    #Criar treino\n\n    \n    df_testando = df.copy()\n    \n\n    #Será gerado uma certa quantidade\n    for q in range(quant_amostras):\n        valores_selecionados = {}\n        for c in colunas:\n            novo_valor = None\n            \n            valores_s = list(set(df_selecionados[c].values))\n            \n            if len(valores_s) <= 5:\n                #novo_valor = valores_s[np.random.randint(0,len(valores_s))]\n                novo_valor = df_selecionados[c].mode()[0]\n            else:\n                media = df_selecionados[c].mean()\n                desvio_padrao = df_selecionados[c].std()\n                novo_valor = np.random.normal(media,desvio_padrao)\n\n            valores_selecionados[c] = novo_valor\n            \n        \n        df_outras = df.copy()\n        df_outras = df_outras.drop(columns=colunas)\n\n        outras_colunas = df_outras.columns\n\n        for o in outras_colunas:\n            valor_selecionado_ = df_outras[o].sample(n=1).iloc[0]\n            valores_selecionados[o] = valor_selecionado_\n\n\n\n        df_novo = pd.DataFrame([valores_selecionados])\n        df_testando = pd.concat([df_testando, df_novo], ignore_index=True)\n\n                \n    return df_testando\n    #Criar amostras com esses valores repetindo todos os outros ou alternando de acordo com valores existentes\n\n    #Treinar primeiro com o df original e calcular a acurária em cima do teste\n\n    #Acrescentar linhas sintéticas e treinar novamente\n\n    #Se a acurácia do segundo for maior igual que o primeiro então irá retornar o novo dataframe com os dados sintéticas gerados\n\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_classificador_novo(X_train,y_train,X_test,y_test):\n    #X = df.drop(['sii'], axis = 1)\n    #y = df['sii']\n    #X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.1,random_state = 42)\n    ##df_teste = pd.concat([X_test,y_test], axis = 1)\n    ##df_teste = df_teste[df_teste.iloc[:,-1].isnull() == False]\n    ##X_test = df_teste.iloc[:,:-1]\n    ##y_test = df_teste.iloc[:,-1]\n    \n    train_data = lgb.Dataset(X_train, label=y_train)\n    test_data = lgb.Dataset(X_test, label=y_test, reference=train_data)\n\n    # Definir os parâmetros do modelo para classificação multiclasse\n    params = {\n        'objective': 'multiclass',     # Tarefa de classificação multiclasse\n        #'boosting_type': 'gbdt',       # Gradient Boosting Decision Tree\n        #'metric': 'multi_logloss',     # Usando log-loss como métrica de avaliação\n        #'learning_rate': 0.01,          # Taxa de aprendizado\n        #'num_leaves': 31,              # Número de folhas no modelo\n        'num_class': 4,                # Número de classes (0, 1, 2, 3)\n        'verbose': -1                  # Suprimir logs\n    }\n\n    \n    # Treinar o modelo\n    model = lgb.train(params, train_data, valid_sets=[test_data], num_boost_round=100)\n    \n    # Fazer previsões no conjunto de teste\n    y_pred = model.predict(X_test)\n    \n    # A função `predict()` retorna probabilidades para cada classe. A previsão final é a classe com maior probabilidade.\n    y_pred = [np.argmax(pred) for pred in y_pred]  # Convertendo para a classe com maior probabilidade\n    \n    # Avaliar a precisão do modelo\n    accuracy = accuracy_score(y_test, y_pred)\n    print(accuracy)\n    return model\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def balance():\n    ros = RandomOverSampler(\n        sampling_strategy='not majority',\n        random_state = 42,\n        shrinkage=2\n    )\n    X_res, y_res = ros.fit_resample(df_train_copia.iloc[:,:-1],df_train_copia.iloc[:,-1])\n    df_train_copia = pd.concat([X_res,y_res], axis = 1)\n\n    ros = ADASYN(\n        sampling_strategy='auto',\n        random_state = 42,\n        n_neighbors=1,\n        n_jobs=4\n    )\n    X_res, y_res = ros.fit_resample(df_train_copia.iloc[:,:-1],df_train_copia.iloc[:,-1])\n    df_train_copia = pd.concat([X_res,y_res], axis = 1)\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def boolean_swap_test(data, boolean_column, target_column=None, model=None, test_threshold=0.05, corr_threshold=0.1, cluster_method='kmeans', n_clusters=3):\n    results = {}\n\n    # 1. Análise de Correlação\n    correlations = {}\n    for col in data.columns:\n        if col != boolean_column and pd.api.types.is_numeric_dtype(data[col]):\n            corr, _ = pointbiserialr(data[boolean_column], data[col])\n            correlations[col] = abs(corr)\n            #print(col)\n            #print(abs(corr))\n            #print('---\\n')\n            \n\n    #Verifica se a correlação dessa coluna passada como argumento é menor do que um limiar\n    #Se for menor então é mais seguro inputar valores sem alterar muito o resultado final\n    results['correlation_pass'] = all(corr < corr_threshold for corr in correlations.values())\n    \n    return results","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_classificador_novo_b(X_train, y_train, X_test, y_test):\n    # Criar e configurar o modelo XGBoost para classificação multiclasse\n    model = XGBClassifier(\n        n_estimators=200,       # Número de árvores\n        random_state=42,        # Para reprodutibilidade\n        n_jobs=-1,              # Uso de todos os núcleos disponíveis\n        use_label_encoder=False, # Evitar mensagens de erro\n        eval_metric='mlogloss'  # Metrica para problemas multiclasse\n    )\n\n    # Treinar o modelo\n    model.fit(X_train, y_train)\n\n    # Fazer previsões no conjunto de teste\n    y_pred = model.predict(X_test)\n\n    # Avaliar a precisão do modelo\n    accuracy = accuracy_score(y_test, y_pred)\n    f1_per_class = f1_score(y_test, y_pred, average=None)  # F1 por classe\n    print(f'Accuracy: {accuracy}')\n    print(f'F1 score: {f1_per_class}')\n\n    \n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_classificador_novo_r(X_train, y_train, X_test, y_test):\n\n    class_weights = {0: 1, 1: 1, 2: 5, 3: 10}  # Exemplo: classe 1 tem custo maior\n    # Criar e treinar o modelo Random Forest\n    model = RandomForestClassifier(n_estimators=200, random_state=42, class_weight = class_weights, n_jobs=-1)\n    \n    # Treinando o modelo\n    model.fit(X_train, y_train)\n    \n    # Fazer previsões no conjunto de teste\n    y_pred = model.predict(X_test)\n    #print(y_pred)\n    # Avaliar a precisão do modelo\n    #accuracy = accuracy_score(y_test, y_pred)\n    #f1_per_class = f1_score(y_test, y_pred, average=None)  # F1 por classe\n    print(balanced_accuracy_score(y_test, y_pred))\n\n    #print(f'Accuracy: {accuracy}')\n    #print(f'F1 score: {f1_per_class}')\n    \n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_classificador_novo_easy(X_train, y_train, X_test, y_test):\n\n    # Criar e treinar o modelo Random Forest\n    model_1 =  EasyEnsembleClassifier(\n                n_estimators=200,\n                sampling_strategy='auto',\n                n_jobs=4,\n                random_state=42)\n    \n    # Treinando o modelo\n    model_1.fit(X_train, y_train)\n\n\n\n    #Balanced RF\n    model_2 = BalancedRandomForestClassifier(\n            n_estimators=100,\n            criterion='gini',\n            max_depth=3,\n            sampling_strategy='auto',\n            n_jobs=4,\n            random_state=42,\n        )\n\n    # Treinando o modelo\n    #model_2.fit(X_train, y_train)\n\n\n\n\n    model_3 = RUSBoostClassifier(\n                #base_estimator=None,\n                n_estimators=40,\n                learning_rate=1.0,\n                sampling_strategy='auto',\n                random_state=42\n            )\n\n    # Treinando o modelo\n    #model_3.fit(X_train, y_train)\n\n\n\n    \n    \n    # Fazer previsões no conjunto de teste\n    y_pred_1 = np.ravel(model_1.predict(X_test))\n    #y_pred_2 = np.ravel(model_2.predict(X_test))\n    #y_pred_3 = np.ravel(model_3.predict(X_test))\n\n\n\n    # Empilhando as previsões\n    #y_preds = np.vstack([y_pred_1, y_pred_2, y_pred_3])\n    \n    # Calculando a moda ao longo do eixo 0 (para cada amostra/coluna)\n    #y_pred_final = mode(y_preds, axis=0).mode  # Pega a moda para cada coluna (amostra)\n    \n    \n    \n    print(balanced_accuracy_score(y_test, y_pred_1))\n\n    #print(f'Accuracy: {accuracy}')\n    #print(f'F1 score: {f1_per_class}')\n    \n    return model_1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_classificador_novo_easy_validacao(model_1,X_validacao):\n    \n    \n    # Fazer previsões no conjunto de teste\n    y_pred_1 = np.ravel(model_1.predict(X_validacao))\n    #y_pred_2 = np.ravel(model_2.predict(X_validacao))\n    #y_pred_3 = np.ravel(model_3.predict(X_validacao))\n\n\n\n    # Empilhando as previsões\n    #y_preds = np.vstack([y_pred_1, y_pred_2, y_pred_3])\n    \n    # Calculando a moda ao longo do eixo 0 (para cada amostra/coluna)\n    #y_pred_final = mode(y_preds, axis=0).mode  # Pega a moda para cada coluna (amostra)\n    \n    \n    return y_pred_1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def correlation(dataset, threshold):\n    col_corr = set()\n    corr_matrix = dataset.corr()\n    for i in range(len(corr_matrix.columns)):\n        for j in range(i):\n            if abs(corr_matrix.iloc[i,j]) > threshold:\n                colname = corr_matrix.columns[i]\n                col_corr.add(colname)\n\n    return col_corr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mudar_hora(coluna):\n    if pd.isna(coluna):\n        return 'Nulo'\n    elif coluna == 0:\n        return 'Pouco'\n    elif coluna == 1:\n        return 'Medio'\n    elif coluna == 2:\n        return 'Alto'\n    else:\n        return 'Bastante'\n\n\ndef calcular_agua(coluna):\n    if pd.isna(coluna):\n        return 'Nulo'\n    elif coluna <= 10:\n        return 'Pouco'\n    elif coluna <= 25:\n        return 'Medio'\n    elif coluna <= 50:\n        return 'Alto'\n    else:\n        return 'Bastante'\n\ndef ajuste_age(coluna):\n    if coluna >= 0 and coluna <= 10:\n        return 0\n    elif coluna > 10 and coluna <= 15:\n        return 1\n    else:\n        return 2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.stats import mode\nimport numpy as np\n\n# Exemplo de previsões (y_pred_1, y_pred_2, y_pred_3)\ny_pred_1 = [1, 2, 2, 1]\ny_pred_2 = [1, 2, 1, 1]\ny_pred_3 = [2, 1, 2, 2]\n\n# Empilhando as previsões\ny_preds = np.vstack([y_pred_1, y_pred_2, y_pred_3])\n\n# Calculando a moda ao longo do eixo 0 (para cada amostra/coluna)\ny_pred_final = mode(y_preds, axis=0).mode  # Pega a moda para cada coluna (amostra)\n\n# Verificando o resultado final\nprint(y_pred_final)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def treinar_classificador_com_cross_validation(X, y):\n    # Definir os parâmetros do modelo para classificação multiclasse\n    params = {\n        'objective': 'multiclass',       # Tarefa de classificação multiclasse\n        'boosting_type': 'gbdt',         # Gradient Boosting Decision Tree\n        'metric': 'multi_logloss',       # Usando log-loss como métrica de avaliação\n        'learning_rate': 0.1,            # Taxa de aprendizado\n        'num_leaves': 31,                # Número de folhas no modelo\n        'num_class': 4,                  # Número de classes (0, 1, 2, 3)\n        'verbose': -1                    # Suprimir logs\n    }\n\n    # Definindo o modelo com os parâmetros especificados\n    model = lgb.LGBMClassifier(**params)\n\n    # Usando cross-validation para avaliar o modelo\n    accuracy_scorer = make_scorer(accuracy_score)\n    scores = cross_val_score(model, X, y, cv=5, scoring=accuracy_scorer)\n\n    # Imprimindo a precisão média\n    print(\"Accuracy médio em cross-validation:\", scores.mean())\n    print(\"Desvio padrão das acurácias:\", scores.std())\n    \n    # Treinando o modelo final com todo o conjunto de dados\n    model.fit(X, y)\n    \n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separar_treinar_cluster(X,y):\n    df = X[pd.isna(y) == False]\n    print(df.shape)\n    km = KMeans(n_clusters = 4, random_state = 42)\n    km.fit(df)\n    return km\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prever_sii_null(km,X,y):\n    df_null = pd.concat([X,y], axis = 1)\n    #df_null = df_null[pd.isna(y)]\n    df_null = df_null[df_null.iloc[:,-1].isnull()]\n    \n    df_null_drop = df_null.iloc[:,:-1]\n    clusters = km.predict(df_null_drop)\n    df = df_null.iloc[:,:-1]\n    df_cluster = pd.DataFrame({'sii':clusters})\n\n    df = df.reset_index(drop=True)\n    df_cluster = df_cluster.reset_index(drop=True)\n    \n    df_new = pd.concat([df,df_cluster], axis = 1)\n    \n    return df_new","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def concatenar_dfs(X,y,df_previstos):\n    df_preenchidos = pd.concat([X,y], axis = 1)\n    df_preenchidos = df_preenchidos[df_preenchidos.iloc[:,-1].isnull() == False]\n    \n    df = pd.concat([df_preenchidos,df_previstos])\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"colunas_valores =  ['CGAS-Season','Basic_Demos-Enroll_Season','PAQ_C-Season','SDS-Season','Physical-Season','FGC-Season','PreInt_EduHx-Season','Basic_Demos-Age','Basic_Demos-Sex','CGAS-CGAS_Score',\n                   'Physical-BMI','Physical-Height','Physical-Weight',\n                  'Physical-Diastolic_BP','Physical-HeartRate','Physical-Systolic_BP',\n                  'Fitness_Endurance-Max_Stage',\n                  'FGC-FGC_CU','FGC-FGC_CU_Zone','FGC-FGC_GSND','FGC-FGC_GSND_Zone','FGC-FGC_GSD','FGC-FGC_SRL','FGC-FGC_SRL_Zone','FGC-FGC_SRR','FGC-FGC_SRR_Zone','FGC-FGC_TL',\n                  'FGC-FGC_TL_Zone','BIA-BIA_Activity_Level_num','BIA-BIA_BMC','BIA-BIA_BMI','BIA-BIA_BMR','BIA-BIA_DEE','BIA-BIA_ECW','BIA-BIA_FFM',\n                  'BIA-BIA_FFMI','BIA-BIA_FMI','BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST','BIA-BIA_SMM',\n                  'BIA-BIA_TBW','PAQ_C-PAQ_C_Total','PreInt_EduHx-computerinternet_hoursday','SDS-SDS_Total_T','SDS-SDS_Total_Raw','Physical-Waist_Circumference',\n                    #'Fitness_Endurance-Time_Sec','Fitness_Endurance-Time_Mins',\n                    'sii','id']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#colunas_classes = ['CGAS-Season','Fitness_Endurance-Season','Basic_Demos-Enroll_Season','BIA-Season','PAQ_C-Season','SDS-Season','Physical-Season','FGC-Season','PreInt_EduHx-Season']\ncolunas_classes = ['CGAS-Season','Basic_Demos-Enroll_Season','PAQ_C-Season','SDS-Season','Physical-Season','FGC-Season','PreInt_EduHx-Season']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def carregar_dados_test():\n    diretorio = \"/kaggle/input/child-mind-institute-problematic-internet-use/\"\n    #diretorio = \"C:\\\\Users\\\\Victor\\\\Documents\\\\computer\\\\\"\n    df = pd.read_csv(diretorio + 'test.csv')\n    colunas_valores_ = colunas_valores.copy()\n    colunas_valores_.remove('sii')\n    df = df[colunas_valores_]\n\n    df_classe = df[colunas_classes]\n    df_classe_numero = pd.get_dummies(df_classe,drop_first = True)\n    df_classe_numero = df_classe_numero.astype(int)\n    \n    df = df.drop(columns = colunas_classes)\n    #df = df.drop(columns = ['sii'])\n    \n    df_novo = pd.concat([df_classe_numero,df], axis = 1)\n    return df_novo\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def carregar_dados_treino():\n    diretorio = \"/kaggle/input/child-mind-institute-problematic-internet-use/\"\n    #diretorio = \"C:\\\\Users\\\\Victor\\\\Documents\\\\computer\\\\\"\n    df = pd.read_csv(diretorio + 'train.csv')\n    df = df.filter(colunas_valores,axis = 1)\n\n    df_classe = df[colunas_classes]\n    df_classe_numero = pd.get_dummies(df_classe,drop_first = True)\n    df_classe_numero = df_classe_numero.astype(int)\n    \n    df = df.drop(columns = colunas_classes)\n    df = df.drop(columns = ['id'])\n    \n    df_novo = pd.concat([df_classe_numero,df], axis = 1)\n    return df_novo\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preencher_weight(coluna):\n    if coluna == 0.0:\n        return None\n    return coluna","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preencher_bmi(linha):\n    if pd.isna(linha['Physical-BMI']) and linha['Physical-Weight'] > 0 and linha['Physical-Height'] > 0:\n        altura_m = linha['Physical-Height'] * 0.0254\n        peso_kg = linha['Physical-Weight'] * 0.453592\n        return peso_kg/(altura_m*altura_m)\n    return linha['Physical-BMI']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Calculando apenas casos mais graves\ndef definir_imc(coluna):\n    if coluna < 18.5:\n        return 0\n    elif coluna < 24.9:\n        return 1\n    elif coluna < 29.9:\n        return 2\n    else:\n        return 3","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Carregar dados treino e teste\ndf_teste = carregar_dados_test()\ndf_treino = carregar_dados_treino()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Preencher valores negativos com nulo\ndf_treino[df_treino<0] = np.nan","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Selecionar colunas para análise\ncolunas_selecionadas_ = [\n                        'Physical-BMI', \n                        'Physical-Height', \n                        'Physical-Weight', \n                        'Physical-Waist_Circumference', \n                        'Physical-Diastolic_BP',\n                        #'Physical-HeartRate', \n                        'Physical-Systolic_BP', \n                        'BIA-BIA_BMC', \n                        'BIA-BIA_BMI',\n                        'Fitness_Endurance-Max_Stage',\n                        'BIA-BIA_BMR', 'BIA-BIA_DEE', \n                        'BIA-BIA_ECW', \n                        'BIA-BIA_FFM',\n                        #'BIA-BIA_Fat',\n                        'BIA-BIA_ICW', \n                        'BIA-BIA_LDM',\n                        \n                        #'SDS-SDS_Total_T','SDS-SDS_Total_Raw','PAQ_C-PAQ_C_Total','BIA-BIA_TBW','BIA-BIA_SMM','BIA-BIA_LST',\n                        #'FGC-FGC_TL','FGC-FGC_SRR','FGC-FGC_GSD','FGC-FGC_GSND','FGC-FGC_CU',\n                        #'Fitness_Endurance-Time_Sec',\n                        #'Fitness_Endurance-Time_Mins'\n                        ]\n\n\n\n# Selecionar colunas para análise\ncolunas_selecionadas = [\n                        'Physical-BMI', \n                        'Physical-Height', \n                        'Physical-Weight', \n                        'Physical-Waist_Circumference', \n                        'Physical-Diastolic_BP',\n                        'Physical-HeartRate', \n                        'Physical-Systolic_BP', \n                        'BIA-BIA_BMC', \n                        'BIA-BIA_BMI',\n                        'Fitness_Endurance-Max_Stage',\n                        'BIA-BIA_BMR', \n                        'BIA-BIA_DEE', \n                        'BIA-BIA_ECW', \n                        'BIA-BIA_FFM',\n                        'BIA-BIA_Fat',\n                        'BIA-BIA_ICW', \n                        'BIA-BIA_LDM',\n                        \n                        #'SDS-SDS_Total_T','SDS-SDS_Total_Raw','PAQ_C-PAQ_C_Total','BIA-BIA_TBW','BIA-BIA_SMM','BIA-BIA_LST',\n                        #'FGC-FGC_TL','FGC-FGC_SRR','FGC-FGC_GSD','FGC-FGC_GSND','FGC-FGC_CU',\n                        #'Fitness_Endurance-Time_Sec',\n                        #'Fitness_Endurance-Time_Mins'\n                        ]\n\n\n\n# Calcular os quartis\nQ1 = df_treino[colunas_selecionadas].quantile(0.25)  # 1º Quartil\nQ3 = df_treino[colunas_selecionadas].quantile(0.75)  # 3º Quartil\n\n# Calcular o IQR\nIQR = Q3 - Q1\n\n# Ajustar o multiplicador do intervalo (ex: 3.0 para considerar valores mais extremos)\nmultiplicador = 3.0\n\n# Identificar outliers\noutliers = ((df_treino[colunas_selecionadas] < (Q1 - multiplicador * IQR)) |\n            (df_treino[colunas_selecionadas] > (Q3 + multiplicador * IQR)))\n\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Substituir os outliers por NaN\n#df_treino[colunas_selecionadas] = df_treino[colunas_selecionadas].where(~outliers, np.nan)\n# Substituir outliers pela média da coluna\nfor coluna in colunas_selecionadas:\n    print(coluna)\n    media = df_treino[coluna][~outliers[coluna]].mean()  # Calcula a média sem considerar os outliers\n    #print(media)\n    df_treino.loc[outliers[coluna], coluna] = media  # Substitui os outliers pela média\n\n# Verificar o resultado\n#print(df_treino[colunas_selecionadas].head())  # Exibe as primeiras linhas para verificar os valores nulos","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_valores = df_treino.iloc[:,:-1]\ny_valores = df_treino.iloc[:,-1]\n\nX_train,X_test,y_train,y_test = train_test_split(X_valores,y_valores, test_size = 0.1, random_state = 42)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 0.72 * X_train.shape[1]\n# Filtrar linhas com mais de 90% de nulos\nfiltered_rows = X_train[X_train.isnull().sum(axis=1) > threshold]\n#filtered_rows.to_csv('C:\\\\Users\\\\Victor\\\\Desktop\\\\aaaaa\\\\teste.csv')\n#filtered_rows\nX_train = X_train.drop(filtered_rows.index)\ny_train = y_train.drop(filtered_rows.index)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"####Aplicar transformações separadamente#####\n##############Train#################\n\nX_train['Physical-Weight'] = X_train['Physical-Weight'].apply(preencher_weight)\nX_train['Physical-BMI'] = X_train['Physical-BMI'].apply(preencher_weight)\nX_train.loc[(X_train['Physical-Height'] > 0) & (X_train['Physical-Weight'].isnull()), 'Physical-Weight'] = 73\nX_train['Physical-BMI'] = X_train[['Physical-BMI','Physical-Weight','Physical-Height']].apply(preencher_bmi, axis = 1)\n\n\n# Substituir zeros em 'Physical-BMI' por um valor muito pequeno para evitar divisão por zero\nX_train['Physical-BMI'].replace(0, 1e-10, inplace=True)\n# Realizar a divisão normalmente\nX_train['risk_score_body'] = X_train['Physical-Waist_Circumference'] / X_train['Physical-BMI']\nX_train['risk_score'] = X_train['Physical-Systolic_BP'] / 120 +  X_train['Physical-Diastolic_BP'] / 80 + X_train['Physical-HeartRate'] / 75\nX_train['fitness_score'] = X_train['Physical-BMI'] /2 +  (X_train['Physical-HeartRate']/100 - X_train['Physical-Waist_Circumference']/10)\n#X_train = X_train.drop(columns=['Physical-Systolic_BP','Physical-Diastolic_BP','Physical-HeartRate'])\n\n#X_train['feature_new_1'] = X_train['BIA-BIA_SMM'] / X_train['BIA-BIA_FMI']\n#X_train['feature_new_2'] = X_train['BIA-BIA_FFM'] / (X_train['BIA-BIA_ECW'] + X_train['BIA-BIA_ICW'])\n#X_train['feature_new_3'] = (X_train['BIA-BIA_FFM'] * X_train['BIA-BIA_BMR'])/100\n#X_train['feature_new_4'] = X_train['BIA-BIA_DEE'] / np.where(\n#    X_train['PreInt_EduHx-computerinternet_hoursday'] == 0, \n#    1e-10, \n#    X_train['PreInt_EduHx-computerinternet_hoursday']\n#)\n\n\n\n\n\n\n\n\n###############Teste###############\n\nX_test['Physical-Weight'] = X_test['Physical-Weight'].apply(preencher_weight)\nX_test['Physical-BMI'] = X_test['Physical-BMI'].apply(preencher_weight)\nX_test.loc[(X_test['Physical-Height'] > 0) & (X_test['Physical-Weight'].isnull()), 'Physical-Weight'] = 73\nX_test['Physical-BMI'] = X_test[['Physical-BMI','Physical-Weight','Physical-Height']].apply(preencher_bmi, axis = 1)\n\nX_test['Physical-BMI'].replace(0, 1e-10, inplace=True)\nX_test['risk_score_body'] = X_test['Physical-Waist_Circumference'] / X_test['Physical-BMI']\nX_test['risk_score'] = X_test['Physical-Systolic_BP'] / 120 +  X_test['Physical-Diastolic_BP'] / 80 + X_test['Physical-HeartRate'] / 75\nX_test['fitness_score'] = X_test['Physical-BMI'] /2 +  (X_test['Physical-HeartRate']/100 - X_test['Physical-Waist_Circumference']/10)\n#X_test = X_test.drop(columns=['Physical-Systolic_BP','Physical-Diastolic_BP','Physical-HeartRate'])\n\n\n\n\n#X_test['feature_new_1'] = X_test['BIA-BIA_SMM'] / X_test['BIA-BIA_FMI']\n#X_test['feature_new_2'] = X_test['BIA-BIA_FFM'] / (X_test['BIA-BIA_ECW'] + X_test['BIA-BIA_ICW'])\n#X_test['feature_new_3'] = (X_test['BIA-BIA_FFM'] * X_test['BIA-BIA_BMR'])/100\n#X_test['feature_new_4'] = X_test['BIA-BIA_DEE'] / np.where(\n#    X_test['PreInt_EduHx-computerinternet_hoursday'] == 0, \n#    1e-10, \n#    X_test['PreInt_EduHx-computerinternet_hoursday']\n#)\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identificar colunas com valores nulos\ncolunas_nulas = X_train.columns[X_train.isnull().any()]\ncolunas_na = []\ncolunas_nome = []\n\n# Criar as colunas binárias que indicam se o valor é nulo\nfor c in colunas_nulas:\n    # Verificar a porcentagem de valores nulos na coluna\n    if X_train[c].isnull().mean() > 0.8:\n        colunas_nome.append(c)\n        colunas_na.append(c + '_na')\n\nX_train[colunas_na] = X_train[colunas_nome].notnull().astype(int)\nX_test[colunas_na] = X_test[colunas_nome].notnull().astype(int)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"####Substituir depois#####\n#Apenas teste em que tem o label preenchido preenchido\nX_train_restante = X_test[y_test.isnull()]\ny_train_restante = y_test[y_test.isnull()]\n\nX_test = X_test[y_test.isnull() == False]\ny_test = y_test[y_test.isnull() == False]\n\nX_train = pd.concat([X_train,X_train_restante])\ny_train = pd.concat([y_train,y_train_restante])\n\nX_train = X_train.reset_index(drop = True)\ny_train = y_train.reset_index(drop = True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sel = VarianceThreshold(threshold=0.1)\nsel.fit(X_train)\nsum(sel.get_support())\nconstant = X_train.columns[~sel.get_support()]\n\nfeat_names = X_train.columns[sel.get_support()]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = sel.transform(X_train)\nX_test = sel.transform(X_test)\n\nX_train = pd.DataFrame(X_train, columns = feat_names)\nX_test = pd.DataFrame(X_test, columns = feat_names)\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train['Physical-BMI'] = X_train['Physical-BMI'].apply(definir_imc)\nX_test['Physical-BMI'] = X_test['Physical-BMI'].apply(definir_imc)\n\nX_train['Basic_Demos-Age'] = X_train['Basic_Demos-Age'].apply(ajuste_age)\nX_test['Basic_Demos-Age'] = X_test['Basic_Demos-Age'].apply(ajuste_age)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vars_to_imput = [var for var in X_train.columns if X_train[var].isnull().mean() <= 0.5 and X_train[var].isnull().mean() > 0]\nvars_to_imput\n\nimputation_dict = {}\nfor i in X_train['Basic_Demos-Age'].unique():\n    imputation_dict[i] = X_train[X_train['Basic_Demos-Age'] == i][vars_to_imput].median().to_dict()\nimputation_dict","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Imputar em valores nulos filtrados\nfor i in imputation_dict.keys():\n    X_train[X_train['Basic_Demos-Age'] == i] = X_train[X_train['Basic_Demos-Age'] == i].fillna(imputation_dict[i])\n    X_test[X_test['Basic_Demos-Age'] == i] = X_test[X_test['Basic_Demos-Age'] == i].fillna(imputation_dict[i])\n\n#Média permanece semelhante um do outro, a distribuição não foi alterada\nX_train.groupby('Basic_Demos-Age')[vars_to_imput].mean()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vars_na = [var for var in X_train.columns if X_train[var].isnull().sum() > 0]\n#vars_na","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train = y_train.reset_index(drop=True)\nX_train = X_train.reset_index(drop=True)\n\nimputer = KNNImputer(\n    n_neighbors = 10,\n    weights = 'uniform',\n    metric = 'nan_euclidean',\n    add_indicator = False\n)\nimputer.fit(X_train,y_train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_t = imputer.transform(X_train)\ntest_t = imputer.transform(X_test)\n\ntrain_t = pd.DataFrame(train_t, columns = X_train.columns)\ntest_t = pd.DataFrame(test_t, columns = X_test.columns)\n\n\ny_train = y_train.reset_index(drop = True)\ny_test = y_test.reset_index(drop = True)\n\n\nX_test_ = test_t\ny_test_ = y_test","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Criando uma cópia do dataset original\ntrain_t_copia = train_t.copy()\n\n# Normalizando os dados apenas para o KMeans\nscaler = StandardScaler()\ntrain_t_scaled = scaler.fit_transform(train_t)\n\n# Convertendo para DataFrame para facilitar operações\ntrain_t_scaled = pd.DataFrame(train_t_scaled, columns=train_t.columns)\n\n# Separar os valores que NÃO são nulos para treino do KMeans\ntrain_t_df_scaled = train_t_scaled[pd.notna(y_train)].reset_index(drop=True)\ny_train_preenchidos = y_train[pd.notna(y_train)].reset_index(drop=True)\n\n# Treinando o modelo KMeans com os dados normalizados\nkm = KMeans(n_clusters=4, random_state=42)\nkm.fit(train_t_df_scaled)\n\n# Separar os valores que SÃO nulos para previsão\ndf_null_scaled = train_t_scaled[pd.isna(y_train)].reset_index(drop=True)\n\n# Prever os labels Y dos valores que estão nulos\nprevisao_labels_ = km.predict(df_null_scaled)\ny_labels_novos = pd.DataFrame({'sii': previsao_labels_})\n\n# Concatenar os dados originais com os valores previstos\n# Usar os dados originais SEM normalização\nX_train_ = pd.concat([\n    train_t_copia[pd.notna(y_train)].reset_index(drop=True),  # Dados originais preenchidos\n    train_t_copia[pd.isna(y_train)].reset_index(drop=True)   # Dados originais nulos\n], ignore_index=True)\n\ny_train_ = pd.concat([\n    y_train_preenchidos,          # Labels preenchidos\n    y_labels_novos['sii']         # Labels previstos\n], ignore_index=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"####Undersampling####\ndf_train = pd.concat([X_train_,y_train_], axis = 1)\n\ndf_train_copia = df_train.copy()\n\n\n\nprint()\nprint(df_train_copia.groupby('sii').size())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corrmat = df_train_copia.corr()\n#fig, ax = plt.subplots()\n#fig.set_size_inches(11,11)\n#sns.heatmap(corrmat)\n\n#Correlation\nX_train_final = df_train_copia.iloc[:,:-1]\ny_train_final = df_train_copia.iloc[:,-1]\n\nX_test_final = X_test_.copy()\ny_test_final = y_test_.copy()\n\ncorr_features = correlation(X_train_final, 0.8)\nprint(len(set(corr_features)))\nX_train_final.drop(labels = corr_features, axis = 1, inplace = True)\nX_test_final.drop(labels = corr_features, axis = 1, inplace = True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#modelo = separar_treinar_classificador_novo_r(X_train_final,y_train_final,X_test_final,y_test_final)\nmodelo_1 = separar_treinar_classificador_novo_easy(X_train_final,y_train_final,X_test_final,y_test_final)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#modelo = separar_treinar_classificador_novo(X_train_final,y_train_final,X_test_final,y_test_final)\n#modelo = treinar_classificador_com_cross_validation(X_train_,y_train_)\nX_validacao = df_teste.iloc[:,:-1]\n\n\n\n\nX_validacao[X_validacao<0] = np.nan\n\nX_validacao['Physical-Weight'] = X_validacao['Physical-Weight'].apply(preencher_weight)\nX_validacao['Physical-BMI'] = X_validacao['Physical-BMI'].apply(preencher_weight)\nX_validacao.loc[(X_validacao['Physical-Height'] > 0) & (X_train['Physical-Weight'].isnull()), 'Physical-Weight'] = 73\nX_validacao['Physical-BMI'] = X_validacao[['Physical-BMI','Physical-Weight','Physical-Height']].apply(preencher_bmi, axis = 1)\n\n\n# Substituir zeros em 'Physical-BMI' por um valor muito pequeno para evitar divisão por zero\nX_validacao['Physical-BMI'].replace(0, 1e-10, inplace=True)\n# Realizar a divisão normalmente\nX_validacao['risk_score_body'] = X_validacao['Physical-Waist_Circumference'] / X_validacao['Physical-BMI']\nX_validacao['risk_score'] = X_validacao['Physical-Systolic_BP'] / 120 +  X_validacao['Physical-Diastolic_BP'] / 80 + X_validacao['Physical-HeartRate'] / 75\nX_validacao['fitness_score'] = X_validacao['Physical-BMI'] /2 +  (X_validacao['Physical-HeartRate']/100 - X_validacao['Physical-Waist_Circumference']/10)\n#X_validacao = X_validacao.drop(columns=['Physical-Systolic_BP','Physical-Diastolic_BP','Physical-HeartRate'])\n\n\n\n\n\nX_validacao[colunas_na] = X_validacao[colunas_nome].notnull().astype(int)\n\n\n\n\nX_validacao = sel.transform(X_validacao)\n\nX_validacao = pd.DataFrame(X_validacao, columns = feat_names)\n\nfor i in imputation_dict.keys():\n    X_validacao[X_validacao['Basic_Demos-Sex'] == i] = X_validacao[X_validacao['Basic_Demos-Sex'] == i].fillna(imputation_dict[i])\n\n\n\n\n\n#Imputar valores nulos\nX_validacao = imputer.transform(X_validacao)\n\nX_validacao = pd.DataFrame(X_validacao, columns = feat_names)\n\n\nX_validacao['Physical-BMI'] = X_validacao['Physical-BMI'].apply(definir_imc)\nX_validacao['Basic_Demos-Age'] = X_validacao['Basic_Demos-Age'].apply(ajuste_age)\n\n#Drop colunas correlacao\nX_validacao.drop(labels = corr_features, axis = 1, inplace = True)\n\n\n\n\n#y_pred = modelo.predict(X_validacao)\ny_pred = separar_treinar_classificador_novo_easy_validacao(modelo_1,X_validacao)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def salvar_submission(y_pred,labels):\n    output = pd.DataFrame({'id':labels, 'sii':y_pred})\n    output.to_csv('submission.csv', index=False)\n\nsalvar_submission(y_pred,list(df_teste.iloc[:,-1].values))\n#salvar_submission(y_pred_final,list(df_teste.iloc[:,-1].values))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}