{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\ndata_path = '/kaggle/input/porto-seguro-safe-driver-prediction/' \n\ntrain = pd.read_csv(data_path + 'train.csv', index_col = 'id')\ntest = pd.read_csv(data_path + 'test.csv', index_col = 'id')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv', index_col = 'id')\n        \n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-05T02:53:03.451078Z","iopub.execute_input":"2022-08-05T02:53:03.451842Z","iopub.status.idle":"2022-08-05T02:53:14.326036Z","shell.execute_reply.started":"2022-08-05T02:53:03.451735Z","shell.execute_reply":"2022-08-05T02:53:14.324479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 피처 엔지니어링\n\n데이터 합치기","metadata":{}},{"cell_type":"code","source":"all_data = pd.concat([train, test], ignore_index = True)\nall_data = all_data.drop('target', axis = 1)\n\nall_features = all_data.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:14.329063Z","iopub.execute_input":"2022-08-05T02:53:14.329994Z","iopub.status.idle":"2022-08-05T02:53:15.697634Z","shell.execute_reply.started":"2022-08-05T02:53:14.329939Z","shell.execute_reply":"2022-08-05T02:53:15.696605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"명목형 피처 원핫 인코딩","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\n\ncat_features = [feature for feature in all_features if 'cat' in feature]\n\nonehot_encoder = OneHotEncoder()\nencoded_cat_matrix = onehot_encoder.fit_transform(all_data[cat_features])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:15.699196Z","iopub.execute_input":"2022-08-05T02:53:15.699979Z","iopub.status.idle":"2022-08-05T02:53:19.097456Z","shell.execute_reply.started":"2022-08-05T02:53:15.699918Z","shell.execute_reply":"2022-08-05T02:53:19.092221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"파생 피처 추가 1; 각각의 데이터 당 결측값 개수를 파생 피처로 만들기","metadata":{}},{"cell_type":"code","source":"all_data['num_missing'] = (all_data == -1).sum(axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:19.107955Z","iopub.execute_input":"2022-08-05T02:53:19.108402Z","iopub.status.idle":"2022-08-05T02:53:19.598285Z","shell.execute_reply.started":"2022-08-05T02:53:19.108362Z","shell.execute_reply":"2022-08-05T02:53:19.596713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remaining_features = [feature for feature in all_features\n                      if ('cat' not in feature and 'calc' not in feature)]\nremaining_features.append('num_missing')\n\n# 빼야할 것들 많았던거 같은데 안빼나??","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:19.600443Z","iopub.execute_input":"2022-08-05T02:53:19.600904Z","iopub.status.idle":"2022-08-05T02:53:19.607640Z","shell.execute_reply.started":"2022-08-05T02:53:19.600858Z","shell.execute_reply":"2022-08-05T02:53:19.606585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"파생 피처 추가 2; ind 피처의 고윳값들을 연결한 다음 이것을 명목형 피처로 간주하고, 다른 명목형 피처들과 함께 명목형 피처의 고윳값별 개수 피처를 만들어내기\n\n(예를 들면, 'ps_ind_02_cat'피처는 1 값을 가진 데이터들이 n개, 2값은 m개... 이런 식)","metadata":{}},{"cell_type":"code","source":"ind_features = [feature for feature in all_features if 'ind' in feature]\n\nis_first_feature = True\nfor ind_feature in ind_features:\n    if is_first_feature: \n        all_data['mix_ind'] = all_data[ind_feature].astype(str) + '_'\n        is_first_feature = False\n    else: \n        all_data['mix_ind'] += all_data[ind_feature].astype(str) + '_'","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:19.609024Z","iopub.execute_input":"2022-08-05T02:53:19.610028Z","iopub.status.idle":"2022-08-05T02:53:48.520285Z","shell.execute_reply.started":"2022-08-05T02:53:19.609969Z","shell.execute_reply":"2022-08-05T02:53:48.518510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['mix_ind']  # 이를 명목형 피처로 간주","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:48.522148Z","iopub.execute_input":"2022-08-05T02:53:48.523214Z","iopub.status.idle":"2022-08-05T02:53:48.538147Z","shell.execute_reply.started":"2022-08-05T02:53:48.523178Z","shell.execute_reply":"2022-08-05T02:53:48.537029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":".value_count() 함수: 피처 속의 고윳값별로 그 고윳값을 가지고 있는 데이터가 몇 개인지 세어보기\n\n.value_count().to_dict 함수: 고윳값의 개수 센 것을 딕셔너리 형태로(고윳값: 개수) 변환\n\n-> 방금 변환한 피처의 고윳값 개수를 딕셔너리형태로 변수에 저장한 다음 (val_counts_dict)\n\n'해당피쳐이름_count'라는 이름의 새로운 피쳐를 만들고 \n\nall_data[해당 변수].apply(lambda x: 딕셔너리변수[x])를 이용하여 해당 변수를 lambda 함수의 x값으로 받은 다음 해당 변수의 딕셔너리를 '해당피쳐이름_count'이라는 새 피쳐에 저장\n\n그리고 그 피처 이름들을 'cat_count_features라는 리스트에 append하기\n","metadata":{}},{"cell_type":"code","source":"cat_count_features = []\nfor feature in cat_features+['mix_ind']:\n    val_counts_dict = all_data[feature].value_counts().to_dict()\n    all_data[f'{feature}_count'] = all_data[feature].apply(lambda x: val_counts_dict[x])\n    cat_count_features.append(f'{feature}_count')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:48.539580Z","iopub.execute_input":"2022-08-05T02:53:48.540040Z","iopub.status.idle":"2022-08-05T02:53:59.225466Z","shell.execute_reply.started":"2022-08-05T02:53:48.540007Z","shell.execute_reply":"2022-08-05T02:53:59.224387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_count_features","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:59.227119Z","iopub.execute_input":"2022-08-05T02:53:59.227809Z","iopub.status.idle":"2022-08-05T02:53:59.236801Z","shell.execute_reply.started":"2022-08-05T02:53:59.227760Z","shell.execute_reply":"2022-08-05T02:53:59.235492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"필요 없는 피처 제거","metadata":{}},{"cell_type":"code","source":"from scipy import sparse\n\ndrop_features = ['ps_ind_14', 'ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_12_bin', 'ps_ind_13_bin', 'ps_car_14']\n\nall_data_remaining = all_data[remaining_features+cat_count_features].drop(drop_features, axis = 1)\n\nall_data_sprs = sparse.hstack([sparse.csr_matrix(all_data_remaining), \n                               encoded_cat_matrix], \n                              format = 'csr')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:53:59.242565Z","iopub.execute_input":"2022-08-05T02:53:59.243240Z","iopub.status.idle":"2022-08-05T02:54:06.064695Z","shell.execute_reply.started":"2022-08-05T02:53:59.243202Z","shell.execute_reply":"2022-08-05T02:54:06.063450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"훈련 데이터와 테스트 데이터 나누기","metadata":{}},{"cell_type":"code","source":"num_train = len(train)\n\nX = all_data_sprs[:num_train]\nX_test = all_data_sprs[num_train:]\n\ny = train['target'].values","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:06.066357Z","iopub.execute_input":"2022-08-05T02:54:06.068042Z","iopub.status.idle":"2022-08-05T02:54:07.669491Z","shell.execute_reply.started":"2022-08-05T02:54:06.067991Z","shell.execute_reply":"2022-08-05T02:54:07.668125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 지니계수 계산 함수 eval_gini(), gini() 정의하기","metadata":{}},{"cell_type":"code","source":"# 실제 타깃값 y_true와 예측 타깃값 y_pred를 이용하여 정규화 지니계수 반환하도록 하기\n\nimport numpy as np\n\ndef eval_gini(y_true, y_pred):\n    assert y_true.shape == y_pred.shape                  # y_true의 크기와 y_pred의 크기가 서로 같은지 확인 (다르면 오류)\n    \n    n_samples = y_true.shape[0]                          # 데이터 개수\n    L_mid = np.linspace(1/n_samples, 1, n_samples)       # 대각선 값 np.linspce(구간 시작점, 구간 끝점, 시작점과 끝점을 균일한 몇 개의 구간으로 나눌 지)\n    \n    # 예측값에 대한 지니계수\n    pred_order = y_true[y_pred.argsort()]                # y_pred 크기순으로 y_true 정렬\n    L_pred = np.cumsum(pred_order) / np.sum(pred_order)  # 로렌츠 곡선\n    G_pred = np.sum(L_mid - L_pred)                      # 예측값에 대한 지니계수 (예측값에 대한 지니계수 * 완전 균등할 때의 삼각형 넓이? 어차피 정규화 때 나눌거라 생략?) \n    \n    # 예측이 완벽할 때 지니계수\n    true_order = y_true[y_true.argsort()]                # y_true 크기순으로 y_true 정렬\n    L_true = np.cumsum(true_order) / np.sum(true_order)  # 로렌츠 곡선\n    G_true = np.sum(L_mid - L_true)                      # 예측이 완벽할 때의 지니계수 (예측이 완벽할 때의 지니계수 * 완전 균등할 때의 삼각형 넓이? 어차피 정규화 때 나눌거라 생략?)  \n    \n    return G_pred / G_true","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:07.671230Z","iopub.execute_input":"2022-08-05T02:54:07.672472Z","iopub.status.idle":"2022-08-05T02:54:07.681121Z","shell.execute_reply.started":"2022-08-05T02:54:07.672421Z","shell.execute_reply":"2022-08-05T02:54:07.680168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 모델 훈련 시 검증 파라미터에 전달하기 위한 함수\n\ndef gini(preds, dtrain):                             # preds = 예측 확률값\n    labels = dtrain.get_label()                      # .get_label() \".\" 앞의 데이터셋의 타깃값 반환 = 실제 타깃값 labels\n    return 'gini', eval_gini(labels, preds), True    # return '평가지표 이름', 평가 점수 계산 함수, 평가 점수가 높을수록 좋은지 여부","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:07.682244Z","iopub.execute_input":"2022-08-05T02:54:07.682965Z","iopub.status.idle":"2022-08-05T02:54:07.698884Z","shell.execute_reply.started":"2022-08-05T02:54:07.682930Z","shell.execute_reply":"2022-08-05T02:54:07.697679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 하이퍼파라미터 최적화\n\n베이지안 최적화 이용 (그리드서치보다 빠르고 효율적)","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.2, random_state = 0)\n\nbayes_dtrain = lgb.Dataset(X_train, y_train)\nbayes_dvalid = lgb.Dataset(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:07.700528Z","iopub.execute_input":"2022-08-05T02:54:07.701006Z","iopub.status.idle":"2022-08-05T02:54:08.952656Z","shell.execute_reply.started":"2022-08-05T02:54:07.700959Z","shell.execute_reply":"2022-08-05T02:54:08.951197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"하이퍼파라미터를 설정할 떄는 1) 하이퍼파라미터 범위를 넓게 설정했다가 최적이 나오면 그 주변으로 다시 범위를 잡아서 최적 찾기, 2) 다른 상위 캐글러 참고","metadata":{}},{"cell_type":"code","source":"param_bounds = {'num_leaves': (30, 40), \n                'lambda_l1': (0.7, 0.9), \n                'lambda_l2': (0.9, 1), \n                'feature_fraction': (0.6, 0.7), \n                'bagging_fraction': (0.6, 0.9), \n                'min_child_samples': (6, 10), \n                'min_child_weight': (10, 40)}\n\nfixed_params = {'objective': 'binary',      # 이진분류\n                'learning_rate': 0.005,    # 학습률은 보통 0.01~ 0.001 사이의 값으로 설정\n                'bagging_freq': 1,         # 배깅은 1로 전달하여 매 이터레이션마다 트리가 새로운 샘플링 데이터로 학습함\n                'force_row_wise': True,    # 경고문구 없애기 위해 \n                'random_state': 1991}      # 다시 수행해도 동일한 결과 나오도록 임의 숫자로 고정","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:08.954501Z","iopub.execute_input":"2022-08-05T02:54:08.954885Z","iopub.status.idle":"2022-08-05T02:54:08.961738Z","shell.execute_reply.started":"2022-08-05T02:54:08.954848Z","shell.execute_reply":"2022-08-05T02:54:08.960416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"베이지안 최적화용 평가지표 계산 함수 작성","metadata":{}},{"cell_type":"code","source":"def eval_function(num_leaves, lambda_l1, lambda_l2, feature_fraction, bagging_fraction, min_child_samples, min_child_weight):\n    \n    params =  {'num_leaves': int(round(num_leaves)), \n               'lambda_l1': lambda_l1, \n               'lambda_l2': lambda_l2, \n               'feature_fraction': feature_fraction, \n               'bagging_fraction': bagging_fraction, \n               'min_child_samples': int(round(min_child_samples)), \n               'min_child_weight': min_child_weight,\n               'feature_pre_filter': False}\n    \n    params.update(fixed_params)                           # 딕셔너리 타입이라 update 함수로 원소 추가하기\n    \n    print('하이퍼파라미터:', params)\n    \n    lgb_model = lgb.train(params = params,                # 하이퍼파라미터 이용하여 모델 훈련\n                          train_set = bayes_dtrain, \n                          num_boost_round = 2500, \n                          valid_sets = bayes_dvalid, \n                          feval = gini, \n                          early_stopping_rounds = 300, \n                          verbose_eval = False)\n    \n    preds = lgb_model.predict(X_valid)                    # 검증 데이터로 예측 수행\n    \n    gini_score = eval_gini(y_valid, preds)\n    print(f'지니계수 {gini_score}\\n')   \n    \n    return gini_score","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:08.964435Z","iopub.execute_input":"2022-08-05T02:54:08.964794Z","iopub.status.idle":"2022-08-05T02:54:08.975670Z","shell.execute_reply.started":"2022-08-05T02:54:08.964756Z","shell.execute_reply":"2022-08-05T02:54:08.974650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"최적화 수행","metadata":{}},{"cell_type":"code","source":"from bayes_opt import BayesianOptimization\n\noptimizer = BayesianOptimization(f = eval_function,       # 평가지표 계산 함수\n                                 pbounds = param_bounds,  # 하이퍼파라미터 범위\n                                 random_state = 0)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:08.977365Z","iopub.execute_input":"2022-08-05T02:54:08.978064Z","iopub.status.idle":"2022-08-05T02:54:09.014058Z","shell.execute_reply.started":"2022-08-05T02:54:08.978016Z","shell.execute_reply":"2022-08-05T02:54:09.012976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer.maximize(init_points = 3, n_iter = 6)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T02:54:09.017177Z","iopub.execute_input":"2022-08-05T02:54:09.017641Z","iopub.status.idle":"2022-08-05T03:25:08.774358Z","shell.execute_reply.started":"2022-08-05T02:54:09.017597Z","shell.execute_reply":"2022-08-05T03:25:08.773184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"결과 확인","metadata":{}},{"cell_type":"code","source":"max_params = optimizer.max['params']\nmax_params","metadata":{"execution":{"iopub.status.busy":"2022-08-05T03:25:08.776205Z","iopub.execute_input":"2022-08-05T03:25:08.777352Z","iopub.status.idle":"2022-08-05T03:25:08.786049Z","shell.execute_reply.started":"2022-08-05T03:25:08.777304Z","shell.execute_reply":"2022-08-05T03:25:08.784488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"num_leaves, min_child_samples는 원래 정수형이므로 정수형으로 변환하여 저장\n\n값이 고정된 하이퍼파라미터들도 추가하기","metadata":{}},{"cell_type":"code","source":"max_params['num_leaves'] = int(round(max_params['num_leaves']))\nmax_params['min_child_samples'] = int(round(max_params['min_child_samples']))\n\nmax_params.update(fixed_params)\n\nmax_params","metadata":{"execution":{"iopub.status.busy":"2022-08-05T03:25:08.787624Z","iopub.execute_input":"2022-08-05T03:25:08.788183Z","iopub.status.idle":"2022-08-05T03:25:08.797550Z","shell.execute_reply.started":"2022-08-05T03:25:08.788140Z","shell.execute_reply":"2022-08-05T03:25:08.796592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 모델 훈련 및 성능 검증","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nfolds = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 1991)\n\noof_val_preds = np.zeros(X.shape[0])    # 나중에 oof 방식으로 훈련된 모델로 검증 데이터 타깃값을 예측한 확률을 담을 1차원 배열\noof_test_preds = np.zeros(X_test.shape[0])    # 나중에 oof 방식으로 훈련된 모델로 테스트 데이터 타깃값을 예측한 확률을 담을 1차원 배열\n\nfor idx, (train_idx, valid_idx) in enumerate(folds.split(X, y)):\n    # 각 폴드를 구분하는 문구 출력\n    print('#'*40, f'폴드{idx+1} / 폴드 {folds.n_splits}', '#'*40)\n    \n    # 훈련용 데이터, 검증용 데이터 설정\n    X_train, y_train = X[train_idx], y[train_idx]\n    X_valid, y_valid = X[valid_idx], y[valid_idx]\n    \n    dtrain = lgb.Dataset(X_train, y_train)\n    dvalid = lgb.Dataset(X_valid, y_valid)\n    \n    # Lightgbm 모델 훈련\n    lgb_model = lgb.train(params = max_params, # 최적화로 찾은 파라미터 값을 적용\n                          train_set = dtrain, \n                          num_boost_round = 2500, \n                          valid_sets = dvalid, \n                          feval = gini, \n                          early_stopping_rounds = 300, \n                          verbose_eval = 100)\n    \n    # 테스트 데이터를 활용해 oof 예측\n    oof_test_preds += lgb_model.predict(X_test)/folds.n_splits\n    \n    # 모델 성능 평가를 위한 검증 데이터 타깃값 예측\n    oof_val_preds[valid_idx] += lgb_model.predict(X_valid)\n    \n    # 검증 데이터 예측 확률에 대한 정규화 지니계수\n    gini_score = eval_gini(y_valid, oof_val_preds[valid_idx])\n    print(f'폴드 {idx+1} 지니계수 : {gini_score}\\n')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T03:25:08.799237Z","iopub.execute_input":"2022-08-05T03:25:08.799935Z","iopub.status.idle":"2022-08-05T03:48:56.407989Z","shell.execute_reply.started":"2022-08-05T03:25:08.799888Z","shell.execute_reply":"2022-08-05T03:48:56.406529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('OOF 검증 데이터 지니계수: ', eval_gini(y, oof_val_preds))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T03:48:56.409645Z","iopub.execute_input":"2022-08-05T03:48:56.410017Z","iopub.status.idle":"2022-08-05T03:48:56.525151Z","shell.execute_reply.started":"2022-08-05T03:48:56.409976Z","shell.execute_reply":"2022-08-05T03:48:56.523792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 예측 및 결과 제출","metadata":{}},{"cell_type":"code","source":"submission['target'] = oof_test_preds\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T03:48:56.526972Z","iopub.execute_input":"2022-08-05T03:48:56.527790Z","iopub.status.idle":"2022-08-05T03:48:58.816926Z","shell.execute_reply.started":"2022-08-05T03:48:56.527741Z","shell.execute_reply":"2022-08-05T03:48:58.815889Z"},"trusted":true},"execution_count":null,"outputs":[]}]}