{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 8.2 안전 운전자 예측 경진대회 탐색적 데이터 분석","metadata":{"papermill":{"duration":0.02457,"end_time":"2021-08-08T12:06:55.857491","exception":false,"start_time":"2021-08-08T12:06:55.832921","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 8.2.1 데이터 둘러보기","metadata":{"papermill":{"duration":0.031413,"end_time":"2021-08-08T12:06:56.037834","exception":false,"start_time":"2021-08-08T12:06:56.006421","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\n\n# 데이터 경로\ndata_path = '/kaggle/input/porto-seguro-safe-driver-prediction/'\n\ntrain = pd.read_csv(data_path + 'train.csv', index_col='id')\ntest = pd.read_csv(data_path + 'test.csv', index_col='id')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv', index_col='id')","metadata":{"papermill":{"duration":11.031626,"end_time":"2021-08-08T12:07:07.096062","exception":false,"start_time":"2021-08-08T12:06:56.064436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-04T15:26:25.319833Z","iopub.execute_input":"2022-03-04T15:26:25.320188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"papermill":{"duration":0.035189,"end_time":"2021-08-08T12:07:07.156047","exception":false,"start_time":"2021-08-08T12:07:07.120858","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"papermill":{"duration":0.059234,"end_time":"2021-08-08T12:07:07.240834","exception":false,"start_time":"2021-08-08T12:07:07.1816","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"papermill":{"duration":0.049964,"end_time":"2021-08-08T12:07:07.317706","exception":false,"start_time":"2021-08-08T12:07:07.267742","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"papermill":{"duration":0.040377,"end_time":"2021-08-08T12:07:07.383308","exception":false,"start_time":"2021-08-08T12:07:07.342931","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"papermill":{"duration":0.119508,"end_time":"2021-08-08T12:07:07.528856","exception":false,"start_time":"2021-08-08T12:07:07.409348","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport missingno as msno\n\n# 훈련 데이터 복사본에서 -1을 np.NaN로 변환\ntrain_copy = train.copy().replace(-1, np.NaN)\n\n# 결측값 시각화(처음 28개만)\nmsno.bar(df=train_copy.iloc[:, 1:29], figsize=(13, 6));","metadata":{"papermill":{"duration":17.070351,"end_time":"2021-08-08T12:07:24.627381","exception":false,"start_time":"2021-08-08T12:07:07.55703","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.bar(df=train_copy.iloc[:, 29:], figsize=(13, 6));","metadata":{"papermill":{"duration":3.762158,"end_time":"2021-08-08T12:07:28.418605","exception":false,"start_time":"2021-08-08T12:07:24.656447","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.matrix(df=train_copy.iloc[:, 1:29], figsize=(13, 6));","metadata":{"papermill":{"duration":5.662209,"end_time":"2021-08-08T12:07:34.111012","exception":false,"start_time":"2021-08-08T12:07:28.448803","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  피처 요약표","metadata":{"papermill":{"duration":0.030951,"end_time":"2021-08-08T12:07:34.17316","exception":false,"start_time":"2021-08-08T12:07:34.142209","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def resumetable(df):\n    print(f'데이터 세트 형상: {df.shape}')\n    summary = pd.DataFrame(df.dtypes, columns=['데이터 타입'])\n    summary['결측값 개수'] = (df == -1).sum().values # 피처별 -1 개수\n    summary['고윳값 개수'] = df.nunique().values\n    summary['데이터 종류'] = None\n    for col in df.columns:\n        if 'bin' in col or col == 'target':\n            summary.loc[col, '데이터 종류'] = '이진형'\n        elif 'cat' in col:\n            summary.loc[col, '데이터 종류'] = '명목형'\n        elif df[col].dtype == float:\n            summary.loc[col, '데이터 종류'] = '연속형'\n        elif df[col].dtype == int:\n            summary.loc[col, '데이터 종류'] = '순서형'\n\n    return summary","metadata":{"papermill":{"duration":0.04241,"end_time":"2021-08-08T12:07:34.246566","exception":false,"start_time":"2021-08-08T12:07:34.204156","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary = resumetable(train)\nsummary","metadata":{"papermill":{"duration":0.598401,"end_time":"2021-08-08T12:07:34.876594","exception":false,"start_time":"2021-08-08T12:07:34.278193","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary[summary['데이터 종류'] == '명목형'].index","metadata":{"papermill":{"duration":0.043164,"end_time":"2021-08-08T12:07:34.952457","exception":false,"start_time":"2021-08-08T12:07:34.909293","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary[summary['데이터 타입'] == 'float64'].index","metadata":{"papermill":{"duration":0.045272,"end_time":"2021-08-08T12:07:35.030564","exception":false,"start_time":"2021-08-08T12:07:34.985292","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.2.2 데이터 시각화","metadata":{"papermill":{"duration":0.032912,"end_time":"2021-08-08T12:07:35.09666","exception":false,"start_time":"2021-08-08T12:07:35.063748","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"papermill":{"duration":0.042873,"end_time":"2021-08-08T12:07:35.172655","exception":false,"start_time":"2021-08-08T12:07:35.129782","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 타깃 값 분포","metadata":{"papermill":{"duration":0.032801,"end_time":"2021-08-08T12:07:35.238185","exception":false,"start_time":"2021-08-08T12:07:35.205384","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def write_percent(ax, total_size):\n    '''도형 객체를 순회하며 막대 그래프 상단에 타깃값 비율 표시'''\n    for patch in ax.patches:\n        height = patch.get_height()     # 도형 높이(데이터 개수)\n        width = patch.get_width()       # 도형 너비\n        left_coord = patch.get_x()      # 도형 왼쪽 테두리의 x축 위치\n        percent = height/total_size*100 # 타깃값 비율\n        \n        # (x, y) 좌표에 텍스트 입력\n        ax.text(left_coord + width/2.0,     # x축 위치\n                height + total_size*0.001,  # y축 위치\n                '{:1.1f}%'.format(percent), # 입력 텍스트\n                ha='center')                # 가운데 정렬\n    \nmpl.rc('font', size=15)\nplt.figure(figsize=(7, 6))\n\nax = sns.countplot(x='target', data=train)\nwrite_percent(ax, len(train)) # 비율 표시\nax.set_title('Target Distribution');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 이진 피처","metadata":{"papermill":{"duration":0.034691,"end_time":"2021-08-08T12:07:35.596922","exception":false,"start_time":"2021-08-08T12:07:35.562231","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import matplotlib.gridspec as gridspec\n\ndef plot_target_ratio_by_features(df, features, num_rows, num_cols, \n                                  size=(12, 18)):\n    mpl.rc('font', size=9) \n    plt.figure(figsize=size)                     # 전체 Figure 크기 설정\n    grid = gridspec.GridSpec(num_rows, num_cols) # 서브플롯 배치\n    plt.subplots_adjust(wspace=0.3, hspace=0.3)  # 서브플롯 좌우/상하 여백 설정\n\n    for idx, feature in enumerate(features):\n        ax = plt.subplot(grid[idx])\n        # ax축에 고윳값별 타깃값 1 비율을 막대 그래프로 그리기\n        sns.barplot(x=feature, y='target', data=df, palette='Set2', ax=ax)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_features = summary[summary['데이터 종류'] == '이진형'].index # 이진 피처\n# 이진 피처 고윳값별 타깃값 1 비율을 막대 그래프로 그리기\nplot_target_ratio_by_features(train, bin_features, 6, 3) # 6행 3열 배치","metadata":{"papermill":{"duration":218.103756,"end_time":"2021-08-08T12:11:13.813058","exception":false,"start_time":"2021-08-08T12:07:35.709302","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 명목형 피처","metadata":{"papermill":{"duration":0.035247,"end_time":"2021-08-08T12:11:13.884668","exception":false,"start_time":"2021-08-08T12:11:13.849421","status":"completed"},"tags":[]}},{"cell_type":"code","source":"nom_features = summary[summary['데이터 종류'] == '명목형'].index # 명목형 피처\n\nplot_target_ratio_by_features(train, nom_features, 7, 2) # 7행 2열","metadata":{"papermill":{"duration":144.018846,"end_time":"2021-08-08T12:13:37.939056","exception":false,"start_time":"2021-08-08T12:11:13.92021","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 순서형 피처","metadata":{"papermill":{"duration":0.037506,"end_time":"2021-08-08T12:13:38.014568","exception":false,"start_time":"2021-08-08T12:13:37.977062","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ord_features = summary[summary['데이터 종류'] == '순서형'].index # 순서형 피처\n\nplot_target_ratio_by_features(train, ord_features, 8, 2, (12, 20)) # 8행 2열","metadata":{"papermill":{"duration":126.269225,"end_time":"2021-08-08T12:15:44.321612","exception":false,"start_time":"2021-08-08T12:13:38.052387","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 연속형 피처","metadata":{"papermill":{"duration":0.040986,"end_time":"2021-08-08T12:15:44.402911","exception":false,"start_time":"2021-08-08T12:15:44.361925","status":"completed"},"tags":[]}},{"cell_type":"code","source":"cont_features = summary[summary['데이터 종류'] == '연속형'].index # 연속형 피처\n\nplt.figure(figsize=(12, 16))                # Figure 크기 설정\ngrid = gridspec.GridSpec(5, 2)              # GridSpec 객체 생성\nplt.subplots_adjust(wspace=0.2, hspace=0.4) # 서브플롯 간 여백 설정\n\nfor idx, cont_feature in enumerate(cont_features):\n    # 값을 5개 구간으로 나누기\n    train[cont_feature] = pd.cut(train[cont_feature], 5)\n\n    ax = plt.subplot(grid[idx])                # 분포도를 그릴 서브플롯 설정\n    sns.barplot(x=cont_feature, y='target', data=train, palette='Set2', ax=ax)\n    ax.tick_params(axis='x', labelrotation=10) # x축 라벨 회전","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_copy = train_copy.dropna() # np.NaN 값 삭제","metadata":{"papermill":{"duration":0.336988,"end_time":"2021-08-08T12:17:14.026297","exception":false,"start_time":"2021-08-08T12:17:13.689309","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\ncont_corr = train_copy[cont_features].corr()     # 연속형 피처 간 상관관계 \nsns.heatmap(cont_corr, annot=True, cmap='OrRd'); # 히트맵 그리기","metadata":{"papermill":{"duration":0.983582,"end_time":"2021-08-08T12:17:15.054615","exception":false,"start_time":"2021-08-08T12:17:14.071033","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}