{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7859511,"sourceType":"datasetVersion","datasetId":4609695}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-16T14:08:05.837415Z","iopub.execute_input":"2024-03-16T14:08:05.837779Z","iopub.status.idle":"2024-03-16T14:08:06.728097Z","shell.execute_reply.started":"2024-03-16T14:08:05.837749Z","shell.execute_reply":"2024-03-16T14:08:06.727375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ライブラリ・データセットのインポート","metadata":{}},{"cell_type":"code","source":"!pip install sweetviz\n!pip install category_encoders","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:08:06.729578Z","iopub.execute_input":"2024-03-16T14:08:06.730101Z","iopub.status.idle":"2024-03-16T14:08:27.759932Z","shell.execute_reply.started":"2024-03-16T14:08:06.730075Z","shell.execute_reply":"2024-03-16T14:08:27.758902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport plotly.express as px\nfrom IPython.display import display\nimport sweetviz as sv\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, ClassifierMixin\n\nimport lightgbm as lgb\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\npd.options.display.float_format = '[:.5f]'.format\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\nplt.style.use('ggplot')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:08:27.761373Z","iopub.execute_input":"2024-03-16T14:08:27.761668Z","iopub.status.idle":"2024-03-16T14:08:30.693531Z","shell.execute_reply.started":"2024-03-16T14:08:27.761642Z","shell.execute_reply":"2024-03-16T14:08:30.692578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.cluster.hierarchy import dendrogram, linkage\nfrom scipy.spatial.distance import squareform\nimport missingno as msno\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom lightgbm import LGBMClassifier\n\nfrom category_encoders import OneHotEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, roc_curve\nfrom sklearn.pipeline import make_pipeline\nfrom xgboost import XGBClassifier","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:08:30.695546Z","iopub.execute_input":"2024-03-16T14:08:30.695841Z","iopub.status.idle":"2024-03-16T14:08:30.946672Z","shell.execute_reply.started":"2024-03-16T14:08:30.695816Z","shell.execute_reply":"2024-03-16T14:08:30.945752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 実行時間を調べるために使う","metadata":{}},{"cell_type":"code","source":"import datetime\nimport time\nimport math\n\nstart_time = time.time()\n\ndef changeHMS(s):\n    h = math.floor(s / 3600)\n    if h > 0:\n        s = s - h * 3600\n        indi_h = str(h) + 'h'\n    else:\n        indi_h = ''\n    m = math.floor(s / 60)\n    if m > 0:\n        indi_m = str(m) + 'm'\n    else:\n        indi_m = ''\n    s = math.floor(s % 60)\n    time = indi_h + indi_m + str(s) + 's'\n    \n    return time","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:08:30.948140Z","iopub.execute_input":"2024-03-16T14:08:30.948882Z","iopub.status.idle":"2024-03-16T14:08:30.956099Z","shell.execute_reply.started":"2024-03-16T14:08:30.948846Z","shell.execute_reply":"2024-03-16T14:08:30.955095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"InputFile = \"/kaggle/input/my2-home-credit-credit-risk-model-stability/\"\n\ntrain = pd.read_csv(InputFile + \"df_train.csv\")\ntest = pd.read_csv(InputFile + \"df_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:08:30.957059Z","iopub.execute_input":"2024-03-16T14:08:30.957327Z","iopub.status.idle":"2024-03-16T14:09:41.685848Z","shell.execute_reply.started":"2024-03-16T14:08:30.957289Z","shell.execute_reply":"2024-03-16T14:09:41.684966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"print(\"Train is duplicated:\\t\", train[\"case_id\"].duplicated().any())\nprint(\"Train Week Range:\\t\", (train[\"WEEK_NUM\"].min(), train[\"WEEK_NUM\"].max()))\n\nprint()\n\nprint(\"Test is duplicated:\\t\", test[\"case_id\"].duplicated().any())\nprint(\"Test Week Range:\\t\", (test[\"WEEK_NUM\"].min(), test[\"WEEK_NUM\"].max()))","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:41.687022Z","iopub.execute_input":"2024-03-16T14:09:41.687343Z","iopub.status.idle":"2024-03-16T14:09:41.729686Z","shell.execute_reply.started":"2024-03-16T14:09:41.687298Z","shell.execute_reply":"2024-03-16T14:09:41.728028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(\n    data=train,\n    x=\"WEEK_NUM\",\n    y=\"target\",\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:41.731222Z","iopub.execute_input":"2024-03-16T14:09:41.731541Z","iopub.status.idle":"2024-03-16T14:09:54.299263Z","shell.execute_reply.started":"2024-03-16T14:09:41.731519Z","shell.execute_reply":"2024-03-16T14:09:54.298228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train['target']\ntarget_col = 'target'\ntarget_name = str(train.iloc[:, [2]].columns.tolist())\nprint(target_name)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:54.300683Z","iopub.execute_input":"2024-03-16T14:09:54.301071Z","iopub.status.idle":"2024-03-16T14:09:54.308181Z","shell.execute_reply.started":"2024-03-16T14:09:54.301039Z","shell.execute_reply":"2024-03-16T14:09:54.307200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データ（DataFrame）の構成を調べる","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:54.311843Z","iopub.execute_input":"2024-03-16T14:09:54.312124Z","iopub.status.idle":"2024-03-16T14:09:54.470838Z","shell.execute_reply.started":"2024-03-16T14:09:54.312102Z","shell.execute_reply":"2024-03-16T14:09:54.470158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:54.471561Z","iopub.execute_input":"2024-03-16T14:09:54.471796Z","iopub.status.idle":"2024-03-16T14:09:54.610556Z","shell.execute_reply.started":"2024-03-16T14:09:54.471775Z","shell.execute_reply":"2024-03-16T14:09:54.609285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train: ')\nprint(train.shape)\nprint(train.info())\nprint('\\n')\nprint('test: ')\nprint(test.shape)\nprint(test.info())","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:54.611820Z","iopub.execute_input":"2024-03-16T14:09:54.612095Z","iopub.status.idle":"2024-03-16T14:09:54.651029Z","shell.execute_reply.started":"2024-03-16T14:09:54.612072Z","shell.execute_reply":"2024-03-16T14:09:54.650108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_columns = train.columns\ntrain_index = train.index\ntrain_data = train.values\n\ntest_columns = test.columns\ntest_index = test.index\ntest_data = test.values\n\nprint('train: ')\nprint(type(train_columns))\nprint(type(train_index))\nprint(type(train_data))\nprint('\\n')\nprint('test:')\nprint(type(test_columns))\nprint(type(test_index))\nprint(type(test_data))","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:09:54.653130Z","iopub.execute_input":"2024-03-16T14:09:54.653630Z","iopub.status.idle":"2024-03-16T14:10:11.218156Z","shell.execute_reply.started":"2024-03-16T14:09:54.653604Z","shell.execute_reply":"2024-03-16T14:10:11.217056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with pd.option_context('display.max_rows', 8):\n    display(train)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:11.219253Z","iopub.execute_input":"2024-03-16T14:10:11.219552Z","iopub.status.idle":"2024-03-16T14:10:11.380014Z","shell.execute_reply.started":"2024-03-16T14:10:11.219527Z","shell.execute_reply":"2024-03-16T14:10:11.379234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with pd.option_context('display.max_rows', 8):\n    display(test)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:11.380933Z","iopub.execute_input":"2024-03-16T14:10:11.381517Z","iopub.status.idle":"2024-03-16T14:10:11.543491Z","shell.execute_reply.started":"2024-03-16T14:10:11.381489Z","shell.execute_reply":"2024-03-16T14:10:11.542800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train: ')\nprint(train.dtypes)\nprint(train.dtypes.value_counts())\nprint('\\n')\nprint('test: ')\nprint(test.dtypes)\nprint(test.dtypes.value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:11.544503Z","iopub.execute_input":"2024-03-16T14:10:11.544876Z","iopub.status.idle":"2024-03-16T14:10:11.565117Z","shell.execute_reply.started":"2024-03-16T14:10:11.544854Z","shell.execute_reply":"2024-03-16T14:10:11.564257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_features = test._get_numeric_data().columns\ncategorical_features = test.drop(numerical_features, axis=1).columns\nnumerical_features, categorical_features","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:11.566252Z","iopub.execute_input":"2024-03-16T14:10:11.566567Z","iopub.status.idle":"2024-03-16T14:10:11.575560Z","shell.execute_reply.started":"2024-03-16T14:10:11.566544Z","shell.execute_reply":"2024-03-16T14:10:11.574619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA(探索的データ分析)\n\nWhat's your EDA?  \n**EDA(探索的データ分析)**: データセットに適宜前処理を施しつつ様々な特徴量を抽出して可視化し、そこに内在する特性・パターン・偏りについて探索的に仮設立案・検証を繰り返して分析すること","metadata":{}},{"cell_type":"markdown","source":"## 要約統計量","metadata":{}},{"cell_type":"code","source":"train.describe().T \\\n    .style.bar(subset=['mean'], color=px.colors.qualitative.G10[0]) \\\n    .background_gradient(subset=['std'], cmap='Greens') \\\n    .background_gradient(subset=['50%'], cmap='BuGn')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:11.576600Z","iopub.execute_input":"2024-03-16T14:10:11.576841Z","iopub.status.idle":"2024-03-16T14:10:25.910382Z","shell.execute_reply.started":"2024-03-16T14:10:11.576821Z","shell.execute_reply":"2024-03-16T14:10:25.909257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe().T \\\n    .style.bar(subset=['mean'], color=px.colors.qualitative.G10[0]) \\\n    .background_gradient(subset=['std'], cmap='Blues') \\\n    .background_gradient(subset=['50%'], cmap='BuGn')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:25.911528Z","iopub.execute_input":"2024-03-16T14:10:25.911824Z","iopub.status.idle":"2024-03-16T14:10:26.349201Z","shell.execute_reply.started":"2024-03-16T14:10:25.911797Z","shell.execute_reply":"2024-03-16T14:10:26.347849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc_train = pd.DataFrame(index=list(train))\ndesc_train['count'] = train.count()\ndesc_train['nunique'] = train.nunique()\ndesc_train['%unique'] = desc_train['nunique'] / len(train) * 100\ndesc_train['null'] = train.isnull().sum()\ndesc_train['type'] = train.dtypes\ndesc_train = desc_train.style.background_gradient(cmap='Greens')\ndesc_train","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:26.350518Z","iopub.execute_input":"2024-03-16T14:10:26.350806Z","iopub.status.idle":"2024-03-16T14:10:41.272493Z","shell.execute_reply.started":"2024-03-16T14:10:26.350781Z","shell.execute_reply":"2024-03-16T14:10:41.271396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc_test = pd.DataFrame(index=list(test))\ndesc_test['count'] = test.count()\ndesc_test['nunique'] = test.nunique()\ndesc_test['%unique'] = desc_test['nunique'] / len(test) * 100\ndesc_test['null'] = test.isnull().sum()\ndesc_test['type'] = test.dtypes\ndesc_test = desc_test.style.background_gradient(cmap='Blues')\ndesc_test","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:41.273537Z","iopub.execute_input":"2024-03-16T14:10:41.274230Z","iopub.status.idle":"2024-03-16T14:10:41.401809Z","shell.execute_reply.started":"2024-03-16T14:10:41.274202Z","shell.execute_reply":"2024-03-16T14:10:41.400832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print('train: ')\n# print(np.var(train[numerical_features], axis=0))\n# print('\\n')\n# print('test: ')\n# print(np.var(test[numerical_features], axis=0))","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:41.403089Z","iopub.execute_input":"2024-03-16T14:10:41.403377Z","iopub.status.idle":"2024-03-16T14:10:41.407679Z","shell.execute_reply.started":"2024-03-16T14:10:41.403353Z","shell.execute_reply":"2024-03-16T14:10:41.406590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 変動係数\n変動係数は、標準偏差を平均値で割った値\n- スケールに依存せず、比較できるようになる","metadata":{}},{"cell_type":"code","source":"# train[numerical_features].std() / train[numerical_features].mean()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:41.409240Z","iopub.execute_input":"2024-03-16T14:10:41.409635Z","iopub.status.idle":"2024-03-16T14:10:41.416745Z","shell.execute_reply.started":"2024-03-16T14:10:41.409605Z","shell.execute_reply":"2024-03-16T14:10:41.416033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 尖度（せんど）と歪度（わいど）\n- 分布が正規分布からどれだけ歪んでいるかを表す統計量で、左右対称性を示す指標のこと\n- 分布が正規分布からどれだけ尖っているかを表す統計量で、山の尖り度と裾の広がり度","metadata":{}},{"cell_type":"code","source":"# print('train: ')\n# print(train[numerical_features].kurtosis())\n# print('\\n')\n# print(test[numerical_features].kurtosis())","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:41.417814Z","iopub.execute_input":"2024-03-16T14:10:41.418040Z","iopub.status.idle":"2024-03-16T14:10:41.425544Z","shell.execute_reply.started":"2024-03-16T14:10:41.418021Z","shell.execute_reply":"2024-03-16T14:10:41.424613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 欠損値","metadata":{}},{"cell_type":"code","source":"print('train: ')\nprint(train.isna().sum().any())\nprint(train.isnull().sum())\nprint('\\n')\nprint('test: ')\nprint(test.isna().sum().any())\nprint(test.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2024-03-16T14:10:41.426711Z","iopub.execute_input":"2024-03-16T14:10:41.426954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(msno.matrix(df=train, figsize=(10, 6), color=(0, .3, .3)))\n# print(msno.matrix(df=test, figsize=(10, 6), color=(0, 0, .3)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 重複データ","metadata":{}},{"cell_type":"code","source":"# total = pd.concat([train.drop(target_col, axis=1), test], axis=0)\n# total.duplicated().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## カラムごとのユニークな値を調べる","metadata":{}},{"cell_type":"code","source":"train[train.columns].nunique().sort_values(ascending=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 目的変数","metadata":{}},{"cell_type":"code","source":"print(target_name + ' Class labels:', np.unique(target))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データセットのプロット（分布）","metadata":{}},{"cell_type":"markdown","source":"## 密度関数（ヒストグラム）","metadata":{}},{"cell_type":"code","source":"# sns.set(style='whitegrid')\n\n# rows = (len(numerical_features) + 1) // 2\n# cols = 2\n# _, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(8 * cols, 6 * rows))\n\n# for i, feature_name in enumerate(numerical_features):\n#     row_idx, col_idx = divmod(i, cols)\n#     sns.histplot(data=train, x=feature_name, kde=True, ax=axes[row_idx, col_idx])\n    \n# plt.tight_layout()\n\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize': (20, 16)})\ntrain.hist(color='mediumseagreen')\nplt.tight_layout()\nplt.suptitle('Feature distributions', y=1.02, fontsize=20)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize': (20, 16)})\ntest.hist(color='royalblue')\nplt.tight_layout()\nplt.suptitle('Feature distributions', y=1.02, fontsize=20)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature = train[numerical_features]\n\n# for feat in feature:\n#     plt.figure(figsize=(12, 3))\n#     ax1 = plt.subplot(1, 2, 1)\n#     train[feat].plot(kind='hist', bins=50, color='blue')\n#     plt.title(feat + ' / train')\n#     ax2 = plt.subplot(1, 2, 2, sharex=ax1)\n#     test[feat].plot(kind='hist', bins=50, color='green')\n#     plt.title(feat + ' / test')\n    \n#     plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データを対数変換する","metadata":{}},{"cell_type":"code","source":"# # 訓練データをX（説明変数）とy（目的変数）に分割する\n# X = train[numerical_features]\n# y = target\n\n# for column in X.columns.tolist():\n#     X[column] = X[column].apply(lambda x: np.log(x + 1))\n    \n# X.hist(color='mediumseagreen')\n# plt.tight_layout()\n# plt.suptitle('Feature distribution', y=1.02, fontsize=20)\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # テストデータも変換する\n# test_log = test[numerical_features]\n# for column in test_log.columns.tolist():\n#     test_log[column] = test_log[column].apply(lambda x: np.log(x + 1))\n    \n# test_log.hist(color='royalblue')\n# plt.tight_layout()\n# plt.suptitle('Feature distributions', y=1.02, fontsize=20)\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 一部の説明変数において分布が確認しやすくなった","metadata":{}},{"cell_type":"markdown","source":"### 数値型の説明変数の分布\nカーネル密度推定(KDE: Kernel Density Estimation)\n- 実データから、正規分布曲線のように下部の面積が１となるような連続した曲線（確率密度関数）を推定する","metadata":{}},{"cell_type":"code","source":"# fig, ax = plt.subplots(len(numerical_features) // 4 + 1, 4)\n# ax = ax.flatten()\n# pal2 = sns.color_palette(\"Set2\")\n# for i, column in enumerate(numerical_features):\n#     sns.kdeplot(train[column], ax=ax[i], color=pal2[0])\n#     sns.kdeplot(test[column], ax=ax[i], color=pal2[1], warn_singular=False)\n    \n#     ax[i].set_title(f'{column}', size=14)\n#     ax[i].set_xlabel(None)\n    \n# fig.suptitle('Distribution of Feature', fontsize=24, fontweight='bold')\n# fig.legend(['Train', 'Test'])\n# plt.tight_layout()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 円グラフと棒グラフ","metadata":{}},{"cell_type":"markdown","source":"### 円グラフと棒グラフ（目的変数）","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(16, 5))\nax = ax.flatten()\n\nax[0].pie(\n    target.value_counts(),\n    shadow=True,\n    explode=[.1 for i in range(target.nunique())],\n    autopct='%1.f%%',\n    textprops={'size': 14, 'color': 'white'}\n)\n\nsns.countplot(data=train, y=target_col, ax=ax[1], palette='viridis', order=train[target_col].value_counts().index)\nax[1].yaxis.label.set_size(20)\nplt.yticks(fontsize=12)\nax[1].set_xlabel('Count', fontsize=20)\nax[1].set_ylabel(None)\nplt.xticks(fontsize=12)\n\nfig.suptitle(target_name + ' Distribution in Train Dataset', fontsize=25, fontweight='bold')\nplt.tight_layout()\n\nprint(f'{train[target_col].value_counts()}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### カテゴリ型変数の分布","metadata":{}},{"cell_type":"code","source":"# fig, ax = plt.subplots(len(categorical_features), 2, figsize=(16, 25))\n\n# for i, column in enumerate(categorical_features):\n#     ax[i][0].pie(\n#         train[column].value_counts(),\n#         shadow=True,\n#         explode=[.1 for i in range(train[column].nunique())],\n#         autopct='%1.f%%',\n#         textprops={'size': 14, 'color': 'white'}\n#     )\n    \n#     sns.countplot(data=train, y=column, ax=[i][1], palette='viridis', order=train[column].value_counts().index)\n#     ax[i][1].yaxis.label.set_size(20)\n#     plt.yticks(fontsize=12)\n#     ax[i][1].set_xlabel('Count in Train', fontsize=15)\n#     ax[i][1].set_ylabel(f'{column}', fontsize=15)\n#     plt.xticks(fontsize=12)\n    \n# fig.suptitle('Distribution of Categorical Features in Train Dataset', fontsize=25, fontweight='bold')\n# plt.tight_layout()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 箱ひげ図\n数値型の説明変数","metadata":{}},{"cell_type":"code","source":"# for feat in numerical_features:\n#     plt.figure(figsize=(12, 4))\n#     ax1 = plt.subplot(1, 2, 1)\n#     sns.boxplot(data=train, x=target, y=feat)\n#     plt.title(target_name + ' vs ' + feat + ' / train')\n#     x1 = plt.subplot(1, 2, 2)\n#     sns.boxplot(data=test, y=feat)\n#     plt.title(feat + ' / test')\n#     plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 特徴量の重要度評価","metadata":{}},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n#     df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols\n\ndf_train, cat_cols = to_pandas(train)\ndf_test, cat_cols = to_pandas(test, cat_cols)\n\nX = df_train.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])\ny = df_train[\"target\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\n# lgb = LGBMClassifier(\n#     random_state=42,\n# )\n\n# lgb.fit(X, y)\n# importance = lgb.feature_importances_\n\n# feature_importance = pd.DataFrame(data=importance, inedx=X.columns, columns=['importance'])\\\n#     .sort_values(ascending=True, by='importance')\n\n# feature_importance.plot(kind='barh', figsize=(10, 15), color='b')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature_importance","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 相関（ヒートマップ）","metadata":{}},{"cell_type":"code","source":"# corr = train[numerical_features].corr(method='spearman')\n# plt.figure()\n# sns.heatmap(corr, linewidth=0.5, square=False, annot=True, cmap=\"coolwarm\", vmin=-1, vmax=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corr_mat_data = train[numerical_features].corr()\n# data_mask = np.triu(np.ones_like(corr_mat_data, dtype=bool))\n\n# cmap = sns.diverging_palette(100, 7, s=75, l=40, n=5, center='light', as_cmap=True)\n\n# plt.figure(figsize=(15, 10))\n# sns.heatmap(corr_mat_data, annot=True, cmap=cmap, fmt='.2f', center=0,\n#            annot_kws={'size': 12}, mask=data_mask).set_titl;e('Correlations Among Features(in Competition Dataset)')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 階層的クラスタリング","metadata":{}},{"cell_type":"markdown","source":"- 特徴間の関係の強さと方向性を調べる\n- 特徴をクラスタリングする","metadata":{}},{"cell_type":"code","source":"# def distance(data, label=''):\n#     corr = data.corr(method='spearman')\n#     dist_linkage = linkage(squareform(1 - abs(corr)), 'complete')\n    \n#     plt.figure(figsize=(10, 8))\n#     dendro = dendrogram(dist_linkage, labels=data.columns, leaf_rotation=90)\n#     plt.title(f'Feature Distance in {label} Dataset', weight='bold', size=20)\n#     plt.show()\n    \n# distance(train[numerical_features], 'Train')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 実行時間を調べるために使う","metadata":{}},{"cell_type":"code","source":"end_time = time.time()\nlap = end_time - start_time\nindi_time = changeHMS(lap)\nindi_time","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}