{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CMI Visualize Importance\n\ninclude train specific columns","metadata":{"papermill":{"duration":0.009105,"end_time":"2022-10-25T14:48:23.51949","exception":false,"start_time":"2022-10-25T14:48:23.510385","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold","metadata":{"papermill":{"duration":8.574502,"end_time":"2022-10-25T14:48:32.101977","exception":false,"start_time":"2022-10-25T14:48:23.527475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:34.538119Z","iopub.execute_input":"2024-08-16T10:18:34.538469Z","iopub.status.idle":"2024-08-16T10:18:52.091052Z","shell.execute_reply.started":"2024-08-16T10:18:34.538442Z","shell.execute_reply":"2024-08-16T10:18:52.090171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.007139,"end_time":"2022-10-25T14:48:32.11684","exception":false,"start_time":"2022-10-25T14:48:32.109701","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data0 = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\nprint(len(data0))\nprint(data0.columns.tolist())","metadata":{"papermill":{"duration":0.130645,"end_time":"2022-10-25T14:48:32.254936","exception":false,"start_time":"2022-10-25T14:48:32.124291","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:52.092641Z","iopub.execute_input":"2024-08-16T10:18:52.093306Z","iopub.status.idle":"2024-08-16T10:18:52.121174Z","shell.execute_reply.started":"2024-08-16T10:18:52.093277Z","shell.execute_reply":"2024-08-16T10:18:52.120135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test0 = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\n#cols = test0.columns.tolist()\n#data0 = data0[cols+['sii']]\ndata0 = data0.dropna(subset=['sii'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(data0.info())","metadata":{"execution":{"iopub.status.busy":"2024-08-16T10:18:52.122145Z","iopub.execute_input":"2024-08-16T10:18:52.122424Z","iopub.status.idle":"2024-08-16T10:18:52.153247Z","shell.execute_reply.started":"2024-08-16T10:18:52.122392Z","shell.execute_reply":"2024-08-16T10:18:52.152247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(data0[0:3])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df):\n    for c in df.columns:\n        if df[c].dtype=='object': \n            df[c] = df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(df[c].values))\n            df[c] = lbl.transform(df[c].values)\n    return df\n\ndata0=labelencoder(data0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(data0[0:3])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0 = data0.sample(frac=1, random_state=42).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T10:18:52.15534Z","iopub.execute_input":"2024-08-16T10:18:52.155663Z","iopub.status.idle":"2024-08-16T10:18:52.165564Z","shell.execute_reply.started":"2024-08-16T10:18:52.155633Z","shell.execute_reply":"2024-08-16T10:18:52.164459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target setting","metadata":{"papermill":{"duration":0.007509,"end_time":"2022-10-25T14:48:32.451206","exception":false,"start_time":"2022-10-25T14:48:32.443697","status":"completed"},"tags":[]}},{"cell_type":"code","source":"target='sii'\ntrainY=data0['sii']\ntrainX=data0.drop('sii',axis=1)","metadata":{"papermill":{"duration":0.031477,"end_time":"2022-10-25T14:48:32.490601","exception":false,"start_time":"2022-10-25T14:48:32.459124","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:52.211363Z","iopub.execute_input":"2024-08-16T10:18:52.21173Z","iopub.status.idle":"2024-08-16T10:18:54.349238Z","shell.execute_reply.started":"2024-08-16T10:18:52.211699Z","shell.execute_reply":"2024-08-16T10:18:54.347051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_columns = list(trainX.columns)\nprint(df_columns)","metadata":{"papermill":{"duration":0.017844,"end_time":"2022-10-25T14:48:32.516164","exception":false,"start_time":"2022-10-25T14:48:32.49832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.350134Z","iopub.status.idle":"2024-08-16T10:18:54.350484Z","shell.execute_reply.started":"2024-08-16T10:18:54.350312Z","shell.execute_reply":"2024-08-16T10:18:54.350327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=trainX","metadata":{"papermill":{"duration":0.016944,"end_time":"2022-10-25T14:48:32.646574","exception":false,"start_time":"2022-10-25T14:48:32.62963","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.351671Z","iopub.status.idle":"2024-08-16T10:18:54.351978Z","shell.execute_reply.started":"2024-08-16T10:18:54.351827Z","shell.execute_reply":"2024-08-16T10:18:54.35184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.016539,"end_time":"2022-10-25T14:48:32.696115","exception":false,"start_time":"2022-10-25T14:48:32.679576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.353484Z","iopub.status.idle":"2024-08-16T10:18:54.35384Z","shell.execute_reply.started":"2024-08-16T10:18:54.353675Z","shell.execute_reply":"2024-08-16T10:18:54.353689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.024259,"end_time":"2022-10-25T14:48:32.728659","exception":false,"start_time":"2022-10-25T14:48:32.7044","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.355873Z","iopub.status.idle":"2024-08-16T10:18:54.356315Z","shell.execute_reply.started":"2024-08-16T10:18:54.356083Z","shell.execute_reply":"2024-08-16T10:18:54.356102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.018733,"end_time":"2022-10-25T14:48:32.755482","exception":false,"start_time":"2022-10-25T14:48:32.736749","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.357761Z","iopub.status.idle":"2024-08-16T10:18:54.358199Z","shell.execute_reply.started":"2024-08-16T10:18:54.357964Z","shell.execute_reply":"2024-08-16T10:18:54.357982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)","metadata":{"papermill":{"duration":0.042183,"end_time":"2022-10-25T14:48:32.805699","exception":false,"start_time":"2022-10-25T14:48:32.763516","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.360148Z","iopub.status.idle":"2024-08-16T10:18:54.360588Z","shell.execute_reply.started":"2024-08-16T10:18:54.360356Z","shell.execute_reply":"2024-08-16T10:18:54.360375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.008218,"end_time":"2022-10-25T14:48:32.822457","exception":false,"start_time":"2022-10-25T14:48:32.814239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgbm\nfrom sklearn.metrics import mean_squared_error\n\ndef fit_lgbm(X, y, cv, \n             params: dict=None, \n             verbose: int=50):\n\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=float)\n\n    for i, (idx_train, idx_valid) in enumerate(cv): \n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = lgbm.LGBMRegressor(**params)\n        \n        with Timer(prefix='fit fold={} '.format(i)):\n            clf.fit(x_train, y_train, \n                    eval_set=[(x_valid, y_valid)])\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        print(f'Fold {i} RMSLE: {mean_squared_error(y_valid, pred_i) ** .5:.4f}')\n        print()\n\n    score = mean_squared_error(y, oof_pred) ** .5\n    print('-' * 50)\n    print('FINISHED | Whole RMSLE: {:.4f}'.format(score))\n    return oof_pred, models","metadata":{"papermill":{"duration":0.022515,"end_time":"2022-10-25T14:48:32.853327","exception":false,"start_time":"2022-10-25T14:48:32.830812","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.362523Z","iopub.status.idle":"2024-08-16T10:18:54.362881Z","shell.execute_reply.started":"2024-08-16T10:18:54.362716Z","shell.execute_reply":"2024-08-16T10:18:54.362732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'objective': 'rmse', \n    'learning_rate': .1,\n    'reg_lambda': 1.,\n    'reg_alpha': .1,\n    'max_depth': 5, \n    'n_estimators': 10000, \n    'colsample_bytree': .5, \n    'min_child_samples': 10,\n    'subsample_freq': 3,\n    'subsample': .9,\n    'importance_type': 'gain', \n    'random_state': 71,\n    'num_leaves': 62\n}","metadata":{"papermill":{"duration":0.018428,"end_time":"2022-10-25T14:48:32.880247","exception":false,"start_time":"2022-10-25T14:48:32.861819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.364321Z","iopub.status.idle":"2024-08-16T10:18:54.364664Z","shell.execute_reply.started":"2024-08-16T10:18:54.364476Z","shell.execute_reply":"2024-08-16T10:18:54.364489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)","metadata":{"papermill":{"duration":0.024445,"end_time":"2022-10-25T14:48:32.913026","exception":false,"start_time":"2022-10-25T14:48:32.888581","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.36639Z","iopub.status.idle":"2024-08-16T10:18:54.36678Z","shell.execute_reply.started":"2024-08-16T10:18:54.366579Z","shell.execute_reply":"2024-08-16T10:18:54.366593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nfor i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params)","metadata":{"papermill":{"duration":6.994676,"end_time":"2022-10-25T14:48:39.916257","exception":false,"start_time":"2022-10-25T14:48:32.921581","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-08-16T10:18:54.368351Z","iopub.status.idle":"2024-08-16T10:18:54.368818Z","shell.execute_reply.started":"2024-08-16T10:18:54.368569Z","shell.execute_reply":"2024-08-16T10:18:54.368588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    fig,ax = plt.subplots(figsize=(6,6))\n    ax.set_title('sii',fontsize=20)\n    ax.set_ylabel('Predicted',fontsize=12)\n    ax.set_xlabel('True',fontsize=12)\n    ax.scatter(y,oof,alpha=0.2)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T10:18:54.370415Z","iopub.status.idle":"2024-08-16T10:18:54.370839Z","shell.execute_reply.started":"2024-08-16T10:18:54.370635Z","shell.execute_reply":"2024-08-16T10:18:54.370659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Importance","metadata":{"papermill":{"duration":0.009622,"end_time":"2022-10-25T14:48:39.936111","exception":false,"start_time":"2022-10-25T14:48:39.926489","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def visualize_importance(models, feat_train_df):\n\n    feature_importance_df = pd.DataFrame()\n    for i, model in enumerate(models):\n        _df = pd.DataFrame()\n        _df['feature_importance'] = model.feature_importances_\n        _df['column'] = feat_train_df.columns\n        _df['fold'] = i + 1\n        feature_importance_df = pd.concat([feature_importance_df, _df], \n                                          axis=0, ignore_index=True)\n\n    order = feature_importance_df.groupby('column')\\\n        .sum()[['feature_importance']]\\\n        .sort_values('feature_importance', ascending=False).index[:50]\n    print(\"Importance Top 25\")\n    print(order[0:25])\n\n    fig, ax = plt.subplots(figsize=(8, max(6, len(order) * .25)))\n    sns.boxenplot(data=feature_importance_df, \n                  x='feature_importance', \n                  y='column', \n                  order=order, \n                  ax=ax, \n                  palette='viridis', \n                  orient='h')\n    \n    ax.tick_params(axis='x', rotation=0)\n    #ax.set_title('Importance')\n    ax.grid()\n    fig.tight_layout()\n    \n    return fig,ax\n\n#fig, ax = visualize_importance(models, train_feat_df)","metadata":{"papermill":{"duration":0.023103,"end_time":"2022-10-25T14:48:39.968971","exception":false,"start_time":"2022-10-25T14:48:39.945868","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-16T10:18:54.372343Z","iopub.status.idle":"2024-08-16T10:18:54.372815Z","shell.execute_reply.started":"2024-08-16T10:18:54.372554Z","shell.execute_reply":"2024-08-16T10:18:54.372574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params)","metadata":{"papermill":{"duration":6.891944,"end_time":"2022-10-25T14:48:46.870562","exception":false,"start_time":"2022-10-25T14:48:39.978618","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-08-16T10:18:54.373905Z","iopub.status.idle":"2024-08-16T10:18:54.374341Z","shell.execute_reply.started":"2024-08-16T10:18:54.374111Z","shell.execute_reply":"2024-08-16T10:18:54.374131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    fig, ax = visualize_importance(models, train_feat_df)\n    ax.set_title('sii',fontsize=20)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T10:18:54.375874Z","iopub.status.idle":"2024-08-16T10:18:54.376225Z","shell.execute_reply.started":"2024-08-16T10:18:54.376061Z","shell.execute_reply":"2024-08-16T10:18:54.376076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets=list(set(data0.columns.tolist())-set(test0.columns.tolist()))\nprint(targets)\nprint(len(targets))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Targets are train specific columns which are not included in test data. \n### This result shows columns of high importance are not included in test data.","metadata":{}}]}