{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MNIST LGBM Predict and Visualize Importance","metadata":{"papermill":{"duration":0.00756,"end_time":"2022-08-05T03:27:53.956337","exception":false,"start_time":"2022-08-05T03:27:53.948777","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:27:53.971716Z","iopub.status.busy":"2022-08-05T03:27:53.970690Z","iopub.status.idle":"2022-08-05T03:28:03.348079Z","shell.execute_reply":"2022-08-05T03:28:03.346572Z"},"papermill":{"duration":9.388788,"end_time":"2022-08-05T03:28:03.351272","exception":false,"start_time":"2022-08-05T03:27:53.962484","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.006031,"end_time":"2022-08-05T03:28:03.364138","exception":false,"start_time":"2022-08-05T03:28:03.358107","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train=pd.read_csv('../input/digit-recognizer/train.csv')\ntest=pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.380449Z","iopub.status.busy":"2022-08-05T03:28:03.378523Z","iopub.status.idle":"2022-08-05T03:28:03.422013Z","shell.execute_reply":"2022-08-05T03:28:03.420726Z"},"papermill":{"duration":0.054374,"end_time":"2022-08-05T03:28:03.424985","exception":false,"start_time":"2022-08-05T03:28:03.370611","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target=['label']\ntrainY=train[target]\ntrainX=train.drop(target,axis=1)\ntestX=test","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.492982Z","iopub.status.busy":"2022-08-05T03:28:03.492168Z","iopub.status.idle":"2022-08-05T03:28:03.507833Z","shell.execute_reply":"2022-08-05T03:28:03.506658Z"},"papermill":{"duration":0.026877,"end_time":"2022-08-05T03:28:03.510903","exception":false,"start_time":"2022-08-05T03:28:03.484026","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_columns = list(trainX.columns)\nprint(df_columns)","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.527018Z","iopub.status.busy":"2022-08-05T03:28:03.526200Z","iopub.status.idle":"2022-08-05T03:28:03.533789Z","shell.execute_reply":"2022-08-05T03:28:03.531970Z"},"papermill":{"duration":0.018744,"end_time":"2022-08-05T03:28:03.536544","exception":false,"start_time":"2022-08-05T03:28:03.517800","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.553065Z","iopub.status.busy":"2022-08-05T03:28:03.552127Z","iopub.status.idle":"2022-08-05T03:28:03.558446Z","shell.execute_reply":"2022-08-05T03:28:03.557366Z"},"papermill":{"duration":0.017854,"end_time":"2022-08-05T03:28:03.561225","exception":false,"start_time":"2022-08-05T03:28:03.543371","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.577966Z","iopub.status.busy":"2022-08-05T03:28:03.577020Z","iopub.status.idle":"2022-08-05T03:28:03.587335Z","shell.execute_reply":"2022-08-05T03:28:03.586337Z"},"papermill":{"duration":0.021666,"end_time":"2022-08-05T03:28:03.590049","exception":false,"start_time":"2022-08-05T03:28:03.568383","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.605807Z","iopub.status.busy":"2022-08-05T03:28:03.605350Z","iopub.status.idle":"2022-08-05T03:28:03.612656Z","shell.execute_reply":"2022-08-05T03:28:03.611248Z"},"papermill":{"duration":0.018214,"end_time":"2022-08-05T03:28:03.615128","exception":false,"start_time":"2022-08-05T03:28:03.596914","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feat_df = to_feature(trainX)\ntest_feat_df = to_feature(testX)","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.630816Z","iopub.status.busy":"2022-08-05T03:28:03.630353Z","iopub.status.idle":"2022-08-05T03:28:03.658409Z","shell.execute_reply":"2022-08-05T03:28:03.656485Z"},"papermill":{"duration":0.039271,"end_time":"2022-08-05T03:28:03.661452","exception":false,"start_time":"2022-08-05T03:28:03.622181","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.007211,"end_time":"2022-08-05T03:28:03.676215","exception":false,"start_time":"2022-08-05T03:28:03.669004","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgbm\nfrom sklearn.metrics import mean_squared_error\n\ndef fit_lgbm(X, y, cv, \n             params: dict=None, \n             verbose: int=50):\n\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=np.float)\n\n    for i, (idx_train, idx_valid) in enumerate(cv): \n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = lgbm.LGBMRegressor(**params)\n        \n        with Timer(prefix='fit fold={} '.format(i)):\n            clf.fit(x_train, y_train, \n                    eval_set=[(x_valid, y_valid)],  \n                    early_stopping_rounds=100,\n                    verbose=verbose)\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        print(f'Fold {i} RMSLE: {mean_squared_error(y_valid, pred_i) ** .5:.4f}')\n        print()\n\n    score = mean_squared_error(y, oof_pred) ** .5\n    print('-' * 50)\n    print('FINISHED | Whole RMSLE: {:.4f}'.format(score))\n    return oof_pred, models","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.693804Z","iopub.status.busy":"2022-08-05T03:28:03.693357Z","iopub.status.idle":"2022-08-05T03:28:03.704443Z","shell.execute_reply":"2022-08-05T03:28:03.703395Z"},"papermill":{"duration":0.022508,"end_time":"2022-08-05T03:28:03.707034","exception":false,"start_time":"2022-08-05T03:28:03.684526","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'objective': 'rmse', \n    'learning_rate': .1,\n    'reg_lambda': 1.,\n    'reg_alpha': .1,\n    'max_depth': 5, \n    'n_estimators': 10000, \n    'colsample_bytree': .5, \n    'min_child_samples': 10,\n    'subsample_freq': 3,\n    'subsample': .9,\n    'importance_type': 'gain', \n    'random_state': 71,\n    'num_leaves': 62\n}","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.724285Z","iopub.status.busy":"2022-08-05T03:28:03.722918Z","iopub.status.idle":"2022-08-05T03:28:03.730029Z","shell.execute_reply":"2022-08-05T03:28:03.729036Z"},"papermill":{"duration":0.018189,"end_time":"2022-08-05T03:28:03.732576","exception":false,"start_time":"2022-08-05T03:28:03.714387","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)\ndisplay(ydf[0:3])","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.749252Z","iopub.status.busy":"2022-08-05T03:28:03.748838Z","iopub.status.idle":"2022-08-05T03:28:03.761352Z","shell.execute_reply":"2022-08-05T03:28:03.759776Z"},"papermill":{"duration":0.023642,"end_time":"2022-08-05T03:28:03.763827","exception":false,"start_time":"2022-08-05T03:28:03.740185","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nfor i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params, verbose=500)\n    \n    fig,ax = plt.subplots(figsize=(6,6))\n    ax.set_title(target[i],fontsize=20)\n    ax.set_ylabel('Train Predicted '+target[i],fontsize=12)\n    ax.set_xlabel('Train Actual '+target[i],fontsize=12)\n    ax.scatter(y,oof)","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:03.780601Z","iopub.status.busy":"2022-08-05T03:28:03.780162Z","iopub.status.idle":"2022-08-05T03:28:06.401551Z","shell.execute_reply":"2022-08-05T03:28:06.400275Z"},"papermill":{"duration":2.632657,"end_time":"2022-08-05T03:28:06.404227","exception":false,"start_time":"2022-08-05T03:28:03.771570","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Importance","metadata":{"papermill":{"duration":0.008992,"end_time":"2022-08-05T03:28:06.422500","exception":false,"start_time":"2022-08-05T03:28:06.413508","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def visualize_importance(models, feat_train_df):\n\n    feature_importance_df = pd.DataFrame()\n    for i, model in enumerate(models):\n        _df = pd.DataFrame()\n        _df['feature_importance'] = model.feature_importances_\n        _df['column'] = feat_train_df.columns\n        _df['fold'] = i + 1\n        feature_importance_df = pd.concat([feature_importance_df, _df], \n                                          axis=0, ignore_index=True)\n\n    order = feature_importance_df.groupby('column')\\\n        .sum()[['feature_importance']]\\\n        .sort_values('feature_importance', ascending=False).index[:50]\n\n    fig, ax = plt.subplots(figsize=(8, max(6, len(order) * .25)))\n    sns.boxenplot(data=feature_importance_df, \n                  x='feature_importance', \n                  y='column', \n                  order=order, \n                  ax=ax, \n                  palette='viridis', \n                  orient='h')\n    \n    ax.tick_params(axis='x', rotation=0)\n    #ax.set_title('Importance')\n    ax.grid()\n    fig.tight_layout()\n    \n    return fig,ax\n\n#fig, ax = visualize_importance(models, train_feat_df)","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:06.442119Z","iopub.status.busy":"2022-08-05T03:28:06.441659Z","iopub.status.idle":"2022-08-05T03:28:06.452480Z","shell.execute_reply":"2022-08-05T03:28:06.451144Z"},"papermill":{"duration":0.024074,"end_time":"2022-08-05T03:28:06.455155","exception":false,"start_time":"2022-08-05T03:28:06.431081","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params, verbose=500)\n    fig, ax = visualize_importance(models, train_feat_df)\n    ax.set_title(target[i]+' Imortance',fontsize=20)","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:06.474598Z","iopub.status.busy":"2022-08-05T03:28:06.474177Z","iopub.status.idle":"2022-08-05T03:28:09.042209Z","shell.execute_reply":"2022-08-05T03:28:09.040852Z"},"papermill":{"duration":2.580626,"end_time":"2022-08-05T03:28:09.044843","exception":false,"start_time":"2022-08-05T03:28:06.464217","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]\nfor i in range(5):\n    preds += [models[i].predict(test_feat_df.values)/5]\npredsT=np.array(preds).T\nprint(predsT.shape)\npreds2=[]\npreds3=[]\nfor item in predsT:\n    value=sum(item)\n    preds2+=[value]\n    preds3+=[int(np.where(value<0.5,0,np.where(value>8.5,9,round(value,0))))]\nprint(preds2[0:3])\nprint(preds3[0:3])","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:09.067124Z","iopub.status.busy":"2022-08-05T03:28:09.066690Z","iopub.status.idle":"2022-08-05T03:28:09.085262Z","shell.execute_reply":"2022-08-05T03:28:09.083714Z"},"papermill":{"duration":0.033887,"end_time":"2022-08-05T03:28:09.089056","exception":false,"start_time":"2022-08-05T03:28:09.055169","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1):\n    fig, ax = plt.subplots(figsize=(10,5))\n    sns.histplot(oof, label='Train Predicted '+target[i], ax=ax, color='C1',bins=30)\n    sns.histplot(preds2, label='Test Predicted '+target[i], ax=ax, color='black',bins=30)\n    ax.legend()\n    ax.grid()","metadata":{"execution":{"iopub.execute_input":"2022-08-05T03:28:09.112787Z","iopub.status.busy":"2022-08-05T03:28:09.111843Z","iopub.status.idle":"2022-08-05T03:28:09.575735Z","shell.execute_reply":"2022-08-05T03:28:09.574476Z"},"papermill":{"duration":0.478488,"end_time":"2022-08-05T03:28:09.578511","exception":false,"start_time":"2022-08-05T03:28:09.100023","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit=pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmit['Label']=preds3\nsubmit.to_csv('submission.csv',index=None)\ndisplay(submit)\ndisplay(submit['Label'].value_counts())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}