{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Jane Street Visualize Importance","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport category_encoders as ce\n\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nfrom sklearn.preprocessing import LabelEncoder\n\nimport polars as pl","metadata":{"papermill":{"duration":8.958811,"end_time":"2021-06-21T06:51:56.66964","exception":false,"start_time":"2021-06-21T06:51:47.710829","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:48.862076Z","iopub.execute_input":"2024-11-01T15:05:48.862578Z","iopub.status.idle":"2024-11-01T15:05:48.870699Z","shell.execute_reply.started":"2024-11-01T15:05:48.862535Z","shell.execute_reply":"2024-11-01T15:05:48.869508Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.024602,"end_time":"2021-06-21T06:51:56.719864","exception":false,"start_time":"2021-06-21T06:51:56.695262","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train0 = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ntest = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet')\n\nprint(len(train0))\nprint(len(test))\ntrain0 = train0.sample(frac=1, random_state=42).reset_index(drop=True)\n\ntcols0=test.columns.tolist()\nprint(tcols0)\ntcols=[ 'date_id', 'time_id', 'symbol_id', 'weight',  'feature_00', 'feature_01', 'feature_02', 'feature_03', 'feature_04', 'feature_05', 'feature_06', 'feature_07', 'feature_08', 'feature_09', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_26', 'feature_27', 'feature_28', 'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_33', 'feature_34', 'feature_35', 'feature_36', 'feature_37', 'feature_38', 'feature_39', 'feature_40', 'feature_41', 'feature_42', 'feature_43', 'feature_44', 'feature_45', 'feature_46', 'feature_47', 'feature_48', 'feature_49', 'feature_50', 'feature_51', 'feature_52', 'feature_53', 'feature_54', 'feature_55', 'feature_56', 'feature_57', 'feature_58', 'feature_59', 'feature_60', 'feature_61', 'feature_62', 'feature_63', 'feature_64', 'feature_65', 'feature_66', 'feature_67', 'feature_68', 'feature_69', 'feature_70', 'feature_71', 'feature_72', 'feature_73', 'feature_74', 'feature_75', 'feature_76', 'feature_77', 'feature_78']\ntarget='responder_6'\ndataX=train0[tcols]\ndataY=train0[target]\nTESTX=test[tcols]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_columns = list(dataX.columns)\nprint(df_columns)\n\nm=len(dataX)\nprint(m)\nM=list(range(m))\nrandom.seed(2021)\nrandom.shuffle(M)\n\ntrainX=dataX.iloc[M[0:(m//5)*4]]\ntrainY=dataY.iloc[M[0:(m//5)*4]]\ntestX=dataX.iloc[M[(m//5)*4:]]\ntestY=dataY.iloc[M[(m//5)*4:]]\n\ntrain_df=trainX\ntest_df=testX\n\ntrain_df.columns=df_columns\ntest_df.columns=df_columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target setting","metadata":{"papermill":{"duration":0.027891,"end_time":"2021-06-21T06:51:57.343874","exception":false,"start_time":"2021-06-21T06:51:57.315983","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_columns = list(trainX.columns)\nprint(df_columns)","metadata":{"papermill":{"duration":0.039658,"end_time":"2021-06-21T06:51:57.556135","exception":false,"start_time":"2021-06-21T06:51:57.516477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.852758Z","iopub.execute_input":"2024-11-01T15:05:49.853098Z","iopub.status.idle":"2024-11-01T15:05:49.865567Z","shell.execute_reply.started":"2024-11-01T15:05:49.853057Z","shell.execute_reply":"2024-11-01T15:05:49.864477Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.037641,"end_time":"2021-06-21T06:51:57.828748","exception":false,"start_time":"2021-06-21T06:51:57.791107","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.867008Z","iopub.execute_input":"2024-11-01T15:05:49.867389Z","iopub.status.idle":"2024-11-01T15:05:49.877393Z","shell.execute_reply.started":"2024-11-01T15:05:49.867351Z","shell.execute_reply":"2024-11-01T15:05:49.876293Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.041214,"end_time":"2021-06-21T06:51:57.899433","exception":false,"start_time":"2021-06-21T06:51:57.858219","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.878719Z","iopub.execute_input":"2024-11-01T15:05:49.879335Z","iopub.status.idle":"2024-11-01T15:05:49.896265Z","shell.execute_reply.started":"2024-11-01T15:05:49.879276Z","shell.execute_reply":"2024-11-01T15:05:49.895064Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.039575,"end_time":"2021-06-21T06:51:57.968414","exception":false,"start_time":"2021-06-21T06:51:57.928839","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.897615Z","iopub.execute_input":"2024-11-01T15:05:49.897969Z","iopub.status.idle":"2024-11-01T15:05:49.914549Z","shell.execute_reply.started":"2024-11-01T15:05:49.897932Z","shell.execute_reply":"2024-11-01T15:05:49.913116Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(trainX)\n#test_feat_df = to_feature(testX)","metadata":{"papermill":{"duration":0.046135,"end_time":"2021-06-21T06:51:58.044269","exception":false,"start_time":"2021-06-21T06:51:57.998134","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.916026Z","iopub.execute_input":"2024-11-01T15:05:49.916411Z","iopub.status.idle":"2024-11-01T15:05:49.935572Z","shell.execute_reply.started":"2024-11-01T15:05:49.916363Z","shell.execute_reply":"2024-11-01T15:05:49.934334Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.030413,"end_time":"2021-06-21T06:51:58.107898","exception":false,"start_time":"2021-06-21T06:51:58.077485","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgbm\nfrom sklearn.metrics import mean_squared_error\n\ndef fit_lgbm(X, y, cv, \n             params: dict=None, \n             verbose: int=50):\n\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=float)\n\n    for i, (idx_train, idx_valid) in enumerate(cv): \n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = lgbm.LGBMRegressor(**params)\n        \n        with Timer(prefix='fit fold={} '.format(i)):\n            clf.fit(x_train, y_train, \n                    eval_set=[(x_valid, y_valid)])\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        print(f'Fold {i} RMSLE: {mean_squared_error(y_valid, pred_i) ** .5:.4f}')\n        print()\n\n    score = mean_squared_error(y, oof_pred) ** .5\n    print('-' * 50)\n    print('FINISHED | Whole RMSLE: {:.4f}'.format(score))\n    return oof_pred, models","metadata":{"papermill":{"duration":0.047557,"end_time":"2021-06-21T06:51:58.186048","exception":false,"start_time":"2021-06-21T06:51:58.138491","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.937642Z","iopub.execute_input":"2024-11-01T15:05:49.937995Z","iopub.status.idle":"2024-11-01T15:05:49.948423Z","shell.execute_reply.started":"2024-11-01T15:05:49.937957Z","shell.execute_reply":"2024-11-01T15:05:49.947216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    'objective': 'rmse', \n    'learning_rate': .1,\n    'reg_lambda': 1.,\n    'reg_alpha': .1,\n    'max_depth': 5, \n    'n_estimators': 10, \n    'colsample_bytree': .5, \n    'min_child_samples': 10,\n    'subsample_freq': 3,\n    'subsample': .9,\n    'importance_type': 'gain', \n    'random_state': 71,\n    'num_leaves': 62\n}","metadata":{"papermill":{"duration":0.04017,"end_time":"2021-06-21T06:51:58.257181","exception":false,"start_time":"2021-06-21T06:51:58.217011","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.949838Z","iopub.execute_input":"2024-11-01T15:05:49.950256Z","iopub.status.idle":"2024-11-01T15:05:49.965247Z","shell.execute_reply.started":"2024-11-01T15:05:49.950199Z","shell.execute_reply":"2024-11-01T15:05:49.964077Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)","metadata":{"papermill":{"duration":0.039211,"end_time":"2021-06-21T06:51:58.32736","exception":false,"start_time":"2021-06-21T06:51:58.288149","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.969459Z","iopub.execute_input":"2024-11-01T15:05:49.969905Z","iopub.status.idle":"2024-11-01T15:05:49.978289Z","shell.execute_reply.started":"2024-11-01T15:05:49.969864Z","shell.execute_reply":"2024-11-01T15:05:49.977245Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nfor i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params)\n    \n    fig,ax = plt.subplots(figsize=(6,6))\n    ax.set_title(target,fontsize=20)\n    ax.set_xlabel('pred',fontsize=12)\n    ax.set_ylabel('true',fontsize=12)\n    ax.scatter(oof,y,alpha=0.2)","metadata":{"papermill":{"duration":3.499601,"end_time":"2021-06-21T06:52:01.94937","exception":false,"start_time":"2021-06-21T06:51:58.449769","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:49.979811Z","iopub.execute_input":"2024-11-01T15:05:49.980288Z","iopub.status.idle":"2024-11-01T15:05:50.387695Z","shell.execute_reply.started":"2024-11-01T15:05:49.980235Z","shell.execute_reply":"2024-11-01T15:05:50.386626Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualize Importance","metadata":{"papermill":{"duration":0.042694,"end_time":"2021-06-21T06:52:02.133811","exception":false,"start_time":"2021-06-21T06:52:02.091117","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def visualize_importance(models, feat_train_df):\n\n    feature_importance_df = pd.DataFrame()\n    for i, model in enumerate(models):\n        _df = pd.DataFrame()\n        _df['feature_importance'] = model.feature_importances_\n        _df['column'] = feat_train_df.columns\n        _df['fold'] = i + 1\n        feature_importance_df = pd.concat([feature_importance_df, _df], \n                                          axis=0, ignore_index=True)\n\n    order = feature_importance_df.groupby('column')\\\n        .sum()[['feature_importance']]\\\n        .sort_values('feature_importance', ascending=False).index[:50]\n\n    fig, ax = plt.subplots(figsize=(8, max(6, len(order) * .25)))\n    sns.boxenplot(data=feature_importance_df, \n                  x='feature_importance', \n                  y='column', \n                  order=order, \n                  ax=ax, \n                  palette='viridis', \n                  orient='h')\n    \n    ax.tick_params(axis='x', rotation=0)\n    #ax.set_title('Importance')\n    ax.grid()\n    fig.tight_layout()\n    \n    return fig,ax\n\n#fig, ax = visualize_importance(models, train_feat_df)","metadata":{"papermill":{"duration":0.053724,"end_time":"2021-06-21T06:52:02.229723","exception":false,"start_time":"2021-06-21T06:52:02.175999","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:50.389265Z","iopub.execute_input":"2024-11-01T15:05:50.389642Z","iopub.status.idle":"2024-11-01T15:05:50.400542Z","shell.execute_reply.started":"2024-11-01T15:05:50.389602Z","shell.execute_reply":"2024-11-01T15:05:50.399246Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params, verbose=500)\n    fig, ax = visualize_importance(models, train_feat_df)\n    ax.set_title(target+' Imortance',fontsize=20)\n","metadata":{"papermill":{"duration":10.556971,"end_time":"2021-06-21T06:52:12.82911","exception":false,"start_time":"2021-06-21T06:52:02.272139","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T15:05:50.40182Z","iopub.execute_input":"2024-11-01T15:05:50.402242Z","iopub.status.idle":"2024-11-01T15:05:51.271559Z","shell.execute_reply.started":"2024-11-01T15:05:50.402167Z","shell.execute_reply":"2024-11-01T15:05:51.269998Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols=df_columns[1:]\nprint(cols)\n\ndef showfig(targ1,targ2): \n    train2=train0[[targ1,targ2]].dropna()\n    \n    df_x=pd.DataFrame(train2[targ1])\n    df_y=pd.DataFrame(train2[targ2])\n\n    fig,ax = plt.subplots(figsize=(4,4))\n    ax.set_title( targ2+' vs '+targ1,fontsize=12)\n    ax.set_xlabel(targ1,fontsize=12)\n    ax.set_ylabel(targ2,fontsize=12)\n    ax.scatter(df_x,df_y,alpha=0.1)\n\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(len(cols)):\n    for j in range(i+1,len(cols)):\n        targ1=cols[i]\n        targ2=cols[j]\n        showfig(targ1,targ2)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}