{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Jane Street Optuna CatBoost","metadata":{"papermill":{"duration":0.009105,"end_time":"2022-10-25T14:48:23.51949","exception":false,"start_time":"2022-10-25T14:48:23.510385","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/jane-street-optuna-lgbm\n\nhttps://www.kaggle.com/code/stpeteishii/jane-street-lgbm-submission","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgb\nimport catboost as catb\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, train_test_split\nimport optuna\nimport yaml\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"papermill":{"duration":8.574502,"end_time":"2022-10-25T14:48:32.101977","exception":false,"start_time":"2022-10-25T14:48:23.527475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-28T07:21:52.259144Z","iopub.execute_input":"2024-10-28T07:21:52.259638Z","iopub.status.idle":"2024-10-28T07:22:15.600110Z","shell.execute_reply.started":"2024-10-28T07:21:52.259570Z","shell.execute_reply":"2024-10-28T07:22:15.599031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.007139,"end_time":"2022-10-25T14:48:32.11684","exception":false,"start_time":"2022-10-25T14:48:32.109701","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train0 = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ntest = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet')\n\nprint(len(train0))\nprint(len(test))\ntrain0 = train0.sample(frac=1, random_state=42).reset_index(drop=True)\n\ntcols0=test.columns.tolist()\nprint(tcols0)\ntcols=[ 'date_id', 'time_id', 'symbol_id', 'weight',  'feature_00', 'feature_01', 'feature_02', 'feature_03', 'feature_04', 'feature_05', 'feature_06', 'feature_07', 'feature_08', 'feature_09', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_26', 'feature_27', 'feature_28', 'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_33', 'feature_34', 'feature_35', 'feature_36', 'feature_37', 'feature_38', 'feature_39', 'feature_40', 'feature_41', 'feature_42', 'feature_43', 'feature_44', 'feature_45', 'feature_46', 'feature_47', 'feature_48', 'feature_49', 'feature_50', 'feature_51', 'feature_52', 'feature_53', 'feature_54', 'feature_55', 'feature_56', 'feature_57', 'feature_58', 'feature_59', 'feature_60', 'feature_61', 'feature_62', 'feature_63', 'feature_64', 'feature_65', 'feature_66', 'feature_67', 'feature_68', 'feature_69', 'feature_70', 'feature_71', 'feature_72', 'feature_73', 'feature_74', 'feature_75', 'feature_76', 'feature_77', 'feature_78']\ntarget='responder_6'\ndataX=train0[tcols]\ndataY=train0[target]\nTESTX=test[tcols]","metadata":{"papermill":{"duration":0.130645,"end_time":"2022-10-25T14:48:32.254936","exception":false,"start_time":"2022-10-25T14:48:32.124291","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-28T07:22:15.602874Z","iopub.execute_input":"2024-10-28T07:22:15.603710Z","iopub.status.idle":"2024-10-28T07:22:21.357308Z","shell.execute_reply.started":"2024-10-28T07:22:15.603667Z","shell.execute_reply":"2024-10-28T07:22:21.356102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataY.min(),dataY.max())","metadata":{"execution":{"iopub.status.busy":"2024-10-28T07:22:21.358701Z","iopub.execute_input":"2024-10-28T07:22:21.359061Z","iopub.status.idle":"2024-10-28T07:22:21.371032Z","shell.execute_reply.started":"2024-10-28T07:22:21.359025Z","shell.execute_reply":"2024-10-28T07:22:21.369652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target setting","metadata":{"papermill":{"duration":0.007509,"end_time":"2022-10-25T14:48:32.451206","exception":false,"start_time":"2022-10-25T14:48:32.443697","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_columns = list(dataX.columns)\nprint(df_columns)\n\nm=len(dataX)\nprint(m)\nM=list(range(m))\nrandom.seed(2021)\nrandom.shuffle(M)\n\ntrainX=dataX.iloc[M[0:(m//5)*4]]\ntrainY=dataY.iloc[M[0:(m//5)*4]]\ntestX=dataX.iloc[M[(m//5)*4:]]\ntestY=dataY.iloc[M[(m//5)*4:]]\n\ntrain_df=trainX\ntest_df=testX\n\ntrain_df.columns=df_columns\ntest_df.columns=df_columns","metadata":{"papermill":{"duration":0.017844,"end_time":"2022-10-25T14:48:32.516164","exception":false,"start_time":"2022-10-25T14:48:32.49832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-28T07:22:21.372267Z","iopub.execute_input":"2024-10-28T07:22:21.372816Z","iopub.status.idle":"2024-10-28T07:22:25.493044Z","shell.execute_reply.started":"2024-10-28T07:22:21.372776Z","shell.execute_reply":"2024-10-28T07:22:25.491858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# valid_df","metadata":{}},{"cell_type":"code","source":"valid_from = 1455 # for private you should change to 1455 (1 year)\nalltraindata = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\nvalid_df = alltraindata.filter(pl.col(\"date_id\")>=valid_from).collect()\nvalid_df = valid_df.with_columns(pl.Series(range(len(valid_df))).alias(\"row_id\"))\nweights = valid_df.select(\"weight\").to_numpy().reshape(-1)\n\nlen(valid_df)\ndisplay(valid_df[0:2])\nprint(weights)\nprint(weights.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# weighted_zero_mean_r2","metadata":{}},{"cell_type":"code","source":"def weighted_zero_mean_r2(y_true, y_pred, weights):\n    \"\"\"\n    Calculate the sample weighted zero-mean R-squared score.\n\n    Parameters:\n    y_true (numpy.ndarray): Ground-truth values for responder_6.\n    y_pred (numpy.ndarray): Predicted values for responder_6.\n    weights (numpy.ndarray): Sample weight vector.\n\n    Returns:\n    float: The weighted zero-mean R-squared score.\n    \"\"\"\n    numerator = np.sum(weights * (y_true - y_pred)**2)\n    denominator = np.sum(weights * y_true**2)\n    \n    r2_score = 1 - numerator / denominator\n    return r2_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Optuna","metadata":{}},{"cell_type":"code","source":"def objective0(trial,data=trainX,target=trainY):\n    train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)\n\n    param =   {\n        'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 3.0,5.0),\n        'random_state': trial.suggest_int('random_state', 56,62),       \n        'learning_rate': trial.suggest_loguniform('learning_rate',0.16,0.20),  \n        'bagging_temperature': trial.suggest_loguniform('bagging_temperature', 0.06,0.1),\n        'random_strength':trial.suggest_loguniform('random_strength', 7.0,10.0),    \n        'iterations': trial.suggest_int('iterations', 600,800),  \n\n        #fixed part            \n        'depth': 3,\n        'border_count': 88,         \n        'verbose': False,\n        #'objective': \"binary:logistic\",\n        #'num_class': 1\n    }\n\n    model = catb.CatBoostRegressor(**param) \n    model.fit(train_x,train_y,eval_set=[(test_x,test_y)])\n    preds = model.predict(test_x)\n    #score = roc_auc_score(test_y, preds)\n    rmse = mean_squared_error(test_y, preds,squared=False)\n    \n    return rmse","metadata":{"execution":{"iopub.status.busy":"2024-10-28T07:22:25.495874Z","iopub.execute_input":"2024-10-28T07:22:25.496258Z","iopub.status.idle":"2024-10-28T07:22:25.504827Z","shell.execute_reply.started":"2024-10-28T07:22:25.496215Z","shell.execute_reply":"2024-10-28T07:22:25.503674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BestTrials=[]\nStudies=[]\nfor i in range(1):\n    objective=[objective0][i]\n    \n    study = optuna.create_study(direction='minimize')\n    study.optimize(objective, n_trials=100)\n    \n    print('Number of finished trials:', len(study.trials))\n    print('Best trial:', study.best_trial.params)\n\n    Best_trial=study.best_trial.params\n    #print(Best_trial)\n\n    with open(f\"Best_trial{i}.yaml\", \"w\") as yaml_file:\n        yaml.dump(Best_trial, yaml_file)\n\n    fix_dict = {\n        #fixed part            \n        'depth': 3,\n        'border_count': 88,         \n        'verbose': False,\n        #'objective': \"binary:logistic\",\n        #'num_class': 1\n      }\n\n    Best_trial.update(fix_dict)\n    BestTrials+=[Best_trial]\n    Studies+=[study]","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-10-28T07:22:25.506351Z","iopub.execute_input":"2024-10-28T07:22:25.506742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.016539,"end_time":"2022-10-25T14:48:32.696115","exception":false,"start_time":"2022-10-25T14:48:32.679576","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.024259,"end_time":"2022-10-25T14:48:32.728659","exception":false,"start_time":"2022-10-25T14:48:32.7044","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.018733,"end_time":"2022-10-25T14:48:32.755482","exception":false,"start_time":"2022-10-25T14:48:32.736749","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\ntest_feat_df = to_feature(test_df)","metadata":{"papermill":{"duration":0.042183,"end_time":"2022-10-25T14:48:32.805699","exception":false,"start_time":"2022-10-25T14:48:32.763516","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.008218,"end_time":"2022-10-25T14:48:32.822457","exception":false,"start_time":"2022-10-25T14:48:32.814239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"params = BestTrials[0]\nprint(params)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\ndef fit_catb(X, y, cv, \n             params: dict=None):\n\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=float)\n\n    for i, (idx_train, idx_valid) in enumerate(cv): \n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = catb.CatBoostRegressor(**params)\n        \n        with Timer(prefix='fit fold={} '.format(i)):\n            clf.fit(x_train, y_train, \n                    eval_set=[(x_valid, y_valid)])\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        \n        # Save each fold's model to .model format\n        clf.save_model(f'model_fold_{i}.model')\n\n\n    weights2=weights[0:len(y)]\n    r2_score=weighted_zero_mean_r2(y, oof_pred, weights2)\n\n    print('-' * 50)\n    print('FINISHED | r2_score: {:.4f}'.format(r2_score))\n    return oof_pred, models","metadata":{"papermill":{"duration":0.022515,"end_time":"2022-10-25T14:48:32.853327","exception":false,"start_time":"2022-10-25T14:48:32.830812","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)","metadata":{"papermill":{"duration":0.018428,"end_time":"2022-10-25T14:48:32.880247","exception":false,"start_time":"2022-10-25T14:48:32.861819","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nfor i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_catb(train_feat_df.values, y, cv, params=params)","metadata":{"papermill":{"duration":6.994676,"end_time":"2022-10-25T14:48:39.916257","exception":false,"start_time":"2022-10-25T14:48:32.921581","status":"completed"},"tags":[],"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]\nfor i in range(5):\n    preds += [models[i].predict(test_feat_df.values)/5]\npredsT=np.array(preds).T\npreds2=[]\nfor item in predsT:\n    value=sum(item)\n    preds2+=[value]\nprint(preds2[0:5])","metadata":{"papermill":{"duration":0.513213,"end_time":"2022-10-25T14:48:47.39475","exception":false,"start_time":"2022-10-25T14:48:46.881537","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submission for API","metadata":{}},{"cell_type":"markdown","source":"    lags_ : pl.DataFrame | None = None\n\n    def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n\n        global lags_,df_columns\n        if lags is not None:\n            lags_ = lags\n\n        predictions = test.select(\n            'row_id',\n            pl.lit(0.0).alias('responder_6'),\n        )\n\n        x_test = test[df_columns].to_pandas()\n        pred = []\n        for i in range(5):\n            pred += [models[i].predict(x_test.values)/5] \n\n        pred = np.mean(pred, axis=0)\n\n        predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n\n        return predictions","metadata":{}},{"cell_type":"markdown","source":"    inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n    if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n        inference_server.serve()\n    else:\n        inference_server.run_local_gateway(\n            (\n                '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n                '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n            )\n        )","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.012162,"end_time":"2022-10-25T14:48:48.951196","exception":false,"start_time":"2022-10-25T14:48:48.939034","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}