{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":204576043,"sourceType":"kernelVersion"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Jane Street LGBM w/trained_model Submission","metadata":{"papermill":{"duration":0.009105,"end_time":"2022-10-25T14:48:23.51949","exception":false,"start_time":"2022-10-25T14:48:23.510385","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/jane-street-optuna-lgbm\n\nhttps://www.kaggle.com/code/stpeteishii/jane-street-lgbm-w-best-trial-submission\n\nhttps://www.kaggle.com/code/stpeteishii/jane-street-lgbm-w-trained-model-submission","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgb\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, train_test_split\nimport optuna\nimport yaml\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server\nimport json","metadata":{"papermill":{"duration":8.574502,"end_time":"2022-10-25T14:48:32.101977","exception":false,"start_time":"2022-10-25T14:48:23.527475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:04.471202Z","iopub.execute_input":"2024-11-01T00:59:04.471642Z","iopub.status.idle":"2024-11-01T00:59:26.630712Z","shell.execute_reply.started":"2024-11-01T00:59:04.471598Z","shell.execute_reply":"2024-11-01T00:59:26.629215Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.007139,"end_time":"2022-10-25T14:48:32.11684","exception":false,"start_time":"2022-10-25T14:48:32.109701","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train0 = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ntest = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet')\n\nprint(len(train0))\nprint(len(test))\ntrain0 = train0.sample(frac=1, random_state=42).reset_index(drop=True)\n\ntcols0=test.columns.tolist()\nprint(tcols0)\ntcols=[ 'date_id', 'time_id', 'symbol_id', 'weight',  'feature_00', 'feature_01', 'feature_02', 'feature_03', 'feature_04', 'feature_05', 'feature_06', 'feature_07', 'feature_08', 'feature_09', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_26', 'feature_27', 'feature_28', 'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_33', 'feature_34', 'feature_35', 'feature_36', 'feature_37', 'feature_38', 'feature_39', 'feature_40', 'feature_41', 'feature_42', 'feature_43', 'feature_44', 'feature_45', 'feature_46', 'feature_47', 'feature_48', 'feature_49', 'feature_50', 'feature_51', 'feature_52', 'feature_53', 'feature_54', 'feature_55', 'feature_56', 'feature_57', 'feature_58', 'feature_59', 'feature_60', 'feature_61', 'feature_62', 'feature_63', 'feature_64', 'feature_65', 'feature_66', 'feature_67', 'feature_68', 'feature_69', 'feature_70', 'feature_71', 'feature_72', 'feature_73', 'feature_74', 'feature_75', 'feature_76', 'feature_77', 'feature_78']\ntarget='responder_6'\ndataX=train0[tcols]\ndataY=train0[target]\nTESTX=test[tcols]","metadata":{"papermill":{"duration":0.130645,"end_time":"2022-10-25T14:48:32.254936","exception":false,"start_time":"2022-10-25T14:48:32.124291","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:26.632664Z","iopub.execute_input":"2024-11-01T00:59:26.633429Z","iopub.status.idle":"2024-11-01T00:59:32.592986Z","shell.execute_reply.started":"2024-11-01T00:59:26.633382Z","shell.execute_reply":"2024-11-01T00:59:32.591510Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target setting","metadata":{"papermill":{"duration":0.007509,"end_time":"2022-10-25T14:48:32.451206","exception":false,"start_time":"2022-10-25T14:48:32.443697","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_columns = list(dataX.columns)\nprint(df_columns)\n\nm=len(dataX)\nprint(m)\nM=list(range(m))\nrandom.seed(2021)\nrandom.shuffle(M)\n\ntrainX=dataX.iloc[M[0:(m//5)*4]]\ntrainY=dataY.iloc[M[0:(m//5)*4]]\ntestX=dataX.iloc[M[(m//5)*4:]]\ntestY=dataY.iloc[M[(m//5)*4:]]\n\ntrain_df=trainX\ntest_df=testX\n\ntrain_df.columns=df_columns\ntest_df.columns=df_columns","metadata":{"papermill":{"duration":0.017844,"end_time":"2022-10-25T14:48:32.516164","exception":false,"start_time":"2022-10-25T14:48:32.49832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:32.594744Z","iopub.execute_input":"2024-11-01T00:59:32.595257Z","iopub.status.idle":"2024-11-01T00:59:36.890883Z","shell.execute_reply.started":"2024-11-01T00:59:32.595206Z","shell.execute_reply":"2024-11-01T00:59:36.889869Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# valid_df","metadata":{}},{"cell_type":"code","source":"valid_from = 1455 # for private you should change to 1455 (1 year)\nalltraindata = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\nvalid_df = alltraindata.filter(pl.col(\"date_id\")>=valid_from).collect()\nvalid_df = valid_df.with_columns(pl.Series(range(len(valid_df))).alias(\"row_id\"))\nweights = valid_df.select(\"weight\").to_numpy().reshape(-1)\n\nlen(valid_df)\ndisplay(valid_df[0:2])\nprint(weights)\nprint(weights.shape)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T00:59:36.893572Z","iopub.execute_input":"2024-11-01T00:59:36.894418Z","iopub.status.idle":"2024-11-01T00:59:46.908873Z","shell.execute_reply.started":"2024-11-01T00:59:36.894374Z","shell.execute_reply":"2024-11-01T00:59:46.907499Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# weighted_zero_mean_r2","metadata":{}},{"cell_type":"code","source":"def weighted_zero_mean_r2(y_true, y_pred, weights):\n    \"\"\"\n    Calculate the sample weighted zero-mean R-squared score.\n\n    Parameters:\n    y_true (numpy.ndarray): Ground-truth values for responder_6.\n    y_pred (numpy.ndarray): Predicted values for responder_6.\n    weights (numpy.ndarray): Sample weight vector.\n\n    Returns:\n    float: The weighted zero-mean R-squared score.\n    \"\"\"\n    numerator = np.sum(weights * (y_true - y_pred)**2)\n    denominator = np.sum(weights * y_true**2)\n    \n    r2_score = 1 - numerator / denominator\n    return r2_score","metadata":{"execution":{"iopub.status.busy":"2024-11-01T00:59:46.910397Z","iopub.execute_input":"2024-11-01T00:59:46.910901Z","iopub.status.idle":"2024-11-01T00:59:46.918005Z","shell.execute_reply.started":"2024-11-01T00:59:46.910848Z","shell.execute_reply":"2024-11-01T00:59:46.916517Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.016539,"end_time":"2022-10-25T14:48:32.696115","exception":false,"start_time":"2022-10-25T14:48:32.679576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:46.919892Z","iopub.execute_input":"2024-11-01T00:59:46.920378Z","iopub.status.idle":"2024-11-01T00:59:46.933192Z","shell.execute_reply.started":"2024-11-01T00:59:46.920326Z","shell.execute_reply":"2024-11-01T00:59:46.931550Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.024259,"end_time":"2022-10-25T14:48:32.728659","exception":false,"start_time":"2022-10-25T14:48:32.7044","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:46.934726Z","iopub.execute_input":"2024-11-01T00:59:46.935821Z","iopub.status.idle":"2024-11-01T00:59:46.948192Z","shell.execute_reply.started":"2024-11-01T00:59:46.935742Z","shell.execute_reply":"2024-11-01T00:59:46.946819Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.018733,"end_time":"2022-10-25T14:48:32.755482","exception":false,"start_time":"2022-10-25T14:48:32.736749","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:46.949626Z","iopub.execute_input":"2024-11-01T00:59:46.950057Z","iopub.status.idle":"2024-11-01T00:59:46.966080Z","shell.execute_reply.started":"2024-11-01T00:59:46.950016Z","shell.execute_reply":"2024-11-01T00:59:46.964634Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\ntest_feat_df = to_feature(test_df)","metadata":{"papermill":{"duration":0.042183,"end_time":"2022-10-25T14:48:32.805699","exception":false,"start_time":"2022-10-25T14:48:32.763516","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:46.968240Z","iopub.execute_input":"2024-11-01T00:59:46.969429Z","iopub.status.idle":"2024-11-01T00:59:47.795071Z","shell.execute_reply.started":"2024-11-01T00:59:46.969369Z","shell.execute_reply":"2024-11-01T00:59:47.793781Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.008218,"end_time":"2022-10-25T14:48:32.822457","exception":false,"start_time":"2022-10-25T14:48:32.814239","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Since a trained model is used, there is no need to retrain it within this notebook, so the execution time is short.\nIf you use best parameters obtained from optuna, you will need to retrain it, which takes time.","metadata":{}},{"cell_type":"code","source":"import joblib\ndir0='/kaggle/input/jane-street-optuna-lgbm'\nnum_folds=5\n\nmodels=[]\nfor i in range(num_folds):\n    clf = joblib.load(f'{dir0}/model_fold_{i}.pkl') \n    models+=[clf]\n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T00:59:47.798944Z","iopub.execute_input":"2024-11-01T00:59:47.799427Z","iopub.status.idle":"2024-11-01T00:59:47.888389Z","shell.execute_reply.started":"2024-11-01T00:59:47.799376Z","shell.execute_reply":"2024-11-01T00:59:47.887254Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    dir0='/kaggle/input/jane-street-optuna-lgbm'\n    num_folds=5\n\n    def load_models(num_folds):\n        models = []\n        for i in range(num_folds):\n            # Load parameters\n            with open(f'{dir0}/model_fold_{i}_params.json', 'r') as f:\n                params = json.load(f)\n\n            # Create a new LGBMRegressor with the saved params\n            clf = lgb.LGBMRegressor(**params)\n\n            # Load Booster model and assign it\n            booster = lgb.Booster(model_file=f'{dir0}/model_fold_{i}.model')\n            clf._Booster = booster  # Assign loaded Booster to LGBMRegressor\n            models.append(clf)\n        return models\n\n    models=load_models(num_folds)","metadata":{}},{"cell_type":"code","source":"models","metadata":{"execution":{"iopub.status.busy":"2024-11-01T00:59:47.889719Z","iopub.execute_input":"2024-11-01T00:59:47.890134Z","iopub.status.idle":"2024-11-01T00:59:47.902982Z","shell.execute_reply.started":"2024-11-01T00:59:47.890094Z","shell.execute_reply":"2024-11-01T00:59:47.901845Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)","metadata":{"papermill":{"duration":0.018428,"end_time":"2022-10-25T14:48:32.880247","exception":false,"start_time":"2022-10-25T14:48:32.861819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:47.904575Z","iopub.execute_input":"2024-11-01T00:59:47.904982Z","iopub.status.idle":"2024-11-01T00:59:47.910842Z","shell.execute_reply.started":"2024-11-01T00:59:47.904944Z","shell.execute_reply":"2024-11-01T00:59:47.909725Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds=[]\nfor i in range(5):\n    preds += [models[i].predict(test_feat_df.values)/5]\npredsT=np.array(preds).T\npreds2=[]\nfor item in predsT:\n    value=sum(item)\n    preds2+=[value]\nprint(preds2[0:5])","metadata":{"papermill":{"duration":0.513213,"end_time":"2022-10-25T14:48:47.39475","exception":false,"start_time":"2022-10-25T14:48:46.881537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-01T00:59:47.912139Z","iopub.execute_input":"2024-11-01T00:59:47.912491Z","iopub.status.idle":"2024-11-01T00:59:48.764942Z","shell.execute_reply.started":"2024-11-01T00:59:47.912454Z","shell.execute_reply":"2024-11-01T00:59:48.763172Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Submission for API","metadata":{}},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n\n    global lags_,df_columns\n    if lags is not None:\n        lags_ = lags\n\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    \n    x_test = test[df_columns].to_pandas()\n    pred = []\n    for i in range(5):\n        pred += [models[i].predict(x_test.values)/5] \n                \n    pred = np.mean(pred, axis=0)\n    \n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2024-11-01T00:59:48.766152Z","iopub.status.idle":"2024-11-01T00:59:48.766581Z","shell.execute_reply.started":"2024-11-01T00:59:48.766384Z","shell.execute_reply":"2024-11-01T00:59:48.766404Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-11-01T00:59:48.768383Z","iopub.status.idle":"2024-11-01T00:59:48.768881Z","shell.execute_reply.started":"2024-11-01T00:59:48.768637Z","shell.execute_reply":"2024-11-01T00:59:48.768662Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.012162,"end_time":"2022-10-25T14:48:48.951196","exception":false,"start_time":"2022-10-25T14:48:48.939034","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}