{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":204224978,"sourceType":"kernelVersion"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":313.680416,"end_time":"2024-10-31T04:37:33.342473","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-31T04:32:19.662057","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Jane Street LGBM Submission","metadata":{"papermill":{"duration":0.009106,"end_time":"2024-10-31T04:32:22.746626","exception":false,"start_time":"2024-10-31T04:32:22.737520","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/jane-street-optuna-lgbm\n\nhttps://www.kaggle.com/code/stpeteishii/jane-street-lgbm-w-best-trial-submission\n\nhttps://www.kaggle.com/code/stpeteishii/jane-street-lgbm-w-trained-model-submission","metadata":{"papermill":{"duration":0.009398,"end_time":"2024-10-31T04:32:22.764962","exception":false,"start_time":"2024-10-31T04:32:22.755564","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgb\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, train_test_split\nimport optuna\nimport yaml\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:32:22.781256Z","iopub.status.busy":"2024-10-31T04:32:22.780816Z","iopub.status.idle":"2024-10-31T04:32:42.447718Z","shell.execute_reply":"2024-10-31T04:32:42.446423Z"},"papermill":{"duration":19.67827,"end_time":"2024-10-31T04:32:42.450483","exception":false,"start_time":"2024-10-31T04:32:22.772213","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.006982,"end_time":"2024-10-31T04:32:42.464954","exception":false,"start_time":"2024-10-31T04:32:42.457972","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train0 = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ntest = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet')\n\nprint(len(train0))\nprint(len(test))\ntrain0 = train0.sample(frac=1, random_state=42).reset_index(drop=True)\n\ntcols0=test.columns.tolist()\nprint(tcols0)\ntcols=[ 'date_id', 'time_id', 'symbol_id', 'weight',  'feature_00', 'feature_01', 'feature_02', 'feature_03', 'feature_04', 'feature_05', 'feature_06', 'feature_07', 'feature_08', 'feature_09', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_26', 'feature_27', 'feature_28', 'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_33', 'feature_34', 'feature_35', 'feature_36', 'feature_37', 'feature_38', 'feature_39', 'feature_40', 'feature_41', 'feature_42', 'feature_43', 'feature_44', 'feature_45', 'feature_46', 'feature_47', 'feature_48', 'feature_49', 'feature_50', 'feature_51', 'feature_52', 'feature_53', 'feature_54', 'feature_55', 'feature_56', 'feature_57', 'feature_58', 'feature_59', 'feature_60', 'feature_61', 'feature_62', 'feature_63', 'feature_64', 'feature_65', 'feature_66', 'feature_67', 'feature_68', 'feature_69', 'feature_70', 'feature_71', 'feature_72', 'feature_73', 'feature_74', 'feature_75', 'feature_76', 'feature_77', 'feature_78']\ntarget='responder_6'\ndataX=train0[tcols]\ndataY=train0[target]\nTESTX=test[tcols]","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:32:42.482685Z","iopub.status.busy":"2024-10-31T04:32:42.480848Z","iopub.status.idle":"2024-10-31T04:32:48.183662Z","shell.execute_reply":"2024-10-31T04:32:48.182687Z"},"papermill":{"duration":5.714225,"end_time":"2024-10-31T04:32:48.186389","exception":false,"start_time":"2024-10-31T04:32:42.472164","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target setting","metadata":{"papermill":{"duration":0.007726,"end_time":"2024-10-31T04:32:48.202278","exception":false,"start_time":"2024-10-31T04:32:48.194552","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_columns = list(dataX.columns)\nprint(df_columns)\n\nm=len(dataX)\nprint(m)\nM=list(range(m))\nrandom.seed(2021)\nrandom.shuffle(M)\n\ntrainX=dataX.iloc[M[0:(m//5)*4]]\ntrainY=dataY.iloc[M[0:(m//5)*4]]\ntestX=dataX.iloc[M[(m//5)*4:]]\ntestY=dataY.iloc[M[(m//5)*4:]]\n\ntrain_df=trainX\ntest_df=testX\n\ntrain_df.columns=df_columns\ntest_df.columns=df_columns","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:32:48.220512Z","iopub.status.busy":"2024-10-31T04:32:48.220061Z","iopub.status.idle":"2024-10-31T04:32:52.814426Z","shell.execute_reply":"2024-10-31T04:32:52.813177Z"},"papermill":{"duration":4.606231,"end_time":"2024-10-31T04:32:52.817305","exception":false,"start_time":"2024-10-31T04:32:48.211074","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# valid_df","metadata":{"papermill":{"duration":0.007639,"end_time":"2024-10-31T04:32:52.832968","exception":false,"start_time":"2024-10-31T04:32:52.825329","status":"completed"},"tags":[]}},{"cell_type":"code","source":"valid_from = 1455 # for private you should change to 1455 (1 year)\nalltraindata = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\nvalid_df = alltraindata.filter(pl.col(\"date_id\")>=valid_from).collect()\nvalid_df = valid_df.with_columns(pl.Series(range(len(valid_df))).alias(\"row_id\"))\nweights = valid_df.select(\"weight\").to_numpy().reshape(-1)\n\nlen(valid_df)\ndisplay(valid_df[0:2])\nprint(weights)\nprint(weights.shape)","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:32:52.850359Z","iopub.status.busy":"2024-10-31T04:32:52.849923Z","iopub.status.idle":"2024-10-31T04:33:03.132944Z","shell.execute_reply":"2024-10-31T04:33:03.131723Z"},"papermill":{"duration":10.294954,"end_time":"2024-10-31T04:33:03.135676","exception":false,"start_time":"2024-10-31T04:32:52.840722","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# weighted_zero_mean_r2","metadata":{"papermill":{"duration":0.007985,"end_time":"2024-10-31T04:33:03.151972","exception":false,"start_time":"2024-10-31T04:33:03.143987","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def weighted_zero_mean_r2(y_true, y_pred, weights):\n    \"\"\"\n    Calculate the sample weighted zero-mean R-squared score.\n\n    Parameters:\n    y_true (numpy.ndarray): Ground-truth values for responder_6.\n    y_pred (numpy.ndarray): Predicted values for responder_6.\n    weights (numpy.ndarray): Sample weight vector.\n\n    Returns:\n    float: The weighted zero-mean R-squared score.\n    \"\"\"\n    numerator = np.sum(weights * (y_true - y_pred)**2)\n    denominator = np.sum(weights * y_true**2)\n    \n    r2_score = 1 - numerator / denominator\n    return r2_score","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:03.170796Z","iopub.status.busy":"2024-10-31T04:33:03.170281Z","iopub.status.idle":"2024-10-31T04:33:03.178258Z","shell.execute_reply":"2024-10-31T04:33:03.176917Z"},"papermill":{"duration":0.020631,"end_time":"2024-10-31T04:33:03.180873","exception":false,"start_time":"2024-10-31T04:33:03.160242","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dir_yaml='/kaggle/input/jane-street-optuna-lgbm/Best_trial0.yaml'\nwith open(dir_yaml, \"r\") as yaml_file:\n    Best_trial = yaml.safe_load(yaml_file)\nprint(Best_trial)","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:03.199718Z","iopub.status.busy":"2024-10-31T04:33:03.199244Z","iopub.status.idle":"2024-10-31T04:33:03.220638Z","shell.execute_reply":"2024-10-31T04:33:03.219311Z"},"papermill":{"duration":0.033867,"end_time":"2024-10-31T04:33:03.223403","exception":false,"start_time":"2024-10-31T04:33:03.189536","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:03.243324Z","iopub.status.busy":"2024-10-31T04:33:03.242275Z","iopub.status.idle":"2024-10-31T04:33:03.248326Z","shell.execute_reply":"2024-10-31T04:33:03.247096Z"},"papermill":{"duration":0.018898,"end_time":"2024-10-31T04:33:03.251055","exception":false,"start_time":"2024-10-31T04:33:03.232157","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:03.270007Z","iopub.status.busy":"2024-10-31T04:33:03.269593Z","iopub.status.idle":"2024-10-31T04:33:03.343135Z","shell.execute_reply":"2024-10-31T04:33:03.342052Z"},"papermill":{"duration":0.085964,"end_time":"2024-10-31T04:33:03.345573","exception":false,"start_time":"2024-10-31T04:33:03.259609","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:03.365363Z","iopub.status.busy":"2024-10-31T04:33:03.364355Z","iopub.status.idle":"2024-10-31T04:33:03.372251Z","shell.execute_reply":"2024-10-31T04:33:03.371055Z"},"papermill":{"duration":0.020364,"end_time":"2024-10-31T04:33:03.374646","exception":false,"start_time":"2024-10-31T04:33:03.354282","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\ntest_feat_df = to_feature(test_df)","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:03.394260Z","iopub.status.busy":"2024-10-31T04:33:03.393299Z","iopub.status.idle":"2024-10-31T04:33:04.184313Z","shell.execute_reply":"2024-10-31T04:33:04.183114Z"},"papermill":{"duration":0.804779,"end_time":"2024-10-31T04:33:04.188090","exception":false,"start_time":"2024-10-31T04:33:03.383311","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.010391,"end_time":"2024-10-31T04:33:04.208802","exception":false,"start_time":"2024-10-31T04:33:04.198411","status":"completed"},"tags":[]}},{"cell_type":"code","source":"params = Best_trial\nprint(params)","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:04.230626Z","iopub.status.busy":"2024-10-31T04:33:04.230172Z","iopub.status.idle":"2024-10-31T04:33:04.236227Z","shell.execute_reply":"2024-10-31T04:33:04.234898Z"},"papermill":{"duration":0.020284,"end_time":"2024-10-31T04:33:04.239280","exception":false,"start_time":"2024-10-31T04:33:04.218996","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from joblib import Parallel, delayed\n\n\ndef fit_lgb(X, y, cv, params: dict = None, n_jobs: int = 1):\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=np.float32)\n\n    def train_fold(i, idx_train, idx_valid):\n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = clf = lgb.LGBMRegressor(**params)\n        with Timer(prefix=f'fit fold={i} '):\n            clf.fit(\n                x_train, y_train,\n                eval_set=[(x_valid, y_valid)],\n            )\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        return clf\n\n    models = Parallel(n_jobs=n_jobs)(delayed(train_fold)(i, idx_train, idx_valid) for i, (idx_train, idx_valid) in enumerate(cv))\n\n    weights2=weights[0:len(y)]\n    r2_score=weighted_zero_mean_r2(y, oof_pred, weights2)\n\n    print('-' * 50)\n    print('FINISHED | r2_score: {:.4f}'.format(r2_score))\n    return oof_pred, models\n","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:04.260881Z","iopub.status.busy":"2024-10-31T04:33:04.260447Z","iopub.status.idle":"2024-10-31T04:33:04.273162Z","shell.execute_reply":"2024-10-31T04:33:04.271834Z"},"papermill":{"duration":0.026756,"end_time":"2024-10-31T04:33:04.275890","exception":false,"start_time":"2024-10-31T04:33:04.249134","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:33:04.297457Z","iopub.status.busy":"2024-10-31T04:33:04.297006Z","iopub.status.idle":"2024-10-31T04:33:04.303271Z","shell.execute_reply":"2024-10-31T04:33:04.302153Z"},"papermill":{"duration":0.019835,"end_time":"2024-10-31T04:33:04.305627","exception":false,"start_time":"2024-10-31T04:33:04.285792","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nfor i in range(1):\n    fold = KFold(n_splits=4, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgb(train_feat_df.values, y, cv, params=params)","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2024-10-31T04:33:04.327368Z","iopub.status.busy":"2024-10-31T04:33:04.326929Z","iopub.status.idle":"2024-10-31T04:37:16.997760Z","shell.execute_reply":"2024-10-31T04:37:16.996410Z"},"papermill":{"duration":252.684709,"end_time":"2024-10-31T04:37:17.000320","exception":false,"start_time":"2024-10-31T04:33:04.315611","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds=[]\nfor i in range(4):\n    preds += [models[i].predict(test_feat_df.values)/4]\npredsT=np.array(preds).T\npreds2=[]\nfor item in predsT:\n    value=sum(item)\n    preds2+=[value]\nprint(preds2[0:5])","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:37:17.102488Z","iopub.status.busy":"2024-10-31T04:37:17.102006Z","iopub.status.idle":"2024-10-31T04:37:29.287460Z","shell.execute_reply":"2024-10-31T04:37:29.286039Z"},"papermill":{"duration":12.238068,"end_time":"2024-10-31T04:37:29.290153","exception":false,"start_time":"2024-10-31T04:37:17.052085","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission for API","metadata":{"papermill":{"duration":0.047692,"end_time":"2024-10-31T04:37:29.387382","exception":false,"start_time":"2024-10-31T04:37:29.339690","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n\n    global lags_,df_columns\n    if lags is not None:\n        lags_ = lags\n\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    \n    x_test = test[df_columns].to_pandas()\n    pred = []\n    for i in range(4):\n        pred += [models[i].predict(x_test.values)/4] \n                \n    pred = np.mean(pred, axis=0)\n    \n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n\n    return predictions","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:37:29.486956Z","iopub.status.busy":"2024-10-31T04:37:29.486418Z","iopub.status.idle":"2024-10-31T04:37:29.495916Z","shell.execute_reply":"2024-10-31T04:37:29.494405Z"},"papermill":{"duration":0.062308,"end_time":"2024-10-31T04:37:29.498218","exception":false,"start_time":"2024-10-31T04:37:29.435910","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"execution":{"iopub.execute_input":"2024-10-31T04:37:29.597166Z","iopub.status.busy":"2024-10-31T04:37:29.596702Z","iopub.status.idle":"2024-10-31T04:37:29.835790Z","shell.execute_reply":"2024-10-31T04:37:29.834509Z"},"papermill":{"duration":0.291595,"end_time":"2024-10-31T04:37:29.838773","exception":false,"start_time":"2024-10-31T04:37:29.547178","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.047795,"end_time":"2024-10-31T04:37:29.936258","exception":false,"start_time":"2024-10-31T04:37:29.888463","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.047854,"end_time":"2024-10-31T04:37:30.032620","exception":false,"start_time":"2024-10-31T04:37:29.984766","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}