{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nimport lightgbm as lgb\nimport joblib\nimport kaggle_evaluation.jane_street_inference_server\nimport os\nfrom sklearn.metrics import r2_score\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T23:24:00.202929Z","iopub.execute_input":"2024-11-23T23:24:00.203302Z","iopub.status.idle":"2024-11-23T23:24:04.830201Z","shell.execute_reply.started":"2024-11-23T23:24:00.203266Z","shell.execute_reply":"2024-11-23T23:24:04.829042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FEAT_COLS = [f'feature_{i:02d}' for i in range(79)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T23:24:04.832410Z","iopub.execute_input":"2024-11-23T23:24:04.832962Z","iopub.status.idle":"2024-11-23T23:24:04.838249Z","shell.execute_reply.started":"2024-11-23T23:24:04.832925Z","shell.execute_reply":"2024-11-23T23:24:04.837058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_data(date_id_range=None, time_id_range=None, columns=None, return_type = 'pl'):\n    data = pl.scan_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet')\n    # pl.scan_parquet得到lazyframe 惰性序列\n    # print(data.collect().shape)\n    if date_id_range is not None:\n        start_date, end_date = date_id_range\n        data = data.filter((pl.col('date_id') >= start_date) & (pl.col('date_id') <= end_date))\n    if time_id_range is not None:\n        start_time, end_time = time_id_range\n        data = data.filter((pl.col('time_id') >= start_time & (pl.col('time_id') <= end_time)))\n    if columns is not None:\n        data = data.select(columns)\n    if return_type == 'pd':\n        return data.collect().to_pandas()\n    else:\n        return data.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T23:24:04.839839Z","iopub.execute_input":"2024-11-23T23:24:04.840322Z","iopub.status.idle":"2024-11-23T23:24:04.868762Z","shell.execute_reply.started":"2024-11-23T23:24:04.840276Z","shell.execute_reply":"2024-11-23T23:24:04.867448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_r2(y_true,y_pred,weight):\n    upper = np.sum(weight*(y_true-y_pred)**2)\n    lower = np.sum(weight*(y_true**2))\n    r2_score = 1 - upper / lower\n    return r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T23:24:04.870877Z","iopub.execute_input":"2024-11-23T23:24:04.871547Z","iopub.status.idle":"2024-11-23T23:24:04.885223Z","shell.execute_reply.started":"2024-11-23T23:24:04.871486Z","shell.execute_reply":"2024-11-23T23:24:04.883774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_validation_spilt_load_model_eval (total_days, n_splits=5, save_model = True):\n    # create the dir for creating model\n    # if save_model and not os.path.exists(save_path):\n    #     os.makedirs(save_path)\n\n    fold_size = total_days // n_splits\n    folds = [(i * fold_size, min((i + 1) * fold_size - 1, total_days - 1)) for i in range(n_splits)]\n    r2_score_adjusted = []\n    model_group = []\n    # print(folds)\n    for fold_index in range(n_splits):\n        valid_range = folds[fold_index]\n        train_ranges = [folds[i] for i in range(n_splits) if i != fold_index]\n        print(f'fold{fold_index}: validation range {valid_range}, training range {train_ranges}')\n        # collect会将lazyframe->pl.df,后续都可以进行立即处理\n        valid_data = load_data(date_id_range=valid_range,columns=['date_id', 'weight','responder_6'] + FEAT_COLS)    # print(type(valid_data))\n        # .to_pandas 转化成df/series,取决于谁使用的它,valid_data本身为df,所以得到df(n,1)\n        # squeeze移除形状中为1的维度,(n,1)->(n,) df->series 变成series后，方便后面series与series的矩阵运算\n        valid_weight = valid_data.select('weight').to_pandas().squeeze()\n        train_data = None\n        for i in train_ranges:\n            partial_range = load_data(date_id_range=i,columns=['date_id','weight','responder_6'] + FEAT_COLS)\n            if train_data is None:\n                train_data = partial_range\n            else:\n                train_data = train_data.vstack(partial_range)\n        train_weight = train_data.select('weight').to_pandas().squeeze()\n        # print(train_weight)\n        # 构建数据集\n        train_ds = lgb.Dataset(data = train_data.select(FEAT_COLS + ['weight']).to_pandas(), label= train_data.select('responder_6').to_pandas(), weight= train_weight)\n        valid_ds = lgb.Dataset(data = valid_data.select(FEAT_COLS + ['weight']).to_pandas(), label= valid_data.select('responder_6').to_pandas(), weight= valid_weight)\n        # 模型训练\n        lgb_params = {\n            'objective':'regression_l2',\n            'metric':'rmse',\n            'num_leaves':31,\n            'max_depth':-1,\n            'random_state':42,\n            'device':'gpu'\n        }\n        early_stopping_callback = lgb.early_stopping(100)\n        verbose_eval_callback = lgb.log_evaluation(period=50)\n        model = lgb.train(\n            lgb_params,\n            train_ds,\n            num_boost_round=1000,\n            valid_sets=[train_ds,valid_ds],\n            valid_names=['train','valid'],\n            callbacks=[early_stopping_callback, verbose_eval_callback]\n        )\n        model_group.append(model)\n        y_valid_pred = model.predict(valid_data.select(['weight'] + FEAT_COLS).to_pandas())\n        r2_score = calculate_r2(valid_data.select('responder_6').to_pandas().squeeze(),y_valid_pred,valid_weight)\n        print(f'Fold{fold_index} validation r2_score{r2_score}')\n        r2_score_adjusted.append(r2_score)\n    print(f'Total trained model:{len(model_group)}')\n    final_model = model_group[0]\n    # save the model to the specific file\n    if save_model:\n        # save the ml model into file\n        joblib.dump(final_model,'lgb_model.pkl')\n        print('saved the model to the lgb_model.pkl')\n    print(f'cross_validation r2_score:{r2_score_adjusted}')\n    print(f'mean r2_score:{np.mean(r2_score_adjusted)}')\n    return final_model, np.mean(r2_score_adjusted), np.std(r2_score_adjusted)\n# lgb_model, r2_score_adjusted_mean, r2_score_adjusted_std = train_validation_spilt_load_model_eval(total_days=100, n_splits=5)\ntrain_validation_spilt_load_model_eval(total_days=800)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T23:24:04.886993Z","iopub.execute_input":"2024-11-23T23:24:04.887375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_model(file_path):\n    model = joblib.load(file_path)\n    return model\nlgb_model = load_model('lgb_model.pkl')\nprint('loaded from lgb_model.pkl')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predict\nlags_: pl.DataFrame | None = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n\n    feat = test[FEAT_COLS + ['weight']].to_pandas()\n\n    pred = lgb_model.predict(feat)\n\n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n    print(predictions)\n\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n\n    assert len(predictions) == len(test)\n\n    return predictions\n\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}