{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Created by <a href=\"https://github.com/yunsuxiaozi/\">yunsuxiaozi </a>  2024/10/24\n\n#### We will use Ridge as the baseline here.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Import Libraries</h1></span>","metadata":{}},{"cell_type":"code","source":"#necessary\nimport polars as pl#similar to pandas, but with better performance when dealing with large datasets.\nimport pandas as pd#read csv,parquet\nimport numpy as np#for scientific computation of matrices\n#model\nfrom sklearn.linear_model import Ridge\nimport os#Libraries that interact with the operating system\nimport gc\nimport warnings#avoid some negligible errors\n#The filterwarnings () method is used to set warning filters, which can control the output method and level of warning information.\nwarnings.filterwarnings('ignore')\n#environment provided by competition hoster\nimport kaggle_evaluation.jane_street_inference_server\n\nimport random#provides some functions for generating random numbers\n#set random seed,make sure model can be recurrented.\ndef seed_everything(seed):\n    np.random.seed(seed)#numpy random seed\n    random.seed(seed)#python built-in random seed\nseed_everything(seed=2025)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Fit and Predict</h1></span>","metadata":{}},{"cell_type":"code","source":"def custom_metric(y_true,y_pred,weight):\n    weighted_r2=1-(np.sum(weight*(y_true-y_pred)**2)/np.sum(weight*y_true**2))\n    return weighted_r2\n    \nprint(\"< read parquet >\")\ndatas=[]\nweights=[]\nfor i in range(6,10):\n    train=pl.read_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\")\n    train=train.to_pandas().sample(frac=0.97, random_state=2025)\n    weights+=list(train['weight'].values)\n    train.drop(['weight'],axis=1,inplace=True)\n    datas.append(train)\ntrain=pd.concat(datas)\ndel datas\ngc.collect()\nprint(f\"train.shape:{train.shape}\")\n\nprint(\"< get X,y >\")\ncols=[f'feature_0{i}' if i<10 else f'feature_{i}' for i in range(79)]\nX=train[cols].fillna(3).values\ny=train['responder_6'].values\ndel train\ngc.collect()\n\nprint(\"< train test split >\")\nsplit=400000#around 2%\ntrain_X,train_y,test_X,test_y,train_weight,test_weight=X[:-split],y[:-split],X[-split:],y[-split:],weights[:-split],weights[-split:]\nprint(f\"train_X.shape:{train_X.shape},test_X.shape:{test_X.shape}\")\n\nprint(\"< fit and predict >\")\nmodel=Ridge()\nmodel.fit(train_X,train_y)\ntrain_pred=model.predict(train_X)\ntest_pred=model.predict(test_X)\nprint(f\"train weighted_r2:{custom_metric(train_y,train_pred,weight=train_weight)}\")\nprint(f\"test weighted_r2:{custom_metric(test_y,test_pred,weight=test_weight)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(test,lags):\n    cols=[f'feature_0{i}' if i<10 else f'feature_{i}' for i in range(79)]\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    test=test.to_pandas()[cols].fillna(3)\n    test_preds=model.predict(test.values)\n    predictions = predictions.with_columns(pl.Series('responder_6', test_preds.ravel()))\n    return predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"####  We can replace Ridge with a more complex neural network.","metadata":{}}]}