{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":4.669361,"end_time":"2024-10-10T13:05:46.686069","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-10T13:05:42.016708","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-12-15T11:44:28.010633Z","iopub.execute_input":"2024-12-15T11:44:28.011101Z","iopub.status.idle":"2024-12-15T11:44:28.847731Z","shell.execute_reply.started":"2024-12-15T11:44:28.011060Z","shell.execute_reply":"2024-12-15T11:44:28.846438Z"},"papermill":{"duration":1.223703,"end_time":"2024-10-10T13:05:45.825911","exception":false,"start_time":"2024-10-10T13:05:44.602208","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The evaluation API requires that you set up a server which will respond to inference requests. We have already defined the server; you just need write the predict function. When we evaluate your submission on the hidden test set the client defined in `jane_street_gateway` will run in a different container with direct access to the hidden test set and hand off the data timestep by timestep.\n\n\n\nYour code will always have access to the published copies of the files.","metadata":{"papermill":{"duration":0.002051,"end_time":"2024-10-10T13:05:45.83073","exception":false,"start_time":"2024-10-10T13:05:45.828679","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"When your notebook is run on the hidden test set, inference_server.serve must be called within 15 minutes of the notebook starting or the gateway will throw an error. If you need more than 15 minutes to load your model you can do so during the very first `predict` call, which does not have the usual 1 minute response deadline.","metadata":{"papermill":{"duration":0.00196,"end_time":"2024-10-10T13:05:45.853279","exception":false,"start_time":"2024-10-10T13:05:45.851319","status":"completed"},"tags":[]}},{"cell_type":"code","source":"        def reduce_mem_usage(self,df, float16_as32=True):\n            #memory_usage()是df每列的内存使用量,sum是对它们求和, B->KB->MB\n            start_mem = df.memory_usage().sum() / 1024**2\n            print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n            for col in df.columns:#遍历每列的列名\n                col_type = df[col].dtype#列名的type\n                if col_type != object and str(col_type)!='category':#不是object也就是说这里处理的是数值类型的变量\n                    c_min,c_max = df[col].min(),df[col].max() #求出这列的最大值和最小值\n                    if str(col_type)[:3] == 'int':#如果是int类型的变量,不管是int8,int16,int32还是int64\n                        #如果这列的取值范围是在int8的取值范围内,那就对类型进行转换 (-128 到 127)\n                        if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                            df[col] = df[col].astype(np.int8)\n                        #如果这列的取值范围是在int16的取值范围内,那就对类型进行转换(-32,768 到 32,767)\n                        elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                            df[col] = df[col].astype(np.int16)\n                        #如果这列的取值范围是在int32的取值范围内,那就对类型进行转换(-2,147,483,648到2,147,483,647)\n                        elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                            df[col] = df[col].astype(np.int32)\n                        #如果这列的取值范围是在int64的取值范围内,那就对类型进行转换(-9,223,372,036,854,775,808到9,223,372,036,854,775,807)\n                        elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                            df[col] = df[col].astype(np.int64)  \n                    else:#如果是浮点数类型.\n                        #如果数值在float16的取值范围内,如果觉得需要更高精度可以考虑float32\n                        if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                            if float16_as32:#如果数据需要更高的精度可以选择float32\n                                df[col] = df[col].astype(np.float32)\n                            else:\n                                df[col] = df[col].astype(np.float16)  \n                        #如果数值在float32的取值范围内，对它进行类型转换\n                        elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                            df[col] = df[col].astype(np.float32)\n                        #如果数值在float64的取值范围内，对它进行类型转换\n                        else:\n                            df[col] = df[col].astype(np.float64)\n            #计算一下结束后的内存\n            end_mem = df.memory_usage().sum() / 1024**2\n            print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n            #相比一开始的内存减少了百分之多少\n            print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n            return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:28.849691Z","iopub.execute_input":"2024-12-15T11:44:28.850195Z","iopub.status.idle":"2024-12-15T11:44:28.864617Z","shell.execute_reply.started":"2024-12-15T11:44:28.850158Z","shell.execute_reply":"2024-12-15T11:44:28.863145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \n\n# Load a data from each file:\nr = range(10)\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pd.read_parquet(file_path)\n    part=part[['date_id','time_id','symbol_id','weight','responder_6']]\n    samples.append(part)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:28.866371Z","iopub.execute_input":"2024-12-15T11:44:28.866979Z","iopub.status.idle":"2024-12-15T11:44:57.232158Z","shell.execute_reply.started":"2024-12-15T11:44:28.866926Z","shell.execute_reply":"2024-12-15T11:44:57.230804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(samples)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:57.234832Z","iopub.execute_input":"2024-12-15T11:44:57.235229Z","iopub.status.idle":"2024-12-15T11:44:57.243322Z","shell.execute_reply.started":"2024-12-15T11:44:57.235191Z","shell.execute_reply":"2024-12-15T11:44:57.242120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df=pd.concat(samples,ignore_index=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:57.244975Z","iopub.execute_input":"2024-12-15T11:44:57.245477Z","iopub.status.idle":"2024-12-15T11:44:57.494475Z","shell.execute_reply.started":"2024-12-15T11:44:57.245426Z","shell.execute_reply":"2024-12-15T11:44:57.493380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:57.495871Z","iopub.execute_input":"2024-12-15T11:44:57.496350Z","iopub.status.idle":"2024-12-15T11:44:57.516702Z","shell.execute_reply.started":"2024-12-15T11:44:57.496278Z","shell.execute_reply":"2024-12-15T11:44:57.515505Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 线下评估","metadata":{}},{"cell_type":"code","source":"train_offline=sample_df[sample_df['date_id']<1698-180]\ntest_offline=sample_df[sample_df['date_id']>=1698-180].reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:57.518076Z","iopub.execute_input":"2024-12-15T11:44:57.518439Z","iopub.status.idle":"2024-12-15T11:44:59.343377Z","shell.execute_reply.started":"2024-12-15T11:44:57.518404Z","shell.execute_reply":"2024-12-15T11:44:59.342182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_offline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:59.344737Z","iopub.execute_input":"2024-12-15T11:44:59.345109Z","iopub.status.idle":"2024-12-15T11:44:59.359857Z","shell.execute_reply.started":"2024-12-15T11:44:59.345072Z","shell.execute_reply":"2024-12-15T11:44:59.358596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_offline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:44:59.361197Z","iopub.execute_input":"2024-12-15T11:44:59.361559Z","iopub.status.idle":"2024-12-15T11:44:59.377588Z","shell.execute_reply.started":"2024-12-15T11:44:59.361524Z","shell.execute_reply":"2024-12-15T11:44:59.376365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_offline_last=train_offline[train_offline['date_id']>=1517-180]\ntrain_offline_last['responder_6'].median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:54:46.786688Z","iopub.execute_input":"2024-12-15T11:54:46.787243Z","iopub.status.idle":"2024-12-15T11:54:47.199442Z","shell.execute_reply.started":"2024-12-15T11:54:46.787196Z","shell.execute_reply":"2024-12-15T11:54:47.198010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_r_squared(y, y_hat, weights):\n    \"\"\"\n    计算样本加权零均值R平方得分。\n\n    参数:\n    y -- 实际值列表或数组\n    y_hat -- 预测值列表或数组\n    weights -- 每个样本的权重列表或数组\n\n    返回:\n    R^2得分\n    \"\"\"\n    # 计算分子和分母\n    numerator = sum(weights[i] * (y[i] - y_hat[i])**2 for i in range(len(y)))\n    denominator = sum(weights[i] * y[i]**2 for i in range(len(y)))\n\n    # 计算R^2\n    r_squared = 1 - numerator / denominator if denominator != 0 else float('inf')  # 防止除以零\n\n    return r_squared","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:45:01.540467Z","iopub.execute_input":"2024-12-15T11:45:01.540864Z","iopub.status.idle":"2024-12-15T11:45:01.548484Z","shell.execute_reply.started":"2024-12-15T11:45:01.540819Z","shell.execute_reply":"2024-12-15T11:45:01.547117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = test_offline['responder_6']\ny_hat = [-0.03230968] * len(test_offline['responder_6'])\nweights = [1.0] * len(test_offline['responder_6'])\n\nr2_score = weighted_r_squared(y, y_hat, weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:46:32.055636Z","iopub.execute_input":"2024-12-15T11:46:32.056100Z","iopub.status.idle":"2024-12-15T11:47:40.345328Z","shell.execute_reply.started":"2024-12-15T11:46:32.056062Z","shell.execute_reply":"2024-12-15T11:47:40.343909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:48:45.710623Z","iopub.execute_input":"2024-12-15T11:48:45.711115Z","iopub.status.idle":"2024-12-15T11:48:45.719828Z","shell.execute_reply.started":"2024-12-15T11:48:45.711072Z","shell.execute_reply":"2024-12-15T11:48:45.718502Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 线上提交","metadata":{}},{"cell_type":"code","source":"_last=sample_df[sample_df['date_id']>=1698-180]\n_last['responder_6'].median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:53:27.770731Z","iopub.execute_input":"2024-12-15T11:53:27.771508Z","iopub.status.idle":"2024-12-15T11:53:28.197451Z","shell.execute_reply.started":"2024-12-15T11:53:27.771448Z","shell.execute_reply":"2024-12-15T11:53:28.196014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\n\n# Replace this function with your inference code.\n# You can return either a Pandas or Polars dataframe, though Polars is recommended.\n# Each batch of predictions (except the very first) must be returned within 1 minute of the batch features being provided.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    # All the responders from the previous day are passed in at time_id == 0. We save them in a global variable for access at every time_id.\n    # Use them as extra features, if you like.\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    # Replace this section with your own predictions\n    predictions = test.select(\n        'row_id',\n        pl.lit(-0.033641677).alias('responder_6'),\n    )\n\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    # Confirm has as many rows as the test data.\n    assert len(predictions) == len(test)\n\n    return predictions","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-12-15T11:53:46.165513Z","iopub.execute_input":"2024-12-15T11:53:46.165954Z","iopub.status.idle":"2024-12-15T11:53:46.175549Z","shell.execute_reply.started":"2024-12-15T11:53:46.165917Z","shell.execute_reply":"2024-12-15T11:53:46.173963Z"},"papermill":{"duration":0.015917,"end_time":"2024-10-10T13:05:45.848958","exception":false,"start_time":"2024-10-10T13:05:45.833041","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-12-15T11:47:40.359082Z","iopub.execute_input":"2024-12-15T11:47:40.359464Z","iopub.status.idle":"2024-12-15T11:47:40.409847Z","shell.execute_reply.started":"2024-12-15T11:47:40.359426Z","shell.execute_reply":"2024-12-15T11:47:40.408651Z"},"papermill":{"duration":0.308219,"end_time":"2024-10-10T13:05:46.163573","exception":false,"start_time":"2024-10-10T13:05:45.855354","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}