{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl \nimport pandas as pd\nimport numpy as np\n#model\nfrom sklearn.linear_model import Ridge\nfrom sklearn.impute import SimpleImputer\nimport kaggle_evaluation.jane_street_inference_server\nimport matplotlib.pyplot as plt\nimport os\nimport warnings\nimport xgboost as xgb\nwarnings.filterwarnings('ignore')\n\nimport random\ndef seed_everything(seed):\n    np.random.seed(seed)\n    random.seed(seed)\nseed_everything(seed=2024)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T11:50:44.159215Z","iopub.execute_input":"2024-10-20T11:50:44.160139Z","iopub.status.idle":"2024-10-20T11:50:47.856988Z","shell.execute_reply.started":"2024-10-20T11:50:44.160075Z","shell.execute_reply":"2024-10-20T11:50:47.855624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_metric(y_true, y_pred, sample_weight):\n    weighted_r2=1-(np.sum(sample_weight*((y_true-y_pred)**2))/np.sum(sample_weight*(y_true**2)))\n    return weighted_r2\nprint(\"read data\")\n\n#train_data = pl.read_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\ntrain_data = pl.read_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\")\n#train_data=pl.read_parquet(r\"C:\\Users\\k.b.anand\\Documents\\personal\\Hackathons\\Jane_Street_Real\\data\\jane-street-real-time-market-data-forecasting\\train.parquet\\partition_id=9\\part-0.parquet\")\nprint (train_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T11:50:47.859662Z","iopub.execute_input":"2024-10-20T11:50:47.860319Z","iopub.status.idle":"2024-10-20T11:50:57.232782Z","shell.execute_reply.started":"2024-10-20T11:50:47.860245Z","shell.execute_reply":"2024-10-20T11:50:57.231629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imputer_func(data):\n    print ('impute func calling')\n    imputer = SimpleImputer(strategy='mean') \n    print ('impute func called')\n    data_imputed = imputer.fit_transform(data)\n    print ('impute func transformed')\n    return data_imputed","metadata":{"execution":{"iopub.status.busy":"2024-10-20T11:50:57.234228Z","iopub.execute_input":"2024-10-20T11:50:57.234677Z","iopub.status.idle":"2024-10-20T11:50:57.241042Z","shell.execute_reply.started":"2024-10-20T11:50:57.234636Z","shell.execute_reply":"2024-10-20T11:50:57.239856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def modeling(train_data, feat_list):\n    train_data_columns=train_data.columns\n    train_data = imputer_func(train_data)\n    print('imputed to data frame')\n    train_data = pd.DataFrame(data=train_data, columns=train_data_columns)\n    \n    unique_dates= sorted(train_data['date_id'].unique())\n    validation_dates = unique_dates[int(len(unique_dates)*0.80):]\n\n    test = train_data.loc[~train_data['date_id'].isin(validation_dates),:]\n    train = train_data.loc[train_data['date_id'].isin(validation_dates),:]\n    print ('train.shape, test.shape', train.shape, test.shape)\n\n\n    train_x = train[feat_list]\n    train_y = train['responder_6']\n    train_weight = train['weight']\n    test_x = test[feat_list]\n    test_y = test['responder_6']\n    test_weight = test['weight']\n\n    model=xgb.XGBRegressor(n_estimators=2000, learning_rate=0.1, max_depth=6, tree_method='hist', device=\"cuda\", objective='reg:squarederror', disable_default_eval_metric=False, early_stopping_rounds=100)\n    model.fit(train_x, train_y, eval_set=[(test_x, test_y)], verbose=10)\n\n    best_iter=model.best_iteration\n    print ('best_iter ', best_iter)\n\n    train_x = train_data[feat_list]\n    train_y = train_data['responder_6']\n\n    model=xgb.XGBRegressor(n_estimators=best_iter, learning_rate=0.1, max_depth=6)\n    model.fit(train_x, train_y)\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-10-20T11:50:57.242643Z","iopub.execute_input":"2024-10-20T11:50:57.243159Z","iopub.status.idle":"2024-10-20T11:50:57.259444Z","shell.execute_reply.started":"2024-10-20T11:50:57.243106Z","shell.execute_reply":"2024-10-20T11:50:57.258289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\n# Replace this function with your inference code.\n# You can return either a Pandas or Polars dataframe, though Polars is recommended.\n# Each batch of predictions (except the very first) must be returned within 10 minutes of the batch features being provided.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    # All the responders from the previous day are passed in at time_id == 0. We save them in a global variable for access at every time_id.\n    # Use them as extra features, if you like.\n    global lags_\n    if lags is not None:\n        lags_ = lags\n    print ('test.shape 1', test.shape)\n\n    test_isna = test.to_pandas().isna().sum()\n    test_not_all_na = [i for i in test_isna.index if test_isna[i]< test.shape[0]]\n    test = test[test_not_all_na]              \n    test_columns=test.columns\n    test = imputer_func(test)\n    print('imputed to test frame')\n    test = pd.DataFrame(data=test, columns=test_columns)\n    feat_list = [i for i in test.columns if i.startswith(\"feature\")]\n    print ('test.shape 2', test.shape)\n    model = modeling(train_data, feat_list)\n\n    predictions = pl.from_pandas(test).select(\n    'row_id',\n    pl.lit(0.0).alias('responder_6'),\n    )\n\n    \n    feat = test[feat_list].to_numpy()\n    \n    # pred = [model.predict(feat) for model in models]\n    # pred = np.mean(pred, axis=0)\n    pred = model.predict(feat)\n    print ('predictions being made')\n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n    print ('predictions made')\n    # The predict function must return a DataFrame\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n    print ('The predict function must return a DataFrame')\n    # with columns 'row_id', 'responer_6'\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n    print (' with columns row_id, responer_6')\n    # and as many rows as the test data.\n    assert len(predictions) == len(test)\n    print ('and as many rows as the test data.')\n\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2024-10-20T11:50:57.261916Z","iopub.execute_input":"2024-10-20T11:50:57.262407Z","iopub.status.idle":"2024-10-20T11:50:57.277257Z","shell.execute_reply.started":"2024-10-20T11:50:57.262366Z","shell.execute_reply":"2024-10-20T11:50:57.276041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print ('inference_server calling')\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\nprint ('inference_server called')\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    print ('os.getenv starts')\n    inference_server.serve()\n    print ('os.getenv ends')\nelse:\n    print ('run_local_gateway ends')\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )\n    print ('run_local_gateway ends')","metadata":{"execution":{"iopub.status.busy":"2024-10-20T11:50:57.278956Z","iopub.execute_input":"2024-10-20T11:50:57.279378Z","iopub.status.idle":"2024-10-20T11:57:12.335117Z","shell.execute_reply.started":"2024-10-20T11:50:57.279325Z","shell.execute_reply":"2024-10-20T11:57:12.332940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}