{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport joblib \n\nimport pandas as pd\nimport polars as pl\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cbt\nimport numpy as np \n\nfrom joblib import Parallel, delayed\n\nimport kaggle_evaluation.jane_street_inference_server\n\n# !pip install lightgbm==4.2.0 -i https://mirrors.aliyun.com/pypi/simple/\n# !pip install catboost==1.2.7 -i https://mirrors.aliyun.com/pypi/simple/\n# !pip install xgboost==2.0.3 -i https://mirrors.aliyun.com/pypi/simple/\n# !pip install joblib==1.4.2 -i https://mirrors.aliyun.com/pypi/simple/\n\n\ndef reduce_mem_usage(self, float16_as32=True):\n    # memory_usage() returns the memory usage of each column in the DataFrame; sum gives the total, in MB\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns: # Iterate through each column name\n        col_type = df[col].dtype # Get the type of the column\n        if col_type != object and str(col_type) != 'category': # If it is not object, i.e., it's a numeric type\n            c_min, c_max = df[col].min(), df[col].max() # Find the min and max of the column\n            if str(col_type)[:3] == 'int': # If the variable type is integer\n                # Convert to int8 if values fall within its range (-128 to 127)\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                # Convert to int16 if values fall within its range (-32,768 to 32,767)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                # Convert to int32 if values fall within its range (-2,147,483,648 to 2,147,483,647)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                # Convert to int64 if values fall within its range (-9,223,372,036,854,775,808 to 9,223,372,036,854,775,807)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else: # If the variable type is float\n                # Convert to float16 if values fall within its range (if higher precision is needed, use float32)\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32: # Use float32 if higher precision is required\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float16)  \n                # Convert to float32 if values fall within its range\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                # Convert to float64 if values fall within its range\n                else:\n                    df[col] = df[col].astype(np.float64)\n    # Calculate the memory usage after optimization\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    # Print the percentage of memory reduced\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df\n\n\nprint(\"done\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the path to the input data directory\nprint(\"started\")\n# If the local directory exists, use it; otherwise, use the Kaggle input directory\ninput_path = './jane-street-realtime-marketdata-forecasting/' if os.path.exists('./jane-street-realtime-marketdata-forecasting') else '/kaggle/input/jane-street-realtime-marketdata-forecasting'\nipss=\"jane_street_realtime_marketdata_forecasting_path = kagglehub.competition_download('jane-street-realtime-marketdata-forecasting')\"\n# Flag to determine if the script is in training mode or not\nTRAINING = True\n\n\n# Define the feature names based on the number of features (79 in this case)\nfeature_names = [f\"feature_{i:02d}\" for i in range(79)]\n\n# Number of validation dates to use\nnum_valid_dates = 100\n\n# Number of dates to skip from the beginning of the dataset\nskip_dates = 500\n\n# Number of folds for cross-validation\nN_fold = 5\n\n# If in training mode, load the training data\nif TRAINING:\n    # Load the training data from a Parquet file\n    df = pd.read_parquet(f'{input_path}/train.parquet/partition_id=0/part-0.parquet')\n    print(\"file imported\")\n    # print(df.head())\n    # Reduce memory usage of the DataFrame (function not provided here)\n    df = reduce_mem_usage(df, False)\n    # print(df.head())\n    # Filter the DataFrame to include only dates greater than or equal to skip_dates\n    # df = df[df['date_id'] >= skip_dates].reset_index(drop=True)\n    # print(df.head())\n    # Get unique dates from the DataFrame\n    dates = df['date_id'].unique()\n    # print(df.head())\n    # Define validation dates as the last `num_valid_dates` dates\n    valid_dates = dates[-num_valid_dates:]\n    \n    # Define training dates as all dates except the last `num_valid_dates` dates\n    train_dates = dates[:-num_valid_dates]\n    print(df.head())\n    # Display the last few rows of the DataFrame (for debugging purposes)\n    print(df.tail())\n\n\nprint(\"ended\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a directory to store the trained models\nos.system('mkdir models')\nprint(\"started\")\n# Define the path to load pre-trained models (if not in training mode)\nmodel_path = '/kaggle/input/jsbaselinezyz'\n\n# If in training mode, prepare validation data\nif TRAINING:\n    print(\"training data prep\")\n    # Extract features, target, and weights for validation dates\n    X_valid = df[feature_names].loc[df['date_id'].isin(valid_dates)]\n    y_valid = df['responder_6'].loc[df['date_id'].isin(valid_dates)]\n    w_valid = df['weight'].loc[df['date_id'].isin(valid_dates)]\n\n# Initialize a list to store trained models\nmodels = []\n\n# Function to train a model or load a pre-trained model\ndef train(model_dict, model_name='lgb'):\n    if TRAINING:\n        # Select dates for training based on the fold number\n        selected_dates = [date for ii, date in enumerate(train_dates) if ii % N_fold != i]\n        \n        # Get the model from the dictionary\n        model = model_dict[model_name]\n        \n        # Extract features, target, and weights for the selected training dates\n        X_train = df[feature_names].loc[df['date_id'].isin(selected_dates)]\n        y_train = df['responder_6'].loc[df['date_id'].isin(selected_dates)]\n        w_train = df['weight'].loc[df['date_id'].isin(selected_dates)]\n\n        # Train the model based on the type (LightGBM, XGBoost, or CatBoost)\n        if model_name == 'lgb':\n            # Train LightGBM model with early stopping and evaluation logging\n            model.fit(X_train, y_train, w_train,  \n                      eval_metric=[r2_lgb],\n                      eval_set=[(X_valid, y_valid, w_valid)], \n                      callbacks=[\n                          lgb.early_stopping(100), \n                          lgb.log_evaluation(10)\n                      ])\n            \n        elif model_name == 'cbt':\n            # Prepare evaluation set for CatBoost\n            evalset = cbt.Pool(X_valid, y_valid, weight=w_valid)\n            \n            # Train CatBoost model with early stopping and verbose logging\n            model.fit(X_train, y_train, sample_weight=w_train, \n                      eval_set=[evalset], \n                      verbose=10, \n                      early_stopping_rounds=100)\n            \n        else:\n            # Train XGBoost model with early stopping and verbose logging\n            model.fit(X_train, y_train, sample_weight=w_train, \n                      eval_set=[(X_valid, y_valid)], \n                      sample_weight_eval_set=[w_valid], \n                      verbose=10, \n                      early_stopping_rounds=100)\n\n        # Append the trained model to the list\n        models.append(model)\n        \n        # Save the trained model to a file\n        joblib.dump(model, f'./models/{model_name}_{i}.model')\n        \n        # Delete training data to free up memory\n        del X_train\n        del y_train\n        del w_train\n        \n        # Collect garbage to free up memory\n        import gc\n        gc.collect()\n        \n    else:\n        # If not in training mode, load the pre-trained model from the specified path\n        models.append(joblib.load(f'{model_path}/{model_name}_{i}.model'))\n        \n    return \n\n# Custom R2 metric for XGBoost\ndef r2_xgb(y_true, y_pred, sample_weight):\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return -r2\n\n# Custom R2 metric for LightGBM\ndef r2_lgb(y_true, y_pred, sample_weight):\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return 'r2', r2, True\n\n# Custom R2 metric for CatBoost\nclass r2_cbt(object):\n    def get_final_error(self, error, weight):\n        return 1 - error / (weight + 1e-38)\n\n    def is_max_optimal(self):\n        return True\n\n    def evaluate(self, approxes, target, weight):\n        assert len(approxes) == 1\n        assert len(target) == len(approxes[0])\n\n        approx = approxes[0]\n\n        error_sum = 0.0\n        weight_sum = 0.0\n\n        for i in range(len(approx)):\n            w = 1.0 if weight is None else weight[i]\n            weight_sum += w * (target[i] ** 2)\n            error_sum += w * ((approx[i] - target[i]) ** 2)\n\n        return error_sum, weight_sum\n\n# Dictionary to store different models with their configurations\nmodel_dict = {\n    'lgb': lgb.LGBMRegressor(n_estimators=500, device='gpu', gpu_use_dp=True, objective='l2'),\n    'xgb': xgb.XGBRegressor(n_estimators=2000, learning_rate=0.1, max_depth=6, tree_method='hist', device=\"cuda\", objective='reg:squarederror', eval_metric=r2_xgb, disable_default_eval_metric=True),\n    'cbt': cbt.CatBoostRegressor(iterations=1000, learning_rate=0.05, task_type='GPU', loss_function='RMSE', eval_metric=r2_cbt()),\n}\n\nprint(\"calling the train fns\")\n# Train models for each fold\nfor i in range(N_fold):\n    print(f\"{i} th fold training \")\n    train(model_dict, 'lgb')\n    train(model_dict, 'xgb')\n    train(model_dict, 'cbt')\n\nprint(\"ended\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\n# Replace this function with your inference code.\n# You can return either a Pandas or Polars dataframe, though Polars is recommended.\n# Each batch of predictions (except the very first) must be returned within 10 minutes of the batch features being provided.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    # All the responders from the previous day are passed in at time_id == 0. We save them in a global variable for access at every time_id.\n    # Use them as extra features, if you like.\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    \n    feat = test[feature_names].to_numpy()\n    \n    pred = [model.predict(feat) for model in models]\n    pred = np.mean(pred, axis=0)\n    \n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n\n    # The predict function must return a DataFrame\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n    # with columns 'row_id', 'responer_6'\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n    # and as many rows as the test data.\n    assert len(predictions) == len(test)\n\n    return predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\n    \nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-realtime-marketdata-forecasting/test.parquet',\n            '/kaggle/input/jane-street-realtime-marketdata-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}