{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":84493,"databundleVersionId":9849268},{"sourceType":"datasetVersion","sourceId":9629618,"datasetId":5878535}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"papermill":{"default_parameters":{},"duration":9.80953,"end_time":"2024-10-15T09:03:19.833973","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-15T09:03:10.024443","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom glob import glob\n\n# Function to read parquet fils\ndef read_parquet(file_path):\n    return pd.read_parquet(file_path)\n\n# Read the CSV files\nfeatures = pd.read_csv('/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv')\nresponders = pd.read_csv('/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv')\nsample_submission = pd.read_csv('/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv')\nlag=read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet')","metadata":{"papermill":{"duration":0.963709,"end_time":"2024-10-15T09:03:13.806120","exception":false,"start_time":"2024-10-15T09:03:12.842411","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-27T13:36:01.568059Z","iopub.execute_input":"2024-10-27T13:36:01.568463Z","iopub.status.idle":"2024-10-27T13:36:02.167063Z","shell.execute_reply.started":"2024-10-27T13:36:01.568421Z","shell.execute_reply":"2024-10-27T13:36:02.166083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading first three partitions in training","metadata":{"papermill":{"duration":0.002097,"end_time":"2024-10-15T09:03:13.810859","exception":false,"start_time":"2024-10-15T09:03:13.808762","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport joblib\nimport pandas as pd\nimport polars as pl\nimport lightgbm as lgb\nimport numpy as np\nfrom glob import glob\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport kaggle_evaluation.jane_street_inference_server\n\n# Define paths\ninput_path = './jane-street-real-time-market-data-forecasting/' if os.path.exists('./jane-street-real-time-market-data-forecasting') else '/kaggle/input/jane-street-real-time-market-data-forecasting/'\nmodel_path = './models'\n\n# Flag to determine if the script is in training mode or not\nTRAINING = True\n\n# Define feature names\nfeature_names = [f\"feature_{i:02d}\" for i in range(79)]\n\ndef reduce_mem_usage(df, float16_as32=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n\n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object and str(col_type) != 'category':\n            c_min, c_max = df[col].min(), df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32 if float16_as32 else np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    print(f'Decreased by {100 * (start_mem - end_mem) / start_mem:.1f}%')\n\n    return df\n\ndef train_model():\n    # Load the first three partition files\n    train_files = glob(f'{input_path}/train.parquet/partition_id=*/*.parquet')[6:9]\n    train_data = pd.concat([pd.read_parquet(f) for f in train_files])\n    \n    # Reduce memory usage\n    train_data = reduce_mem_usage(train_data, float16_as32=False)\n\n    # Create lagged features\n    train_data['lag_1'] = train_data.groupby('date_id')['responder_6'].shift(1)\n    train_data.dropna(inplace=True)\n\n    # Split features and target\n    X = train_data[feature_names + ['lag_1']]\n    y = train_data['responder_6']\n\n    # Create time-based split\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, shuffle=False)\n\n    # Initialize and train the model\n    model = lgb.LGBMRegressor(n_estimators=500, device='gpu', gpu_use_dp=True, objective='l2')\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='mse',\n        callbacks=[lgb.early_stopping(stopping_rounds=100), lgb.log_evaluation(period=100)]\n    )\n\n    # Make predictions\n    y_val_pred = model.predict(X_val)\n    y_val_pred_clipped = np.clip(y_val_pred, -5, 5)\n\n    # Evaluate the model\n    mse = mean_squared_error(y_val, y_val_pred_clipped)\n    print(f'Mean Squared Error: {mse}')\n\n    # Save the model\n    os.makedirs(model_path, exist_ok=True)\n    joblib.dump(model, f'{model_path}/lgb_model.joblib')\n\n    return model\n\nlags_: pl.DataFrame | None = None\nmodel: lgb.LGBMRegressor | None = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    global lags_, model\n    if lags is not None:\n        lags_ = lags\n\n    # Load the model if it hasn't been loaded yet\n    if model is None:\n        model_file = f'/kaggle/input/models/lgb_model.joblib'\n        if os.path.exists(model_file):\n            model = joblib.load(model_file)\n        else:\n            raise FileNotFoundError(f\"Model file not found: {model_file}\")\n    \n    # Create lagged feature\n    if lags_ is not None:\n        test = test.join(lags_.select(['row_id', 'responder_6']), on='row_id', how='left')\n        test = test.with_columns(pl.col('responder_6').alias('lag_1'))\n    else:\n        test = test.with_columns(pl.lit(None).alias('lag_1'))\n\n    # Make predictions\n    feat = test[feature_names + ['lag_1']].to_numpy()\n    pred = model.predict(feat)\n    pred_clipped = np.clip(pred, -5, 5)\n\n    # Create predictions dataframe\n    predictions = test.select(\n        'row_id',\n        pl.Series('responder_6', pred_clipped)\n    )\n\n    return predictions","metadata":{"papermill":{"duration":5.186833,"end_time":"2024-10-15T09:03:18.999799","exception":false,"start_time":"2024-10-15T09:03:13.812966","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-27T13:42:23.529931Z","iopub.execute_input":"2024-10-27T13:42:23.530515Z","iopub.status.idle":"2024-10-27T13:42:27.597478Z","shell.execute_reply.started":"2024-10-27T13:42:23.530455Z","shell.execute_reply":"2024-10-27T13:42:27.596668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAINING:\n    model = train_model()","metadata":{"papermill":{"duration":0.002538,"end_time":"2024-10-15T09:03:19.005225","exception":false,"start_time":"2024-10-15T09:03:19.002687","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-27T13:42:43.134146Z","iopub.execute_input":"2024-10-27T13:42:43.135332Z","iopub.status.idle":"2024-10-27T13:44:24.494228Z","shell.execute_reply.started":"2024-10-27T13:42:43.135275Z","shell.execute_reply":"2024-10-27T13:44:24.493120Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n# if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n#     inference_server.serve()\n# else:\n#     inference_server.run_local_gateway(\n#         (\n#             f'{input_path}/test.parquet',\n#             f'{input_path}/lags.parquet',\n#         )\n#     )","metadata":{"execution":{"iopub.status.busy":"2024-10-27T13:44:31.705142Z","iopub.execute_input":"2024-10-27T13:44:31.706404Z","iopub.status.idle":"2024-10-27T13:44:32.638390Z","shell.execute_reply.started":"2024-10-27T13:44:31.706354Z","shell.execute_reply":"2024-10-27T13:44:32.636915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}