{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport pandas as pd\nfrom scipy.stats import linregress\nfrom scipy.stats import  ks_2samp\nimport random","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_keep = []\n\nfor i in range(10):\n\n    print(i)\n    \n    file_path = f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pl.read_parquet(file_path)\n    \n    chunk = pl.read_parquet(file_path)\n    \n    # randomly shuffle rows\n    num_rows = chunk.shape[0]\n    chunk = chunk.sample(n=num_rows, shuffle=True, seed=random.randint(0, 1000))\n  \n\n    # keep 30%\n    split_point = int(len(chunk) * 0.3)\n    chunk_sample = chunk[:split_point]\n    to_keep.append(chunk_sample)\n\n\n\n\n#     # split into 20% and 80% -- keep 20%\n#     split_point = int(len(chunk) * 0.2)\n#     chunk_20 = chunk[:split_point]\n#     chunk_80 = chunk[split_point:]\n#     to_keep.append(chunk_20)\n\n\n\n#     split into 10% and 90% -- keep 10%\n#     split_point = int(len(chunk) * 0.1)\n#     chunk_10 = chunk[:split_point]\n#     chunk_90 = chunk[split_point:]\n#     to_keep.append(chunk_10)\n\n    \n    # # save the 80% subset to \"unused_so_far\"\n    # output_path = f\"unused_so_far/unused_{i}.parquet\"\n    # chunk_80.write_parquet(output_path)\n\n\ntraining_df = pl.concat(to_keep)\n\ntraining_df = training_df.sample(n=training_df.shape[0], shuffle=True, seed=random.randint(0, 1000))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## lasso attempt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import Lasso\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n\nX = training_df[\"feature_01\",\"feature_02\",\"feature_04\",\"feature_05\",\"feature_06\",\"feature_07\",\"feature_08\",\n                            \"feature_12\",\"feature_33\",\"feature_34\",\"feature_35\",\"feature_36\",\"feature_40\",\"feature_45\",\"feature_49\",\n                            \"feature_51\",\"feature_52\",\"feature_54\",\"feature_55\",\"feature_56\",\"feature_58\",\"feature_59\",\"feature_60\",\n                            \"feature_66\",\"feature_67\",\"feature_68\",\"feature_70\"]\n\nX = X.to_numpy()\nX = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\n\ny = training_df[\"responder_6\"].to_numpy()\n\n\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n\n\nmodel = Lasso(alpha=0.00001, random_state=42)  \n\nmodel.fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    \n    global lags_\n    if lags is not None:\n        lags_ = lags\n        \n    # # Extract the features for the model input\n    # feature_columns = [col for col in test.columns if col.startswith(\"feature_\")]\n    # features = test.select(feature_columns).to_numpy()  # Convert to numpy array for model input\n\n\n    features = test[\"feature_01\",\"feature_02\",\"feature_04\",\"feature_05\",\"feature_06\",\"feature_07\",\"feature_08\",\n                                \"feature_12\",\"feature_33\",\"feature_34\",\"feature_35\",\"feature_36\",\"feature_40\",\"feature_45\",\"feature_49\",\n                                \"feature_51\",\"feature_52\",\"feature_54\",\"feature_55\",\"feature_56\",\"feature_58\",\"feature_59\",\"feature_60\",\n                                \"feature_66\",\"feature_67\",\"feature_68\",\"feature_70\"]\n\n    features = features.to_numpy()\n    features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    # Generate predictions using the model\n    responder_6_predictions = model.predict(features)\n    # responder_6_predictions = model_predictions[:, 6]  # Assuming responder_6 is at index 6\n\n    # responder_6_predictions = model_predictions[:, 6]  # Assuming responder_6 is at index 6\n\n    # Create a new Polars DataFrame with row_id and responder_6 predictions\n    predictions = test.select(\"row_id\").with_columns(\n        pl.Series(\"responder_6\", responder_6_predictions)\n    )\n    # Ensure the output format and length requirements\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    \n    assert len(predictions) == len(test)\n    return predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import kaggle_evaluation.jane_street_inference_server\nimport os\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet'\n        )\n    )","metadata":{},"execution_count":null,"outputs":[]}]}