{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n# Initialize a list to hold samples from each file\nsamples = []\n# Load a sample from each file\nfor i in range(10):\n    file_path = f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pd.read_parquet(file_path)\n    \n    # Take a sample of the data (adjust sample size as needed)\n    sample_chunk = chunk.sample(n=100000, random_state=42)  # For example, 100 rows\n    samples.append(sample_chunk)\n# Concatenate all samples into one DataFrame if needed\nsample_df = pd.concat(samples, ignore_index=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-16T13:18:47.333253Z","iopub.execute_input":"2024-12-16T13:18:47.334241Z","iopub.status.idle":"2024-12-16T13:20:16.940623Z","shell.execute_reply.started":"2024-12-16T13:18:47.334165Z","shell.execute_reply":"2024-12-16T13:20:16.939434Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-16T13:20:16.942709Z","iopub.execute_input":"2024-12-16T13:20:16.943134Z","iopub.status.idle":"2024-12-16T13:20:16.973796Z","shell.execute_reply.started":"2024-12-16T13:20:16.943100Z","shell.execute_reply":"2024-12-16T13:20:16.972825Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import Lasso\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n# 1. Carregando e preparando os dados\nfeatures = sample_df.filter(regex='^feature_')  # Features (preditores)\nresponders = sample_df.filter(regex='^responder_')  # Responders (variáveis-alvo)\n\n# Focando no responder_6\nX = features.values  # Features como entrada\ny = responders['responder_6'].values  # Responder_6 como saída\n\n# Dividindo os dados em treino e teste\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nmedianas = np.nanmedian(X_train, axis=0)  # Calcula a mediana de cada coluna no treino\nX_train = np.where(np.isnan(X_train), medianas, X_train)  # Substitui NaNs no treino\n\n# Substituindo NaNs no conjunto de teste com as medianas calculadas no treino\nX_test = np.where(np.isnan(X_test), medianas, X_test)  # Aplica medianas diretamente no teste\n\n# Garantindo substituição de infinitos por 0.0 em ambos os conjuntos\nX_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\nX_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# 2. Criando e ajustando o modelo de regressão LASSO\nmodel = Lasso(alpha=0.001, random_state=42)  # Alpha controla a penalização\nmodel.fit(X_train, y_train)\n\n# 3. Avaliação do modelo\ny_pred = model.predict(X_test)\n\nmse = mean_squared_error(y_test, y_pred)\nr2 = r2_score(y_test, y_pred)\n\nprint(f\"Mean Squared Error (MSE): {mse:.4f}\")\nprint(f\"R^2 Score: {r2:.4f}\")\n\n# 4. Exibindo previsões para as primeiras 5 amostras\npredictions = pd.DataFrame({\n    'Actual': y_test[:5],\n    'Predicted': y_pred[:5]\n})\nprint(predictions)\n\n# 5. Exibindo os coeficientes aprendidos\ncoefficients = pd.DataFrame({\n    'Feature': features.columns,\n    'Coefficient': model.coef_\n}).sort_values(by='Coefficient', ascending=False)\n\nprint(coefficients)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T13:27:38.267935Z","iopub.execute_input":"2024-12-16T13:27:38.269133Z","iopub.status.idle":"2024-12-16T13:27:43.186687Z","shell.execute_reply.started":"2024-12-16T13:27:38.269092Z","shell.execute_reply":"2024-12-16T13:27:43.185528Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission\n\nSee [Jane Street RMF Demo Submission](https://www.kaggle.com/code/ryanholbrook/jane-street-rmf-demo-submission) for details","metadata":{}},{"cell_type":"code","source":"import os\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"execution":{"iopub.status.busy":"2024-12-16T13:20:21.477290Z","iopub.execute_input":"2024-12-16T13:20:21.477962Z","iopub.status.idle":"2024-12-16T13:20:21.919053Z","shell.execute_reply.started":"2024-12-16T13:20:21.477912Z","shell.execute_reply":"2024-12-16T13:20:21.918171Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\n# Assuming `model` is your trained model\n# Assuming features required by the model are named 'feature_00', 'feature_01', etc.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_\n    if lags is not None:\n        lags_ = lags\n    # Extract the features for the model input\n    feature_columns = [col for col in test.columns if col.startswith(\"feature_\")]\n    features = test.select(feature_columns).to_numpy()  # Convert to numpy array for model input\n    features = np.where(np.isnan(features), medianas, features)\n    features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    #features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    # Generate predictions using the model\n    # model_predictions = model.predict(features)\n    # responder_6_predictions = model_predictions[:, 6]  # Assuming responder_6 is at index 6\n    responder_6_predictions = model.predict(features)\n    # Create a new Polars DataFrame with row_id and responder_6 predictions\n    predictions = test.select(\"row_id\").with_columns(\n        pl.Series(\"responder_6\", responder_6_predictions)\n    )\n    # Ensure the output format and length requirements\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    assert len(predictions) == len(test)\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T13:20:21.920924Z","iopub.execute_input":"2024-12-16T13:20:21.921417Z","iopub.status.idle":"2024-12-16T13:20:21.929802Z","shell.execute_reply.started":"2024-12-16T13:20:21.921381Z","shell.execute_reply":"2024-12-16T13:20:21.928800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-12-16T13:20:21.931190Z","iopub.execute_input":"2024-12-16T13:20:21.931812Z","iopub.status.idle":"2024-12-16T13:20:22.216677Z","shell.execute_reply.started":"2024-12-16T13:20:21.931768Z","shell.execute_reply":"2024-12-16T13:20:22.215748Z"},"trusted":true},"outputs":[],"execution_count":null}]}