{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n# Initialize a list to hold samples from each file\nsamples = []\n# Load a sample from each file\nfor i in range(10):\n    file_path = f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pd.read_parquet(file_path)\n    \n    # Take a sample of the data (adjust sample size as needed)\n    sample_chunk = chunk.sample(n=100000, random_state=42)  # For example, 100 rows\n    samples.append(sample_chunk)\n# Concatenate all samples into one DataFrame if needed\nsample_df = pd.concat(samples, ignore_index=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-17T22:26:45.842092Z","iopub.execute_input":"2024-12-17T22:26:45.842474Z","iopub.status.idle":"2024-12-17T22:28:08.199850Z","shell.execute_reply.started":"2024-12-17T22:26:45.842436Z","shell.execute_reply":"2024-12-17T22:28:08.193795Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-17T22:28:08.207465Z","iopub.execute_input":"2024-12-17T22:28:08.208192Z","iopub.status.idle":"2024-12-17T22:28:08.284153Z","shell.execute_reply.started":"2024-12-17T22:28:08.208131Z","shell.execute_reply":"2024-12-17T22:28:08.282694Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\n# Preparação de dados\nfeatures = sample_df.filter(regex='^feature_')\nresponders = sample_df.filter(regex='^responder_')\n\nX = features.values\ny = responders.values\n\n# Substituir valores NaN e infinitos\nX = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\ny = np.nan_to_num(y, nan=0.0, posinf=0.0, neginf=0.0)\n\n# Converter para tensores\nX_tensor = torch.tensor(X, dtype=torch.float32)\ny_tensor = torch.tensor(y, dtype=torch.float32)\n\n# Criar DataLoader\ndataset = TensorDataset(X_tensor, y_tensor)\ndataloader = DataLoader(dataset, batch_size=32, shuffle=True)\n\n# Dimensões\ninput_dim = X.shape[1]  # Número de features (79)\noutput_dim = y.shape[1]  # Número de responders (9)\n\n# Definição do modelo\nclass Autoencoder(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(Autoencoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, 64),\n            nn.ReLU(),\n            nn.Linear(64, 32),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(32, 64),\n            nn.ReLU(),\n            nn.Linear(64, output_dim)\n        )\n    \n    def forward(self, x):\n        x = self.encoder(x)\n        x = self.decoder(x)\n        return x\n\nmodel = Autoencoder(input_dim, output_dim)\n\n# Configuração do otimizador e da função de perda\noptimizer = optim.Adam(model.parameters(), lr=0.001)\ncriterion = nn.MSELoss()\n\n# Treinamento\nepochs = 5\nearly_stopping_patience = 5\nbest_loss = float('inf')\npatience_counter = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T22:28:08.285982Z","iopub.execute_input":"2024-12-17T22:28:08.286550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Dividir em conjuntos de treinamento e validação\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.2, random_state=42  # 20% para validação\n)\n\n# Converter para tensores\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32)\ny_train_tensor = torch.tensor(y_train, dtype=torch.float32)\nX_val_tensor = torch.tensor(X_val, dtype=torch.float32)\ny_val_tensor = torch.tensor(y_val, dtype=torch.float32)\n\n# Criar DataLoaders\ntrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\nval_dataset = TensorDataset(X_val_tensor, y_val_tensor)\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)\n\n# Treinamento com validação\nfor epoch in range(epochs):\n    # Treinamento\n    model.train()\n    running_loss = 0.0\n    for batch_X, batch_y in train_dataloader:\n        optimizer.zero_grad()\n        outputs = model(batch_X)\n        loss = criterion(outputs, batch_y)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n    \n    avg_train_loss = running_loss / len(train_dataloader)\n\n    # Validação\n    model.eval()\n    val_loss = 0.0\n    with torch.no_grad():\n        for batch_X, batch_y in val_dataloader:\n            outputs = model(batch_X)\n            loss = criterion(outputs, batch_y)\n            val_loss += loss.item()\n    \n    avg_val_loss = val_loss / len(val_dataloader)\n    print(f\"Epoch {epoch+1}/{epochs}, Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}\")\n\n    # Early stopping\n    if avg_val_loss < best_loss - 0.001:\n        best_loss = avg_val_loss\n        best_model_state = model.state_dict()\n        patience_counter = 0\n    else:\n        patience_counter += 1\n        if patience_counter >= early_stopping_patience:\n            print(\"Early stopping triggered.\")\n            break\n\n# Restaura os melhores pesos\nmodel.load_state_dict(best_model_state)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T22:28:08.285982Z","iopub.execute_input":"2024-12-17T22:28:08.286550Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission\n\nSee [Jane Street RMF Demo Submission](https://www.kaggle.com/code/ryanholbrook/jane-street-rmf-demo-submission) for details","metadata":{}},{"cell_type":"code","source":"import os\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport torch\n\n# Supondo que `model` seja o modelo PyTorch treinado e carregado\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    # Extrair as colunas de features\n    feature_columns = [col for col in test.columns if col.startswith(\"feature_\")]\n    features = test.select(feature_columns).to_numpy()  # Converter para numpy\n    features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    \n    # Converter as features para tensores PyTorch\n    features_tensor = torch.tensor(features, dtype=torch.float32)\n    \n    # Colocar o modelo em modo de avaliação\n    model.eval()\n    \n    # Fazer predições com PyTorch sem calcular gradiente\n    with torch.no_grad():\n        model_predictions = model(features_tensor).numpy()\n    \n    # Selecionar as predições do `responder_6` (índice 6)\n    responder_6_predictions = model_predictions[:, 6]  # Supondo que responder_6 esteja no índice 6\n\n    # Criar um novo DataFrame Polars com `row_id` e `responder_6` predictions\n    predictions = test.select(\"row_id\").with_columns(\n        pl.Series(\"responder_6\", responder_6_predictions)\n    )\n\n    # Garantir o formato de saída e os requisitos de comprimento\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    \n    assert len(predictions) == len(test)\n    return predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}