{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)  \n\n# Add this line to adjust Dask memory\nimport dask; dask.config.set({'distributed.worker.memory.target': '2GB'})\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Using Dask with frac\n# You can define frac as a fraction (e.g. 0.1 for 10% of the data):\nimport os\nimport dask.dataframe as dd  # Importando Dask para manipulação de dados\n\n# Diretório dos arquivos de entrada\ninput_dir = '/kaggle/input'\n\n# Definindo o caminho para o arquivo train.parquet\ntrain_path = os.path.join(input_dir, 'jane-street-real-time-market-data-forecasting', 'train.parquet')\n\n# Lendo o arquivo Parquet usando Dask\ntry:\n    train_data = dd.read_parquet(train_path)\n    \n    # Visualiza apenas as primeiras 1000 linhas para evitar o uso excessivo de memória\n    train_data_sample = train_data.head(1000)\n\n    # Amostrando 10% dos dados\n    # train_data_sample = train_data.sample(frac=0.1, random_state=42).compute()  # Computa a amostra\n\n    # Mostrando as primeiras linhas da amostra\n    print(train_data_sample.head())\n\nexcept Exception as e:\n    print(f\"Ocorreu um erro ao ler o arquivo: {e}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-31T18:34:23.952271Z","iopub.execute_input":"2024-10-31T18:34:23.952778Z","iopub.status.idle":"2024-10-31T18:34:27.001093Z","shell.execute_reply.started":"2024-10-31T18:34:23.952722Z","shell.execute_reply":"2024-10-31T18:34:26.999402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load data with valid columns:**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Definindo o caminho para os arquivos Parquet\ntrain_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet'\n\n# Ler os dados com colunas válidas\ntrain_data_sample = pd.read_parquet(train_path, engine='pyarrow', columns=['date_id', 'time_id', 'symbol_id', 'weight', 'feature_00'])\n\n# Exibir as primeiras linhas do DataFrame\nprint(train_data_sample.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.003781Z","iopub.execute_input":"2024-10-31T18:34:27.004195Z","iopub.status.idle":"2024-10-31T18:34:27.073339Z","shell.execute_reply.started":"2024-10-31T18:34:27.004154Z","shell.execute_reply":"2024-10-31T18:34:27.072109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1. Explore the Data**\n* You can start by exploring the data to better understand the characteristics of the set. Some useful operations include:","metadata":{}},{"cell_type":"code","source":"# Verificar o tamanho do DataFrame:\nprint(train_data_sample.shape)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.074872Z","iopub.execute_input":"2024-10-31T18:34:27.075255Z","iopub.status.idle":"2024-10-31T18:34:27.082041Z","shell.execute_reply.started":"2024-10-31T18:34:27.075214Z","shell.execute_reply":"2024-10-31T18:34:27.080445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verificar tipos de dados e informações gerais:\nprint(train_data_sample.info())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.084869Z","iopub.execute_input":"2024-10-31T18:34:27.085278Z","iopub.status.idle":"2024-10-31T18:34:27.104487Z","shell.execute_reply.started":"2024-10-31T18:34:27.085234Z","shell.execute_reply":"2024-10-31T18:34:27.103154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Estatísticas descritivas:\nprint(train_data_sample.describe())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.106466Z","iopub.execute_input":"2024-10-31T18:34:27.106970Z","iopub.status.idle":"2024-10-31T18:34:27.422326Z","shell.execute_reply.started":"2024-10-31T18:34:27.106926Z","shell.execute_reply":"2024-10-31T18:34:27.420641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2. Analyze the Data**\n* After exploration, you can start analyzing the data. Here are some ideas:","metadata":{}},{"cell_type":"code","source":"# Verificar valores ausentes:\nprint(train_data_sample.isnull().sum())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.424178Z","iopub.execute_input":"2024-10-31T18:34:27.424653Z","iopub.status.idle":"2024-10-31T18:34:27.443604Z","shell.execute_reply.started":"2024-10-31T18:34:27.424608Z","shell.execute_reply":"2024-10-31T18:34:27.442252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizar a distribuição de weight: Você pode usar bibliotecas como Matplotlib ou Seaborn para visualização:\n# import plotly.express as px\n\n# Criar um histograma usando Plotly com cores personalizadas\n# fig = px.histogram(\n#    train_data_sample, \n#    x='weight', \n#    nbins=30, \n#    title='Distribuição de Peso', \n#    labels={'weight': 'Peso'},\n#    color_discrete_sequence=['#636EFA']  # Cor personalizada\n# )\n\n# fig.update_layout(xaxis_title='Peso', yaxis_title='Frequência')\n# fig.show()\n\nimport matplotlib.pyplot as plt\n\n# Criar um histograma usando Matplotlib\nplt.figure(figsize=(10, 6))\nplt.hist(train_data_sample['weight'], bins=30, color='#636EFA')  # Cor personalizada e número de bins\n\n# Adicionar título e rótulos\nplt.title('Distribuição de Peso')\nplt.xlabel('Peso')\nplt.ylabel('Frequência')\n\n# Mostrar o gráfico\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.445131Z","iopub.execute_input":"2024-10-31T18:34:27.445545Z","iopub.status.idle":"2024-10-31T18:34:27.887672Z","shell.execute_reply.started":"2024-10-31T18:34:27.445502Z","shell.execute_reply":"2024-10-31T18:34:27.886275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import plotly.graph_objects as go\n\n# Criar um histograma usando Plotly com uma escala de cores contínuas\n# fig = go.Figure(data=[go.Histogram(\n#    x=train_data_sample['weight'],\n#    nbinsx=30,\n#    marker=dict(\n#        color=train_data_sample['weight'],  # Usar 'weight' como cor\n#        colorscale='Viridis',  # Escala de cores\n#        showscale=True  # Mostrar a barra de escala\n#    )\n# )])\n\n# fig.update_layout(title='Distribuição de Peso', xaxis_title='Peso', yaxis_title='Frequência')\n# fig.show()\n# import matplotlib.pyplot as plt\n# import numpy as np\n\n# Definir os dados\n# data = train_data_sample['weight']\n\n# Criar o histograma com uma coloração contínua usando Matplotlib\n# plt.figure(figsize=(10, 6))\n# counts, bins, patches = plt.hist(data, bins=30, color='#636EFA')\n\n# Aplicar a escala de cores 'viridis' ao histograma\n# bin_centers = 0.5 * (bins[:-1] + bins[1:])\n# colormap = plt.cm.viridis\n# normalize = plt.Normalize(vmin=min(data), vmax=max(data))\n# for c, p in zip(bin_centers, patches):\n#    plt.setp(p, 'facecolor', colormap(normalize(c)))\n\n# Adicionar título e rótulos\n# plt.title('Distribuição de Peso')\n# plt.xlabel('Peso')\n# plt.ylabel('Frequência')\n# plt.colorbar(plt.cm.ScalarMappable(norm=normalize, cmap=colormap), label='Peso')  # Barra de escala\n\n# Mostrar o gráfico\n# plt.show()\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n# Definir os dados\ndata = train_data_sample['weight']\n\n# Criar categorias (bins) para o peso\nbins = np.linspace(min(data), max(data), 6)  # 5 intervalos\nlabels = [f\"{int(bins[i])} - {int(bins[i+1])}\" for i in range(len(bins)-1)]\n\n# Contar a quantidade de dados em cada bin\ncounts, _ = np.histogram(data, bins=bins)\n\n# Criar o gráfico de pizza\nplt.figure(figsize=(8, 8))\nplt.pie(counts, labels=labels, autopct='%1.1f%%', startangle=140, colors=plt.cm.viridis(np.linspace(0, 1, len(counts))))\n\n# Adicionar título\nplt.title('Distribuição de Peso em Gráfico de Pizza')\n\n# Mostrar o gráfico\nplt.show()\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:27.889506Z","iopub.execute_input":"2024-10-31T18:34:27.889962Z","iopub.status.idle":"2024-10-31T18:34:28.825801Z","shell.execute_reply.started":"2024-10-31T18:34:27.889917Z","shell.execute_reply":"2024-10-31T18:34:28.824391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 1: Preparing the DataFrame**\n* First, you need to have a DataFrame that you want to divide into partitions. For example purposes, I will show how to create a simple DataFrame and then how to divide it into partitions.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Criando um DataFrame de exemplo\nnum_rows = 4500000  # Exemplo: 4,5 milhões de linhas\ndata = {\n    'col1': np.random.rand(num_rows),\n    'col2': np.random.rand(num_rows),\n    'responder_6': np.random.randint(0, 2, size=num_rows)  # Supondo que 'responder_6' é binário\n}\n\ndf = pd.DataFrame(data)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:28.827454Z","iopub.execute_input":"2024-10-31T18:34:28.827907Z","iopub.status.idle":"2024-10-31T18:34:29.023368Z","shell.execute_reply.started":"2024-10-31T18:34:28.827861Z","shell.execute_reply":"2024-10-31T18:34:29.021916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 2: Create Partitions**\n* Now, you can split the DataFrame into multiple partitions and save each partition as a Parquet file.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# Definindo o caminho base para salvar as partições\nbase_path = r\"C:\\Users\\lglucena\\jane_street_data\\jane-street-real-time-market-data-forecasting\"\n\n# Supondo que df já tenha sido carregado anteriormente e contenha os dados\n# Verifique se df está definido e contém dados\nif 'df' not in locals():\n    raise ValueError(\"O DataFrame 'df' não foi definido. Certifique-se de carregá-lo antes de executar este código.\")\n\n# Obtendo o número de linhas do DataFrame\nnum_rows = df.shape[0]  # Número total de linhas\n\nnum_partitions = 10  # Número de partições desejadas\nrows_per_partition = num_rows // num_partitions\n\n# Salvando as partições\nfor i in range(num_partitions):\n    partition_df = df.iloc[i * rows_per_partition:(i + 1) * rows_per_partition]\n    partition_path = f\"partition_id={i}\"\n    # Criar o diretório se não existir\n    os.makedirs(os.path.join(base_path, partition_path), exist_ok=True)\n    # Caminho do arquivo Parquet\n    file_path = os.path.join(base_path, partition_path, f\"data_partition_{i}.parquet\")\n    # Salvando a partição em um arquivo Parquet\n    partition_df.to_parquet(file_path, index=False)\n\nprint(\"Partições criadas com sucesso!\")\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:29.028255Z","iopub.execute_input":"2024-10-31T18:34:29.028737Z","iopub.status.idle":"2024-10-31T18:34:30.352110Z","shell.execute_reply.started":"2024-10-31T18:34:29.028693Z","shell.execute_reply":"2024-10-31T18:34:30.350414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 3: Load Partitions into a DataFrame**\n* You can load partitions to perform analysis or train your model.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# Defina o caminho base onde as partições estão localizadas\nbase_path = r\"C:\\Users\\lglucena\\jane_street_data\\jane-street-real-time-market-data-forecasting\"\n\n# Lista para armazenar os DataFrames de cada partição\ndataframes = []\n\n# Percorre as partições de 0 a 9\nfor i in range(10):\n    partition_path = os.path.join(base_path, f\"partition_id={i}\")\n    \n    # Verifique se a partição existe\n    if os.path.exists(partition_path):\n        # Carregar todos os arquivos Parquet na partição\n        for file in os.listdir(partition_path):\n            if file.endswith('.parquet'):\n                file_path = os.path.join(partition_path, file)\n                df = pd.read_parquet(file_path)\n                dataframes.append(df)\n    else:\n        print(f\"Partição {partition_path} não encontrada.\")\n\n# Concatenar todos os DataFrames em um único DataFrame\nfinal_df = pd.concat(dataframes, ignore_index=True)\n\n# Exibir as primeiras linhas do DataFrame final\nprint(final_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:30.353862Z","iopub.execute_input":"2024-10-31T18:34:30.354413Z","iopub.status.idle":"2024-10-31T18:34:30.624802Z","shell.execute_reply.started":"2024-10-31T18:34:30.354352Z","shell.execute_reply":"2024-10-31T18:34:30.623181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 4: Checking for Missing Values:**\n\n* Identify if there are missing values ​​in the DataFrame and decide how to handle them (e.g. fill them with media or remove rows/columns).","metadata":{}},{"cell_type":"code","source":"# Verificar valores ausentes\nmissing_values = final_df.isnull().sum()\nprint(\"Valores ausentes por coluna:\\n\", missing_values)\n\n# Tratamento de valores ausentes\n# Exemplo: Preencher com a média para colunas numéricas\nfor col in final_df.columns:\n    if final_df[col].isnull().any():\n        final_df[col].fillna(final_df[col].mean(), inplace=True)\n\n# Verificar duplicatas\nduplicates = final_df.duplicated().sum()\nprint(f\"Número de duplicatas: {duplicates}\")\n\n# Se necessário, remover duplicatas\nif duplicates > 0:\n    final_df.drop_duplicates(inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:30.626848Z","iopub.execute_input":"2024-10-31T18:34:30.627438Z","iopub.status.idle":"2024-10-31T18:34:33.876475Z","shell.execute_reply.started":"2024-10-31T18:34:30.627371Z","shell.execute_reply":"2024-10-31T18:34:33.874825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 5: Exploratory Data Analysis (EDA):**\n\n* Explore descriptive statistics and visualizations to better understand data.","metadata":{}},{"cell_type":"code","source":"# 1. Descriptive Statistics:\n# View a statistical summary of numerical variables.\n# Estatísticas descritivas\nprint(final_df.describe())","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:33.878363Z","iopub.execute_input":"2024-10-31T18:34:33.878927Z","iopub.status.idle":"2024-10-31T18:34:34.514124Z","shell.execute_reply.started":"2024-10-31T18:34:33.878842Z","shell.execute_reply":"2024-10-31T18:34:34.512693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2. Variable Distributions:\n# Visualize the distribution of variables to understand their shape and characteristics.\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Distribuição de col1\nplt.figure(figsize=(12, 6))\nsns.histplot(final_df['col1'], kde=True)\nplt.title('Distribuição de col1')\nplt.show()\n\n# Distribuição de col2\nplt.figure(figsize=(12, 6))\nsns.histplot(final_df['col2'], kde=True)\nplt.title('Distribuição de col2')\nplt.show()\n\n# Distribuição de responder_6\nplt.figure(figsize=(12, 6))\nsns.countplot(x='responder_6', data=final_df)\nplt.title('Contagem de responder_6')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:34:34.515996Z","iopub.execute_input":"2024-10-31T18:34:34.516554Z","iopub.status.idle":"2024-10-31T18:35:42.155694Z","shell.execute_reply.started":"2024-10-31T18:34:34.516496Z","shell.execute_reply":"2024-10-31T18:35:42.154341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3. Correlation between Variables:\n# Calculate and visualize the correlation matrix to identify relationships between variables\n# Matriz de correlação\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Calcular e visualizar a matriz de correlação\ncorrelation_matrix = final_df.corr()\n\nplt.figure(figsize=(8, 6))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\")\nplt.title('Matriz de Correlação')\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:35:42.157498Z","iopub.execute_input":"2024-10-31T18:35:42.158006Z","iopub.status.idle":"2024-10-31T18:35:42.653650Z","shell.execute_reply.started":"2024-10-31T18:35:42.157957Z","shell.execute_reply":"2024-10-31T18:35:42.652403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. Choose a Template\n# To begin with, you can opt for Logistic Regression, which is a simple and effective model for binary classification problems. Let's implement this first.\n\n# 2. Split the Data\n# Split your dataset into training and validation sets. scikit-learn's train_test_split is an easy way to do this.\n\n# 3. Model Training\n# After splitting the data, you can train the Logistic Regression model and evaluate its performance.\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report, confusion_matrix\n\n# Supondo que 'final_df' seja o DataFrame que você criou\n# Separar as features (X) e o target (y)\nX = final_df[['col1', 'col2']]  # Ajuste para as suas colunas de features\ny = final_df['responder_6']\n\n# Dividir os dados em conjuntos de treinamento e validação\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Inicializar o modelo de Regressão Logística\nmodel = LogisticRegression()\n\n# Treinar o modelo\nmodel.fit(X_train, y_train)\n\n# Fazer previsões no conjunto de validação\ny_pred = model.predict(X_val)\n\n# Avaliar o desempenho\nprint(\"Matriz de Confusão:\")\nprint(confusion_matrix(y_val, y_pred))\n\nprint(\"\\nRelatório de Classificação:\")\nprint(classification_report(y_val, y_pred))","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:35:42.655569Z","iopub.execute_input":"2024-10-31T18:35:42.655981Z","iopub.status.idle":"2024-10-31T18:35:48.339791Z","shell.execute_reply.started":"2024-10-31T18:35:42.655938Z","shell.execute_reply":"2024-10-31T18:35:48.338276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport joblib\n\n# Carregar o DataFrame diretamente com as colunas necessárias (ajuste o caminho do arquivo Parquet)\nfile_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet'\nfinal_df = pd.read_parquet(file_path, columns=['feature_00', 'feature_01', 'responder_6'])\n\n# Remover ou preencher valores ausentes\nfinal_df = final_df.fillna(final_df.mean())\n\n# Transformar responder_6 em uma variável categórica (0 ou 1)\nfinal_df['responder_6'] = (final_df['responder_6'] > 0).astype(int)\n\n# Separar as features (X) e o target (y)\nX = final_df[['feature_00', 'feature_01']]\ny = final_df['responder_6']\n\n# Dividir os dados em conjuntos de treinamento e validação\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Inicializar e treinar o modelo de Regressão Logística\nmodel = LogisticRegression()\nmodel.fit(X_train, y_train)\n\n# Fazer previsões e avaliar o desempenho\ny_pred = model.predict(X_val)\nprint(\"Matriz de Confusão:\")\nprint(confusion_matrix(y_val, y_pred))\nprint(\"\\nRelatório de Classificação:\")\nprint(classification_report(y_val, y_pred))\n\n# Salvar o modelo treinado\nmodel_filename = 'logistic_regression_model.pkl'\njoblib.dump(model, model_filename)\nprint(f\"Modelo salvo como '{model_filename}'\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:35:48.341719Z","iopub.execute_input":"2024-10-31T18:35:48.342245Z","iopub.status.idle":"2024-10-31T18:36:48.606525Z","shell.execute_reply.started":"2024-10-31T18:35:48.342188Z","shell.execute_reply":"2024-10-31T18:36:48.605150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Verifica o diretório e lista arquivos\ndef list_files_in_directory(directory):\n    print(f\"Listando arquivos em {directory}...\")\n    try:\n        files = os.listdir(directory)\n        if not files:\n            print(\"O diretório está vazio.\")\n        else:\n            for file in files:\n                print(file)\n    except Exception as e:\n        print(f\"Erro ao acessar o diretório: {e}\")\n\n# Teste a listagem de arquivos\npartitions_directory = '/kaggle/input/jane-street-real-time-market-data-forecasting'  # Altere conforme necessário\nlist_files_in_directory(partitions_directory)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:36:48.608385Z","iopub.execute_input":"2024-10-31T18:36:48.608874Z","iopub.status.idle":"2024-10-31T18:36:48.620019Z","shell.execute_reply.started":"2024-10-31T18:36:48.608770Z","shell.execute_reply":"2024-10-31T18:36:48.618507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyarrow as pa\nimport pyarrow.parquet as pq\n\n# Definindo um esquema para os arquivos Parquet\nschema = pa.schema([\n    ('date_id', pa.int32()),  # Definindo como int32 para compatibilidade\n    ('time_id', pa.int16()),\n    ('symbol_id', pa.int8()),\n    ('weight', pa.float32()),\n    # Adicione os outros campos conforme necessário...\n])\n\n# Função para ler arquivos Parquet individualmente\ndef read_parquet_file(file_path):\n    try:\n        print(f\"Lendo {file_path}...\")\n        table = pq.read_table(file_path, schema=schema)\n        print(table)\n        return table\n    except Exception as e:\n        print(f\"Erro ao ler {file_path}: {e}\")\n\n# Caminho completo dos arquivos Parquet\npartitions_directory = '/kaggle/input/jane-street-real-time-market-data-forecasting'  # Altere conforme necessário\ntrain_file = f\"{partitions_directory}/train.parquet\"\nlags_file = f\"{partitions_directory}/lags.parquet\"\ntest_file = f\"{partitions_directory}/test.parquet\"\n\n# Testando a leitura dos arquivos\ntrain_table = read_parquet_file(train_file)\nlags_table = read_parquet_file(lags_file)\ntest_table = read_parquet_file(test_file)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:36:48.622910Z","iopub.execute_input":"2024-10-31T18:36:48.623398Z","iopub.status.idle":"2024-10-31T18:36:49.135434Z","shell.execute_reply.started":"2024-10-31T18:36:48.623354Z","shell.execute_reply":"2024-10-31T18:36:49.133798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyarrow as pa\nimport pyarrow.parquet as pq\n\n# Definindo um esquema para os arquivos Parquet\nschema = pa.schema([\n    ('date_id', pa.int32()),  # Definindo como int32 para compatibilidade\n    ('time_id', pa.int16()),\n    ('symbol_id', pa.int8()),\n    ('weight', pa.float32()),\n    # Adicione os outros campos conforme necessário...\n])\n\n# Função para ler arquivos Parquet individualmente\ndef read_parquet_file(file_path):\n    try:\n        print(f\"Lendo {file_path}...\")\n        table = pq.read_table(file_path, schema=schema)\n        print(table)\n        return table\n    except Exception as e:\n        print(f\"Erro ao ler {file_path}: {e}\")\n\n# Caminho completo dos arquivos Parquet\npartitions_directory = '/kaggle/input/jane-street-real-time-market-data-forecasting'  # Altere conforme necessário\ntrain_file = f\"{partitions_directory}/train.parquet\"\nlags_file = f\"{partitions_directory}/lags.parquet\"\ntest_file = f\"{partitions_directory}/test.parquet\"\n\n# Testando a leitura dos arquivos\ntrain_table = read_parquet_file(train_file)\nlags_table = read_parquet_file(lags_file)\ntest_table = read_parquet_file(test_file)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:36:49.137013Z","iopub.execute_input":"2024-10-31T18:36:49.137483Z","iopub.status.idle":"2024-10-31T18:36:49.672216Z","shell.execute_reply.started":"2024-10-31T18:36:49.137438Z","shell.execute_reply":"2024-10-31T18:36:49.671300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Função para converter tabelas PyArrow em DataFrames do pandas\ndef convert_to_dataframe(table):\n    df = table.to_pandas()\n    return df\n\n# Convertendo as tabelas para DataFrames\ntrain_df = convert_to_dataframe(train_table)\nlags_df = convert_to_dataframe(lags_table)\ntest_df = convert_to_dataframe(test_table)\n\n# Exibir as primeiras linhas dos DataFrames\nprint(\"Train DataFrame:\")\nprint(train_df.head())\nprint(\"\\nLags DataFrame:\")\nprint(lags_df.head())\nprint(\"\\nTest DataFrame:\")\nprint(test_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:36:49.673596Z","iopub.execute_input":"2024-10-31T18:36:49.674578Z","iopub.status.idle":"2024-10-31T18:36:49.983886Z","shell.execute_reply.started":"2024-10-31T18:36:49.674530Z","shell.execute_reply":"2024-10-31T18:36:49.982450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 6: Configure the Inference Server**\n* Define a prediction function predict(features): This function will use the trained model to make predictions on new data. It must be ready before starting inference_server.\n* Initialize inference_server with the predict function: The inference_server will call the predict function to make inferences with new data that is received.\n* Code example: Let's integrate inference_server into your training code, adding the predict function and inference server initialization.","metadata":{}},{"cell_type":"code","source":"import kaggle_evaluation\nprint(dir(kaggle_evaluation))\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:36:49.985294Z","iopub.execute_input":"2024-10-31T18:36:49.985688Z","iopub.status.idle":"2024-10-31T18:36:49.993177Z","shell.execute_reply.started":"2024-10-31T18:36:49.985648Z","shell.execute_reply":"2024-10-31T18:36:49.991569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Colunas do test_df:\", test_df.columns.tolist())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T19:07:58.888061Z","iopub.execute_input":"2024-10-31T19:07:58.888908Z","iopub.status.idle":"2024-10-31T19:07:58.901903Z","shell.execute_reply.started":"2024-10-31T19:07:58.888809Z","shell.execute_reply":"2024-10-31T19:07:58.900039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['row_id'] = test_df['date_id'].astype(str) + '_' + test_df['time_id'].astype(str) + '_' + test_df['symbol_id'].astype(str)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T19:08:40.595592Z","iopub.execute_input":"2024-10-31T19:08:40.596081Z","iopub.status.idle":"2024-10-31T19:08:40.605500Z","shell.execute_reply.started":"2024-10-31T19:08:40.596035Z","shell.execute_reply":"2024-10-31T19:08:40.603745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef predict(test_df):\n    # Verifica as colunas do DataFrame\n    print(\"Colunas do test_df:\", test_df.columns.tolist())\n    \n    # Se 'row_id' não existir, crie uma coluna 'row_id'\n    if 'row_id' not in test_df.columns:\n        test_df['row_id'] = test_df['date_id'].astype(str) + '_' + test_df['time_id'].astype(str) + '_' + test_df['symbol_id'].astype(str)\n    \n    predictions = np.clip(np.random.rand(len(test_df)) * 10 - 5, -5, 5)\n\n    prediction_df = pd.DataFrame({\n        'row_id': test_df['row_id'],\n        'pred': predictions\n    })\n\n    # Verificações adicionais\n    print(f\"Número de linhas no conjunto de teste: {len(test_df)}\")\n    print(f\"Número de linhas no DataFrame de previsões: {len(prediction_df)}\")\n    print(prediction_df.isnull().sum())\n    print(prediction_df.dtypes)\n\n    return prediction_df\n\n# Gerar o DataFrame de previsões\nprediction_df = predict(test_df)\n\n# Salvar o arquivo de submissão\nsubmission_file_path = 'submission.csv'\nprediction_df.to_csv(submission_file_path, index=False)\n\n# Verificar o arquivo gerado\nsubmission_check = pd.read_csv(submission_file_path)\nprint(submission_check.head())\nprint(submission_check.isnull().sum())\nprint(submission_check.dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T19:08:50.524428Z","iopub.execute_input":"2024-10-31T19:08:50.524946Z","iopub.status.idle":"2024-10-31T19:08:50.562908Z","shell.execute_reply.started":"2024-10-31T19:08:50.524899Z","shell.execute_reply":"2024-10-31T19:08:50.561284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verificar a forma do arquivo CSV gerado\nsubmission_check = pd.read_csv(submission_file_path)\n\n# Checar se possui apenas as colunas necessárias\nexpected_columns = ['row_id', 'pred']\nprint(\"Colunas esperadas:\", expected_columns)\nprint(\"Colunas do arquivo de submissão:\", submission_check.columns.tolist())\n\n# Verificar o número de linhas\nprint(f\"Número total de linhas no arquivo de submissão: {len(submission_check)} (incluindo cabeçalho)\")\n\n# Verificar se há linhas em branco\nprint(\"Linhas em branco:\", submission_check.isnull().sum())\n\n# Verificar os tipos de dados\nprint(\"Tipos de dados no arquivo de submissão:\")\nprint(submission_check.dtypes)\n\n# Exibir as primeiras linhas do arquivo\nprint(submission_check.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T19:09:43.157068Z","iopub.execute_input":"2024-10-31T19:09:43.157603Z","iopub.status.idle":"2024-10-31T19:09:43.177641Z","shell.execute_reply.started":"2024-10-31T19:09:43.157553Z","shell.execute_reply":"2024-10-31T19:09:43.176169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef predict(test_df, lags_df):\n    combined_data = test_df  # Combine com lags_df se necessário\n    predictions = np.clip(np.random.rand(len(combined_data)) * 10 - 5, -5, 5)\n\n    prediction_df = pd.DataFrame({\n        'row_id': combined_data['row_id'],\n        'pred': predictions\n    })\n\n    # Verificações adicionais\n    print(f\"Número de linhas no conjunto de teste: {len(test_df)}\")\n    print(f\"Número de linhas no DataFrame de previsões: {len(prediction_df)}\")\n    print(prediction_df.isnull().sum())\n    print(prediction_df.dtypes)\n\n    return prediction_df\n\n# Gerar o DataFrame de previsões\nprediction_df = predict(test_df, lags_df)\n\n# Salvar o arquivo de submissão\nsubmission_file_path = 'submission.csv'\nprediction_df.to_csv(submission_file_path, index=False)\n\n# Verificar o arquivo gerado\nsubmission_check = pd.read_csv(submission_file_path)\nprint(submission_check.head())\nprint(submission_check.isnull().sum())\nprint(submission_check.dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T19:10:09.696402Z","iopub.execute_input":"2024-10-31T19:10:09.696949Z","iopub.status.idle":"2024-10-31T19:10:09.721078Z","shell.execute_reply.started":"2024-10-31T19:10:09.696905Z","shell.execute_reply":"2024-10-31T19:10:09.719265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom kaggle_evaluation.jane_street_inference_server import JSInferenceServer\nimport os\n\ndef predict(test_df, lags_df):\n    # Exemplo de pré-processamento dos dados, caso seja necessário\n    combined_data = test_df  # Combine com lags_df se necessário\n\n    # Previsões ajustadas para o responder_6\n    predictions = np.clip(np.random.rand(len(combined_data)) * 10 - 5, -5, 5)  # Previsões aleatórias dentro do intervalo [-5, 5]\n\n    # Criação do DataFrame de submissão\n    prediction_df = pd.DataFrame({\n        'row_id': combined_data['row_id'],  # Certifique-se de que 'row_id' existe no DataFrame de teste\n        'pred': predictions  # 'pred' é o nome da coluna de previsões\n    })\n\n    return prediction_df\n\n# Configuração do servidor de inferência usando JSInferenceServer diretamente\ninference_server = JSInferenceServer(predict)\n\n# Executando o servidor de inferência\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T19:10:13.739646Z","iopub.execute_input":"2024-10-31T19:10:13.740119Z","iopub.status.idle":"2024-10-31T19:10:13.816008Z","shell.execute_reply.started":"2024-10-31T19:10:13.740079Z","shell.execute_reply":"2024-10-31T19:10:13.814339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step 7:** Creating and Saving the Submission DataFrame","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Assuming 'y_pred' contains your predictions and you have an ID column\n# Create a DataFrame with the predictions\nsubmission_df = pd.DataFrame({\n    'id': range(len(y_pred)),  # Or use an existing ID column if you have one\n    'responder_6': y_pred      # The predictions you generated\n})\n\n# Save the DataFrame as a Parquet file\nsubmission_df.to_parquet('submission.parquet', index=False)\n\nprint(\"File 'submission.parquet' created successfully!\")\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-31T18:36:50.126146Z","iopub.execute_input":"2024-10-31T18:36:50.127428Z","iopub.status.idle":"2024-10-31T18:36:51.161175Z","shell.execute_reply.started":"2024-10-31T18:36:50.127351Z","shell.execute_reply":"2024-10-31T18:36:51.159899Z"},"trusted":true},"execution_count":null,"outputs":[]}]}