{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":6.532692,"end_time":"2024-12-30T13:35:51.368157","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-30T13:35:44.835465","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"9961d8ff","cell_type":"markdown","source":"**Features CSV File Analysis**","metadata":{"papermill":{"duration":0.003055,"end_time":"2024-12-30T13:35:47.093584","exception":false,"start_time":"2024-12-30T13:35:47.090529","status":"completed"},"tags":[]}},{"id":"832688e6","cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Load the dataset\nfile_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv'\ndf = pd.read_csv(file_path)\n\n# Inspect the dataset\nprint(\"Dataset Shape:\", df.shape)\nprint(\"Column Types Overview:\")\nprint(df.dtypes.value_counts())\nprint(\"\\nFirst Few Rows of the Dataset:\")\nprint(df.head())\n\n# Check for numeric columns\nnumeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()\nnon_numeric_columns = df.select_dtypes(exclude=[np.number]).columns.tolist()\nprint(f\"Numeric Columns: {numeric_columns}\")\nprint(f\"Non-Numeric Columns: {non_numeric_columns}\")\n\n# Attempt to convert non-numeric columns to numeric\nfor col in non_numeric_columns:\n    try:\n        df[col] = pd.to_numeric(df[col], errors='coerce')\n    except Exception as e:\n        print(f\"Could not convert column {col} to numeric. Error: {e}\")\n\n# Re-check numeric columns after conversion\nnumeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()\nprint(f\"Updated Numeric Columns: {numeric_columns}\")\n\n# Check for columns with variance\nvalid_numeric_columns = [col for col in numeric_columns if df[col].nunique() > 1]\nprint(f\"Valid Numeric Columns (with variance): {valid_numeric_columns}\")\n\n# If no valid numeric columns, inspect non-numeric data\nif not valid_numeric_columns:\n    print(\"No valid numeric columns found. Investigating non-numeric columns...\")\n    print(df[non_numeric_columns].head())\nelse:\n    # Proceed with analysis (e.g., correlation matrix, heatmaps)\n    correlation_matrix = df[valid_numeric_columns].corr()\n    print(\"Correlation Matrix:\")\n    print(correlation_matrix)\n","metadata":{"execution":{"iopub.execute_input":"2024-12-30T13:35:47.100515Z","iopub.status.busy":"2024-12-30T13:35:47.100157Z","iopub.status.idle":"2024-12-30T13:35:47.543026Z","shell.execute_reply":"2024-12-30T13:35:47.541266Z"},"papermill":{"duration":0.448774,"end_time":"2024-12-30T13:35:47.545205","exception":false,"start_time":"2024-12-30T13:35:47.096431","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"89d3f9f5","cell_type":"markdown","source":"**Responder CSV File Analysis**","metadata":{"papermill":{"duration":0.002202,"end_time":"2024-12-30T13:35:47.550322","exception":false,"start_time":"2024-12-30T13:35:47.548120","status":"completed"},"tags":[]}},{"id":"543b28f5","cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Load the dataset\nfile_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv'\ndf = pd.read_csv(file_path)\n\n# Check basic info about the dataset\nprint(\"Dataset Shape:\", df.shape)\nprint(\"\\nColumn Data Types:\")\nprint(df.dtypes)\nprint(\"\\nFirst Few Rows:\")\nprint(df.head())\nprint(\"\\nMissing Values Per Column:\")\nprint(df.isnull().sum())\n\n# Debug: Ensure DataFrame is not empty\nif df.empty:\n    raise ValueError(\"The dataset is empty. Please check the file.\")\n\n# Identify numeric and categorical columns\nnumeric_columns = df.select_dtypes(include=[np.number]).columns\ncategorical_columns = df.select_dtypes(include=['object', 'category']).columns\n\n# Debug: Ensure there are numeric columns\nif numeric_columns.empty:\n    print(\"No numeric columns found in the dataset.\")\nelse:\n    print(\"\\nSummary of Numeric Columns:\")\n    print(df[numeric_columns].describe())\n\n# Debug: Ensure there are categorical columns\nif categorical_columns.empty:\n    print(\"No categorical columns found in the dataset.\")\nelse:\n    print(\"\\nSummary of Categorical Columns:\")\n    for col in categorical_columns:\n        print(f\"{col}: {df[col].nunique()} unique values\")\n\n# Handle Missing Values\nif not numeric_columns.empty:\n    df[numeric_columns] = df[numeric_columns].fillna(df[numeric_columns].median())\nfor col in categorical_columns:\n    df[col] = df[col].fillna(df[col].mode()[0])\n\n# Visualizations (if applicable)\nif not numeric_columns.empty:\n    for col in numeric_columns:\n        plt.figure(figsize=(8, 4))\n        sns.histplot(df[col], kde=True, bins=30)\n        plt.title(f\"Distribution of {col}\")\n        plt.show()\n\nif not categorical_columns.empty:\n    for col in categorical_columns:\n        plt.figure(figsize=(8, 4))\n        sns.countplot(data=df, x=col, order=df[col].value_counts().index)\n        plt.title(f\"Counts of {col}\")\n        plt.xticks(rotation=45)\n        plt.show()\n\n# Correlation Matrix (if numeric columns exist)\nif len(numeric_columns) > 1:\n    correlation_matrix = df[numeric_columns].corr()\n    plt.figure(figsize=(10, 6))\n    sns.heatmap(correlation_matrix, cmap=\"coolwarm\", annot=False)\n    plt.title(\"Correlation Matrix of Numeric Features\")\n    plt.show()\nelse:\n    print(\"Not enough numeric columns for correlation analysis.\")\n\n","metadata":{"execution":{"iopub.execute_input":"2024-12-30T13:35:47.556554Z","iopub.status.busy":"2024-12-30T13:35:47.556125Z","iopub.status.idle":"2024-12-30T13:35:49.538943Z","shell.execute_reply":"2024-12-30T13:35:49.537483Z"},"papermill":{"duration":1.988292,"end_time":"2024-12-30T13:35:49.540939","exception":false,"start_time":"2024-12-30T13:35:47.552647","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"de984b82","cell_type":"markdown","source":"Sample Submission CSV File Analysis","metadata":{"papermill":{"duration":0.002727,"end_time":"2024-12-30T13:35:49.546960","exception":false,"start_time":"2024-12-30T13:35:49.544233","status":"completed"},"tags":[]}},{"id":"9f92431f","cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Load the dataset\nfile_path = \"/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\"\ndf = pd.read_csv(file_path)\n\n# Display basic information\nprint(\"Dataset Information:\")\nprint(df.info())\n\n# Display first few rows\nprint(\"\\nFirst few rows:\")\nprint(df.head())\n\n# Replace inf and -inf values with NaN\ndf.replace([float('inf'), float('-inf')], pd.NA, inplace=True)\n\n# 1. Summary of numeric columns\nnumeric_columns = df.select_dtypes(include=[np.number]).columns\nprint(\"\\nSummary of Numeric Columns:\")\nif not numeric_columns.empty:\n    print(df[numeric_columns].describe())\nelse:\n    print(\"No numeric columns found in the dataset.\")\n\n# 2. Summary of categorical columns\ncategorical_columns = df.select_dtypes(include=['object', 'category']).columns\nprint(\"\\nSummary of Categorical Columns:\")\nif not categorical_columns.empty:\n    print(df[categorical_columns].describe())\nelse:\n    print(\"No categorical columns found in the dataset.\")\n\n# 3. Check for missing values\nprint(\"\\nMissing Values:\")\nprint(df.isnull().sum())\n\n# 4. Visualize the distribution of numeric columns\nif not numeric_columns.empty:\n    for col in numeric_columns:\n        plt.figure(figsize=(8, 4))\n        sns.histplot(df[col], kde=True, bins=30)\n        plt.title(f\"Distribution of {col}\")\n        plt.show()\nelse:\n    print(\"No numeric columns to visualize.\")\n\n# 5. Correlation analysis for numeric columns\nif len(numeric_columns) > 1:\n    correlation_matrix = df[numeric_columns].corr()\n    plt.figure(figsize=(12, 8))\n    sns.heatmap(correlation_matrix, cmap=\"coolwarm\", annot=True)\n    plt.title(\"Feature Correlation Heatmap\")\n    plt.show()\nelse:\n    print(\"Not enough numeric columns for correlation analysis.\")\n\n# 6. Countplot for categorical columns (if any)\nif not categorical_columns.empty:\n    for col in categorical_columns:\n        plt.figure(figsize=(8, 4))\n        sns.countplot(data=df, x=col, palette=\"viridis\")\n        plt.title(f\"Countplot of {col}\")\n        plt.xticks(rotation=45)\n        plt.show()\nelse:\n    print(\"No categorical columns to visualize.\")\n\n# Additional Insights\nprint(\"\\nDataset Shape:\")\nprint(df.shape)\nprint(\"\\nUnique Values in Each Column:\")\nprint(df.nunique())\n\n\n","metadata":{"execution":{"iopub.execute_input":"2024-12-30T13:35:49.554229Z","iopub.status.busy":"2024-12-30T13:35:49.553849Z","iopub.status.idle":"2024-12-30T13:35:50.438792Z","shell.execute_reply":"2024-12-30T13:35:50.437358Z"},"papermill":{"duration":0.891143,"end_time":"2024-12-30T13:35:50.440944","exception":false,"start_time":"2024-12-30T13:35:49.549801","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}