{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#this cell shows that feature_21, feature_26, feature_27, feature_31 are totally missing for first three partitions\n#this cell also shows that feature_42, feature_39, feature_53, feature_50 are top features with missing values in the other partitions  \n\nimport pandas as pd\nimport glob\n\n# Define the path pattern to all partition files\npath_pattern = '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=*/part-0.parquet'\n\n# List of all parquet files based on the path pattern\nparquet_files = glob.glob(path_pattern)\n\n# Loop through each parquet file (partition) and calculate top 5 features with missing values\nfor file in parquet_files:\n    # Load the parquet file\n    df = pd.read_parquet(file)\n    \n    # Calculate the percentage of missing values for each feature\n    missing_percentage = df.isna().sum() * 100 / len(df)\n    \n    # Sort features by missing percentage and select the top 5\n    top_5_missing_features = missing_percentage.sort_values(ascending=False).head(5)\n    \n    # Display the top 5 features with the highest missing value percentages for the current partition\n    partition_id = file.split(\"=\")[1]  # Extract partition ID from the file path\n    print(f\"Top 5 features with missing values for partition {partition_id}:\")\n    print(top_5_missing_features)\n    print(\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-10-21T12:18:35.565330Z","iopub.execute_input":"2024-10-21T12:18:35.565704Z","iopub.status.idle":"2024-10-21T12:20:35.842587Z","shell.execute_reply.started":"2024-10-21T12:18:35.565668Z","shell.execute_reply":"2024-10-21T12:20:35.841538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this cell shows that around 82.5% of missing values are under time_id 0-67\nimport pandas as pd\nimport numpy as np\n\n# Define the file paths and partitions you want to process\npartitions = [4, 5, 6, 7, 8, 9]\nfile_paths = [f'/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={p}/part-0.parquet' for p in partitions]\n\n# Initialize a list to store all percentages for final average calculation\nall_part_percentages = []\n\n# Loop through each file\nfor file_path, partition in zip(file_paths, partitions):\n    # Read the data\n    df = pd.read_parquet(file_path)\n    \n    # Keep only the rows that contain NaN values\n    df_nan = df[df.isna().any(axis=1)]\n    \n    # Initialize a list to store the missing value percentages for this partition\n    part_percentages = []\n    \n    # Calculate the percentage of NaN values for each part (67 time_id values)\n    total_nan_rows = len(df_nan)  # Total NaN rows for normalization\n    \n    for i in range(0, df['time_id'].max(), 67):\n        # Filter rows where time_id is within the current range\n        time_part = df_nan[(df_nan['time_id'] >= i) & (df_nan['time_id'] < i + 67)]\n        \n        # Calculate the percentage of NaN rows for this time_id range\n        nan_rows = len(time_part)\n        part_percentage = (nan_rows / total_nan_rows) * 100 if total_nan_rows > 0 else 0\n        \n        # Append the percentage to the list for this partition\n        part_percentages.append((i, i + 67, part_percentage))\n    \n    # Ensure percentages sum to 100% and print results\n    total_percentage = sum(p[2] for p in part_percentages)\n    scaling_factor = 100 / total_percentage if total_percentage > 0 else 0\n    scaled_percentages = [(part[0], part[1], part[2] * scaling_factor) for part in part_percentages]\n    \n    # Store the scaled percentages for final average calculation\n    all_part_percentages.append([p[2] for p in scaled_percentages])\n    \n    print(f\"Partition {partition}:\")\n    for part in scaled_percentages:\n        print(f\"  {part[2]:.2f}% of missing values are under time_id {part[0]}-{part[1]}\")\n    print()  # Add a newline between partitions\n\n# Calculate the average percentage for each time_id range across all partitions\naverage_percentages = np.mean(np.array(all_part_percentages), axis=0)\n\n# Print the average percentages\nprint(\"Average percentages across all partitions:\")\nfor idx, avg in enumerate(average_percentages):\n    start_time = idx * 67\n    end_time = start_time + 67\n    print(f\"  {avg:.2f}% of missing values are under time_id {start_time}-{end_time}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-21T12:41:00.497523Z","iopub.execute_input":"2024-10-21T12:41:00.497963Z","iopub.status.idle":"2024-10-21T12:41:33.403662Z","shell.execute_reply.started":"2024-10-21T12:41:00.497925Z","shell.execute_reply":"2024-10-21T12:41:33.397894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}