{"metadata":{"kernelspec":{"display_name":"pytorch","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.8"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"4263c58d","cell_type":"markdown","source":"# test data's colmun mean and variance are seemingly affine transformed. look at the image and you can see it.","metadata":{}},{"id":"f2f0a665","cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nprint(\"numpy version: \", np.__version__)\nprint(\"pandas version: \", pd.__version__)\ndf = pd.read_parquet('drw-crypto-market-prediction/train.parquet')\ndf_test = pd.read_parquet('drw-crypto-market-prediction/test.parquet')","metadata":{},"outputs":[],"execution_count":null},{"id":"34ab5583","cell_type":"markdown","source":"## type inspection","metadata":{}},{"id":"49ceaf56","cell_type":"code","source":"print(df.dtypes.value_counts())\nprint(df_test.dtypes.value_counts())","metadata":{},"outputs":[],"execution_count":null},{"id":"346bb06f","cell_type":"code","source":"# Find columns with NaN and their percentage\nfor col in df.columns:\n   if df[col].isna().any():\n       nan_count = df[col].isna().sum()\n       total_count = len(df[col])\n       nan_percentage = (nan_count / total_count) * 100\n       print(f\"Column '{col}': {nan_percentage:.2f}% NaN ({nan_count}/{total_count})\")\n\n# Find columns with NaN and their percentage in test set\nfor col in df_test.columns:\n    if df_test[col].isna().any():\n        nan_count = df_test[col].isna().sum()\n        total_count = len(df_test[col])\n        nan_percentage = (nan_count / total_count) * 100\n        print(f\"Column '{col}': {nan_percentage:.2f}% NaN ({nan_count}/{total_count})\")","metadata":{},"outputs":[],"execution_count":null},{"id":"734a251d","cell_type":"code","source":"print(\"\\nInf values in train set:\")\n# Find columns with inf and their percentage\nfor col in df.columns:\n   if np.isinf(df[col]).any():\n       inf_count = np.isinf(df[col]).sum()\n       total_count = len(df[col])\n       inf_percentage = (inf_count / total_count) * 100\n       print(f\"Column '{col}': {inf_percentage:.2f}% inf ({inf_count}/{total_count})\")\n\nprint(\"\\nInf values in test set:\")\n# Find columns with inf and their percentage in test set\nfor col in df_test.columns:\n    if np.isinf(df_test[col]).any():\n        inf_count = np.isinf(df_test[col]).sum()\n        total_count = len(df_test[col])\n        inf_percentage = (inf_count / total_count) * 100\n        print(f\"Column '{col}': {inf_percentage:.2f}% inf ({inf_count}/{total_count})\")","metadata":{},"outputs":[],"execution_count":null},{"id":"4e190d68","cell_type":"code","source":"df.describe()","metadata":{},"outputs":[],"execution_count":null},{"id":"cfb1511f","cell_type":"code","source":"df_test.describe()","metadata":{},"outputs":[],"execution_count":null},{"id":"de1c254c","cell_type":"code","source":"def plot_x_columns(df, title_suffix=''):\n    # Filter columns whose names start with 'X'\n    x_cols = df.columns[df.columns.str.startswith('X')]\n\n    # Calculate all statistics for these columns\n    x_means = df[x_cols].mean()\n    x_vars = df[x_cols].var()  # Changed back to variance\n    x_ranges = df[x_cols].max() - df[x_cols].min()\n    \n    # Quantiles\n    x_q0 = df[x_cols].min()  # 0th percentile (minimum)\n    x_q25 = df[x_cols].quantile(0.25)  # 25th percentile\n    x_q50 = df[x_cols].median()  # 50th percentile (median)\n    x_q75 = df[x_cols].quantile(0.75)  # 75th percentile\n    x_q100 = df[x_cols].max()  # 100th percentile (maximum)\n    \n    # IQR\n    x_iqr = x_q75 - x_q25\n    \n    # Second moment (variance)\n    x_second_moment = df[x_cols].var()\n    \n    # Third moment (for skewness calculation)\n    x_third_moment = df[x_cols].apply(lambda x: ((x - x.mean()) ** 3).mean())\n    \n    # Fourth moment (for kurtosis calculation)\n    x_fourth_moment = df[x_cols].apply(lambda x: ((x - x.mean()) ** 4).mean())\n\n    # Create colors based on column position (red to blue)\n    num_cols = len(x_cols)\n    colors = range(num_cols)\n\n    # Create figure with single column layout (now 12 subplots)\n    fig, axs = plt.subplots(12, 1, figsize=(10, 32))\n\n    # Plot means\n    sc = axs[0].scatter(range(num_cols), x_means.values, c=colors, cmap='coolwarm')\n    axs[0].set_title(f'Mean - {title_suffix}')\n    axs[0].set_ylabel('Mean Value')\n    axs[0].set_ylim(-1, 1)\n    fig.colorbar(sc, ax=axs[0])\n\n    # Plot variances\n    sc = axs[1].scatter(range(num_cols), x_vars.values, c=colors, cmap='coolwarm')\n    axs[1].set_title(f'Variance - {title_suffix}')\n    axs[1].set_ylabel('Variance')\n    axs[1].set_ylim(0, 2)\n    fig.colorbar(sc, ax=axs[1])\n\n    # Plot ranges\n    sc = axs[2].scatter(range(num_cols), x_ranges.values, c=colors, cmap='coolwarm')\n    axs[2].set_title(f'Range - {title_suffix}')\n    axs[2].set_ylabel('Range')\n    axs[2].set_ylim(0, 300)\n    fig.colorbar(sc, ax=axs[2])\n\n    # Plot Q0 (min)\n    sc = axs[3].scatter(range(num_cols), x_q0.values, c=colors, cmap='coolwarm')\n    axs[3].set_title(f'Q0 (Min) - {title_suffix}')\n    axs[3].set_ylabel('Q0')\n    axs[3].set_ylim(-160, 10)\n    fig.colorbar(sc, ax=axs[3])\n\n    # Plot Q25\n    sc = axs[4].scatter(range(num_cols), x_q25.values, c=colors, cmap='coolwarm')\n    axs[4].set_title(f'Q25 - {title_suffix}')\n    axs[4].set_ylabel('Q25')\n    axs[4].set_ylim(-2, 1)\n    fig.colorbar(sc, ax=axs[4])\n\n    # Plot Q50 (median)\n    sc = axs[5].scatter(range(num_cols), x_q50.values, c=colors, cmap='coolwarm')\n    axs[5].set_title(f'Q50 (Median) - {title_suffix}')\n    axs[5].set_ylabel('Q50')\n    axs[5].set_ylim(-1, 1)\n    fig.colorbar(sc, ax=axs[5])\n\n    # Plot Q75\n    sc = axs[6].scatter(range(num_cols), x_q75.values, c=colors, cmap='coolwarm')\n    axs[6].set_title(f'Q75 - {title_suffix}')\n    axs[6].set_ylabel('Q75')\n    axs[6].set_ylim(-0.75, 1.5)\n    fig.colorbar(sc, ax=axs[6])\n\n    # Plot Q100 (max)\n    sc = axs[7].scatter(range(num_cols), x_q100.values, c=colors, cmap='coolwarm')\n    axs[7].set_title(f'Q100 (Max) - {title_suffix}')\n    axs[7].set_ylabel('Q100')\n    axs[7].set_ylim(-1, 300)\n    fig.colorbar(sc, ax=axs[7])\n\n    # Plot IQR\n    sc = axs[8].scatter(range(num_cols), x_iqr.values, c=colors, cmap='coolwarm')\n    axs[8].set_title(f'IQR - {title_suffix}')\n    axs[8].set_ylabel('IQR')\n    axs[8].set_ylim(-0.5, 2.5)\n    fig.colorbar(sc, ax=axs[8])\n\n    # Plot Second Moment\n    sc = axs[9].scatter(range(num_cols), x_second_moment.values, c=colors, cmap='coolwarm')\n    axs[9].set_title(f'Second Moment - {title_suffix}')\n    axs[9].set_ylabel('Second Moment')\n    axs[9].set_ylim(0, 2.5)\n    fig.colorbar(sc, ax=axs[9])\n\n    # Plot Third Moment\n    sc = axs[10].scatter(range(num_cols), x_third_moment.values, c=colors, cmap='coolwarm')\n    axs[10].set_title(f'Third Moment - {title_suffix}')\n    axs[10].set_ylabel('Third Moment')\n    axs[10].set_ylim(-10, 300)\n    fig.colorbar(sc, ax=axs[10])\n\n    # Plot Fourth Moment\n    sc = axs[11].scatter(range(num_cols), x_fourth_moment.values, c=colors, cmap='coolwarm')\n    axs[11].set_title(f'Fourth Moment - {title_suffix}')\n    axs[11].set_ylabel('Fourth Moment')\n    axs[11].set_ylim(0, 5000)\n    fig.colorbar(sc, ax=axs[11])\n\n    # Set x-axis labels for all plots\n    for i in range(12):\n        axs[i].set_xticks(range(num_cols))\n        axs[i].set_xticklabels(x_cols, rotation=45)\n        axs[i].set_xlabel('Column Index')\n\n    # Layout so plots do not overlap\n    fig.tight_layout()\n\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"0c9cb62a","cell_type":"code","source":"plot_x_columns(df, title_suffix='Training Set')","metadata":{},"outputs":[],"execution_count":null},{"id":"5d27388d","cell_type":"code","source":"plot_x_columns(df_test, title_suffix='Test Set')","metadata":{},"outputs":[],"execution_count":null},{"id":"02419af6","cell_type":"code","source":"import numpy as np\nfrom sklearn.linear_model import LinearRegression\n\ndef find_affine_transform(train_stats, test_stats, stat_name):\n    \"\"\"\n    Find the best affine transformation: test = a * train + b\n    \"\"\"\n    \n    # Fit linear regression: test = a * train + b\n    X = train_stats.values.reshape(-1, 1)\n    y = test_stats.values\n    \n    reg = LinearRegression().fit(X, y)\n    a = reg.coef_[0]  # slope\n    b = reg.intercept_  # intercept\n    r_squared = reg.score(X, y)\n    \n    # Calculate mean absolute error\n    predicted_test = a * train_stats + b\n    mae = np.mean(np.abs(test_stats - predicted_test))\n    \n    # Calculate correlation\n    correlation = np.corrcoef(train_stats, test_stats)[0, 1]\n    \n    print(f\"\\n{stat_name} Affine Transform Analysis:\")\n    print(f\"  Best affine transform: test = {a:.6f} * train + {b:.6f}\")\n    print(f\"  R-squared: {r_squared:.6f}\")\n    print(f\"  Correlation: {correlation:.6f}\")\n    print(f\"  Mean Absolute Error: {mae:.6f}\")\n    \n    # Interpret the transformation\n    if abs(a + 1) < 0.01:  # Close to -1\n        print(f\"  → Near-perfect reflection around y = {b/2:.6f}\")\n    elif abs(a - 1) < 0.01:  # Close to 1\n        print(f\"  → Near-perfect shift by {b:.6f}\")\n    elif abs(a) < 0.01:  # Close to 0\n        print(f\"  → Near-constant value {b:.6f}\")\n    else:\n        print(f\"  → General linear scaling and shift\")\n    \n    return a, b, r_squared, mae, correlation\n\ndef analyze_affine_transform(df_train, df_test, title_suffix=''):\n    x_cols = df_train.columns[df_train.columns.str.startswith('X')]\n    \n    # Calculate statistics\n    train_means = df_train[x_cols].mean()\n    test_means = df_test[x_cols].mean()\n    \n    train_vars = df_train[x_cols].var()\n    test_vars = df_test[x_cols].var()\n    \n    # Find affine transforms\n    mean_a, mean_b, mean_r2, mean_mae, mean_corr = find_affine_transform(train_means, test_means, \"Mean\")\n    var_a, var_b, var_r2, var_mae, var_corr = find_affine_transform(train_vars, test_vars, \"Variance\")\n    \n    # Plot the analysis\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n    \n    # Mean affine transform plot\n    ax1.scatter(train_means, test_means, alpha=0.7, label='Actual', s=50)\n    x_range = np.linspace(train_means.min(), train_means.max(), 100)\n    y_pred = mean_a * x_range + mean_b\n    ax1.plot(x_range, y_pred, 'r-', linewidth=2, label=f'Fit: y = {mean_a:.3f}x + {mean_b:.3f}')\n    ax1.plot(x_range, x_range, 'k--', alpha=0.5, label='y = x (identity)')\n    ax1.set_xlabel('Train Mean')\n    ax1.set_ylabel('Test Mean')\n    ax1.set_title(f'Mean Affine Transform\\nR² = {mean_r2:.6f}, r = {mean_corr:.3f}')\n    ax1.legend()\n    ax1.grid(True, alpha=0.3)\n    \n    # Variance affine transform plot\n    ax2.scatter(train_vars, test_vars, alpha=0.7, label='Actual', s=50)\n    x_range = np.linspace(train_vars.min(), train_vars.max(), 100)\n    y_pred = var_a * x_range + var_b\n    ax2.plot(x_range, y_pred, 'r-', linewidth=2, label=f'Fit: y = {var_a:.3f}x + {var_b:.3f}')\n    ax2.plot(x_range, x_range, 'k--', alpha=0.5, label='y = x (identity)')\n    ax2.set_xlabel('Train Variance')\n    ax2.set_ylabel('Test Variance')\n    ax2.set_title(f'Variance Affine Transform\\nR² = {var_r2:.6f}, r = {var_corr:.3f}')\n    ax2.legend()\n    ax2.grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return {\n        'mean': {'a': mean_a, 'b': mean_b, 'r2': mean_r2, 'mae': mean_mae, 'correlation': mean_corr},\n        'variance': {'a': var_a, 'b': var_b, 'r2': var_r2, 'mae': var_mae, 'correlation': var_corr}\n    }\n\n# Usage:\naffine_results = analyze_affine_transform(df, df_test, 'Your Data')","metadata":{},"outputs":[],"execution_count":null},{"id":"f423f21c","cell_type":"code","source":"def align_training_to_test(df_train, df_test):\n    \"\"\"\n    Transform training data to match test data statistics\n    Much better for deployment!\n    \"\"\"\n    x_cols = df_train.columns[df_train.columns.str.startswith('X')]\n    df_train_aligned = df_train.copy()\n    \n    for col in x_cols:\n        # Target statistics (from TEST - the \"reality\")\n        target_mean = df_test[col].mean()\n        target_std = df_test[col].std()\n        \n        # Current statistics (from TRAIN - what we modify)\n        current_mean = df_train[col].mean()\n        current_std = df_train[col].std()\n        \n        # Transform train column to match test column's mean/std\n        if current_std > 0:\n            df_train_aligned[col] = ((df_train[col] - current_mean) / current_std) * target_std + target_mean\n        else:\n            df_train_aligned[col] = df_train[col] - current_mean + target_mean\n    \n    return df_train_aligned\n\n# Transform training data instead\ndf_train_aligned = align_training_to_test(df, df_test)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"85dce160","cell_type":"code","source":"plot_x_columns(df_train_aligned, title_suffix='Train Set Aligned')","metadata":{},"outputs":[],"execution_count":null}]}