{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":96164,"databundleVersionId":12993472}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:24:07.792609Z","iopub.execute_input":"2026-02-23T06:24:07.792905Z","iopub.status.idle":"2026-02-23T06:24:08.797539Z","shell.execute_reply.started":"2026-02-23T06:24:07.792885Z","shell.execute_reply":"2026-02-23T06:24:08.796481Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1️⃣ Linear Regression\n2️⃣ Ridge\n3️⃣ Lasso\n4️⃣ Elastic Net\n5️⃣ Random Forest\n6️⃣ XGBoost\n7️⃣ LightGBM\n8️⃣ SVR\n9️⃣ ANN (MLP)\n🔟 LSTM (optional structure note)","metadata":{}},{"cell_type":"markdown","source":"# **Understanding the Dataset & Requirements**","metadata":{}},{"cell_type":"code","source":"train_path = '/kaggle/input/drw-crypto-market-prediction/train.parquet'\ntest_path  = '/kaggle/input/drw-crypto-market-prediction/test.parquet'\n\ndf_train = pd.read_parquet(train_path)\ndf_train = df_train.tail(250000)\n\ny_train_DRW = df_train[['label']]\nx_train_DRW = df_train.drop(['label'], axis=1)\n\ndf_test = pd.read_parquet(test_path)\ndf_test = df_test.tail(250000)\n\ny_test_DRW = df_test[['label']]\nx_test_DRW = df_test.drop(['label'], axis=1)\n\nx_train_DRW.shape, y_train_DRW.shape, x_test_DRW.shape, y_test_DRW.shape\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:24:12.763360Z","iopub.execute_input":"2026-02-23T06:24:12.763800Z","iopub.status.idle":"2026-02-23T06:24:48.093587Z","shell.execute_reply.started":"2026-02-23T06:24:12.763776Z","shell.execute_reply":"2026-02-23T06:24:48.092922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train_DRW.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:41:05.094988Z","iopub.execute_input":"2026-02-23T06:41:05.095806Z","iopub.status.idle":"2026-02-23T06:41:05.116265Z","shell.execute_reply.started":"2026-02-23T06:41:05.095767Z","shell.execute_reply":"2026-02-23T06:41:05.115346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:24:52.375070Z","iopub.execute_input":"2026-02-23T06:24:52.375405Z","iopub.status.idle":"2026-02-23T06:24:52.402875Z","shell.execute_reply.started":"2026-02-23T06:24:52.375381Z","shell.execute_reply":"2026-02-23T06:24:52.401569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train_DRW.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:24:55.728304Z","iopub.execute_input":"2026-02-23T06:24:55.728627Z","iopub.status.idle":"2026-02-23T06:24:55.745144Z","shell.execute_reply.started":"2026-02-23T06:24:55.728605Z","shell.execute_reply":"2026-02-23T06:24:55.744291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" \"\"\"duplicate_cols = df.T.duplicated()\n\n    if duplicate_cols.any():\n        print(\"Duplicate columns found.\")\n        cols_to_drop = df.columns[duplicate_cols]\n        print(\"Duplicate columns:\", list(cols_to_drop))\n        \n        df.drop(columns=cols_to_drop, inplace=True)\n        print(\"Duplicate columns removed.\")\n    else:\n        print(\"No duplicate columns found.\")\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:26:04.527570Z","iopub.execute_input":"2026-02-23T06:26:04.528146Z","iopub.status.idle":"2026-02-23T06:26:04.532656Z","shell.execute_reply.started":"2026-02-23T06:26:04.528121Z","shell.execute_reply":"2026-02-23T06:26:04.532159Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataframes = {\n    'x_train_DRW': x_train_DRW,\n    'x_test_DRW': x_test_DRW\n}\n\nfor name, df in dataframes.items():\n    print(\"\\n==========================================\")\n    print(f\"Properties for DataFrame: '{name}'\")\n    print(\"==========================================\")\n\n    print(\"Shape:\", df.shape)\n\n    print(\"\\nInfo:\")\n    df.info()\n\n    print(\"\\nDescriptive Statistics:\")\n    print(df.describe())\n\n    print(\"\\nTotal missing values:\", df.isna().sum().sum())\n\n    # Duplicate row check\n    if df.duplicated().any():\n        print(\"Duplicate rows found. Removing duplicates.\")\n        df.drop_duplicates(inplace=True)\n    else:\n        print(\"No duplicate rows found.\")\n\n    # Duplicate column check\n \n\n    # Infinite values check\n    has_inf = df.isin([np.inf, -np.inf]).any().any()\n\n    if has_inf:\n        print(\"Infinite values detected.\")\n        cols_with_inf = df.columns[df.isin([np.inf, -np.inf]).any()]\n        print(\"Columns with infinite values:\", list(cols_with_inf))\n\n        for col in cols_with_inf:\n            if df[col].value_counts().values[0] == len(df):\n                df.drop(col, axis=1, inplace=True)\n                print(f\"Dropped column: {col}\")\n    else:\n        print(\"No infinite values found.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:26:14.343062Z","iopub.execute_input":"2026-02-23T06:26:14.343352Z","iopub.status.idle":"2026-02-23T06:27:14.029390Z","shell.execute_reply.started":"2026-02-23T06:26:14.343335Z","shell.execute_reply":"2026-02-23T06:27:14.028380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"duplicate_columns = [\n    'X104', 'X110', 'X116', 'X122', 'X128', 'X134', 'X140',\n    'X146', 'X152', 'X158', 'X164', 'X170', 'X176', 'X182',\n    'X351', 'X357', 'X363', 'X369', 'X375', 'X381', 'X387',\n    'X393', 'X399', 'X405', 'X411', 'X417', 'X423', 'X429'\n]\n\nfor base_col in duplicate_columns:\n    duplicates = []\n    \n    for col in df_train.columns:\n        if col != base_col:\n            if df_train[base_col].equals(df_train[col]):\n                duplicates.append(col)\n    \n    print(f\"{base_col} is duplicate with: {duplicates}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:27:18.326118Z","iopub.execute_input":"2026-02-23T06:27:18.326361Z","iopub.status.idle":"2026-02-23T06:27:34.028159Z","shell.execute_reply.started":"2026-02-23T06:27:18.326343Z","shell.execute_reply":"2026-02-23T06:27:34.027111Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport statsmodels.api as sm\n\nleft_skewed_cols = []\nright_skewed_cols = []\n\nfor name, df in dataframes.items():\n\n    print(\"\\nChecking skewness for:\", name)\n\n    for col in df.columns:\n        if df[col].mean() < df[col].median():\n            left_skewed_cols.append(col)\n        elif df[col].mean() > df[col].median():\n            right_skewed_cols.append(col)\n\n    # Simple histogram for one column\n    plt.hist(df['X247'], bins=100)\n    plt.title(f\"Histogram of X247 ({name})\")\n    plt.xlabel(\"X247\")\n    plt.ylabel(\"Frequency\")\n    plt.show()\n\n    # Q-Q plot\n    sm.qqplot(df['X247'], line='s')\n    plt.title(f\"Q-Q Plot of X247 ({name})\")\n    plt.show()\n\nprint(\"\\nLeft skewed columns:\")\nprint(left_skewed_cols)\n\nprint(\"\\nRight skewed columns:\")\nprint(right_skewed_cols)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:27:40.896127Z","iopub.execute_input":"2026-02-23T06:27:40.896375Z","iopub.status.idle":"2026-02-23T06:27:50.999238Z","shell.execute_reply.started":"2026-02-23T06:27:40.896361Z","shell.execute_reply":"2026-02-23T06:27:50.998465Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Only on Train","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train_original, x_test_original, y_train_original, y_test_original = train_test_split(\n    x_train_DRW,\n    y_train_DRW,\n    test_size=0.2,\n    random_state=42,\n    shuffle=False\n)\n\nprint(\"Train-test split created\")\nx_train_original.shape, x_test_original.shape, y_train_original.shape, y_test_original.shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:28:00.346476Z","iopub.execute_input":"2026-02-23T06:28:00.346910Z","iopub.status.idle":"2026-02-23T06:28:00.857601Z","shell.execute_reply.started":"2026-02-23T06:28:00.346888Z","shell.execute_reply":"2026-02-23T06:28:00.856819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc\nfrom sklearn.preprocessing import StandardScaler, PowerTransformer\nfrom scipy.stats import kurtosis\n\n# ---------------- PARAMETERS ----------------\nLOW_Q = 0.05\nHIGH_Q = 0.95\nSKEW_TH = 1.0\nKURT_TH = 1.0\nSTD_TH = 1e-9\n\nscaler = StandardScaler()\npt = PowerTransformer(method=\"yeo-johnson\", standardize=False)\n\n# --------------------------------------------------\n# TRAIN = fit, TEST = transform only\n# --------------------------------------------------\n\n# ----------- 1. IQR / PERCENTILE CLIPPING (fit on train) ----------\nfor df in [x_train_original, x_test_original]:\n    for col in x_train_original.columns:\n        lower = x_train_original[col].quantile(LOW_Q)\n        upper = x_train_original[col].quantile(HIGH_Q)\n        df[col] = df[col].clip(lower, upper)\n\n# ----------- 2. FIND SKEWED COLUMNS (train only) ----------\nskewed_cols = []\n\nfor col in x_train_original.columns:\n    skew_val = x_train_original[col].skew()\n    kurt_val = kurtosis(x_train_original[col], fisher=True)\n\n    if abs(skew_val) > SKEW_TH or abs(kurt_val) > KURT_TH:\n        skewed_cols.append(col)\n\n# Remove zero / near-zero variance columns\nfinal_skewed_cols = [\n    col for col in skewed_cols\n    if x_train_original[col].nunique() > 1\n    and x_train_original[col].std() > 1e-6\n]\n\nprint(\"Power transform columns:\", final_skewed_cols)\n\n# ----------- 3. POWER TRANSFORMATION ----------\nif final_skewed_cols:\n    pt.fit(x_train_original[final_skewed_cols])\n\n    x_train_original[final_skewed_cols] = pt.transform(\n        x_train_original[final_skewed_cols]\n    )\n    x_test_original[final_skewed_cols] = pt.transform(\n        x_test_original[final_skewed_cols]\n    )\n\n# ----------- 4. HANDLE INF / NAN ----------\nfor df in [x_train_original, x_test_original]:\n    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n    df.fillna(0, inplace=True)   # safe fallback\n\n# ----------- 5. STANDARD SCALING ----------\nfinal_scale_cols = []\n\nfor col in x_train_original.columns:\n    if x_train_original[col].std() > STD_TH:\n        final_scale_cols.append(col)\n\nscaler.fit(x_train_original[final_scale_cols])\n\nx_train_original = pd.DataFrame(\n    scaler.transform(x_train_original[final_scale_cols]),\n    columns=final_scale_cols,\n    index=x_train_original.index\n)\n\nx_test_original = pd.DataFrame(\n    scaler.transform(x_test_original[final_scale_cols]),\n    columns=final_scale_cols,\n    index=x_test_original.index\n)\n\n# ----------- 6. MEMORY OPTIMIZATION ----------\nfor df in [x_train_original, x_test_original]:\n    for col in df.columns:\n        df[col] = df[col].astype(\"float32\")\n\n    df.replace(\n        [np.inf, -np.inf],\n        [np.finfo(np.float32).max, np.finfo(np.float32).min],\n        inplace=True\n    )\n\n# REMOVE DUPLICATE COLUMNS (CORRECT WAY)\nx_train_original = x_train_original.loc[:, ~x_train_original.columns.duplicated()]\nx_test_original  = x_test_original.loc[:, ~x_test_original.columns.duplicated()]\n\ngc.collect()\n\n# ----------- FINAL OUTPUT ----------\nx_train_original_processed = x_train_original\nx_test_original_processed  = x_test_original\n\nprint(\"✅ Outliers handled, data transformed, scaled, and memory optimized\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:28:13.398484Z","iopub.execute_input":"2026-02-23T06:28:13.398760Z","iopub.status.idle":"2026-02-23T06:28:51.194537Z","shell.execute_reply.started":"2026-02-23T06:28:13.398742Z","shell.execute_reply":"2026-02-23T06:28:51.193489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_stats = x_train_original_processed.describe().T[['mean', 'std']]\ntest_stats = x_test_original_processed.describe().T[['mean', 'std']]\ndrift_df = train_stats.join(\n    test_stats,\n    lsuffix='_train',\n    rsuffix='_test'\n)\n\ndrift_df['mean_diff'] = (\n    drift_df['mean_test'] - drift_df['mean_train']\n).abs()\n\ndrift_df['std_ratio'] = (\n    drift_df['std_test'] / drift_df['std_train']\n)\n\ndrift_df = drift_df.sort_values(by='mean_diff', ascending=False)\n\ndrift_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:29:55.320796Z","iopub.execute_input":"2026-02-23T06:29:55.321632Z","iopub.status.idle":"2026-02-23T06:30:01.399105Z","shell.execute_reply.started":"2026-02-23T06:29:55.321597Z","shell.execute_reply":"2026-02-23T06:30:01.397919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# ---------------- SETTINGS ----------------\nNUM_FEATURES = 10\nSAMPLE_ROWS = 50\nLABEL_COL = 'label'\n\n# ---------------- MERGE X & y ----------------\nfeatures_to_plot = x_train_original_processed.iloc[:, :NUM_FEATURES]\n\nmerged_df = pd.concat(\n    [features_to_plot, y_train_original[[LABEL_COL]]],\n    axis=1\n)\n\nprint(\"Merged dataframe shape:\", merged_df.shape)\n\n# ---------------- PAIRPLOT (RELATIONSHIP CHECK) ----------------\nsns.pairplot(merged_df.iloc[:SAMPLE_ROWS, :])\nplt.show()\n\n# ---------------- PEARSON CORRELATION ----------------\npearson_corr = merged_df.corr(method='pearson')\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(pearson_corr, annot=True, cmap='coolwarm', fmt=\".2f\")\nplt.title(\"Pearson Correlation Heatmap\")\nplt.show()\n\n# ---------------- SPEARMAN CORRELATION ----------------\nspearman_corr = merged_df.corr(method='spearman')\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(spearman_corr, annot=True, cmap='coolwarm', fmt=\".2f\")\nplt.title(\"Spearman Correlation Heatmap\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:30:07.870966Z","iopub.execute_input":"2026-02-23T06:30:07.871249Z","iopub.status.idle":"2026-02-23T06:30:31.360784Z","shell.execute_reply.started":"2026-02-23T06:30:07.871232Z","shell.execute_reply":"2026-02-23T06:30:31.359345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.feature_selection import mutual_info_regression\n\n# ---------------- SETTINGS ----------------\nCORR_THRESHOLD = 0.80\nROWS_TO_INCLUDE = 50000\nLABEL_COL = 'label'\nCORR_METHOD = 'pearson'\n\n# ---------------- STEP 1: CORRELATION MATRIX (X only) ----------------\ncorr_matrix = x_train_original_processed.tail(ROWS_TO_INCLUDE).corr(method=CORR_METHOD)\n\n# ---------------- STEP 2: MUTUAL INFORMATION (X vs y) ----------------\nmi_scores = mutual_info_regression(\n    x_train_original_processed.tail(ROWS_TO_INCLUDE),\n    y_train_original[LABEL_COL].tail(ROWS_TO_INCLUDE)\n)\n\nmi_df = pd.DataFrame({\n    'Feature': x_train_original_processed.columns,\n    'MI_Score': mi_scores\n}).sort_values(by='MI_Score', ascending=False)\n\nmi_dict = mi_df.set_index('Feature')['MI_Score'].to_dict()\n\n# ---------------- STEP 3: FIND HIGHLY CORRELATED PAIRS ----------------\nhigh_corr_pairs = []\n\ncols = corr_matrix.columns\n\nfor i in range(len(cols)):\n    for j in range(i + 1, len(cols)):\n        corr_val = corr_matrix.iloc[i, j]\n        if abs(corr_val) > CORR_THRESHOLD:\n            high_corr_pairs.append((cols[i], cols[j], corr_val))\n\nprint(f\"Highly correlated pairs (|corr| > {CORR_THRESHOLD}):\")\nfor c1, c2, val in high_corr_pairs:\n    print(f\"{c1} & {c2} : {val:.3f}\")\n\n# ---------------- STEP 4: DECIDE WHICH FEATURES TO DROP (USING MI) ----------------\nfeatures_to_drop = set()\n\nfor c1, c2, _ in high_corr_pairs:\n    mi_1 = mi_dict.get(c1, -np.inf)\n    mi_2 = mi_dict.get(c2, -np.inf)\n\n    # Drop the less informative feature\n    if mi_1 >= mi_2:\n        features_to_drop.add(c2)\n    else:\n        features_to_drop.add(c1)\n\nprint(\"\\nNumber of features dropped:\", len(features_to_drop))\n\n# ---------------- STEP 5: DROP FROM TRAIN ----------------\nx_train_reduced_post_pearson = x_train_original_processed.drop(\n    columns=list(features_to_drop),\n    errors='ignore'\n)\n\nprint(\"Train shape after multicollinearity removal:\",\n      x_train_reduced_post_pearson.shape)\n\n# ---------------- STEP 6: APPLY SAME COLUMNS TO TEST ----------------\nx_test_reduced_post_pearson = x_test_original_processed[\n    x_train_reduced_post_pearson.columns\n]\n\nprint(\"Test shape after multicollinearity removal:\",\n      x_test_reduced_post_pearson.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:30:36.916266Z","iopub.execute_input":"2026-02-23T06:30:36.916579Z","iopub.status.idle":"2026-02-23T06:34:48.982252Z","shell.execute_reply.started":"2026-02-23T06:30:36.916559Z","shell.execute_reply":"2026-02-23T06:34:48.981173Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.feature_selection import mutual_info_regression\n\n# ---------------- SETTINGS ----------------\nROWS_TO_INCLUDE = 50000\nLABEL_COL = 'label'\nMI_THRESHOLD = 0.05\n\n# ---------------- STEP 1: COMPUTE MUTUAL INFORMATION ----------------\nmi_scores = mutual_info_regression(\n    x_train_reduced_post_pearson.tail(ROWS_TO_INCLUDE),\n    y_train_original[LABEL_COL].tail(ROWS_TO_INCLUDE)\n)\n\nmi_df = pd.DataFrame({\n    'Feature': x_train_reduced_post_pearson.columns,\n    'MI_Score': mi_scores\n}).sort_values(by='MI_Score', ascending=False)\n\nprint(\"Total features:\", mi_df.shape)\n\n# ---------------- STEP 2: VISUALIZE MI SCORES ----------------\nplt.figure(figsize=(14, 4))\nsns.barplot(\n    data=mi_df.head(50),\n    x='Feature',\n    y='MI_Score'\n)\nplt.xticks(rotation=90)\nplt.title(\"Top 50 Features by Mutual Information\")\nplt.tight_layout()\nplt.show()\n\nplt.figure(figsize=(6, 4))\nsns.lineplot(x=range(len(mi_df)), y=mi_df['MI_Score'])\nplt.xlabel(\"Feature Rank\")\nplt.ylabel(\"MI Score\")\nplt.title(\"Mutual Information Scores (Descending)\")\nplt.grid(linestyle=':')\nplt.show()\n\n# ---------------- STEP 3: SELECT FEATURES ABOVE THRESHOLD ----------------\nselected_features = mi_df[mi_df['MI_Score'] >= MI_THRESHOLD]['Feature'].tolist()\ndropped_features  = mi_df[mi_df['MI_Score'] < MI_THRESHOLD]['Feature'].tolist()\n\nprint(\"Features kept after MI filtering:\", len(selected_features))\nprint(\"Features dropped after MI filtering:\", len(dropped_features))\n\n# ---------------- STEP 4: APPLY FEATURE SELECTION ----------------\nx_train_reduced_post_pearson_MI = x_train_reduced_post_pearson[selected_features]\nx_test_reduced_post_pearson_MI  = x_test_reduced_post_pearson[selected_features]\n\nprint(\"Train shape after MI filtering:\", x_train_reduced_post_pearson_MI.shape)\nprint(\"Test shape after MI filtering:\", x_test_reduced_post_pearson_MI.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T06:36:02.462926Z","iopub.execute_input":"2026-02-23T06:36:02.463236Z","iopub.status.idle":"2026-02-23T06:36:32.867703Z","shell.execute_reply.started":"2026-02-23T06:36:02.463219Z","shell.execute_reply":"2026-02-23T06:36:32.866830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n x_train_reduced_post_pearson_MI\nx_test_reduced_post_pearson_MI\ny_train_original\ny_test_original","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#-------------------------------------------------------------------------------------------\nimport pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt                  # To perform data visualisation\nimport seaborn as sns                                 # To perform data visualisation\nimport plotly.express as px                           # To perform data visualisation\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport statsmodels.api as sm\nfrom scipy.stats import skew, kurtosis\n%matplotlib inline\n                                             \nfrom sklearn.linear_model import LinearRegression     # To perform prediction\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import Lasso, LassoCV, RidgeCV, ElasticNetCV\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.svm import SVR\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import LSTM, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\n\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.preprocessing import StandardScaler\nimport math\n\nfrom scipy.stats import chi2_contingency              # To perform chi sqr test\nfrom scipy import stats\nfrom scipy.stats import skewtest, norm\nfrom sklearn.preprocessing import PowerTransformer\n\nfrom sklearn.preprocessing import StandardScaler      # To perform feature scaling\nfrom sklearn.model_selection import train_test_split  # To perform train test split\n\nfrom sklearn.model_selection import GridSearchCV      # To perform hyperparameter tuning\nfrom sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit\nfrom scipy.stats import pearsonr                       # To perform pearson correlation\nfrom scipy.stats import loguniform\nfrom scipy.stats import uniform\nfrom random import randint\nimport copy\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import r2_score\nfrom sklearn.inspection import permutation_importance\nimport gc\n#import mlflow\n\nsns.set_theme(style=\"white\", palette=\"muted\")\n%matplotlib inline\n\n#-------------------------------------------------------------------------------------------\nimport warnings                                       # Importing warning to disable runtime warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-23T06:45:55.004994Z","iopub.execute_input":"2026-02-23T06:45:55.005293Z","iopub.status.idle":"2026-02-23T06:46:17.030705Z","shell.execute_reply.started":"2026-02-23T06:45:55.005279Z","shell.execute_reply":"2026-02-23T06:46:17.030022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport numpy as np\nimport pandas as pd\n\n# Ensure target is 1D\ny_train = y_train_original['label']\ny_test  = y_test_original['label']\n\n# -----------------------------\n# TRAIN LINEAR REGRESSION\n# -----------------------------\nlr_model = LinearRegression()\n\nlr_model.fit(x_train_reduced_post_pearson_MI, y_train)\n\n# -----------------------------\n# PREDICT\n# -----------------------------\nlr_pred = lr_model.predict(x_test_reduced_post_pearson_MI)\n\n# -----------------------------\n# METRICS\n# -----------------------------\n\n# Pearson Correlation\ncorr = pd.Series(lr_pred).corr(y_test.reset_index(drop=True))\nprint(f\"Pearson Correlation: {corr:.6f}\")\n\n# RMSE\nrmse = np.sqrt(mean_squared_error(y_test, lr_pred))\nprint(f\"RMSE: {rmse:.6f}\")\n\n# R2 Score\nr2 = r2_score(y_test, lr_pred)\nprint(f\"R2 Score: {r2:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T08:05:45.447185Z","iopub.execute_input":"2026-02-23T08:05:45.447698Z","iopub.status.idle":"2026-02-23T08:05:48.739548Z","shell.execute_reply.started":"2026-02-23T08:05:45.447645Z","shell.execute_reply":"2026-02-23T08:05:48.736554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import LassoCV\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport numpy as np\nimport pandas as pd\n\n# Make sure y is 1D\ny_train = y_train_original['label']\ny_test  = y_test_original['label']\n\n# Define alpha grid\ncustom_alphas = np.logspace(-3, 0, 100)\n\n# -----------------------------\n# TRAIN LASSO DIRECTLY\n# -----------------------------\nlasso_model = LassoCV(\n    cv=5,\n    alphas=custom_alphas,\n    max_iter=10000,\n    random_state=42\n)\n\nlasso_model.fit(x_train_reduced_post_pearson_MI, y_train)\n\nprint(f\"Optimal alpha: {lasso_model.alpha_}\")\n\n# -----------------------------\n# PREDICT\n# -----------------------------\nlasso_pred = lasso_model.predict(x_test_reduced_post_pearson_MI)\n\n# -----------------------------\n# METRICS\n# -----------------------------\n\n# Pearson Correlation\ncorr = pd.Series(lasso_pred).corr(y_test.reset_index(drop=True))\nprint(f\"Pearson Correlation: {corr:.6f}\")\n\n# RMSE\nrmse = np.sqrt(mean_squared_error(y_test, lasso_pred))\nprint(f\"RMSE: {rmse:.6f}\")\n\n# R2\nr2 = r2_score(y_test, lasso_pred)\nprint(f\"R2 Score: {r2:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T07:18:21.945063Z","iopub.execute_input":"2026-02-23T07:18:21.945417Z","iopub.status.idle":"2026-02-23T07:18:23.993182Z","shell.execute_reply.started":"2026-02-23T07:18:21.945388Z","shell.execute_reply":"2026-02-23T07:18:23.992451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import RidgeCV\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport numpy as np\nimport pandas as pd\n\n# Ensure target is 1D\ny_train = y_train_original['label']\ny_test  = y_test_original['label']\n\n# Alpha grid\nalphas = np.logspace(-3, 0, 100)\n\n# -----------------------------\n# TRAIN RIDGE\n# -----------------------------\nridge_model = RidgeCV(alphas=alphas, cv=5)\n\nridge_model.fit(x_train_reduced_post_pearson_MI, y_train)\n\nprint(f\"Optimal alpha: {ridge_model.alpha_}\")\n\n# -----------------------------\n# PREDICT\n# -----------------------------\nridge_pred = ridge_model.predict(x_test_reduced_post_pearson_MI)\n\n# -----------------------------\n# METRICS\n# -----------------------------\n\n# Pearson Correlation\ncorr = pd.Series(ridge_pred).corr(y_test.reset_index(drop=True))\nprint(f\"Pearson Correlation: {corr:.6f}\")\n\n# RMSE\nrmse = np.sqrt(mean_squared_error(y_test, ridge_pred))\nprint(f\"RMSE: {rmse:.6f}\")\n\n# R2 Score\nr2 = r2_score(y_test, ridge_pred)\nprint(f\"R2 Score: {r2:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T07:21:14.439717Z","iopub.execute_input":"2026-02-23T07:21:14.440048Z","iopub.status.idle":"2026-02-23T07:21:43.170416Z","shell.execute_reply.started":"2026-02-23T07:21:14.440029Z","shell.execute_reply":"2026-02-23T07:21:43.169234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import ElasticNetCV\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport numpy as np\nimport pandas as pd\n\n# Ensure target is 1D\ny_train = y_train_original['label']\ny_test  = y_test_original['label']\n\n# Define alpha grid\nalphas = np.logspace(-3, 0, 100)\n\n# -----------------------------\n# TRAIN ELASTIC NET\n# -----------------------------\nelastic_model = ElasticNetCV(\n    alphas=alphas,\n    cv=5,\n    l1_ratio=0.5,      # 0 = Ridge, 1 = Lasso, 0.5 = balanced\n    max_iter=10000,\n    random_state=42\n)\n\nelastic_model.fit(x_train_reduced_post_pearson_MI, y_train)\n\nprint(f\"Optimal alpha: {elastic_model.alpha_}\")\n\n# -----------------------------\n# PREDICT\n# -----------------------------\nelastic_pred = elastic_model.predict(x_test_reduced_post_pearson_MI)\n\n# -----------------------------\n# METRICS\n# -----------------------------\n\n# Pearson Correlation\ncorr = pd.Series(elastic_pred).corr(y_test.reset_index(drop=True))\nprint(f\"Pearson Correlation: {corr:.6f}\")\n\n# RMSE\nrmse = np.sqrt(mean_squared_error(y_test, elastic_pred))\nprint(f\"RMSE: {rmse:.6f}\")\n\n# R2\nr2 = r2_score(y_test, elastic_pred)\nprint(f\"R2 Score: {r2:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T07:25:30.687296Z","iopub.execute_input":"2026-02-23T07:25:30.687604Z","iopub.status.idle":"2026-02-23T07:25:32.738234Z","shell.execute_reply.started":"2026-02-23T07:25:30.687586Z","shell.execute_reply":"2026-02-23T07:25:32.737081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport numpy as np\nimport pandas as pd\n\n# Ensure target is 1D\ny_train = y_train_original['label']\ny_test  = y_test_original['label']\n\n# -----------------------------\n# TRAIN RANDOM FOREST\n# -----------------------------\nrf_model = RandomForestRegressor(\n    n_estimators=200,      # number of trees\n    max_depth=None,        # allow full growth\n    min_samples_split=2,\n    min_samples_leaf=1,\n    random_state=42,\n    n_jobs=-1\n)\n\nrf_model.fit(x_train_reduced_post_pearson_MI, y_train)\n\n# -----------------------------\n# PREDICT\n# -----------------------------\nrf_pred = rf_model.predict(x_test_reduced_post_pearson_MI)\n\n# -----------------------------\n# METRICS\n# -----------------------------\n\n# Pearson Correlation\ncorr = pd.Series(rf_pred).corr(y_test.reset_index(drop=True))\nprint(f\"Pearson Correlation: {corr:.6f}\")\n\n# RMSE\nrmse = np.sqrt(mean_squared_error(y_test, rf_pred))\nprint(f\"RMSE: {rmse:.6f}\")\n\n# R2 Score\nr2 = r2_score(y_test, rf_pred)\nprint(f\"R2 Score: {r2:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T08:00:07.755546Z","iopub.execute_input":"2026-02-23T08:00:07.755873Z","execution_failed":"2026-02-23T08:05:10.416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Pearson Correlation for Random Forest from RF's best features - original: -0.009571749228437102\nRoot Mean Squared Error for Random Forest from RF's best features - original: 1.3119\nR2 score for RF from RF's best features - original: -0.5359","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Features Importances List - RF Regressor: (31, 2)\nAbove Threshold Features Importances  List - RF Regressor: (31, 2)\nPearson Correlation for Random Forest from RF's best features - original: 0.028684649250007015\nRoot Mean Squared Error for Random Forest from RF's best features - original: 1.4222\nR2 score for RF from RF's best features - original: -3.9047","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.ensemble import RandomForestRegressor\n\n# -----------------------------\n# INPUT DATA (ALREADY PROCESSED)\n# -----------------------------\nX_train = x_train_reduced_post_pearson_MI\ny_train = y_train_original['label']   # ✅ FIXED HERE\n\nX_test  = x_test_reduced_post_pearson_MI\n\nROWS_TO_INCLUDE = 50000\nTHRESHOLD = 0.015\n\n# -----------------------------\n# RANDOM FOREST HYPERPARAMETERS\n# -----------------------------\nrf_params = {\n    'n_estimators': 150,\n    'max_depth': 15,\n    'min_samples_split': 10,\n    'min_samples_leaf': 5,\n    'max_features': 0.7,\n    'n_jobs': 2,\n    'random_state': 42\n}\n\n# -----------------------------\n# TRAIN RANDOM FOREST REGRESSOR\n# -----------------------------\nrf_model = RandomForestRegressor(**rf_params)\n\nrf_model.fit(\n    X_train.tail(ROWS_TO_INCLUDE),\n    y_train.tail(ROWS_TO_INCLUDE)\n)\n\n# -----------------------------\n# FEATURE IMPORTANCES\n# -----------------------------\nimportance_df = pd.DataFrame({\n    'Feature': X_train.columns,\n    'Importance': rf_model.feature_importances_\n}).sort_values(by='Importance', ascending=False)\n\nprint(\"Total features:\", importance_df.shape)\n\n# -----------------------------\n# VISUALIZATION\n# -----------------------------\nfig, ax = plt.subplots(1, 2, figsize=(16, 7))\n\nsns.barplot(\n    data=importance_df.head(50),\n    x='Feature',\n    y='Importance',\n    ax=ax[0]\n)\nax[0].set_title('Top 50 Features - Random Forest')\nax[0].tick_params(axis='x', rotation=90)\n\nsns.lineplot(\n    x=range(len(importance_df)),\n    y=importance_df['Importance'],\n    ax=ax[1]\n)\nax[1].set_title('RF Feature Importance (Ranked)')\nax[1].set_xlabel('Feature Rank')\n\nplt.tight_layout()\nplt.show()\n\n# -----------------------------\n# SELECT FEATURES ABOVE THRESHOLD\n# -----------------------------\nabove_threshold_features_RF = importance_df[\n    importance_df['Importance'] >= THRESHOLD\n]\n\nprint(\"Features above threshold:\", above_threshold_features_RF.shape)\n\n# -----------------------------\n# REDUCE TRAIN & TEST DATA\n# -----------------------------\nX_train_reduced_RF = X_train[above_threshold_features_RF['Feature']]\nX_test_reduced_RF  = X_test[X_train_reduced_RF.columns]\n\nprint(\"Reduced Train Shape:\", X_train_reduced_RF.shape)\nprint(\"Reduced Test Shape :\", X_test_reduced_RF.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T07:28:25.042254Z","iopub.execute_input":"2026-02-23T07:28:25.042587Z","iopub.status.idle":"2026-02-23T07:30:27.813068Z","shell.execute_reply.started":"2026-02-23T07:28:25.042565Z","shell.execute_reply":"2026-02-23T07:30:27.812086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport numpy as np\nimport pandas as pd\n\n# Ensure target is 1D\ny_train = y_train_original['label']\ny_test  = y_test_original['label']\n\n# -----------------------------\n# TRAIN RANDOM FOREST\n# -----------------------------\nrf_model_two = RandomForestRegressor(\n    tuned_hyperparameters = {\n    \"n_estimators\": 300,\n    \"max_depth\": 10,\n    \"min_samples_split\": 5,\n    \"min_samples_leaf\": 2\n},\n    random_state=42,\n    n_jobs=-1\n)\n\nrf_model_two.fit(x_train_reduced_post_pearson_MI_RF, y_train)\n\n# -----------------------------\n# PREDICTIONS\n# -----------------------------\nRF_pred = rf_model_two.predict(x_test_reduced_post_pearson_MI_RF)\n\nRF_pred_df = pd.DataFrame(\n    RF_pred,\n    columns=['pred_label from RF top features - Original']\n)\n\nm1_RF = RF_pred_df['pred_label from RF top features - Original'].round(3)\nm2_RF = y_test.reset_index(drop=True).round(3)\n\n# -----------------------------\n# METRICS (UNCHANGED ORDER)\n# -----------------------------\n\n# Pearson Correlation\ncorr_pandas_RF = m1_RF.corr(m2_RF)\nprint(f\"Pearson Correlation for Random Forest from RF's best features - original: {corr_pandas_RF}\")\n\n# RMSE (same order as your code)\nrmse_RF = np.sqrt(mean_squared_error(m1_RF, m2_RF))\nprint(f\"Root Mean Squared Error for Random Forest from RF's best features - original: {rmse_RF:.4f}\")\n\n# R2 (same order as your code)\nr2_RF = r2_score(m1_RF, m2_RF)\nprint(f\"R2 score for RF from RF's best features - original: {r2_RF:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T07:34:18.570770Z","iopub.execute_input":"2026-02-23T07:34:18.571139Z","iopub.status.idle":"2026-02-23T07:39:44.139537Z","shell.execute_reply.started":"2026-02-23T07:34:18.571119Z","shell.execute_reply":"2026-02-23T07:39:44.138592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport xgboost as xgb\n\n# -----------------------------\n# INPUT DATA (ALREADY PROCESSED)\n# -----------------------------\nX_train = x_train_reduced_post_pearson_MI\ny_train = y_train_original['label']\n\nX_test  = x_test_reduced_post_pearson_MI\n\nTHRESHOLD = 0.015\n\n# -----------------------------\n# XGBOOST HYPERPARAMETERS\n# -----------------------------\nxg_model = xgb.XGBRegressor(\n    random_state=42,\n    n_estimators=2000,\n    learning_rate=0.05,\n    max_depth=5,\n    gamma=0.1,\n    subsample=0.8,\n    colsample_bytree=0.7,\n    n_jobs=-1\n)\n\n# -----------------------------\n# TRAIN XGBOOST REGRESSOR\n# -----------------------------\nxg_model.fit(X_train, y_train)\n\n# -----------------------------\n# FEATURE IMPORTANCES\n# -----------------------------\nimportance_df = pd.DataFrame({\n    'Feature': X_train.columns,\n    'Importance': xg_model.feature_importances_\n}).sort_values(by='Importance', ascending=False)\n\nprint(\"Total features:\", importance_df.shape)\n\n# -----------------------------\n# VISUALIZATION\n# -----------------------------\nfig, ax = plt.subplots(1, 2, figsize=(16, 7))\n\nsns.barplot(\n    data=importance_df.head(50),\n    x='Feature',\n    y='Importance',\n    ax=ax[0]\n)\nax[0].set_title('Top 50 Features - XGBoost')\nax[0].tick_params(axis='x', rotation=90)\n\nsns.lineplot(\n    x=range(len(importance_df)),\n    y=importance_df['Importance'],\n    ax=ax[1]\n)\nax[1].set_title('XGBoost Feature Importance (Ranked)')\nax[1].set_xlabel('Feature Rank')\n\nplt.tight_layout()\nplt.show()\n\n# -----------------------------\n# SELECT FEATURES ABOVE THRESHOLD\n# -----------------------------\nabove_threshold_features_XG = importance_df[\n    importance_df['Importance'] >= THRESHOLD\n]\n\nprint(\"Features above threshold:\", above_threshold_features_XG.shape)\n\n# -----------------------------\n# REDUCE TRAIN & TEST DATA\n# -----------------------------\nX_train_reduced_XG = X_train[above_threshold_features_XG['Feature']]\nX_test_reduced_XG  = X_test[X_train_reduced_XG.columns]\n\nprint(\"Reduced Train Shape:\", X_train_reduced_XG.shape)\nprint(\"Reduced Test Shape :\", X_test_reduced_XG.shape)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n# -----------------------------\n# INPUT DATA (CORRECT VARIABLES)\n# -----------------------------\nX_train = X_train_reduced_XG        # XG-selected features\ny_train = y_train_original['label']\n\nX_test  = X_test_reduced_XG\ny_test  = y_test_original['label']\n\n# -----------------------------\n# TRAIN XGBOOST REGRESSOR\n# -----------------------------\nmodel_XG = xgb.XGBRegressor(\n    random_state=42,\n    n_estimators=2000,\n    learning_rate=0.05,\n    max_depth=5,\n    gamma=0.1,\n    subsample=0.8,\n    colsample_bytree=0.7,\n    n_jobs=-1\n)\n\nmodel_XG.fit(X_train, y_train)\n\n# -----------------------------\n# PREDICTION\n# -----------------------------\nXG_pred = model_XG.predict(X_test)\n\nXG_pred_df = pd.DataFrame(\n    XG_pred,\n    columns=['pred_label from XG top features - Original']\n)\n\nm1_XG = XG_pred_df['pred_label from XG top features - Original'].round(3)\nm2_XG = y_test.reset_index(drop=True).round(3)\n\n# -----------------------------\n# METRICS\n# -----------------------------\ncorr_pandas_XG = m1_XG.corr(m2_XG)\nrmse_XG = np.sqrt(mean_squared_error(m2_XG, m1_XG))\nr2_XG = r2_score(m2_XG, m1_XG)\n\nprint(f\"Pearson Correlation for XG Regressor from XG's best features - original: {corr_pandas_XG}\")\nprint(f\"Root Mean Squared Error for XG Regressor from XG's best features - original: {rmse_XG:.4f}\")\nprint(f\"R2 score for XG from XG's best features - original: {r2_XG:.4f}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.svm import SVR\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n# -----------------------------\n# SETTINGS\n# -----------------------------\nTIMESTAMPS_NUM = 10000\nTHRESHOLD = 0          # keep all, or change later\nN_REPEATS = 5\nRANDOM_STATE = 42\n\n# -----------------------------\n# PREPARE DATA (TIME-AWARE)\n# -----------------------------\nX_train = x_train_reduced_post_pearson_MI.tail(TIMESTAMPS_NUM)\ny_train = y_train_original['label'].tail(TIMESTAMPS_NUM)\n\nX_test  = x_test_reduced_post_pearson_MI.tail(TIMESTAMPS_NUM)\ny_test  = y_test_original['label'].tail(TIMESTAMPS_NUM)\n\n# -----------------------------\n# TRAIN SVR MODEL\n# -----------------------------\nsvr_model = SVR(\n    kernel='rbf',\n    C=1,\n    epsilon=0.1,\n    cache_size=500\n)\n\nsvr_model.fit(X_train, y_train)\n\n# -----------------------------\n# PREDICTION & METRICS\n# -----------------------------\ny_pred = pd.Series(svr_model.predict(X_test), name=\"Prediction from SVR\").round(3)\ny_true = y_test.reset_index(drop=True).round(3)\n\npearson_svr = y_pred.corr(y_true)\nrmse_svr = np.sqrt(mean_squared_error(y_true, y_pred))\nr2_svr = r2_score(y_true, y_pred)\n\nprint(\"\\n--- SVR Model Results (RBF Kernel) ---\")\nprint(f\"Pearson Correlation: {pearson_svr:.4f}\")\nprint(f\"RMSE: {rmse_svr:.4f}\")\nprint(f\"R2 Score: {r2_svr:.4f}\")\n\n# -----------------------------\n# PERMUTATION IMPORTANCE\n# -----------------------------\nperm_result = permutation_importance(\n    svr_model,\n    X_test,\n    y_test,\n    n_repeats=N_REPEATS,\n    random_state=RANDOM_STATE,\n    n_jobs=-1\n)\n\nimportance_df = pd.DataFrame({\n    \"Feature\": X_train.columns,\n    \"Importance\": perm_result.importances_mean\n}).sort_values(by=\"Importance\", ascending=False)\n\nprint(\"\\nTotal features:\", importance_df.shape[0])\n\n# -----------------------------\n# VISUALIZATION\n# -----------------------------\nfig, ax = plt.subplots(1, 2, figsize=(16, 7))\n\nsns.barplot(\n    data=importance_df.head(50),\n    x=\"Feature\",\n    y=\"Importance\",\n    ax=ax[0]\n)\nax[0].set_title(\"Top 50 Features - SVR (Permutation Importance)\")\nax[0].tick_params(axis=\"x\", rotation=90)\n\nsns.lineplot(\n    x=range(len(importance_df)),\n    y=importance_df[\"Importance\"],\n    ax=ax[1]\n)\nax[1].set_title(\"SVR Feature Importance (Descending)\")\nax[1].set_xlabel(\"Feature Rank\")\n\nplt.tight_layout()\nplt.show()\n\n# -----------------------------\n# FEATURE SELECTION\n# -----------------------------\nabove_threshold_features_svr = importance_df[\n    importance_df[\"Importance\"] >= THRESHOLD\n]\n\nX_train_reduced_SVR = X_train[above_threshold_features_svr[\"Feature\"]]\nX_test_reduced_SVR  = X_test[X_train_reduced_SVR.columns]\n\nprint(\"Selected features:\", X_train_reduced_SVR.shape[1])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.svm import SVR\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n# -----------------------------\n# INPUT DATA (USE AVAILABLE VARIABLES)\n# -----------------------------\nX_train = X_train_reduced_SVR\ny_train = y_train_original['label']\n\nX_test  = X_test_reduced_SVR\ny_test  = y_test_original['label']\n\n# -----------------------------\n# TRAIN SVR REGRESSOR\n# -----------------------------\nmodel_SVR = SVR(\n    kernel='rbf',\n    C=1,\n    epsilon=0.1,\n    cache_size=500\n)\n\nmodel_SVR.fit(X_train, y_train)\n\n# -----------------------------\n# PREDICTION\n# -----------------------------\nSVR_pred = model_SVR.predict(X_test)\n\nm1_SVR = pd.Series(SVR_pred).round(3)\nm2_SVR = y_test.reset_index(drop=True).round(3)\n\n# -----------------------------\n# METRICS\n# -----------------------------\ncorr_pandas_SVR = m1_SVR.corr(m2_SVR)\nrmse_SVR = np.sqrt(mean_squared_error(m2_SVR, m1_SVR))\nr2_SVR = r2_score(m2_SVR, m1_SVR)\n\nprint(\"Pearson Correlation:\", corr_pandas_SVR)\nprint(\"RMSE:\", rmse_SVR)\nprint(\"R2 Score:\", r2_SVR)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\n\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom scipy.stats import pearsonr\n\n# -----------------------------\n# SETTINGS\n# -----------------------------\nTIMESTAMPS_NUM = 10000\nTHRESHOLD = -0.025\n\nEPOCHS = 100\nBATCH_SIZE = 256\nVALIDATION_SPLIT = 0.2\n\n# -----------------------------\n# PREPARE DATA (TIME-AWARE)\n# -----------------------------\nX_train = x_train_reduced_post_pearson_MI.tail(TIMESTAMPS_NUM)\ny_train = y_train_original['label'].tail(TIMESTAMPS_NUM)\n\nX_test  = x_test_reduced_post_pearson_MI\ny_test  = y_test_original['label']\n\ninput_dim = X_train.shape[1]\n\n# -----------------------------\n# BUILD ANN MODEL\n# -----------------------------\nann_model = Sequential([\n    Dense(64, activation='relu', input_shape=(input_dim,)),\n    Dropout(0.2),\n    Dense(32, activation='relu'),\n    Dropout(0.2),\n    Dense(16, activation='relu'),\n    Dense(1, activation='linear')\n])\n\nann_model.compile(\n    optimizer=Adam(),\n    loss='mse',\n    metrics=['mae']\n)\n\n# -----------------------------\n# TRAIN ANN\n# -----------------------------\nearly_stopping = EarlyStopping(\n    monitor='val_loss',\n    patience=15,\n    restore_best_weights=True,\n    verbose=0\n)\n\nann_model.fit(\n    X_train,\n    y_train,\n    validation_split=VALIDATION_SPLIT,\n    epochs=EPOCHS,\n    batch_size=BATCH_SIZE,\n    callbacks=[early_stopping],\n    verbose=0\n)\n\n# -----------------------------\n# PREDICTION & METRICS\n# -----------------------------\ny_pred = ann_model.predict(X_test, verbose=0).flatten()\n\nrmse_ann = np.sqrt(mean_squared_error(y_test, y_pred))\nr2_ann = r2_score(y_test, y_pred)\npearson_ann, _ = pearsonr(y_test, y_pred)\n\nprint(f\"\\nANN Results:\")\nprint(f\"RMSE    : {rmse_ann:.4f}\")\nprint(f\"R2      : {r2_ann:.4f}\")\nprint(f\"Pearson : {pearson_ann:.4f}\")\n\n# -----------------------------\n# PERMUTATION IMPORTANCE\n# -----------------------------\n# sklearn needs a wrapper-like interface → use lambda\ndef ann_predict(X):\n    return ann_model.predict(X, verbose=0).flatten()\n\nperm_result = permutation_importance(\n    estimator=ann_model,\n    X=X_test.tail(TIMESTAMPS_NUM),\n    y=y_test.tail(TIMESTAMPS_NUM),\n    n_repeats=3,\n    random_state=42,\n    n_jobs=-1,\n    scoring='r2'\n)\n\nimportance_df = pd.DataFrame({\n    \"Feature\": X_train.columns,\n    \"Importance\": perm_result.importances_mean\n}).sort_values(by=\"Importance\", ascending=False)\n\nprint(\"\\nFeature Importances (ANN):\")\nprint(importance_df.head())\n\n# -----------------------------\n# VISUALIZATION\n# -----------------------------\nfig, ax = plt.subplots(1, 2, figsize=(16, 7))\n\nsns.barplot(\n    data=importance_df.head(50),\n    x='Feature',\n    y='Importance',\n    ax=ax[0]\n)\nax[0].set_title('Top 50 Features - ANN (Permutation Importance)')\nax[0].tick_params(axis='x', rotation=90)\n\nsns.lineplot(\n    x=range(len(importance_df)),\n    y=importance_df['Importance'],\n    ax=ax[1]\n)\nax[1].set_title('ANN Feature Importance (Descending)')\nax[1].set_xlabel('Feature Rank')\n\nplt.tight_layout()\nplt.show()\n\n# -----------------------------\n# FEATURE SELECTION\n# -----------------------------\nabove_threshold_features_ANN = importance_df[\n    importance_df['Importance'] >= THRESHOLD\n]\n\nX_train_reduced_ANN = X_train[above_threshold_features_ANN['Feature']]\nX_test_reduced_ANN  = X_test[X_train_reduced_ANN.columns]\n\nprint(\"\\nSelected features:\", X_train_reduced_ANN.shape[1])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom scipy.stats import pearsonr\n\n# -----------------------------\n# INPUT DATA (CORRECT VARIABLES)\n# -----------------------------\nX_train = x_train_reduced_post_pearson_MI_ANN\ny_train = y_train_original['label']\n\nX_test  = x_test_reduced_post_pearson_MI_ANN\ny_test  = y_test_original['label']\n\ninput_dim = X_train.shape[1]\n\n# -----------------------------\n# BUILD ANN MODEL\n# -----------------------------\nmodel_ANN = Sequential([\n    Dense(64, activation='relu', input_shape=(input_dim,)),\n    Dropout(0.2),\n    Dense(32, activation='relu'),\n    Dropout(0.2),\n    Dense(16, activation='relu'),\n    Dense(1, activation='linear')\n])\n\nmodel_ANN.compile(\n    optimizer=Adam(),\n    loss='mse',\n    metrics=['mae']\n)\n\n# -----------------------------\n# TRAIN ANN\n# -----------------------------\nearly_stopping = EarlyStopping(\n    monitor='val_loss',\n    patience=15,\n    restore_best_weights=True,\n    verbose=0\n)\n\nmodel_ANN.fit(\n    X_train,\n    y_train,\n    validation_split=0.2,\n    epochs=100,\n    batch_size=256,\n    callbacks=[early_stopping],\n    verbose=0\n)\n\n# -----------------------------\n# PREDICTION\n# -----------------------------\nANN_pred = model_ANN.predict(X_test, verbose=0).flatten()\n\nANN_pred_df = pd.DataFrame(\n    ANN_pred,\n    columns=['Predictions from ANN top features - Original']\n)\n\nm1_ANN = ANN_pred_df['Predictions from ANN top features - Original'].round(3)\nm2_ANN = y_test.reset_index(drop=True).round(3)\n\n# -----------------------------\n# METRICS\n# -----------------------------\ncorr_pandas_ANN, _ = pearsonr(m1_ANN, m2_ANN)\nrmse_ANN = np.sqrt(mean_squared_error(m2_ANN, m1_ANN))\nr2_ANN = r2_score(m2_ANN, m1_ANN)\n\nprint(f\"Pearson Correlation from ANN's best features - Original: {corr_pandas_ANN}\")\nprint(f\"Root Mean Squared Error from ANN's best features - Original: {rmse_ANN:.4f}\")\nprint(f\"R2 score for ANN from ANN's best features - original: {r2_ANN:.4f}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n7. INITIAL FEATURE EXPLORATION - REDUCING DIMENSIONALITY\n\n1.RELATIVE IMPORTANCE OF FEATURES - PEARSON'S CORRELATION AND SPEARMAN'S CORRELATION\n2.RELATIVE IMPORTANCE OF FEATURES - MUTUAL INFORMATION SCORE\n3.RELATIVE IMPORTANCE OF FEATURES - DECISION TREE AND RANDOM FOREST APPROACH✅ Data Drift\n\nMore precisely, in ML / data science terms, what you are seeing is:\n\n🔹 Covariate Shift (a type of Data Drift)\n multicollinear features (can be tested by pearson's correlation)\n#4.Even though features like X1,X2 etc. percieved to have a linear relationships with label, their correlation percentage i\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport statsmodels.api as sm\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n# ------------------------------\n# DataFrames dictionary\n# ------------------------------\ndataframes = {\n    'x_train': x_train,\n    'x_test': x_test\n}\n\n# ==============================\n# DATASET DESCRIPTION\n# ==============================\nfor name, df in dataframes.items():\n    print(\"\\n\" + \"=\" * 40)\n    print(f\"DataFrame: {name}\")\n    print(\"=\" * 40)\n\n    # Shape\n    print(\"Shape:\", df.shape)\n\n    # Info\n    print(\"\\nInfo:\")\n    df.info()\n\n    # Descriptive statistics\n    print(\"\\nDescriptive Statistics:\")\n    print(df.describe())\n\n    # Missing values\n    print(\"\\nTotal Missing Values:\", df.isna().sum().sum())\n\n    # Duplicates\n    if df.duplicated().any():\n        print(\"Duplicates found. Removing duplicates...\")\n        df.drop_duplicates(inplace=True)\n    else:\n        print(\"No duplicates found.\")\n\n    # Infinite values\n    if df.isin([np.inf, -np.inf]).any().any():\n        print(\"Infinite values found.\")\n        cols_with_inf = df.columns[df.isin([np.inf, -np.inf]).any()]\n        print(\"Columns with infinite values:\", list(cols_with_inf))\n    else:\n        print(\"No infinite values found.\")\n\n# ==============================\n# SKEWNESS CHECK\n# ==============================\nleft_skewed = []\nright_skewed = []\n\nfor name, df in dataframes.items():\n\n    for col in df.columns:\n        if df[col].mean() < df[col].median():\n            left_skewed.append(col)\n        elif df[col].mean() > df[col].median():\n            right_skewed.append(col)\n\n    # Histogram & Violin plot for X247\n    print(f\"\\nSkewness visualization for X247 in {name}\")\n\n    fig = make_subplots(\n        rows=1, cols=2,\n        subplot_titles=(\"Histogram of X247\", \"Violin Plot of X247\")\n    )\n\n    fig.add_trace(go.Histogram(x=df['X247'], nbinsx=500), row=1, col=1)\n    fig.add_trace(\n        go.Violin(\n            y=df['X247'],\n            box_visible=True,\n            meanline_visible=True\n        ),\n        row=1, col=2\n    )\n\n    fig.update_layout(title=\"Distribution of X247\", showlegend=False)\n    fig.show()\n\n    # Q-Q plot\n    sm.qqplot(df['X247'], line='s')\n    plt.title(f\"Q-Q Plot of X247 ({name})\")\n    plt.show()\n\nprint(\"\\nLeft skewed columns:\", left_skewed)\nprint(\"Right skewed columns:\", right_skewed)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train_original, x_test_original, y_train_original, y_test_original = train_test_split(\n    x_train,\n    y_train,\n    test_size=0.3,\n    random_state=42,\n    shuffle=False\n)\n\nprint(\"Train-test split created\")\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# HANDLING OUTLIERS\n\nimport numpy as np\nimport pandas as pd\nimport gc\nfrom sklearn.preprocessing import StandardScaler, PowerTransformer\nfrom scipy.stats import kurtosis\n\n\nclass OutlierTreatment:\n\n    # Step 1: Handle outliers and skewness\n    def handle_outliers(self, data, fit_data):\n\n        data = data.copy()\n        fit_data = fit_data.copy()\n\n        # Clip extreme values (5% to 95%)\n        for col in fit_data.columns:\n            lower = fit_data[col].quantile(0.05)\n            upper = fit_data[col].quantile(0.95)\n            data[col] = data[col].clip(lower, upper)\n            fit_data[col] = fit_data[col].clip(lower, upper)\n\n        # Find skewed columns\n        skewed_cols = []\n        for col in fit_data.columns:\n            if abs(fit_data[col].skew()) > 1 or abs(kurtosis(fit_data[col])) > 1:\n                if fit_data[col].std() > 1e-6:\n                    skewed_cols.append(col)\n\n        # Apply power transformation\n        if skewed_cols:\n            pt = PowerTransformer(method=\"yeo-johnson\", standardize=False)\n            pt.fit(fit_data[skewed_cols])\n\n            data[skewed_cols] = pt.transform(data[skewed_cols])\n            fit_data[skewed_cols] = pt.transform(fit_data[skewed_cols])\n\n            # Clip again to avoid extreme values\n            for col in skewed_cols:\n                data[col] = data[col].clip(\n                    fit_data[col].quantile(0.10),\n                    fit_data[col].quantile(0.90)\n                )\n\n        return data, fit_data\n\n    # Step 2: Standard scaling\n    def scale_features(self, data, fit_data, std_threshold=1e-9):\n\n        scaler = StandardScaler()\n\n        valid_cols = [\n            col for col in fit_data.columns\n            if fit_data[col].std() > std_threshold\n        ]\n\n        scaler.fit(fit_data[valid_cols])\n        data[valid_cols] = scaler.transform(data[valid_cols])\n\n        return data[valid_cols]\n\n    # Step 3: Memory optimization\n    def optimize_memory(self, data):\n\n        for col in data.columns:\n            data[col] = data[col].astype(\"float32\")\n\n        data.replace(\n            [np.inf, -np.inf],\n            [np.finfo(np.float32).max, np.finfo(np.float32).min],\n            inplace=True\n        )\n\n        gc.collect()\n        return data\n\n\n# ===================== USAGE =====================\n\nprocessor = OutlierTreatment()\n\n# X train\nx_train_clean, x_train_fit = processor.handle_outliers(\n    x_train_original, x_train_original\n)\nx_train_original_processed = processor.optimize_memory(\n    processor.scale_features(x_train_clean, x_train_fit)\n)\n\n# X test (fit on train)\nx_test_clean, _ = processor.handle_outliers(\n    x_test_original, x_train_original\n)\nx_test_original_processed = processor.optimize_memory(\n    processor.scale_features(x_test_clean, x_train_fit)\n)\n\n# y train\ny_train_clean, y_train_fit = processor.handle_outliers(\n    y_train_original, y_train_original\n)\ny_train_original_processed = processor.optimize_memory(\n    processor.scale_features(y_train_clean, y_train_fit)\n)\n\n# y test\ny_test_clean, _ = processor.handle_outliers(\n    y_test_original, y_train_original\n)\ny_test_original_processed = processor.optimize_memory(\n    processor.scale_features(y_test_clean, y_train_fit)\n)\n\nprint(\"All data processed and converted to float32\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nfrom scipy.stats import skew, kurtosis\n\n# function to check distribution of a numeric column\ndef check_distribution(df, col):\n    data = df[col].dropna()\n\n    # basic stats\n    mean = np.mean(data)\n    median = np.median(data)\n    skewness = skew(data)\n    kurt = kurtosis(data)\n\n    print(\"Distribution for:\", col)\n    print(\"Mean    =\", round(mean, 4))\n    print(\"Median  =\", round(median, 4))\n    print(\"Skewness=\", round(skewness, 4))\n    print(\"Kurtosis=\", round(kurt, 4))\n    print(\"Note: skew > 1 means right skewed, < -1 means left skewed\")\n\n    # plotting\n    fig, ax = plt.subplots(1, 2, figsize=(12, 5))\n\n    # histogram + kde\n    sns.histplot(data, bins=40, kde=True, ax=ax[0], color=\"skyblue\")\n    ax[0].set_title(\"Histogram + KDE: \" + col)\n\n    # boxplot\n    sns.boxplot(x=data, ax=ax[1], color=\"lightcoral\")\n    ax[1].set_title(\"Boxplot: \" + col)\n\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"check_distribution(df,\"label\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ** Data Preprocessing****","metadata":{}},{"cell_type":"code","source":"var=df.var().sort_values(ascending=False)\ninfo=df.info()\nnull=df.isnull().sum()\nduplicate_columns = df.T.duplicated()\nduplicate_rows = df.duplicated()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dup=duplicate_columns.sort_values(ascending=False)\nprint(dup.head(29))\nrowdup=duplicate_rows.sort_values(ascending=False)\nprint(rowdup.head(20))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\nbad_cols = ['X716', 'X711', 'X697', 'X698', 'X699', 'X700', 'X701', 'X702', 'X703', 'X704',\n            'X705', 'X706', 'X707', 'X708', 'X710', 'X709', 'X712', 'X713', 'X714', 'X715',\n            'X717', 'X864', 'X867', 'X869', 'X870', 'X871', 'X872']\n#df = df.drop(columns=bad_cols)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\npd.set_option('display.max_rows', 10) \n\npd.set_option('display.max_columns', 15)    \npd.set_option('display.width', 1000)        \npd.set_option('display.expand_frame_repr', False) \nmissing_summary = []\n\nfor col in df.columns:\n    total = len(df[col])\n    missing = df[col].isnull().sum()\n    percent_missing = (missing / total) * 100\n\n    missing_summary.append({\n        'column': col,\n        'missing_count': missing,\n        'percent_missing': percent_missing\n    })\n\nmissing_df = pd.DataFrame(missing_summary).sort_values('percent_missing', ascending=True)\nprint(missing_df.head(20))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nnumeric_cols = df.select_dtypes(include=['float64', 'int64']).columns\n\noutlier_summary = []\n\nfor col in numeric_cols:\n    Q1 = df[col].quantile(0.25)\n    Q3 = df[col].quantile(0.75)\n    IQR = Q3 - Q1\n    lower = Q1 - 1.5*IQR\n    upper = Q3 + 1.5*IQR\n    \n    total = len(df[col])\n    outliers = df[(df[col] < lower) | (df[col] > upper)][col].count()\n    percent_outliers = outliers / total * 100\n    \n    outlier_summary.append({\n        'column': col,\n        'outlier_count': outliers,\n        'percent_outlier': percent_outliers\n    })\n\noutlier_df = pd.DataFrame(outlier_summary).sort_values('percent_outlier', ascending=False)\nprint(outlier_df.head(20))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering and Selection\n","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import VarianceThreshold\nselector = VarianceThreshold(threshold=0.0099)\ndf_new= selector.fit_transform(df.drop('label', axis=1))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import PowerTransformer, QuantileTransformer, FunctionTransformer\n\ndef auto_transform_numeric(df):\n    \"\"\"\n    Automatically transform numeric columns based on skewness and kurtosis.\n    Returns a new DataFrame with transformed numeric columns only.\n    \"\"\"\n    transformed_df = pd.DataFrame(index=df.index)\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n\n    for col in numeric_cols:\n        skew = df[col].skew()\n        kurt = df[col].kurtosis()\n        \n        # Decide transformer\n        if abs(skew) < 1 and abs(kurt) < 3:\n            # roughly normal → no transform\n            transformer = FunctionTransformer(func=None)\n        elif skew > 1:\n            # right-skewed\n            if (df[col] <= 0).any():\n                transformer = PowerTransformer(method='yeo-johnson', standardize=True)\n            else:\n                transformer = FunctionTransformer(func=np.log1p)\n        elif skew < -1:\n            # left-skewed → Yeo-Johnson\n            transformer = PowerTransformer(method='yeo-johnson', standardize=True)\n        else:\n            # heavy-tailed → QuantileTransformer\n            transformer = QuantileTransformer(output_distribution='normal', random_state=42)\n        \n        # Fit and transform\n        transformed = transformer.fit_transform(df[[col]])\n        \n        # Ensure it's 1D\n        if isinstance(transformed, pd.DataFrame):\n            transformed = transformed.values\n        transformed_df[col] = transformed.ravel()  # safer than flatten()\n    \n    return transformed_df\n\n\ndf_transformed = auto_transform_numeric(df)\ndf_transformed.head(5)\n ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"var=df_transformed.var().sort_values(ascending=False)\nvar.tail(20)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"means=df.mean()\nstd=df.std()\ncoefficient_variation=std/means\nvar=df.var()\nvar.sort_values(ascending=False) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = df.corr().abs()  \nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\nto_drop = [column for column in upper.columns if any(upper[column] > 0.9)]\nprint(\"Highly correlated features to consider dropping:\")\nprint(to_drop)\ndf_reduced = df.drop(columns=to_drop)\nprint(\"Shape before dropping:\", df.shape)\nprint(\"Shape after dropping:\", df_reduced.shape)\ndf_reduced.head()\ndf_reduced.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = df.corr(numeric_only=True).abs()\nplt.figure(figsize=(12, 10))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm')\nplt.title(\"Correlation Matrix\")\nplt.show()\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfeatures = [col for col in df.columns if col != 'target']\nX = df[features]\ny = df['target']\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\npca = PCA(n_components=0.95) \nX_pca = pca.fit_transform(X_scaled)\n\nprint(f\"Original number of features: {X.shape[1]}\")\nprint(f\"Reduced number of features after PCA: {X_pca.shape[1]}\")\npca_columns = [f'PC{i+1}' for i in range(X_pca.shape[1])]\ndf_pca = pd.DataFrame(X_pca, columns=pca_columns)\ndf_pca['target'] = y.values\nprint(df_pca.head())\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.feature_selection import mutual_info_regression\nX = df.drop(columns=[\"label\"])\ny = df[\"label\"]\nmi_scores = mutual_info_regression(X, y, random_state=42)\nmi_df = pd.DataFrame({\n    \"Feature\": X.columns,\n    \"Mutual_Information\": mi_scores\n})\nmi_df = mi_df.sort_values(by=\"Mutual_Information\", ascending=False)\n\nprint(\"Top 15 features by Mutual Information with label:\")\nprint(mi_df.head(15))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_feature_label = df.corrwith(df[\"label\"])\ncorr_df = corr_feature_label.reset_index()\ncorr_df.columns = [\"Feature\", \"Correlation_with_Label\"]\ncorr_df = corr_df.sort_values(by=\"Correlation_with_Label\", key=abs, ascending=False)\nprint(\"Top 15 correlated features with label:\")\nprint(corr_df.head(15))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Evaluation & Comparison**","metadata":{}},{"cell_type":"code","source":"top_corr = corr_df.head(20).set_index(\"Feature\")\nplt.figure(figsize=(6,8))\nsns.heatmap(top_corr, annot=True, cmap=\"coolwarm\", cbar=False)\nplt.title(\"Top 20 Features vs Label Correlation\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df1=df_transformed.head(5)\ndf2=df.head(5)\nprint(df1)\nprint(df2)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Model Training & Hyperparameter Tuning and Evaluation & Comparison**","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n\n\ndata_sample=df.head(2000)\nX = data_sample.drop(\"label\", axis=1)\ny = data_sample[\"label\"]\ndata_test = df.iloc[2000:2500]\nX_test = data_test .drop(\"label\", axis=1)\ny_test = data_test [\"label\"]\nrf = RandomForestRegressor(\n    n_estimators=300,\n    random_state=42,\n    n_jobs=-1\n)\nrf.fit(X, y)\ny_pred = rf.predict(X_test)\nmse = mean_squared_error(y_test, y_pred)\nmae = mean_absolute_error(y_test, y_pred)\nr2 = r2_score(y_test, y_pred)\n\nprint(\"MSE:\", mse)\nprint(\"MAE:\", mae)\nprint(\"R2 score:\", r2)\nimportances = pd.Series(rf.feature_importances_, index=X.columns)\nprint(\"\\nTop 10 important features:\\n\", importances.sort_values(ascending=False).head(10))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_error\n\n\nif 'label' in df_reduced.columns:\n    X = df_reduced.drop('label', axis=1)\n    y = df_reduced['label']\nelse:\n   \n    X = df_reduced\n    y = df['label']\n\n\n\nX_train = X.iloc[:split_point]\nX_test  = X.iloc[split_point:]  \ny_train = y.iloc[:split_point]\ny_test  = y.iloc[split_point:]\n\nprint(f\"Training on {len(X_train)} rows (Past)\")\nprint(f\"Testing on  {len(X_test)} rows (Future)\")\n\n\nprint(\"\\nTraining LightGBM...\")\nmodel = LGBMRegressor(n_estimators=100, learning_rate=0.1, random_state=42)\nmodel.fit(X_train, y_train)\n\npredictions = model.predict(X_test)\nrmse = mean_squared_error(y_test, predictions, squared=False)\n\nprint(f\"\\n--- TIME SERIES RESULTS ---\")\nprint(f\"Model Error (RMSE): {rmse}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}