{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":96164,"databundleVersionId":12993472},{"sourceType":"datasetVersion","sourceId":12502193,"datasetId":7890532,"databundleVersionId":13081681}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Core libraries\nimport numpy as np\nimport pandas as pd\nimport warnings\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import StandardScaler\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\n\n# Configuration\nDATA_PATHS = {\n    'train': \"/kaggle/input/drw-crypto-market-prediction/train.parquet\",\n    'test': \"/kaggle/input/drw-crypto-market-prediction/test.parquet\",\n    'submission': \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\",\n    'best_time_training':\"/kaggle/input/best-time-training-unique-model-01/datetime_index_Q_model_uniquemodel.csv\"\n}\n\n# Memory optimization\ndef reduce_memory_usage(df):\n    \"\"\"Optimize DataFrame memory usage by downcasting numeric columns.\"\"\"\n    start_mem = df.memory_usage().sum() / 1024**3\n    print(f\"Starting memory: {start_mem:.2f} GB\")\n    \n    for col in df.select_dtypes(include=['float']).columns:\n        col_min = df[col].min()\n        col_max = df[col].max()\n        \n        if col_min > np.finfo(np.float16).min and col_max < np.finfo(np.float16).max:\n            df[col] = df[col].astype(np.float16)\n        elif col_min > np.finfo(np.float32).min and col_max < np.finfo(np.float32).max:\n            df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**3\n    reduction = 100 * (start_mem - end_mem) / start_mem\n    print(f\"Memory reduced to: {end_mem:.2f} GB ({reduction:.1f}% reduction)\")\n    return df\n\n# Feature engineering\ndef add_features(df):\n    \"\"\"Create additional features from the raw data.\"\"\"\n    # Original features\n    df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n    df['log_volume'] = np.log1p(df['volume'])\n\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-10)\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-10)\n    \n    # === NEW MICROSTRUCTURE FEATURES ===\n    \n    # Price Pressure Indicators\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n    df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n    \n    # Liquidity Depth Measures\n    df['total_depth'] = df['bid_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['log_depth'] = np.log1p(df['total_depth'])\n    \n    # Order Flow Toxicity Proxies\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity'] = np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    \n    # Market Activity Indicators\n    df['volume_depth_ratio'] = df['volume'] / (df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['bid_qty'])\n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    # Microstructure Volatility Proxies\n    df['realized_spread_proxy'] = 2 * np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['quote_volatility_proxy'] = np.abs(df['depth_imbalance'])\n    \n    # Complex Interaction Terms\n    df['flow_depth_interaction'] = df['net_order_flow'] * df['total_depth']\n    df['imbalance_volume_interaction'] = df['order_flow_imbalance'] * df['volume']\n    df['depth_volume_interaction'] = df['total_depth'] * df['volume']\n    df['buy_sell_spread'] = np.abs(df['buy_qty'] - df['sell_qty'])\n    df['bid_ask_spread'] = np.abs(df['bid_qty'] - df['ask_qty'])\n    \n    # Information Asymmetry Measures\n    df['trade_informativeness'] = df['net_order_flow'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['execution_shortfall_proxy'] = df['buy_sell_spread'] / (df['volume'] + 1e-10)\n    df['adverse_selection_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10) * df['volume']\n    \n    # Market Efficiency Indicators\n    df['fill_probability'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['execution_rate'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_efficiency'] = df['volume'] / (df['bid_ask_spread'] + 1e-10)\n    \n    # Non-linear Transformations\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['sqrt_depth'] = np.sqrt(df['total_depth'])\n    df['volume_squared'] = df['volume'] ** 2\n    df['imbalance_squared'] = df['order_flow_imbalance'] ** 2\n    \n    # Relative Measures\n    df['bid_ratio'] = df['bid_qty'] / (df['total_depth'] + 1e-10)\n    df['ask_ratio'] = df['ask_qty'] / (df['total_depth'] + 1e-10)\n    df['buy_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['sell_ratio'] = df['sell_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    # Market Stress Indicators\n    df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_stress'] = df['volume'] / (df['total_depth'] + 1e-10) * np.abs(df['order_flow_imbalance'])\n    df['depth_depletion'] = df['volume'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    \n    # Directional Indicators\n    df['net_buying_ratio'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['directional_volume'] = df['net_order_flow'] * np.log1p(df['volume'])\n    df['signed_volume'] = np.sign(df['net_order_flow']) * df['volume']\n    # Replace infinities and NaNs\n    return df.replace([np.inf, -np.inf], 0).fillna(0)\n\ndef create_aggregated_features(df, feature_list, prefix):\n    \"\"\"Create aggregated statistics for a list of features.\"\"\"\n    df[f'{prefix}_sum'] = df[feature_list].sum(axis=1)\n    df[f'{prefix}_mean'] = df[feature_list].mean(axis=1)\n    df[f'{prefix}_median'] = df[feature_list].median(axis=1)\n    df[f'{prefix}_max'] = df[feature_list].max(axis=1)\n    df[f'{prefix}_min'] = df[feature_list].min(axis=1)\n    df[f'{prefix}_std'] = df[feature_list].std(axis=1)\n    return df\n\ndef extract_time_features(df):\n    \"\"\"Extract time-based features from timestamp.\"\"\"\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n    df['hour'] = df['timestamp'].dt.hour\n    df['minute'] = df['timestamp'].dt.minute\n    df['dayofweek'] = df['timestamp'].dt.dayofweek\n    df['Y_M_D_H'] = df['timestamp'].dt.strftime('%Y-%m-%d-%H')\n    df['Y_M_D_H_M'] = df['timestamp'].dt.strftime('%Y-%m-%d-%H-%M')\n    return df\n\n# Main pipeline\ndef main():\n    # Load data\n    train_df = pd.read_parquet(DATA_PATHS['train'])\n    test_df = pd.read_parquet(DATA_PATHS['test'])\n    sample_submission = pd.read_csv(DATA_PATHS['submission'])\n    best_timeseries = pd.read_csv(DATA_PATHS['best_time_training'])\n    # Memory optimization\n    train_df = reduce_memory_usage(train_df)\n    test_df = reduce_memory_usage(test_df)\n    \n    # Feature engineering\n    train_df = add_features(train_df)\n    test_df = add_features(test_df)\n    \n    # Define feature lists (truncated for brevity - use your full lists)\n    Negative_features_list = ['X563', 'X560', 'X486', 'X215', 'X7', 'X290', 'X581', 'X492',\n       'X569', 'X590', 'X201', 'X625', 'X247', 'X283', 'X261', 'X626',\n       'X193', 'X545', 'X195', 'X291', 'X703', 'X243', 'X297', 'X38',\n       'X490', 'X252', 'X42', 'X41', 'X254', 'X706', 'X237', 'X40', 'X16',\n       'X331', 'X47', 'X571', 'X302', 'X711', 'X280',\n       'activity_intensity', 'X412', 'X580', 'X54', 'X679', 'X245',\n       'X700', 'X622', 'X646', 'X223', 'X608', 'X695', 'X48', 'X319',\n       'X621', 'X125', 'X539', 'X259', 'X651', 'X648', 'X278', 'X682',\n       'X441', 'X5', 'X378', 'X565', 'X343', 'X654', 'X629', 'X14',\n       'X448', 'ask_qty', 'bid_qty', 'X95', 'X127', 'X463', 'X698',\n       'X482', 'X123', 'X46', 'X488', 'X572', 'X521', 'X30', 'X562',\n       'X421', 'X578', 'X380', 'X303', 'X171', 'X731', 'X452', 'X690',\n       'X221', 'X226', 'X456', 'X32', 'X643', 'X115', 'X676', 'X217',\n       'X602', 'X329', 'X765', 'X507', 'X722', 'X708', 'X179', 'X636',\n       'X210', 'X670', 'X767', 'X416', 'X218', 'X317', 'X204', 'X461',\n       'X250', 'X208', 'X159', 'X385', 'X198', 'X450', 'X28', 'X457',\n       'X400', 'X373', 'X272', 'X77', 'X154', 'X777', 'X256', 'X203',\n       'X20', 'X326', 'X407', 'X476', 'X274', 'X424', 'X33', 'X267',\n       'X443', 'X499', 'X113', 'X547', 'X382', 'X554', 'X583', 'X76',\n       'X639', 'X81']\n    Positive_features_list = ['X557', 'X566', 'X485', 'X194', 'X575', 'X584', 'X587', 'X493',\n       'X216', 'X44', 'X253', 'X627', 'X50', 'X707', 'X36', 'X6', 'X287',\n       'X262', 'X551', 'X244', 'X8', 'X286', 'X214', 'X236', 'X624',\n       'X623', 'X483', 'X337', 'X15', 'X240', 'X200', 'X289', 'X39',\n       'X699', 'X533', 'X222', 'X260', 'X325', 'X647', 'X299', 'X285',\n       'X650', 'X577', 'X284', 'X559', 'X43', 'X202', 'X702', 'X251',\n       'X704', 'X246', 'X418', 'X710', 'X683', 'X55', 'X620', 'X694',\n       'X298', 'X31', 'X292', 'X574', 'X239', 'X605', 'X288', 'X494',\n       'X121', 'X449', 'X462', 'X89', 'X696', 'X489', 'total_depth',\n       'X678', 'X165', 'X628', 'X406', 'X675', 'X568', 'X294', 'X384',\n       'X131', 'X481', 'X374', 'X119', 'X56', 'X133', 'X442', 'X277',\n       'X422', 'X655', 'X9', 'X451', 'X455', 'X209', 'X440', 'X652',\n       'X379', 'X323', 'X279', 'X611', 'X735', 'X766', 'X527', 'X205',\n       'X129', 'X644', 'X21', 'X224', 'X196', 'X642', 'X313', 'X293',\n       'X427', 'X258', 'X586', 'X225', 'X726', 'X691', 'X219', 'X335',\n       'X4', 'X635', 'X506', 'X415', 'X173', 'X458', 'X117', 'X372',\n       'X197', 'X27', 'X281', 'X672', 'X275', 'X500', 'X775', 'X469',\n       'X680', 'X599', 'X35', 'X29', 'X477', 'X401', 'X330', 'X160',\n       'X370', 'X266', 'X408', 'X640', 'X87', 'X70']\n    \n    # Create aggregated features\n    for i in [5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150]:\n        train_df = create_aggregated_features(train_df, Negative_features_list[:i], f\"Negative_features_{i}\")\n        test_df = create_aggregated_features(test_df, Negative_features_list[:i], f\"Negative_features_{i}\")\n        train_df = create_aggregated_features(train_df, Positive_features_list[:i], f\"Positive_features_{i}\")\n        test_df = create_aggregated_features(test_df, Positive_features_list[:i], f\"Positive_features_{i}\")\n    \n    # Time features\n    train_df = train_df.reset_index().rename(columns={'index': 'timestamp'})\n    train_df = extract_time_features(train_df)\n\n    # Define selected features (truncated - use your full list)\n    Features = ['X455', 'X557', 'X6', 'X259', 'X627', 'X450', 'X48', 'X44', 'X448', 'X291', 'X563', 'X218', 'X574', 'X695', 'X41', 'X418', 'X644', 'X621', 'X648', 'X451', 'X443', 'X643', 'Negative_features_10_sum', 'X722', 'X290', 'X200', 'Positive_features_10_sum', 'X458', 'X286', 'X571', 'X640', 'X559', 'X625', 'X222', 'X201', 'X214', 'X193', 'X639', 'X197', 'X31', 'X323', 'X456', 'X319', 'X278', 'X260', 'X251', 'X651', 'X5', 'X652', 'X285', 'X584', 'X317', 'X412', 'X647', 'Positive_features_100_std', 'X42', 'X440', 'X225', 'X575', 'X203', 'X683', 'X289', 'X204', 'X35', 'X562', 'X566', 'X605', 'X33', 'X293', 'Negative_features_5_sum', 'X219', 'X45', 'X587', 'X36', 'X16', 'X449', 'X569', 'X287', 'X246', 'X56', 'X726', 'X8', 'X379', 'X292', 'X679', 'X507', 'X406', 'Negative_features_150_std', 'X165', 'X329', 'X376', 'X15', 'X608', 'X626', 'X678', 'X628', 'X696', 'X457', 'X539', 'X469', 'X442', 'Positive_features_5_sum', 'Positive_features_60_std', 'Positive_features_90_std', 'X580', 'X694', 'X491', 'X258', 'X454', 'X46', 'X196', 'X123', 'X765', 'X370', 'X215', 'Negative_features_100_std', 'X331', 'X125', 'X119', 'X254', 'X279', 'X4', 'X298', 'X551', 'X768', 'X675', 'X226', 'X223', 'X43', 'X568', 'X444', 'X337', 'X320', 'Negative_features_60_std', 'X335', 'X581', 'X623', 'X294', 'X325', 'X572', 'X554', 'X416', 'X486', 'X602', 'X484', 'X577', 'X735', 'X288', 'X250', 'X691', 'X776', 'X766', 'X113', 'X611', 'X635', 'X666', 'X401', 'X489', 'X326', 'X699', 'X718', 'X636', 'X767', 'X476', 'Negative_features_70_std', 'X154', 'X221', 'X706', 'X506', 'X470', 'Positive_features_30_sum', 'X34', 'X21', 'X284', 'X38', 'X245', 'X700', 'X620', 'X415', 'X407', 'Positive_features_90_median', 'X32', 'X244', 'X670', 'X95', 'X343', 'X299', 'X129', 'X421', 'X482', 'X14', 'X668', 'X704', 'X9', 'X216', 'X77', 'X247', 'X155', 'Negative_features_90_std', 'Negative_features_50_sum', 'X477', 'X642', 'X465', 'X160', 'X373', 'X629', 'X777', 'X609', 'X724', 'X422', 'X672', 'X441', 'X302', 'X646', 'X527', 'X707', 'X71', 'X39', 'X194', 'X728', 'X410', 'X682', 'Positive_features_70_std', 'X49', 'X127', 'X198', 'X229', 'X579', 'X599', 'X622', 'X601', 'X461', 'fill_probability', 'X121', 'Positive_features_20_std', 'X676', 'Negative_features_30_sum', 'X573', 'X720', 'X50', 'X209', 'X690', 'X173', 'X89', 'X313', 'X439', 'X275', 'X171', 'X20', 'X30', 'X708', 'X237', 'X710', 'X217', 'X488', 'Negative_features_50_std', 'X133', 'X404', 'X281', 'X427', 'X28', 'X70', 'X650', 'X311', 'X731', 'X764', 'X385', 'X274', 'X51', 'Negative_features_150_median', 'X205', 'X283', 'X266', 'X408', 'X424', 'X610', 'X277', 'X261', 'X362', 'X230', 'X256', 'X280', 'X624', 'X447', 'X303', 'X175', 'X485', 'X462', 'X262', 'X115', 'Positive_features_50_sum', 'X117', 'Negative_features_30_std', 'X775', 'X565', 'X27', 'Negative_features_80_std', 'X377', 'X400', 'X272', 'X698', 'X578', 'Negative_features_60_sum', 'X159', 'X664', 'X269', 'X87', 'X680', 'X179', 'X508', 'X481', 'X499', 'X114', 'X545', 'X604', 'Positive_features_40_median', 'Positive_features_50_std', 'X267', 'X253', 'X459', 'X118', 'X547', 'X76', 'X464', 'Positive_features_150_std', 'X479', 'X295', 'X161', 'X192', 'X7', 'X316', 'Positive_features_90_max', 'X23', 'X548', 'X494', 'X500', 'X174', 'X521', 'X533', 'X655', 'X236', 'X24', 'ask_ratio', 'X382', 'X492', 'X380', 'X368', 'X29', 'X242', 'X148', 'X463', 'X238', 'X662', 'X598', 'X149', 'X124', 'X81', 'Negative_features_70_sum', 'X606', 'Negative_features_40_sum', 'X243', 'X723', 'X169', 'X314', 'X157', 'X84', 'X692', 'Negative_features_5_mean', 'X402', 'X328', 'X612', 'X613', 'X252', 'X769', 'Positive_features_5_mean', 'X210', 'X322', 'X235', 'X334', 'X271', 'X656', 'X409', 'Positive_features_5_std', 'X395', 'X255', 'X213', 'X107', 'X498', 'X446', 'X338', 'X530', 'X600', 'X112', 'X151', 'X583', 'X228', 'X78', 'X384', 'X372', 'X139', 'Negative_features_80_sum', 'X371', 'sell_ratio', 'X211', 'X120', 'X341', 'net_order_flow', 'X550', 'X90', 'Positive_features_40_std', 'X300', 'X667', 'X344', 'X40', 'X65', 'X497', 'X413', 'X490', 'X483', 'Negative_features_40_median', 'X212', 'X398', 'X73', 'X231', 'buy_qty', 'X734', 'Positive_features_30_std', 'Positive_features_80_max', 'X475', 'X163', 'X386', 'X542', 'X472', 'Negative_features_90_sum', 'X619', 'X282', 'activity_intensity', 'X576', 'X336', 'X241', 'X560', 'X375', 'X417', 'X333', 'X330', 'Positive_features_50_median', 'X590', 'Negative_features_10_mean', 'X240', 'Negative_features_40_std']\n    \n    \n    # Filter training data by best times (truncated - use your full list)\n    best_time_training = pd.to_datetime(best_timeseries.values.ravel()).strftime('%Y-%m-%d-%H-%M')\n\n    train_clean = train_df.loc[train_df[\"Y_M_D_H_M\"].isin(best_time_training)]\n    \n    # Prepare data\n    X_train = train_clean[Features].values\n    y_train = train_clean[\"label\"]\n    X_test = test_df[Features].values\n    \n    # Train model and predict\n    model = LinearRegression()\n    model.fit(X_train, y_train)\n    test_df[\"prediction\"] = model.predict(X_test)\n    \n    # Create submission\n    sample_submission[\"prediction\"] = test_df[\"prediction\"].values\n    sample_submission.to_csv('submission.csv', index=False)\n    print(\"✅ Submission saved to submission.csv\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-31T16:34:12.469451Z","iopub.execute_input":"2025-07-31T16:34:12.469849Z","iopub.status.idle":"2025-07-31T16:42:52.076352Z","shell.execute_reply.started":"2025-07-31T16:34:12.469826Z","shell.execute_reply":"2025-07-31T16:42:52.075185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}