{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":11951556,"sourceType":"datasetVersion","datasetId":1346},{"sourceId":11952364,"sourceType":"datasetVersion","datasetId":3010373}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Description**\n\nA list of visualizations to better understand features. **work in progress**\n\n**Observations**\n\n* target variable `label` closesly resembles BTC minute close price change (refer to **section 2.**).\n* cumulative sum charts of hidden features indicate that features contain info about other crypto prices on different exchanges (see **section 6**). I could not identify which cryto currency those are (see **section 6.2**).\n* No NULL values among proprietary features.\n* there are 30 feature pairs, i.e. 31 features, which absolute correlation of 1.\n* there are 27 features with single unique value.\n\n**Changelog**\n\n* **v1** loaded data and created  basic visualization.\n* **v2** added correlation scater plots.\n* **v3** added BTC historical price trend charts.\n* **v4** expanded correlation analysis of proprietary features.\n* **v5** added correlation charts with target.\n* **v6** added analysis how many features have single unique value.\n* **v7** added reduce memory function and trend lines for proprietery features.\n* **v8** tried matching X363, X405 and X321 features with random cryto currencies by adding trend analysis (6.2 section).","metadata":{}},{"cell_type":"code","source":"# data processing libraries\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom datetime import datetime\nimport os\n\n# for monitoring progress\nfrom tqdm import tqdm\n\nimport seaborn as sns # plots for statistical analysis\nimport matplotlib.pyplot as plt # for data visualization\n\n# define default colors for plots in notebook\nfrom matplotlib import cycler\nfrom matplotlib.colors import LinearSegmentedColormap\ncolors = [\"#068D9D\", \"#53599A\", \"#607BB0\", \"#6D9DC5\", \"#77BECF\", \"#80DED9\", \"#AEECEF\"]\n\nplt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=True, prop_cycle=cycler('color', colors))\n\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:02:01.476341Z","iopub.execute_input":"2025-05-26T15:02:01.476609Z","iopub.status.idle":"2025-05-26T15:02:06.553921Z","shell.execute_reply.started":"2025-05-26T15:02:01.476588Z","shell.execute_reply":"2025-05-26T15:02:06.552940Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Load and format data","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(dataframe, dataset):\n    \"\"\"\n    Function taken from: https://www.kaggle.com/code/ravaghi/drw-crypto-market-prediction-ensemble\n    \"\"\"\n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:02:06.555607Z","iopub.execute_input":"2025-05-26T15:02:06.555995Z","iopub.status.idle":"2025-05-26T15:02:06.566505Z","shell.execute_reply.started":"2025-05-26T15:02:06.555972Z","shell.execute_reply":"2025-05-26T15:02:06.565513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf_train = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ndf_test = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\ndf_train = reduce_mem_usage(df_train, \"train\")\ndf_test = reduce_mem_usage(df_test, \"test\")\n\ndf_train = df_train.reset_index()\n\nproprietary_features = [col for col in df_train.columns if col.startswith('X')]\nprint(f\"There are {len(proprietary_features)} anonymized market proprietary features.\")\n\nbasic_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\nprint(f\"There are {len(basic_features)} basic features.\\n\")\n\nall_features = proprietary_features + basic_features\ntarget = 'label'\n\n# convert from pandas to polars\ndf_train = pl.from_pandas(df_train)\ndf_test = pl.from_pandas(df_test)\n\nprint(f\"Train dataset contains {df_train.shape[0]} rows and {df_train.shape[1]} columns.\" )\nprint(f\"Test dataset contains {df_test.shape[0]} rows and {df_test.shape[1]} columns.\" )","metadata":{"execution":{"iopub.status.busy":"2025-05-26T15:02:06.567495Z","iopub.execute_input":"2025-05-26T15:02:06.567788Z","iopub.status.idle":"2025-05-26T15:03:32.679791Z","shell.execute_reply.started":"2025-05-26T15:02:06.567767Z","shell.execute_reply":"2025-05-26T15:03:32.678669Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Target\n\n## 2.1. train dataset trend line","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 1, figsize=(18, 6), sharex=True)\n\nax[0].plot(df_train[\"timestamp\"], df_train[target])\nax[0].set_ylabel(target)\n\nax[0].set_title(\"Train dataset\")\n\nax[1].plot(df_train[\"timestamp\"], np.cumsum(df_train[target]), color=colors[1])\nax[1].set_xlabel(\"timestamp\")\nax[1].set_ylabel(f\"{target} cumulative sum\")\n\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:03:32.681436Z","iopub.execute_input":"2025-05-26T15:03:32.681798Z","iopub.status.idle":"2025-05-26T15:03:33.843744Z","shell.execute_reply.started":"2025-05-26T15:03:32.681771Z","shell.execute_reply":"2025-05-26T15:03:33.842614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 BTC historical price","metadata":{}},{"cell_type":"code","source":"%%time\n# import BTC historical data \nbtc_data = pd.read_csv('../input/bitcoin-historical-data/btcusd_1-min_data.csv')\nbtc_data['Timestamp'] = [datetime.fromtimestamp(x) for x in btc_data['Timestamp']]\n\n# filter out range\nbtc_data = btc_data.loc[btc_data['Timestamp'] >= \"2023-03-01\"]\nbtc_data = btc_data.loc[btc_data['Timestamp'] < \"2024-03-01\"].reset_index(drop=True)\n\nbtc_data[\"close_norm\"] = btc_data[\"Close\"] - btc_data[\"Close\"].values[0]\nbtc_data[\"close_change\"] = btc_data[\"Close\"].pct_change()\nbtc_data[\"close_change\"] = btc_data[\"close_change\"].fillna(0)\nbtc_data[\"close_change\"] *= 1000\nbtc_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:03:33.844852Z","iopub.execute_input":"2025-05-26T15:03:33.845653Z","iopub.status.idle":"2025-05-26T15:04:01.045838Z","shell.execute_reply.started":"2025-05-26T15:03:33.845607Z","shell.execute_reply":"2025-05-26T15:04:01.044877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 1, figsize=(18, 6), sharex=True)\n\nax[0].plot(btc_data[\"Timestamp\"], btc_data[\"close_change\"])\nax[0].set_ylabel(\"pct change of close price\")\n\nax[0].set_title(\"BTC historical price\")\n\nax[1].plot(btc_data[\"Timestamp\"], btc_data[\"close_norm\"], color=colors[1])\nax[1].set_xlabel(\"timestamp\")\nax[1].set_ylabel(\"norm. close price\")\n\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:01.046891Z","iopub.execute_input":"2025-05-26T15:04:01.047185Z","iopub.status.idle":"2025-05-26T15:04:02.173334Z","shell.execute_reply.started":"2025-05-26T15:04:01.047156Z","shell.execute_reply":"2025-05-26T15:04:02.172290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# compare charts\nfig, ax = plt.subplots(2, 2, figsize=(18, 6), sharex=True)\nax = ax.flatten()\n\nax[0].set_title(\"Train dataset\")\nax[0].plot(df_train[\"timestamp\"], np.cumsum(df_train[target]), color=colors[0])\nax[0].set_xlabel(\"timestamp\")\nax[0].set_ylabel(f\"{target} cumulative sum\")\n\nax[1].set_title(\"BTC historical price\")\nax[1].plot(btc_data[\"Timestamp\"], btc_data[\"close_norm\"], color=colors[1])\nax[1].set_xlabel(\"timestamp\")\nax[1].set_ylabel(\"norm. close price\")\n\nax[2].plot(df_train[\"timestamp\"], df_train[target])\nax[2].set_ylabel(target)\n\nax[3].plot(btc_data[\"Timestamp\"], btc_data[\"close_change\"], color=colors[1])\nax[3].set_ylabel(\"pct change of close price\")\n\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:02.177063Z","iopub.execute_input":"2025-05-26T15:04:02.177368Z","iopub.status.idle":"2025-05-26T15:04:04.031882Z","shell.execute_reply.started":"2025-05-26T15:04:02.177345Z","shell.execute_reply":"2025-05-26T15:04:04.030657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.3. distributions","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(18, 6))\nax = ax.flatten()\n\nsns.boxplot(x=df_train[target], ax=ax[0])\n\nsns.boxplot(x=btc_data[\"close_change\"], ax=ax[1], color=colors[1])\n\nsns.histplot(data=df_train, x=target, kde=True, ax=ax[2])\nsns.histplot(data=btc_data, x=\"close_change\", kde=True, ax=ax[3])\n\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:04.032935Z","iopub.execute_input":"2025-05-26T15:04:04.033181Z","iopub.status.idle":"2025-05-26T15:04:21.965596Z","shell.execute_reply.started":"2025-05-26T15:04:04.033162Z","shell.execute_reply":"2025-05-26T15:04:21.964654Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. desriptive stats\n\n## 3.1. missing data","metadata":{}},{"cell_type":"code","source":"# Get count of null values per column\nnull_counts  = df_train.select(proprietary_features).null_count()\n\n# Filter to show only columns with missing values\ncolumns_with_nulls = null_counts.select([\n    col for col in null_counts.columns \n    if null_counts[col].item() > 0\n])\n\nif columns_with_nulls.is_empty():\n    print(\"There are no columns with NA values.\")\nelse:\n    print(columns_with_nulls)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:21.966602Z","iopub.execute_input":"2025-05-26T15:04:21.966940Z","iopub.status.idle":"2025-05-26T15:04:22.031723Z","shell.execute_reply.started":"2025-05-26T15:04:21.966913Z","shell.execute_reply":"2025-05-26T15:04:22.030482Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2. single unique value","metadata":{}},{"cell_type":"code","source":"%%time\nsingle_unique_value = []\nprint(f\"feature | unique value count\")\nfor col in all_features:\n    _cnt = df_train[col].n_unique()\n    if _cnt < 10:\n        single_unique_value.append(col)\n        print(f\"{col} | {_cnt}\")\n\nprint(f\"There are {len(single_unique_value)} features with single unique value.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:22.032842Z","iopub.execute_input":"2025-05-26T15:04:22.033205Z","iopub.status.idle":"2025-05-26T15:04:29.251366Z","shell.execute_reply.started":"2025-05-26T15:04:22.033173Z","shell.execute_reply":"2025-05-26T15:04:29.250281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. correlation analysis between features","metadata":{}},{"cell_type":"code","source":"%%time\ncorr_data = df_train.select(all_features).corr()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:29.252687Z","iopub.execute_input":"2025-05-26T15:04:29.253045Z","iopub.status.idle":"2025-05-26T15:04:45.412536Z","shell.execute_reply.started":"2025-05-26T15:04:29.253016Z","shell.execute_reply":"2025-05-26T15:04:45.411432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ncolumns = corr_data.columns\ncorr_long = []\n\nfor i, col1 in enumerate(columns):\n    for j, col2 in enumerate(columns):\n        # Only upper triangle to avoid duplicates\n        if i < j:  \n            corr_value = corr_data[col1][j]\n            corr_long.append({\n                'feature_1': col1,\n                'feature_2': col2,\n                'correlation': corr_value,\n                'abs_correlation': abs(corr_value)\n            })\n\ncorr_long = pd.DataFrame(corr_long).dropna().sort_values(\"correlation\").reset_index(drop=True)\ncorr_long['abs_correlation_rounded'] = corr_long['abs_correlation'].round(2)\ncorr_long.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:45.413512Z","iopub.execute_input":"2025-05-26T15:04:45.413835Z","iopub.status.idle":"2025-05-26T15:04:46.817282Z","shell.execute_reply.started":"2025-05-26T15:04:45.413800Z","shell.execute_reply":"2025-05-26T15:04:46.816512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_long.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:46.818185Z","iopub.execute_input":"2025-05-26T15:04:46.818528Z","iopub.status.idle":"2025-05-26T15:04:46.829963Z","shell.execute_reply.started":"2025-05-26T15:04:46.818497Z","shell.execute_reply":"2025-05-26T15:04:46.828934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_no_prairs = len(corr_long)\n_df = corr_long.groupby(\"abs_correlation_rounded\")['correlation'].count().tail(11).reset_index()\n_df.columns = [\"abs_correlation_rounded\", \"feature_pair_cnt\"]\n_df['share, %'] = (_df['feature_pair_cnt'] / _no_prairs * 100).round(3)\n_df['cum. share, %'] = _df['share, %'].cumsum()\n_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:46.830779Z","iopub.execute_input":"2025-05-26T15:04:46.831025Z","iopub.status.idle":"2025-05-26T15:04:46.869424Z","shell.execute_reply.started":"2025-05-26T15:04:46.831008Z","shell.execute_reply":"2025-05-26T15:04:46.868510Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.1. perfect correlation feature pairs","metadata":{}},{"cell_type":"code","source":"_df = corr_long[corr_long[\"abs_correlation\"] == 1]\nprint(f\"{_df.shape[0]} feature pairs have perfect correlation.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:46.870331Z","iopub.execute_input":"2025-05-26T15:04:46.870619Z","iopub.status.idle":"2025-05-26T15:04:46.877438Z","shell.execute_reply.started":"2025-05-26T15:04:46.870599Z","shell.execute_reply":"2025-05-26T15:04:46.876474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(6, 5, figsize=(18, 18))\nax = ax.flatten()\n\nfor i, idx in enumerate(_df.index):\n    _col_x = _df.loc[idx, \"feature_1\"]\n    _col_y = _df.loc[idx, \"feature_2\"]\n    _corr = _df.loc[idx, \"correlation\"]\n\n    _df_plot = df_train.select([_col_x, _col_y]).sample(1_000)\n\n    ax[i].scatter(_df_plot.select(_col_x), _df_plot.select(_col_y))\n\n    ax[i].set_title(f\"Corr: {_corr:.3f}\")\n    ax[i].set_xlabel(_col_x)\n    ax[i].set_ylabel(_col_x)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:46.878402Z","iopub.execute_input":"2025-05-26T15:04:46.878822Z","iopub.status.idle":"2025-05-26T15:04:52.783707Z","shell.execute_reply.started":"2025-05-26T15:04:46.878791Z","shell.execute_reply":"2025-05-26T15:04:52.782483Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.2. most positive correlated feature pairs\n\n * exclude perfect correlation features;\n * plot random sample of 1_000 data points.","metadata":{}},{"cell_type":"code","source":"# Scater plot of TOP 20 most positive correlated features\n\n# select feature pairs\n_df = corr_long[corr_long[\"abs_correlation\"].round(2) < 1]\n_df = _df.tail(20)\n\nfig, ax = plt.subplots(5, 4, figsize=(18, 16))\nax = ax.flatten()\n\nfor i, idx in enumerate(_df.index):\n    _col_x = _df.loc[idx, \"feature_1\"]\n    _col_y = _df.loc[idx, \"feature_2\"]\n    _corr = _df.loc[idx, \"correlation\"]\n\n    _df_plot = df_train.select([_col_x, _col_y]).sample(1_000)\n\n    ax[i].scatter(_df_plot.select(_col_x), _df_plot.select(_col_y))\n\n    ax[i].set_title(f\"Corr: {_corr:.3f}\")\n    ax[i].set_xlabel(_col_x)\n    ax[i].set_ylabel(_col_x)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:52.784765Z","iopub.execute_input":"2025-05-26T15:04:52.785034Z","iopub.status.idle":"2025-05-26T15:04:56.949966Z","shell.execute_reply.started":"2025-05-26T15:04:52.785013Z","shell.execute_reply":"2025-05-26T15:04:56.948911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.3. most negative correlated feature pairs\n\n * exclude perfect correlation features;\n * plot random samples of 1_000 data points.","metadata":{}},{"cell_type":"code","source":"# Scater plot of TOP 20 most negative correlated features\n\n# select feature pairs\n_df = corr_long[corr_long[\"abs_correlation\"].round(2) < 1]\n_df = _df.head(20)\n\nfig, ax = plt.subplots(5, 4, figsize=(18, 16))\nax = ax.flatten()\n\nfor i, idx in enumerate(_df.index):\n    _col_x = _df.loc[idx, \"feature_1\"]\n    _col_y = _df.loc[idx, \"feature_2\"]\n    _corr = _df.loc[idx, \"correlation\"]\n\n    _df_plot = df_train.select([_col_x, _col_y]).sample(1_000)\n\n    ax[i].scatter(_df_plot.select(_col_x), _df_plot.select(_col_y))\n\n    ax[i].set_title(f\"Corr: {_corr:.3f}\")\n    ax[i].set_xlabel(_col_x)\n    ax[i].set_ylabel(_col_x)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:04:56.951154Z","iopub.execute_input":"2025-05-26T15:04:56.951462Z","iopub.status.idle":"2025-05-26T15:05:01.106761Z","shell.execute_reply.started":"2025-05-26T15:04:56.951439Z","shell.execute_reply":"2025-05-26T15:05:01.105428Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.4. most un-correlated feature pairs\n\n * plot random samples of 1_000 data points.","metadata":{}},{"cell_type":"code","source":"# select feature pairs\n_df = corr_long[corr_long[\"abs_correlation\"].round(2) < 0.05]\n_df = _df.sample(20)\n\nfig, ax = plt.subplots(5, 4, figsize=(18, 16))\nax = ax.flatten()\n\nfor i, idx in enumerate(_df.index):\n    _col_x = _df.loc[idx, \"feature_1\"]\n    _col_y = _df.loc[idx, \"feature_2\"]\n    _corr = _df.loc[idx, \"correlation\"]\n\n    _df_plot = df_train.select([_col_x, _col_y]).sample(1_000)\n\n    ax[i].scatter(_df_plot.select(_col_x), _df_plot.select(_col_y))\n\n    ax[i].set_title(f\"Corr: {_corr:.3f}\")\n    ax[i].set_xlabel(_col_x)\n    ax[i].set_ylabel(_col_x)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:01.107784Z","iopub.execute_input":"2025-05-26T15:05:01.108071Z","iopub.status.idle":"2025-05-26T15:05:06.550889Z","shell.execute_reply.started":"2025-05-26T15:05:01.108049Z","shell.execute_reply":"2025-05-26T15:05:06.549989Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. correlation analysis on target","metadata":{}},{"cell_type":"code","source":"%%time\ncorr_data = df_train.select(all_features + [target]).corr()\ndf_target_corr = pd.DataFrame({\"feature\": corr_data.columns, \"corr\": np.array(corr_data.select(\"label\")).reshape(-1)})\ndf_target_corr = df_target_corr.dropna()\ndf_target_corr = df_target_corr.loc[df_target_corr[\"feature\"] != \"label\"].reset_index(drop=True)\ndf_target_corr.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:06.552486Z","iopub.execute_input":"2025-05-26T15:05:06.552778Z","iopub.status.idle":"2025-05-26T15:05:18.258327Z","shell.execute_reply.started":"2025-05-26T15:05:06.552757Z","shell.execute_reply":"2025-05-26T15:05:18.257253Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.1. most positive correlated features\n\n * plot random sample of 1_000 data points.","metadata":{}},{"cell_type":"code","source":"# select feature pairs\n_df = df_target_corr.sort_values(\"corr\")\n_df = _df.tail(20)\n_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:18.259530Z","iopub.execute_input":"2025-05-26T15:05:18.259869Z","iopub.status.idle":"2025-05-26T15:05:18.273579Z","shell.execute_reply.started":"2025-05-26T15:05:18.259839Z","shell.execute_reply":"2025-05-26T15:05:18.272323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 4, figsize=(18, 16))\nax = ax.flatten()\n\n_df_sample = df_train.sample(1_000)\n\nfor i, idx in enumerate(_df.index):\n    _feat = _df.loc[idx, \"feature\"]\n    _corr = _df.loc[idx, \"corr\"]\n\n    ax[i].scatter(_df_sample.select(_feat), _df_sample.select(target))\n\n    ax[i].set_title(f\"Corr: {_corr:.3f}\")\n    ax[i].set_xlabel(_feat)\n    ax[i].set_ylabel(target)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:18.278001Z","iopub.execute_input":"2025-05-26T15:05:18.278280Z","iopub.status.idle":"2025-05-26T15:05:22.902029Z","shell.execute_reply.started":"2025-05-26T15:05:18.278260Z","shell.execute_reply":"2025-05-26T15:05:22.900979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.2. most negative correlated features\n\n * plot random sample of 1_000 data points.","metadata":{}},{"cell_type":"code","source":"# select feature pairs\n_df = df_target_corr.sort_values(\"corr\")\n_df = _df.head(20)\n_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:22.903634Z","iopub.execute_input":"2025-05-26T15:05:22.903976Z","iopub.status.idle":"2025-05-26T15:05:22.916821Z","shell.execute_reply.started":"2025-05-26T15:05:22.903950Z","shell.execute_reply":"2025-05-26T15:05:22.915878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 4, figsize=(18, 16))\nax = ax.flatten()\n\n_df_sample = df_train.sample(1_000)\n\nfor i, idx in enumerate(_df.index):\n    _feat = _df.loc[idx, \"feature\"]\n    _corr = _df.loc[idx, \"corr\"]\n\n    ax[i].scatter(_df_sample.select(_feat), _df_sample.select(target))\n\n    ax[i].set_title(f\"Corr: {_corr:.3f}\")\n    ax[i].set_xlabel(_feat)\n    ax[i].set_ylabel(target)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:22.917735Z","iopub.execute_input":"2025-05-26T15:05:22.917957Z","iopub.status.idle":"2025-05-26T15:05:27.146511Z","shell.execute_reply.started":"2025-05-26T15:05:22.917940Z","shell.execute_reply":"2025-05-26T15:05:27.145465Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. trendlines from unknown features\n\n## 6.1 TOP 20 features with most unique values\n\nRound values before searching for unique values count.","metadata":{}},{"cell_type":"code","source":"%%time\ndata = list()\n\nfor col in df_train.columns[6:]:\n    _cnt = df_train.select(pl.col(col).round(4).n_unique()).item()\n    data.append([col, _cnt])\n\ndf_unique_cnts = pd.DataFrame(data, columns = [\"Feature\", \"n_unique\"])\ndf_unique_cnts = df_unique_cnts.sort_values(\"n_unique\").reset_index(drop=True)\ndf_unique_cnts.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:27.148125Z","iopub.execute_input":"2025-05-26T15:05:27.148586Z","iopub.status.idle":"2025-05-26T15:05:35.722924Z","shell.execute_reply.started":"2025-05-26T15:05:27.148556Z","shell.execute_reply":"2025-05-26T15:05:35.722180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_features = df_unique_cnts[\"Feature\"][-10:]\n\nfig, ax = plt.subplots(5, 2, figsize=(18, 16), sharex=True)\nax = ax.flatten()\n\nfor i, _feat in enumerate(top_features):\n\n    ax[i].plot(df_train[\"timestamp\"], df_train[_feat])\n    ax[i].set_xlabel(\"Timestamp\")\n    ax[i].set_ylabel(_feat)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:35.723801Z","iopub.execute_input":"2025-05-26T15:05:35.724045Z","iopub.status.idle":"2025-05-26T15:05:40.573155Z","shell.execute_reply.started":"2025-05-26T15:05:35.724026Z","shell.execute_reply":"2025-05-26T15:05:40.572266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 2, figsize=(18, 16))\nax = ax.flatten()\n\nfor i, _feat in enumerate(top_features):\n\n    ax[i].plot(df_train[\"timestamp\"], df_train[_feat].cum_sum())\n    ax[i].set_xlabel(\"Timestamp\")\n    ax[i].set_ylabel(_feat)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:40.574304Z","iopub.execute_input":"2025-05-26T15:05:40.574603Z","iopub.status.idle":"2025-05-26T15:05:44.558610Z","shell.execute_reply.started":"2025-05-26T15:05:40.574582Z","shell.execute_reply":"2025-05-26T15:05:44.557690Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6.2 find most probable crypto tockens for features X363, X405 and X321.\n\n### 6.2.1. load historical data","metadata":{}},{"cell_type":"code","source":"folder_path = \"/kaggle/input/crypto-currencies-daily-prices\"\nfilenames = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]\n\ndf_crypto = pd.DataFrame()\n\nfor filename in tqdm(filenames):\n    _df = pd.read_csv(f\"{folder_path}/{filename}\")\n    _df['date'] = pd.to_datetime(_df['date'])\n\n    # filter out train date range\n    _df = _df.loc[_df['date'] >= \"2023-03-01\"]\n    _df = _df.loc[_df['date'] < \"2024-03-01\"].reset_index(drop=True)\n\n    df_crypto = pd.concat([df_crypto, _df]).reset_index(drop=True)\n\ndf_crypto.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:44.559890Z","iopub.execute_input":"2025-05-26T15:05:44.560158Z","iopub.status.idle":"2025-05-26T15:05:46.746640Z","shell.execute_reply.started":"2025-05-26T15:05:44.560139Z","shell.execute_reply":"2025-05-26T15:05:46.745446Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 6.2.1. plot timeseries\n\nselect random 20 tickers","metadata":{}},{"cell_type":"code","source":"_tickers = df_crypto[\"ticker\"].sample(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:46.747495Z","iopub.execute_input":"2025-05-26T15:05:46.747776Z","iopub.status.idle":"2025-05-26T15:05:46.756466Z","shell.execute_reply.started":"2025-05-26T15:05:46.747755Z","shell.execute_reply":"2025-05-26T15:05:46.755144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 4, figsize=(18, 16))\nax = ax.flatten()\n\nfor i, _ticker in enumerate(_tickers):\n\n    _df_plot = df_crypto.loc[df_crypto[\"ticker\"] == _ticker]\n\n    ax[i].plot(pd.to_datetime(_df_plot.date), _df_plot[\"close\"])\n    ax[i].set_xlabel(\"date\")\n    ax[i].set_ylabel(\"close\")\n    ax[i].set_title(_ticker)\n    \nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:46.758860Z","iopub.execute_input":"2025-05-26T15:05:46.759211Z","iopub.status.idle":"2025-05-26T15:05:52.259594Z","shell.execute_reply.started":"2025-05-26T15:05:46.759174Z","shell.execute_reply":"2025-05-26T15:05:52.258280Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 6.2.2. find most similiar timeseries","metadata":{}},{"cell_type":"code","source":"df_benckmark = df_train.select([\"timestamp\", \"X363\"])\ndf_benckmark = df_benckmark.to_pandas()\ndf_benckmark = df_benckmark.set_index('timestamp')\ndf_benckmark = df_benckmark.resample('D').agg({\n    'X363': ['min', 'max', 'first', 'last']})\ndf_benckmark = df_benckmark.reset_index()\ndf_benckmark.columns = ['timestamp', 'low', 'high', 'open', 'close']\ndf_benckmark['cum_sum'] = df_benckmark['close'].cumsum()\ndf_benckmark.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:52.260411Z","iopub.execute_input":"2025-05-26T15:05:52.260744Z","iopub.status.idle":"2025-05-26T15:05:52.341541Z","shell.execute_reply.started":"2025-05-26T15:05:52.260718Z","shell.execute_reply":"2025-05-26T15:05:52.340302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = list()\n\n_tickers = df_crypto[\"ticker\"].unique()\n\nfor i, _ticker in enumerate(_tickers):\n    _df = df_crypto.loc[df_crypto[\"ticker\"] == _ticker].copy()\n    \n    # join tables\n    _df = pd.merge(\n        df_benckmark[[\"timestamp\", \"cum_sum\"]],\n        _df[[\"date\", \"close\"]],\n        left_on='timestamp',\n        right_on='date',\n        how='left').dropna()\n    \n    # normalize\n    # _df['cum_sum'] /= _df['cum_sum'].values[0]\n    # _df['close'] /= _df['close'].values[0]\n    \n    _df['ratio'] = _df['cum_sum'] / _df['close']\n    _corr = _df[[\"cum_sum\", \"close\"]].corr()\n\n    data.append([_ticker, _df['ratio'].std(), _corr.values[0][1]])\n\ndf_ratios = pd.DataFrame(data, columns=[\"ticker\", 'ratio_std', 'corr']).sort_values(by=\"corr\")\ndf_ratios.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:52.342803Z","iopub.execute_input":"2025-05-26T15:05:52.343145Z","iopub.status.idle":"2025-05-26T15:05:53.239423Z","shell.execute_reply.started":"2025-05-26T15:05:52.343123Z","shell.execute_reply":"2025-05-26T15:05:53.238283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_ratios.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:53.240504Z","iopub.execute_input":"2025-05-26T15:05:53.240973Z","iopub.status.idle":"2025-05-26T15:05:53.251215Z","shell.execute_reply.started":"2025-05-26T15:05:53.240876Z","shell.execute_reply":"2025-05-26T15:05:53.250488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot most matching prices\n\nfig, ax = plt.subplots(3, 2, figsize=(18, 10))\nax = ax.flatten()\n\nfor i, _ticker in enumerate(df_ratios[\"ticker\"].values[-5:]):\n\n    _df_plot = df_crypto.loc[df_crypto[\"ticker\"] == _ticker]\n\n    ax[i].plot(pd.to_datetime(_df_plot.date), _df_plot[\"close\"].values, label=f\"{_ticker}\")\n    \n    ax[i].set_xlabel(\"date\")\n    ax[i].set_ylabel(\"close\")\n    ax[i].set_title(_ticker)\n\nax[5].plot(pd.to_datetime(df_benckmark.timestamp), df_benckmark[\"cum_sum\"], label=\"X363\", color=colors[1])\nax[5].set_xlabel(\"date\")\nax[5].set_ylabel(\"cum. sum of X363\")\nax[5].set_title(\"Benchmark: X363 feature\")\n\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T15:05:53.252129Z","iopub.execute_input":"2025-05-26T15:05:53.252466Z","iopub.status.idle":"2025-05-26T15:05:55.034946Z","shell.execute_reply.started":"2025-05-26T15:05:53.252445Z","shell.execute_reply":"2025-05-26T15:05:55.033912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}