{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":5456.26661,"end_time":"2025-05-24T10:59:14.597955","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-24T09:28:18.331345","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.004584,"end_time":"2025-05-24T09:28:33.615072","exception":false,"start_time":"2025-05-24T09:28:33.610488","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from scipy.stats import pearsonr\nfrom xgboost import XGBClassifier\nfrom sklearn.base import clone\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import classification_report\n\nimport warnings\nimport random\nimport gc\nimport os\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.023958,"end_time":"2025-05-24T09:28:43.643567","exception":false,"start_time":"2025-05-24T09:28:33.619609","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:04:02.618791Z","iopub.execute_input":"2025-05-30T03:04:02.619129Z","iopub.status.idle":"2025-05-30T03:04:05.333109Z","shell.execute_reply.started":"2025-05-30T03:04:02.619100Z","shell.execute_reply":"2025-05-30T03:04:05.332000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    target = \"label\"\n    n_folds = 5\n    seed = 42","metadata":{"papermill":{"duration":0.012751,"end_time":"2025-05-24T09:28:43.661332","exception":false,"start_time":"2025-05-24T09:28:43.648581","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:04:05.334536Z","iopub.execute_input":"2025-05-30T03:04:05.334925Z","iopub.status.idle":"2025-05-30T03:04:05.340249Z","shell.execute_reply.started":"2025-05-30T03:04:05.334899Z","shell.execute_reply":"2025-05-30T03:04:05.339239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"papermill":{"duration":0.0045,"end_time":"2025-05-24T09:28:43.670566","exception":false,"start_time":"2025-05-24T09:28:43.666066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def reduce_mem_usage(dataframe, dataset):    \n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.017076,"end_time":"2025-05-24T09:28:43.692076","exception":false,"start_time":"2025-05-24T09:28:43.675","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:04:05.551511Z","iopub.execute_input":"2025-05-30T03:04:05.551786Z","iopub.status.idle":"2025-05-30T03:04:05.561335Z","shell.execute_reply.started":"2025-05-30T03:04:05.551765Z","shell.execute_reply":"2025-05-30T03:04:05.560618Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engeerning","metadata":{}},{"cell_type":"code","source":"# Unnecessary features\ncols_to_drop = [\n    'X697', 'X698', 'X699', 'X700', 'X701', 'X702', 'X703', 'X704', 'X705', 'X706', \n    'X707', 'X708', 'X709', 'X710', 'X711', 'X712', 'X713', 'X714', 'X715', 'X716',\n    'X717', 'X864', 'X867', 'X869', 'X870', 'X871', 'X872', 'X104', 'X110', 'X116',\n    'X122', 'X128', 'X134', 'X140', 'X146', 'X152', 'X158', 'X164', 'X170', 'X176',\n    'X182', 'X351', 'X357', 'X363', 'X369', 'X375', 'X381', 'X387', 'X393', 'X399',\n    'X405', 'X411', 'X417', 'X423', 'X429'\n]","metadata":{"papermill":{"duration":0.012692,"end_time":"2025-05-24T09:28:43.709591","exception":false,"start_time":"2025-05-24T09:28:43.696899","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:04:07.493265Z","iopub.execute_input":"2025-05-30T03:04:07.493572Z","iopub.status.idle":"2025-05-30T03:04:07.498991Z","shell.execute_reply.started":"2025-05-30T03:04:07.493554Z","shell.execute_reply":"2025-05-30T03:04:07.498228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet(CFG.train_path)\ntest = pd.read_parquet(CFG.test_path).reset_index(drop=True)\n\ntime_idx = train.index\ntrain.reset_index(drop=True, inplace=True)\n\ntrain = train.drop(columns=cols_to_drop)\ntest = test.drop(columns=[\"label\"] + cols_to_drop)\n\nFEATURES = [i for i in test.columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:04:08.903890Z","iopub.execute_input":"2025-05-30T03:04:08.904160Z","iopub.status.idle":"2025-05-30T03:05:02.335814Z","shell.execute_reply.started":"2025-05-30T03:04:08.904140Z","shell.execute_reply":"2025-05-30T03:05:02.334824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'label' in FEATURES","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:05:02.337126Z","iopub.execute_input":"2025-05-30T03:05:02.337412Z","iopub.status.idle":"2025-05-30T03:05:02.343817Z","shell.execute_reply.started":"2025-05-30T03:05:02.337378Z","shell.execute_reply":"2025-05-30T03:05:02.342134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\nX = train.drop(CFG.target, axis=1)\ny = train[CFG.target]\nX_test = test","metadata":{"papermill":{"duration":74.723597,"end_time":"2025-05-24T09:29:58.438005","exception":false,"start_time":"2025-05-24T09:28:43.714408","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:05:02.344871Z","iopub.execute_input":"2025-05-30T03:05:02.345195Z","iopub.status.idle":"2025-05-30T03:05:12.483648Z","shell.execute_reply.started":"2025-05-30T03:05:02.345167Z","shell.execute_reply":"2025-05-30T03:05:12.482743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training base models","metadata":{"papermill":{"duration":0.004536,"end_time":"2025-05-24T09:29:58.44762","exception":false,"start_time":"2025-05-24T09:29:58.443084","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def set_seed(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\nset_seed()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:05:12.484840Z","iopub.execute_input":"2025-05-30T03:05:12.485058Z","iopub.status.idle":"2025-05-30T03:05:12.490141Z","shell.execute_reply.started":"2025-05-30T03:05:12.485037Z","shell.execute_reply":"2025-05-30T03:05:12.488952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def multi_class_data(time_idx, train_ratio=0.4, valid_ratio=0.3, test_ratio=0.3):\n    parts_month = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 1, 2]\n    parts_idx = []\n    for it in parts_month:\n        parts_idx.append(np.where(time_idx.month==it)[0])\n    for i in range(len(parts_idx)):\n        np.random.shuffle(parts_idx[i])\n\n    parts_train_end_idx = []\n    parts_valid_end_idx = []\n    for it in parts_idx:\n        parts_train_end_idx.append(int(len(it) * train_ratio))\n        parts_valid_end_idx.append(int(len(it) *(train_ratio+valid_ratio)))\n\n    train_idx = []\n    valid_idx = []\n    test_idx = []\n    for i in range(len(parts_idx)):\n        train_idx.append(parts_idx[i][:parts_train_end_idx[i]])\n        valid_idx.append(parts_idx[i][parts_train_end_idx[i]:parts_valid_end_idx[i]])\n        test_idx.append(parts_idx[i][parts_valid_end_idx[i]:])\n    return parts_idx, np.concatenate(train_idx), np.concatenate(valid_idx), np.concatenate(test_idx)\n    \n\ndef make_train_valid(time_idx, train_ratio=0.4, valid_ratio=0.3, test_ratio=0.3):\n    part1 = [3, 4, 5, 6, 7]\n    part2 = [10, 11, 12, 1, 2]\n    \n    part1_cond = False\n    for it in part1:\n        part1_cond = part1_cond | (time_idx.month == it)\n        \n    part2_cond = False\n    for it in part2:\n        part2_cond = part2_cond | (time_idx.month == it)\n        \n    part1_idx = np.where(part1_cond)[0]\n    part2_idx = np.where(part2_cond)[0]\n\n    np.random.shuffle(part1_idx)\n    np.random.shuffle(part2_idx)\n\n    part1_bnd1 = int(len(part1_idx) * train_ratio)\n    part1_bnd2 = int(len(part1_idx) *(train_ratio+valid_ratio))\n    \n    part2_bnd1 = int(len(part2_idx) * train_ratio)\n    part2_bnd2 = int(len(part2_idx) *(train_ratio+valid_ratio))\n    \n\n    train_idx = np.concatenate([\n        part1_idx[:part1_bnd1],\n        part2_idx[:part2_bnd1]\n    ])\n    valid_idx = np.concatenate([\n        part1_idx[part1_bnd1:part1_bnd2],\n        part2_idx[part2_bnd1:part2_bnd2]\n    ])\n    test_idx = np.concatenate([\n        part1_idx[part1_bnd2:],\n        part2_idx[part2_bnd2:]\n    ])\n    \n    print(len(part1_idx)+len(part2_idx), len(train_idx)+len(valid_idx)+len(test_idx))\n    return part1_idx, part2_idx, train_idx, valid_idx, test_idx\n\n# Trainer with selectable features\nclass Trainer_with_FE(object):\n    def __init__(self, model, features):\n        self.model = model\n        self.features = features\n        \n    def fit_predict(self, X, y, fold_idx):\n        print(f\"Training {self.model.__class__.__name__}\\n\")\n\n        train_idx, val_idx, test_idx = fold_idx\n        \n        X_train, X_val = X.loc[train_idx, self.features].copy(), X.loc[val_idx, self.features].copy()\n        X_test = X.loc[test_idx, self.features].copy()\n        \n        y_train, y_val, y_test = y[train_idx], y[val_idx], y[test_idx]\n\n        model = clone(self.model)\n        model.fit(X_train, y_train)\n\n        print(f\"--- Train Classification Report ---\")\n        print(classification_report(y_train, model.predict(X_train)))\n\n        print(f\"--- Valid Classification Report ---\")\n        print(classification_report(y_val, model.predict(X_val)))\n\n        print(f\"--- Test Classification Report ---\")\n        print(classification_report(y_test, model.predict(X_test)))\n\n        del X_train, y_train, X_test, X_val, y_val, y_test\n        gc.collect()\n\n        return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:05:24.132102Z","iopub.execute_input":"2025-05-30T03:05:24.132411Z","iopub.status.idle":"2025-05-30T03:05:24.144388Z","shell.execute_reply.started":"2025-05-30T03:05:24.132393Z","shell.execute_reply":"2025-05-30T03:05:24.143492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:05:47.332033Z","iopub.execute_input":"2025-05-30T03:05:47.333112Z","iopub.status.idle":"2025-05-30T03:05:47.474359Z","shell.execute_reply.started":"2025-05-30T03:05:47.333086Z","shell.execute_reply":"2025-05-30T03:05:47.473542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Multi Classification","metadata":{}},{"cell_type":"code","source":"parts_idx, train_idx, valid_idx, test_idx = multi_class_data(time_idx)\n\nfor i, it in enumerate(parts_idx):\n    y[it] = i\n\nprint(\"-- Train\\n\", y[train_idx].value_counts()) \nprint(\"-- Valid\\n\", y[valid_idx].value_counts())\nprint(\"-- Test\\n\", y[test_idx].value_counts())\n\nxgb_params = {\n    \"objective\": \"multiclass\",\n    \"learning_rate\": 0.05,\n    \"max_depth\": 10,\n    \"max_leaves\": 19,\n    \"n_estimators\": 100,\n    \"n_jobs\": -1,\n    \"device\":\"cuda\",\n    \"random_state\": 42,\n    \"verbosity\": 0\n}\n\nxgb_trainer = Trainer_with_FE(\n    XGBClassifier(**xgb_params),\n    features = FEATURES\n)\n\nmodel_xgb = xgb_trainer.fit_predict(X, y, (train_idx, valid_idx, test_idx))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:08:30.925911Z","iopub.execute_input":"2025-05-30T03:08:30.926166Z","iopub.status.idle":"2025-05-30T03:16:40.198210Z","shell.execute_reply.started":"2025-05-30T03:08:30.926149Z","shell.execute_reply":"2025-05-30T03:16:40.197530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_importance = model_xgb.feature_importances_\nimportance_df = pd.DataFrame({\n    \"Feature\": FEATURES,  # Replace FEATURES with your list of feature names\n    \"Importance\": feature_importance\n}).sort_values(by=\"Importance\", ascending=False)\nplt.figure(figsize=(10, 15))\nplt.barh(importance_df.iloc[:100, :][\"Feature\"], importance_df.iloc[:100, :][\"Importance\"])\nplt.xlabel(\"Importance\")\nplt.ylabel(\"Feature\")\nplt.title(\"XGBoost Feature Importance\")\nplt.gca().invert_yaxis()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:20:21.818715Z","iopub.execute_input":"2025-05-30T03:20:21.818962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"need_feature = importance_df[importance_df['Importance'] == 0]['Feature']\nprint('[', end='')\ncnt = 1\nfor it in need_feature:\n    print(f\"'{it}'\", end=',')\n    if cnt % 12 == 0:\n        print()\n    cnt += 1\nprint(']')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T03:24:07.046174Z","iopub.execute_input":"2025-05-30T03:24:07.047117Z","iopub.status.idle":"2025-05-30T03:24:07.055025Z","shell.execute_reply.started":"2025-05-30T03:24:07.047082Z","shell.execute_reply":"2025-05-30T03:24:07.054087Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Binary Classification","metadata":{}},{"cell_type":"code","source":"part1_idx, part2_idx, train_idx, valid_idx, test_idx = make_train_valid(time_idx)\ny[part1_idx] = 0\ny[part2_idx] = 1\n\nprint(\"-- Train\\n\", y[train_idx].value_counts()) \nprint(\"-- Valid\\n\", y[valid_idx].value_counts())\n\nxgb_params = {\n    \"learning_rate\": 0.05,\n    \"max_depth\": 10,\n    \"max_leaves\": 19,\n    \"n_estimators\": 100,\n    \"n_jobs\": -1,\n    \"device\":\"cuda\",\n    \"random_state\": 42,\n    \"verbosity\": 0\n}\n\nxgb_trainer = Trainer_with_FE(\n    XGBClassifier(**xgb_params),\n    features = FEATURES\n)\n\nmodel_xgb = xgb_trainer.fit_predict(X, y, (train_idx, valid_idx, test_idx))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T16:38:35.290839Z","iopub.execute_input":"2025-05-29T16:38:35.291229Z","iopub.status.idle":"2025-05-29T16:39:57.708117Z","shell.execute_reply.started":"2025-05-29T16:38:35.291196Z","shell.execute_reply":"2025-05-29T16:39:57.707101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_importance = model_xgb.feature_importances_\nimportance_df = pd.DataFrame({\n    \"Feature\": FEATURES,  # Replace FEATURES with your list of feature names\n    \"Importance\": feature_importance\n}).sort_values(by=\"Importance\", ascending=False)\nplt.figure(figsize=(10, 15))\nplt.barh(importance_df.iloc[:100, :][\"Feature\"], importance_df.iloc[:100, :][\"Importance\"])\nplt.xlabel(\"Importance\")\nplt.ylabel(\"Feature\")\nplt.title(\"XGBoost Feature Importance\")\nplt.gca().invert_yaxis()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T16:40:05.924157Z","iopub.execute_input":"2025-05-29T16:40:05.924438Z","iopub.status.idle":"2025-05-29T16:40:06.735094Z","shell.execute_reply.started":"2025-05-29T16:40:05.924419Z","shell.execute_reply":"2025-05-29T16:40:06.734104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"importance_df[importance_df['Importance'] > 0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T16:40:10.637115Z","iopub.execute_input":"2025-05-29T16:40:10.637404Z","iopub.status.idle":"2025-05-29T16:40:10.648840Z","shell.execute_reply.started":"2025-05-29T16:40:10.637386Z","shell.execute_reply":"2025-05-29T16:40:10.647877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"importance_df[importance_df['Importance'] == 0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T16:40:16.073695Z","iopub.execute_input":"2025-05-29T16:40:16.074004Z","iopub.status.idle":"2025-05-29T16:40:16.086334Z","shell.execute_reply.started":"2025-05-29T16:40:16.073973Z","shell.execute_reply":"2025-05-29T16:40:16.085291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"need_feature = importance_df[importance_df['Importance'] == 0]['Feature']\nprint('[', end='')\ncnt = 1\nfor it in need_feature:\n    print(f\"'{it}'\", end=',')\n    if cnt % 12 == 0:\n        print()\n    cnt += 1\nprint(']')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T16:40:31.501099Z","iopub.execute_input":"2025-05-29T16:40:31.501408Z","iopub.status.idle":"2025-05-29T16:40:31.507892Z","shell.execute_reply.started":"2025-05-29T16:40:31.501389Z","shell.execute_reply":"2025-05-29T16:40:31.506915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}