{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":12489060,"sourceType":"datasetVersion","datasetId":7881104},{"sourceId":12529837,"sourceType":"datasetVersion","datasetId":7909511},{"sourceId":483307,"sourceType":"modelInstanceVersion","modelInstanceId":386569,"modelId":405696},{"sourceId":485240,"sourceType":"modelInstanceVersion","modelInstanceId":387389,"modelId":406465}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## See Input Folders","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-25T02:45:05.548197Z","iopub.execute_input":"2025-07-25T02:45:05.548714Z","iopub.status.idle":"2025-07-25T02:45:05.568264Z","shell.execute_reply.started":"2025-07-25T02:45:05.548686Z","shell.execute_reply":"2025-07-25T02:45:05.566974Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Settings","metadata":{}},{"cell_type":"code","source":"#getting data analysis and ML packages\nimport pandas\nfrom pandas.plotting import scatter_matrix\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import median_absolute_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer #transform different types\nfrom sklearn.datasets import fetch_openml\nimport numpy\nfrom numpy import sqrt\nfrom numpy import sum\nfrom numpy import square\nimport seaborn\nimport matplotlib\nimport statsmodels\nimport time\nimport keras\nfrom sklearn.decomposition import PCA\nimport xgboost as xgb\nfrom mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier\nfrom mlxtend.regressor import StackingRegressor\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom matplotlib.ticker import LinearLocator, FormatStrFormatter\nfrom mpl_toolkits.mplot3d import Axes3D","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T02:45:06.006353Z","iopub.execute_input":"2025-07-25T02:45:06.006705Z","iopub.status.idle":"2025-07-25T02:45:21.236596Z","shell.execute_reply.started":"2025-07-25T02:45:06.006680Z","shell.execute_reply":"2025-07-25T02:45:21.235378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n%load_ext cuml.accel\n%load_ext cudf.pandas","metadata":{"execution":{"iopub.status.busy":"2025-07-24T14:36:05.395249Z","iopub.execute_input":"2025-07-24T14:36:05.396047Z","iopub.status.idle":"2025-07-24T14:36:05.400895Z","shell.execute_reply.started":"2025-07-24T14:36:05.396027Z","shell.execute_reply":"2025-07-24T14:36:05.399970Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install ipympl\n\n# Restart the kernel after installing for the changes to take effect.\n# After restarting, you can run the cell again.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:39:49.647897Z","iopub.status.idle":"2025-07-23T15:39:49.648241Z","shell.execute_reply.started":"2025-07-23T15:39:49.648085Z","shell.execute_reply":"2025-07-23T15:39:49.648098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Processing stages\n### - Convert parquet into dataframe\n### - Descriptive (Done, me and Lija)\n#### -- Descriptive statistics (Done)\n#### -- Distribution graphs (hist(Done), KDE (Done), boxplot, violin)\n### - Exploratory (Done, me and Liying)\n#### -- Pivot table\n#### -- Scatter (Done), corrplot (Done but shit), bubble plot, time series (need), ribbon (line with confidence interval)\nhttps://r-graph-gallery.com/318-custom-dygraphs-time-series-example.html\nhttps://r-graph-gallery.com/104-plot-lines-with-error-envelopes-ggplot2.html\n#### -- Treemap, stacked bar or heatmap\nhttps://r-graph-gallery.com/237-interactive-treemap.html\nhttps://r-graph-gallery.com/235-treemap-with-subgroups.html\n### - Statistical/Inferential\n#### -- Feature Selection/Dimension Reduction (FA, PCA, Simple Linear Reg(Gone) ) (Done Liying)\n#### -- Test effect of insignificant variables (n-way ANOVA) (Gone)\n#### -- Test independence of variables (Bartlett test of sphericity, X2 test of independence, correlations t test/sample correlation test)\n### - Predictive\n#### -- Feature Engineering\n#### -- Transform\n#### -- HPT with and without transform\n#### -- Post-fit tuning (Gone)","metadata":{}},{"cell_type":"code","source":"df_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\n#replace = pandas.read_csv('/kaggle/input/x1-x780/pls-762-to-5.csv')\n#df_test = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet', engine='pyarrow')\n#df_sample = pandas.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T02:45:21.238590Z","iopub.execute_input":"2025-07-25T02:45:21.239361Z","iopub.status.idle":"2025-07-25T02:45:45.779364Z","shell.execute_reply.started":"2025-07-25T02:45:21.239331Z","shell.execute_reply":"2025-07-25T02:45:45.778354Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## X1 - X780 Drop","metadata":{}},{"cell_type":"code","source":"df_train = df_train.drop(df_train.loc[:,'X1':'X780'].columns, axis=1)","metadata":{"execution":{"iopub.status.busy":"2025-07-23T15:22:39.065722Z","iopub.execute_input":"2025-07-23T15:22:39.066248Z","iopub.status.idle":"2025-07-23T15:22:39.474810Z","shell.execute_reply.started":"2025-07-23T15:22:39.066152Z","shell.execute_reply":"2025-07-23T15:22:39.473662Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"df_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:39:49.651885Z","iopub.status.idle":"2025-07-23T15:39:49.652353Z","shell.execute_reply.started":"2025-07-23T15:39:49.652122Z","shell.execute_reply":"2025-07-23T15:39:49.652141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train['year']   = df_train.index.year\ndf_train['month']  = df_train.index.month\ndf_train['day']    = df_train.index.day\ndf_train['hour']   = df_train.index.hour\ndf_train['minute'] = df_train.index.minute\ndf_train['weekday'] = df_train.index.weekday","metadata":{"execution":{"iopub.status.busy":"2025-07-22T12:26:30.053093Z","iopub.execute_input":"2025-07-22T12:26:30.053346Z","iopub.status.idle":"2025-07-22T12:26:30.099303Z","shell.execute_reply.started":"2025-07-22T12:26:30.053327Z","shell.execute_reply":"2025-07-22T12:26:30.09873Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train['second_cont'] = (df_train.index - df_train.index[0]).total_seconds()\ndf_train['minute_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 60\ndf_train['hour_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 3600\n","metadata":{"execution":{"iopub.status.busy":"2025-07-22T12:26:30.100065Z","iopub.execute_input":"2025-07-22T12:26:30.10036Z","iopub.status.idle":"2025-07-22T12:26:30.242676Z","shell.execute_reply.started":"2025-07-22T12:26:30.100337Z","shell.execute_reply":"2025-07-22T12:26:30.241775Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Lag k","metadata":{}},{"cell_type":"code","source":"df_train['lag_1'] = df_train['label'].shift(periods=1)\ndf_train['lag_2'] = df_train['label'].shift(periods=2)\ndf_train['lag_3'] = df_train['label'].shift(periods=3)\ndf_train['lag_4'] = df_train['label'].shift(periods=4)\ndf_train['lag_5'] = df_train['label'].shift(periods=5)\ndf_train['lag_6'] = df_train['label'].shift(periods=6)\ndf_train['lag_7'] = df_train['label'].shift(periods=7)\ndf_train['lag_8'] = df_train['label'].shift(periods=8)\ndf_train['lag_9'] = df_train['label'].shift(periods=9)\ndf_train['lag_10'] = df_train['label'].shift(periods=10)\ndf_train['lag_11'] = df_train['label'].shift(periods=11)\ndf_train['lag_12'] = df_train['label'].shift(periods=12)\ndf_train['lag_13'] = df_train['label'].shift(periods=13)\ndf_train['lag_14'] = df_train['label'].shift(periods=14)\ndf_train['lag_15'] = df_train['label'].shift(periods=15)\ndf_train['lag_16'] = df_train['label'].shift(periods=16)\ndf_train.loc[:,'lag_1':'lag_16'] = df_train.loc[:,'lag_1':'lag_16'].replace({np.nan: df_train['label'].mean()})","metadata":{"execution":{"iopub.status.busy":"2025-07-23T08:12:49.513597Z","iopub.execute_input":"2025-07-23T08:12:49.514025Z","iopub.status.idle":"2025-07-23T08:12:50.06026Z","shell.execute_reply.started":"2025-07-23T08:12:49.513985Z","shell.execute_reply":"2025-07-23T08:12:50.058751Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dropping Infs","metadata":{}},{"cell_type":"code","source":"## Dropping infs\ninfcol = df_train.columns[np.isinf(df_train).any(axis=0)].tolist()\nprint(infcol)\n\ndf_train = df_train.drop(columns=infcol)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:39:49.653750Z","iopub.status.idle":"2025-07-23T15:39:49.654171Z","shell.execute_reply.started":"2025-07-23T15:39:49.653958Z","shell.execute_reply":"2025-07-23T15:39:49.653976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## converting index from datetime to numbers. Cannot use pipeline bc not applying to entire df, would need columnTransformer, too lazy\ndf_train.index = range(0,len(df_train.index))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:39:49.655659Z","iopub.status.idle":"2025-07-23T15:39:49.655959Z","shell.execute_reply.started":"2025-07-23T15:39:49.655826Z","shell.execute_reply":"2025-07-23T15:39:49.655838Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Join (must same range index, not datetime)","metadata":{}},{"cell_type":"code","source":"df_train = df_train.join(replace)","metadata":{"execution":{"iopub.status.busy":"2025-07-23T15:22:39.557223Z","iopub.execute_input":"2025-07-23T15:22:39.557692Z","iopub.status.idle":"2025-07-23T15:22:39.607685Z","shell.execute_reply.started":"2025-07-23T15:22:39.557660Z","shell.execute_reply":"2025-07-23T15:22:39.606216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train.rename(columns={'0':'PLS_1','1':'PLS_2','2':'PLS_3','3':'PLS_4','4':'PLS_5'})","metadata":{"execution":{"iopub.status.busy":"2025-07-23T15:22:39.610872Z","iopub.execute_input":"2025-07-23T15:22:39.611342Z","iopub.status.idle":"2025-07-23T15:22:39.674368Z","shell.execute_reply.started":"2025-07-23T15:22:39.611316Z","shell.execute_reply":"2025-07-23T15:22:39.673123Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Descriptive","metadata":{}},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.126053Z","iopub.status.idle":"2025-07-08T05:54:02.126402Z","shell.execute_reply.started":"2025-07-08T05:54:02.126226Z","shell.execute_reply":"2025-07-08T05:54:02.126241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.columns.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.127871Z","iopub.status.idle":"2025-07-08T05:54:02.128128Z","shell.execute_reply.started":"2025-07-08T05:54:02.128013Z","shell.execute_reply":"2025-07-08T05:54:02.128023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.128959Z","iopub.status.idle":"2025-07-08T05:54:02.129718Z","shell.execute_reply.started":"2025-07-08T05:54:02.129525Z","shell.execute_reply":"2025-07-08T05:54:02.129541Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.131265Z","iopub.status.idle":"2025-07-08T05:54:02.131681Z","shell.execute_reply.started":"2025-07-08T05:54:02.131455Z","shell.execute_reply":"2025-07-08T05:54:02.131471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Correlation (Done)\ntemp = df_train.iloc[0:44640,0:900]\nseaborn.heatmap(temp.corr(), annot=True, cmap='coolwarm', vmin=-1, vmax=1)\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-07-08T05:54:02.133149Z","iopub.status.idle":"2025-07-08T05:54:02.13347Z","shell.execute_reply.started":"2025-07-08T05:54:02.133339Z","shell.execute_reply":"2025-07-08T05:54:02.133351Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Histogram (Done)\noutput_dir = \"/kaggle/working/\"\nos.makedirs(output_dir, exist_ok=True)\n\nfor col in df_train.columns:\n    if df_train[col].isin([numpy.inf, -numpy.inf]).any().any() == False:\n        fig, ax = plt.subplots()\n        df_train[col].plot(kind='hist', ax=ax,bins=20)\n        plt.title(f\"{col} histogram\")\n        plt.tight_layout()\n        plt.savefig(f\"{output_dir}/{col}_histogram.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.134202Z","iopub.status.idle":"2025-07-08T05:54:02.13509Z","shell.execute_reply.started":"2025-07-08T05:54:02.134436Z","shell.execute_reply":"2025-07-08T05:54:02.134454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Histogram and KDE but with separated y axis by twinning the x axis (Done)\noutput_dir = \"/kaggle/working/\"\nos.makedirs(output_dir, exist_ok=True)\n\nfor col in df_train.columns:\n    if df_train[col].isin([numpy.inf, -numpy.inf]).any().any() == False:\n        fig, ax = plt.subplots()\n        ax2 = ax.twinx()\n        df_train[col].plot(kind='hist', bins=20, ax=ax)\n        df_train[col].plot(kind='kde',ax=ax2,color='red', linestyle='--')\n        plt.title(f\"{col} Density and Histogram Plot\")\n        plt.tight_layout()\n        plt.savefig(f\"{output_dir}/{col}_Density and histogram.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.136717Z","iopub.status.idle":"2025-07-08T05:54:02.137068Z","shell.execute_reply.started":"2025-07-08T05:54:02.136902Z","shell.execute_reply":"2025-07-08T05:54:02.136914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tell histogram to change y axis to density bc I intend to plot kde also, and use histogram as plotted axis\nax = df_train['X354'].plot(kind='hist',stacked=False, density=True)\ndf_train['X354'].plot.kde(ax=ax)\nplt.title(f\"X354 Density Plot\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.138824Z","iopub.status.idle":"2025-07-08T05:54:02.139249Z","shell.execute_reply.started":"2025-07-08T05:54:02.139055Z","shell.execute_reply":"2025-07-08T05:54:02.139073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# KDE (Done)\noutput_dir = \"/kaggle/working/\"\nos.makedirs(output_dir, exist_ok=True)\n\nfor col in df_train.columns:\n    ax = df_train[col].plot.kde()\n    plt.title(f\"{col} Density Plot\")\n    plt.tight_layout()\n    plt.savefig(f\"{output_dir}/{col}_KDE.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.140524Z","iopub.status.idle":"2025-07-08T05:54:02.140892Z","shell.execute_reply.started":"2025-07-08T05:54:02.140716Z","shell.execute_reply":"2025-07-08T05:54:02.140729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Which ones are normal?\nfrom scipy import stats\n\nfor col in df_train.columns:\n    if df_train[col].isin([numpy.inf, -numpy.inf]).any().any() == False:\n        p = stats.kstest(df_train[col],stats.norm.cdf).pvalue\n        pl = stats.kstest(df_train[col],stats.norm.cdf, alternative='less').pvalue\n        pg = stats.kstest(df_train[col],stats.norm.cdf, alternative='greater').pvalue\n        if p > 0.005:\n            print(col, ':Not rejected, RV is normally distributed')\n        elif pl > 0.005:\n            print(col, ':Not rejected, RV is normally distributed, leaning towards greater value')\n        elif pg > 0.005:\n            print(col, ':Not rejected, RV is normally distributed, leaning towards lesser value \\n=====')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.142886Z","iopub.status.idle":"2025-07-08T05:54:02.14329Z","shell.execute_reply.started":"2025-07-08T05:54:02.143146Z","shell.execute_reply":"2025-07-08T05:54:02.143159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Which ones are normal (Detailed)?\nfrom scipy import stats\n\nfor col in df_train.columns:\n    if df_train[col].isin([numpy.inf, -numpy.inf]).any().any() == False:\n        p = stats.kstest(df_train[col],stats.norm.cdf).pvalue\n        pl = stats.kstest(df_train[col],stats.norm.cdf, alternative='less').pvalue\n        pg = stats.kstest(df_train[col],stats.norm.cdf, alternative='greater').pvalue\n        if p <= 0.005:\n            print(col, ':Rejected, Not normal @ 0.5% α')\n        elif p > 0.005:\n            print(col, ':Not rejected, RV is normally distributed')\n        if pl <= 0.005:\n            print(col, ':Rejected, Not normal @ 0.5% α (less test)')\n        elif pl > 0.005:\n            print(col, ':Not rejected, RV is normally distributed, leaning towards greater value')\n        if pg <= 0.005:\n            print(col, ':Rejected, Not normal @ 0.5% α (greater test) \\n=====')\n        elif pg > 0.005:\n            print(col, ':Not rejected, RV is normally distributed, leaning towards lesser value \\n=====')","metadata":{"trusted":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.144176Z","iopub.status.idle":"2025-07-08T05:54:02.144459Z","shell.execute_reply.started":"2025-07-08T05:54:02.144334Z","shell.execute_reply":"2025-07-08T05:54:02.144347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Which ones are correlated significantly?\nfrom scipy import stats\n\nfor col in df_train.columns:\n    if df_train[col].isin([numpy.inf, -numpy.inf]).any().any() == False:\n        p = stats.pearsonr(df_train[col],df_train['label']).pvalue\n        pl = stats.pearsonr(df_train[col],df_train['label'], alternative='less').pvalue\n        pg = stats.pearsonr(df_train[col],df_train['label'], alternative='greater').pvalue\n        if p <= 0.005:\n            print(col, ':Rejected, Rho != 0 \\n=====')\n        if pl <= 0.005:\n            print(col, ':Rejected, Rho < 0, Rho is likely negative \\n=====')\n        if pg <= 0.005:\n            print(col, ':Rejected, Rho > 0, Rho is likely positive \\n=====')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.145878Z","iopub.status.idle":"2025-07-08T05:54:02.146235Z","shell.execute_reply.started":"2025-07-08T05:54:02.146107Z","shell.execute_reply":"2025-07-08T05:54:02.146121Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## defunc\ncols=[]\nfor col in df_train.columns:\n    if df_train[col].isin([numpy.inf, -numpy.inf]).any().any() == True:\n        cols.append(col)\ncols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.147127Z","iopub.status.idle":"2025-07-08T05:54:02.147488Z","shell.execute_reply.started":"2025-07-08T05:54:02.14731Z","shell.execute_reply":"2025-07-08T05:54:02.147326Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exploratory","metadata":{}},{"cell_type":"code","source":"import os\n\noutput_dir = \"/kaggle/working/\"\nos.makedirs(output_dir, exist_ok=True)\n\nfor col in df_train.columns:\n    ax = df_train.plot.scatter(x=col, y='label')\n    plt.title(f\"{col} vs label\")\n    plt.tight_layout()\n    plt.savefig(f\"{output_dir}/{col}_vs_label.png\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.149204Z","iopub.status.idle":"2025-07-08T05:54:02.149464Z","shell.execute_reply.started":"2025-07-08T05:54:02.149347Z","shell.execute_reply":"2025-07-08T05:54:02.149358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### output all the graphs to a zip file\n!zip -r output_folder.zip  /kaggle/working/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.150303Z","iopub.status.idle":"2025-07-08T05:54:02.150661Z","shell.execute_reply.started":"2025-07-08T05:54:02.150465Z","shell.execute_reply":"2025-07-08T05:54:02.150483Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transformation (Pre-split) (risk of leakage) (Defunc)","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler().fit(df_train[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890','year','month','day']])\ntemp = scaler.transform(df_train[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890','year','month','day']])\ntemp = pandas.DataFrame(temp).rename(columns={0:'bid_qty_std',1:'ask_qty_std',2:'buy_qty_std',3:'sell_qty_std',4:'volume_std',5:'X17_std',6:'X25_std',7:'X198_std',8:'X205_std',9:'X413_std',10:'X415_std',11:'X421_std',12:'X594_std',13:'X596_std',14:'X606_std',15:'X610_std',16:'X663_std',17:'X667_std',18:'X675_std',19:'X681_std',20:'X687_std',21:'X693_std',22:'X865_std',23:'X875_std',24:'X876_std',25:'X879_std',26:'X880_std',27:'X881_std',28:'X882_std',29:'X883_std',30:'X890_std',31:'year_std',32:'month_std',33:'day_std'})\ndf_train = df_train.join(temp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.151801Z","iopub.status.idle":"2025-07-08T05:54:02.152133Z","shell.execute_reply.started":"2025-07-08T05:54:02.151999Z","shell.execute_reply":"2025-07-08T05:54:02.152014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Auto Encoder and DenseNet","metadata":{}},{"cell_type":"code","source":"df_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\n\nX = df_train.loc[:, df_train.columns != 'label']\ny = df_train['label'].values\ndel df_train\n#X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T04:11:14.568812Z","iopub.execute_input":"2025-07-25T04:11:14.569196Z","iopub.status.idle":"2025-07-25T04:11:45.612800Z","shell.execute_reply.started":"2025-07-25T04:11:14.569170Z","shell.execute_reply":"2025-07-25T04:11:45.602338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#### from sklearn.preprocessing import StandardScaler\nX_scaled = StandardScaler().fit_transform(X)\n\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense\n\nimport gc\ngc.collect()\n\ninput_dim = X_scaled.shape[1]\nlatent_dim = 25  # You choose how many dimensions to reduce to\n\n# Encoder\ninput_layer = Input(shape=(input_dim,))\nencoded = Dense(64, activation='relu')(input_layer)\nencoded = Dense(32, activation='relu')(encoded)\nlatent = Dense(latent_dim, activation='linear')(encoded)\n\n# Decoder\ndecoded = Dense(32, activation='relu')(latent)\ndecoded = Dense(64, activation='relu')(decoded)\noutput_layer = Dense(input_dim, activation='linear')(decoded)\n\nautoencoder = Model(inputs=input_layer, outputs=output_layer)\nautoencoder.compile(optimizer='adam', loss='mse',  metrics=['r2_score'])\n\nautoencoder.fit(X_scaled, X_scaled, epochs=50, batch_size=256, shuffle=True)\n\nencoder = Model(inputs=input_layer, outputs=latent)\nX_latent = encoder.predict(X_scaled)\n\n# save\nautoencoder.save(\"/kaggle/working/my_autoencoder.keras\")  # or use .h5 for HDF5 format\nencoder.save(\"/kaggle/working/my_encoder.keras\")  # or use .h5 for HDF5 format","metadata":{"execution":{"iopub.status.busy":"2025-07-25T04:29:16.282010Z","iopub.execute_input":"2025-07-25T04:29:16.283783Z","iopub.status.idle":"2025-07-25T04:43:58.930069Z","shell.execute_reply.started":"2025-07-25T04:29:16.283742Z","shell.execute_reply":"2025-07-25T04:43:58.929011Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Perceptron, Lasso, LinearRegression, ElasticNet\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\n\nhgbr =  Pipeline([('std', StandardScaler()), ('hgbr',HistGradientBoostingRegressor())])\nlsr = Pipeline([('std', StandardScaler()), ('lsr',Lasso())])\nlr = Pipeline([('std', StandardScaler()), ('lr',LinearRegression())])\nelst = Pipeline([('std', StandardScaler()), ('elst',ElasticNet())])\nrd = Pipeline([('std', StandardScaler()), ('rd',Ridge())])\n\nmodels = [('Hist GB Reg',hgbr), ('Lasso', lsr), ('Linear', lr), ('ElasticNet', elst), ('Ridge', rd)]\n\nfor model_name, model in models:\n    ave_r2, ave_mae, ave_mape, ave_rmse, ave_mdae, ave_time = [],[],[],[],[],[]\n    for i in range(0,5):\n        kfold = model_selection.KFold(n_splits=5, random_state=i, shuffle=True) \n\t\n    \t# execute cross val to est skill of ML model (cross_validate faster)\n        start = time.time()\n        cv_results = model_selection.cross_validate(model, X_latent, y_train, cv=kfold, scoring = ('r2','neg_root_mean_squared_error','neg_mean_absolute_error','neg_mean_absolute_percentage_error','neg_median_absolute_error'),return_train_score=False)\n        end = time.time() \n        duration =  end-start\n        \n        ave_r2.append(cv_results['test_r2'].mean())\n        ave_mae.append(cv_results['test_neg_mean_absolute_error'].mean())\n        ave_mape.append(cv_results['test_neg_mean_absolute_percentage_error'].mean())\n        ave_rmse.append(cv_results['test_neg_root_mean_squared_error'].mean())\n        ave_mdae.append(cv_results['test_neg_median_absolute_error'].mean())\n        ave_time.append(duration)\n\n    # print results\n    import statistics\n    msg1 = \"%s: mean R2: %f \\nMAE: %f \\nMAPE: %f \\nRMSE: %f \\nMDAE: %f\" % (model_name, statistics.mean(ave_r2), statistics.mean(ave_mae), statistics.mean(ave_mape), statistics.mean(ave_rmse), statistics.mean(ave_mdae))\n    print(msg1, \"\\n Time ~\", statistics.mean(ave_time))\n    gc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Perceptron, Lasso, LinearRegression, ElasticNet\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\n\nhgbr =  Pipeline([('std', StandardScaler()), ('hgbr',HistGradientBoostingRegressor())])\nlsr = Pipeline([('std', StandardScaler()), ('lsr',Lasso())])\nlr = Pipeline([('std', StandardScaler()), ('lr',LinearRegression())])\nelst = Pipeline([('std', StandardScaler()), ('elst',ElasticNet())])\nrd = Pipeline([('std', StandardScaler()), ('rd',Ridge())])\n#svr_rbf = SVR(kernel='rbf')\n\nmodels = [('Hist GB Reg',hgbr), ('Linear', lr), ('Ridge', rd)]\n\n\nkfold = model_selection.KFold(n_splits=5) \n\nstk2 = StackingRegressor(estimators=models, final_estimator=hgbr, cv=kfold)\n\n# Training the stacking classifier\nstart1 = time.time()\nstk2.fit(X_train, y_train)\nend1 = time.time()\n# Evaluate and visualize the fit\n\nprint(\"Stack \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,stk2.predict(X_test))), r2_score(y_test,stk2.predict(X_test)), mean_absolute_percentage_error(y_test,stk2.predict(X_test)),median_absolute_error(y_test,stk2.predict(X_test)),mean_absolute_error(y_test,stk2.predict(X_test)),end1-start1 ))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense\nimport gc\ngc.collect()\nX_scaled = StandardScaler().fit_transform(X)\nX_latent = encoder.predict(X_scaled)\n\nmodel = Sequential([\n    Dense(64, activation='relu', input_shape=(X_latent.shape[1],)),\n    Dense(32, activation='relu'),\n    Dense(1)  # No activation for regression\n])\n\nmodel.compile(optimizer='adam', loss='mse', metrics=['mae','r2_score', 'root_mean_squared_error'])\n\nhistory = model.fit(X_latent, y, epochs=100, batch_size=256, validation_split=0.2)\n#loss, mae, r2, rmse = model.evaluate(encoder.predict(StandardScaler().fit_transform(X_test)), y_test)\nloss, mae, r2, rmse = model.evaluate(X_latent, y)\nprint(f\"Test MAE: {mae:.4f} R2: {r2:.4f} RMSE: {rmse:.4f}\")\ny_pred = model.predict(X_latent)\n\n## save\nmodel.save(\"/kaggle/working/my_model.keras\")  # or use .h5 for HDF5 format","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T04:43:58.934678Z","iopub.execute_input":"2025-07-25T04:43:58.935036Z","iopub.status.idle":"2025-07-25T04:55:22.032737Z","shell.execute_reply.started":"2025-07-25T04:43:58.935009Z","shell.execute_reply":"2025-07-25T04:55:22.031654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet', engine='pyarrow')\nX = df_test.loc[:, df_test.columns != 'label']\ndel df_test\nprint('input done!')\n\nX_scaled = StandardScaler().fit_transform(X)\nX_latent = encoder.predict(X_scaled)\ny_pred = model.predict(X_latent)\nprint('pred done!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T04:56:09.894339Z","iopub.execute_input":"2025-07-25T04:56:09.894733Z","iopub.status.idle":"2025-07-25T04:57:51.796649Z","shell.execute_reply.started":"2025-07-25T04:56:09.894705Z","shell.execute_reply":"2025-07-25T04:57:51.795487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pandas.DataFrame({'ID': range(1,len(X)+1)})\ny_pred = pandas.DataFrame(y_pred)\nsubmission = submission.join(y_pred).rename({0:'prediction'}, axis = 1)\nsubmission\nsubmission.to_csv('/kaggle/working/submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T04:57:51.798384Z","iopub.execute_input":"2025-07-25T04:57:51.798704Z","iopub.status.idle":"2025-07-25T04:57:52.853866Z","shell.execute_reply.started":"2025-07-25T04:57:51.798680Z","shell.execute_reply":"2025-07-25T04:57:52.852506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T04:03:15.005630Z","iopub.execute_input":"2025-07-25T04:03:15.005963Z","iopub.status.idle":"2025-07-25T04:03:15.017665Z","shell.execute_reply.started":"2025-07-25T04:03:15.005941Z","shell.execute_reply":"2025-07-25T04:03:15.016588Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Interaction Variables (Defunc)","metadata":{}},{"cell_type":"code","source":"columns=['X345-int-X387',\n        'X345-int-X429',\n        'X387-int-X429',\n        'X339-int-X381',\n        'X339-int-X423',\n        'X381-int-X423',\n        'X333-int-X375',\n        'X333-int-X417',\n        'X375-int-X417',\n        'X92-int-X134',\n        'X92-int-X176',\n        'X134-int-X176',\n        'X68-int-X110',\n        'X68-int-X152',\n        'X110-int-X152',\n        'X435-int-X438',\n        'X188-int-X191',\n        'X657-int-X660',\n        'X669-int-X672',\n        'X685-int-X691',\n        'X688-int-X694',\n        'X681-int-X684',\n        'X651-int-X654',\n        'X676-int-X682',\n        'X673-int-X679',\n        'X664-int-X670',\n        'X645-int-X648',\n        'X663-int-X666',\n        'X639-int-X642',\n        'X675-int-X678',\n        'X652-int-X658',\n        'X661-int-X667',\n        'X693-int-X696',\n        'X687-int-X690',\n        'X640-int-X646',\n        'X649-int-X655',\n        'X637-int-X643',\n        'X615-int-X618',\n        'X621-int-X624',\n        'X628-int-X634',\n        'X627-int-X630',\n        'X298-int-X300',\n        'X625-int-X631',\n        'X299-int-X301',\n        'X633-int-X636',\n        'X51-int-X53',\n        'X432-int-X435',\n        'X482-int-X489',\n        'X45-int-X47',\n        'X52-int-X54',\n        'X300-int-X302',\n        'X613-int-X619',\n        'X483-int-X490',\n        'X481-int-X488',\n        'X616-int-X622',\n        'X185-int-X188',\n        'X432-int-X438',\n        'X301-int-X303',\n        'X431-int-X434',\n        'X7-int-X8',\n        'X484-int-X491',\n        'X292-int-X294',\n        'X185-int-X191',\n        'X46-int-X48',\n        'X434-int-X437',\n        'X293-int-X295',\n        'X286-int-X288',\n        'X39-int-X41',\n        'X487-int-X494',\n        'X235-int-X242',\n        'X239-int-X246',\n        'X240-int-X247',\n        'X486-int-X493',\n        'X485-int-X492',\n        'X236-int-X243',\n        'X40-int-X42',\n        'X234-int-X241',\n        'X287-int-X289',\n        'X238-int-X245',\n        'X237-int-X244',\n        'X53-int-X55',\n        'X6-int-X7',\n        'X280-int-X282',\n        'X281-int-X283',\n        'X54-int-X56',\n        'X254-int-X255',\n        'X47-int-X49',\n        'X298-int-X302',\n        'X756-int-X757',\n        'X885-int-X886',\n        'X299-int-X303',\n        'X15-int-X16',\n        'X752-int-X753',\n        'X873-int-X874',\n        'X253-int-X254',\n        'X33-int-X35',\n        'X294-int-X296',\n        'X295-int-X297',\n        'X48-int-X50',\n        'X431-int-X437']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.153269Z","iopub.status.idle":"2025-07-08T05:54:02.153572Z","shell.execute_reply.started":"2025-07-08T05:54:02.153419Z","shell.execute_reply":"2025-07-08T05:54:02.153434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## make interaction variables\nfor ip in columns:\n    c1, c2 = ip.split('-int-')\n    df_train[ip] = df_train[c1]*df_train[c2]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.154683Z","iopub.status.idle":"2025-07-08T05:54:02.154956Z","shell.execute_reply.started":"2025-07-08T05:54:02.15482Z","shell.execute_reply":"2025-07-08T05:54:02.154833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Dropping perfectly one-half of all perfectly correlated pairs of RVs\ndf_train = df_train.drop(['X104',\n                            'X62',\n                            'X116',\n                            'X74',\n                            'X122',\n                            'X80',\n                            'X128',\n                            'X86',\n                            'X140',\n                            'X98',\n                            'X351',\n                            'X309',\n                            'X357',\n                            'X315',\n                            'X363',\n                            'X321',\n                            'X369',\n                            'X327',\n                            'date'], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.155935Z","iopub.status.idle":"2025-07-08T05:54:02.156299Z","shell.execute_reply.started":"2025-07-08T05:54:02.156137Z","shell.execute_reply":"2025-07-08T05:54:02.156152Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split (Defunc)","metadata":{}},{"cell_type":"code","source":"X = df_train.loc[:, df_train.columns != 'label']\ny = df_train['label'].values\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)\n\nscaler = StandardScaler().fit(X_train[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890']])\ntemp = scaler.transform(X_train[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890']])\ntemp = pandas.DataFrame(temp).rename(columns={0:'bid_qty_std',1:'ask_qty_std',2:'buy_qty_std',3:'sell_qty_std',4:'volume_std',5:'X17_std',6:'X25_std',7:'X198_std',8:'X205_std',9:'X413_std',10:'X415_std',11:'X421_std',12:'X594_std',13:'X596_std',14:'X606_std',15:'X610_std',16:'X663_std',17:'X667_std',18:'X675_std',19:'X681_std',20:'X687_std',21:'X693_std',22:'X865_std',23:'X875_std',24:'X876_std',25:'X879_std',26:'X880_std',27:'X881_std',28:'X882_std',29:'X883_std',30:'X890_std'})\ntemp.index = X_train.index\nX_train = X_train.join(temp)\nX_train = X_train.drop(columns=['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890'])\n\nscaler = StandardScaler().fit(X_test[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890','year','month','day']])\ntemp = scaler.transform(X_test[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890','year','month','day']])\ntemp = pandas.DataFrame(temp).rename(columns={0:'bid_qty_std',1:'ask_qty_std',2:'buy_qty_std',3:'sell_qty_std',4:'volume_std',5:'X17_std',6:'X25_std',7:'X198_std',8:'X205_std',9:'X413_std',10:'X415_std',11:'X421_std',12:'X594_std',13:'X596_std',14:'X606_std',15:'X610_std',16:'X663_std',17:'X667_std',18:'X675_std',19:'X681_std',20:'X687_std',21:'X693_std',22:'X865_std',23:'X875_std',24:'X876_std',25:'X879_std',26:'X880_std',27:'X881_std',28:'X882_std',29:'X883_std',30:'X890_std',31:'year_std',32:'month_std',33:'day_std'})\ntemp.index = X_test.index\nX_test = X_test.join(temp)\nX_test = X_test.drop(columns=['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890'])\n\n#X_train, X_test, y_train, y_test = X_train.values, X_test.values, y_train.values, y_test.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:39.672552Z","iopub.execute_input":"2025-07-08T05:54:39.673284Z","iopub.status.idle":"2025-07-08T05:54:40.645686Z","shell.execute_reply.started":"2025-07-08T05:54:39.673256Z","shell.execute_reply":"2025-07-08T05:54:40.644803Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Selection (Defunc)","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\n\nsgd = SGDRegressor()\npar = PassiveAggressiveRegressor()\n\nmodels = [('SGD Reg', sgd), ('PA Reg', par)]\n\n\nfor model_name, model in models:\n    ave_r2, ave_mae, ave_mape, ave_rmse, ave_mdae = [],[],[],[],[]\n    for i in range(0,2):\n        kfold = model_selection.KFold(n_splits=5, random_state=i, shuffle=True) \n\t\n    \t# execute cross val to est skill of ML model (cross_validate faster)\n        start = time.time()\n        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold, scoring = ('r2','neg_root_mean_squared_error','neg_mean_absolute_error','neg_mean_absolute_percentage_error','neg_median_absolute_error'),return_train_score=False)\n        end = time.time() \n        duration =  end-start\n        \n        ave_r2.append(cv_results['test_r2'].mean())\n        ave_mae.append(cv_results['test_neg_mean_absolute_error'].mean())\n        ave_mape.append(cv_results['test_neg_mean_absolute_percentage_error'].mean())\n        ave_rmse.append(cv_results['test_neg_root_mean_squared_error'].mean())\n        ave_mdae.append(cv_results['test_neg_median_absolute_error'].mean())\n\n    # print results\n    import statistics\n    msg1 = \"%s: mean R2: %f (SD: %f) \\nMAE: %f (SD: %f) \\nMAPE: %f (SD: %f) \\nRMSE: %f (SD: %f) \\nMDAE: %f (SD: %f)\" % (model_name, statistics.mean(ave_r2),statistics.stdev(ave_r2), statistics.mean(ave_mae),statistics.stdev(ave_mae), statistics.mean(ave_mape),statistics.stdev(ave_mape), statistics.mean(ave_rmse),statistics.stdev(ave_rmse), statistics.mean(ave_mdae),statistics.stdev(ave_mdae))\n    print(msg1, \"\\n Time ~\", duration)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:55:18.91149Z","iopub.execute_input":"2025-07-08T05:55:18.912286Z","iopub.status.idle":"2025-07-08T05:59:24.279224Z","shell.execute_reply.started":"2025-07-08T05:55:18.912242Z","shell.execute_reply":"2025-07-08T05:59:24.278123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\n\netr = ExtraTreesRegressor()\nhgbr =  HistGradientBoostingRegressor()\nrfr = RandomForestRegressor()\n\nmodels = [('Extra Trees Reg Ensemble', etr),('Random Forest Reg', rfr),('Hist GB Reg',hgbr)]\n\n\nfor model_name, model in models:\n    ave_r2, ave_mae, ave_mape, ave_rmse, ave_mdae = [],[],[],[],[]\n    for i in range(0,2):\n        kfold = model_selection.KFold(n_splits=5, random_state=i, shuffle=True) # stratified for classing\n\t\n    \t# execute cross val to est skill of ML model (cross_validate faster)\n        start = time.time()\n        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold, scoring = ('r2','neg_root_mean_squared_error','neg_mean_absolute_error','neg_mean_absolute_percentage_error','neg_median_absolute_error'),return_train_score=False)\n        end = time.time() \n        duration =  end-start\n        \n        ave_r2.append(cv_results['test_r2'].mean())\n        ave_mae.append(cv_results['test_neg_mean_absolute_error'].mean())\n        ave_mape.append(cv_results['test_neg_mean_absolute_percentage_error'].mean())\n        ave_rmse.append(cv_results['test_neg_root_mean_squared_error'].mean())\n        ave_mdae.append(cv_results['test_neg_median_absolute_error'].mean())\n\n    # print results\n    import statistics\n    msg1 = \"%s: mean R2: %f (SD: %f) \\nMAE: %f (SD: %f) \\nMAPE: %f (SD: %f) \\nRMSE: %f (SD: %f) \\nMDAE: %f (SD: %f)\" % (model_name, statistics.mean(ave_r2),statistics.stdev(ave_r2), statistics.mean(ave_mae),statistics.stdev(ave_mae), statistics.mean(ave_mape),statistics.stdev(ave_mape), statistics.mean(ave_rmse),statistics.stdev(ave_rmse), statistics.mean(ave_mdae),statistics.stdev(ave_mdae))\n    print(msg1, \"\\n Time ~\", duration)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import Lasso, LinearRegression, ElasticNet\n\nlsr = Lasso()\nlr = LinearRegression()\nelst = ElasticNet()\n\nmodels = [('Lasso', lsr), ('Linear', lr), ('ElasticNet', elst)]\n\n\nfor model_name, model in models:\n    ave_r2, ave_mae, ave_mape, ave_rmse, ave_mdae = [],[],[],[],[]\n    for i in range(0,6):\n        kfold = model_selection.KFold(n_splits=5) # not stratified, no class\n\t\n    \t# execute cross val to est skill of ML model (cross_validate faster)\n        start = time.time()\n        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold, scoring = ('r2','neg_root_mean_squared_error','neg_mean_absolute_error','neg_mean_absolute_percentage_error','neg_median_absolute_error'),return_train_score=False)\n        end = time.time() \n        duration =  end-start\n        \n        ave_r2.append(cv_results['test_r2'].mean())\n        ave_mae.append(cv_results['test_neg_mean_absolute_error'].mean())\n        ave_mape.append(cv_results['test_neg_root_mean_squared_error'].mean())\n        ave_rmse.append(cv_results['test_neg_mean_absolute_percentage_error'].mean())\n        ave_mdae.append(cv_results['test_neg_median_absolute_error'].mean())\n\n    # print results\n    import statistics\n    msg1 = \"%s: mean R2: %f (SD: %f) \\nMAE: %f (SD: %f) \\nMAPE: %f (SD: %f) \\nRMSE: %f (SD: %f) \\nMDAE: %f (SD: %f)\" % (model_name, statistics.mean(ave_r2),statistics.stdev(ave_r2), statistics.mean(ave_mae),statistics.stdev(ave_mae), statistics.mean(ave_mape),statistics.stdev(ave_mape), statistics.mean(ave_rmse),statistics.stdev(ave_rmse), statistics.mean(ave_mdae),statistics.stdev(ave_mdae))\n    print(msg1, \"\\n Time ~\", duration)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T07:48:44.997301Z","iopub.status.idle":"2025-07-08T07:48:44.997705Z","shell.execute_reply.started":"2025-07-08T07:48:44.997534Z","shell.execute_reply":"2025-07-08T07:48:44.997548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Perceptron, Lasso, LinearRegression, ElasticNet\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\n\nlsr = Lasso()\nlr = LinearRegression()\nelst = ElasticNet()\nsvr_rbf = SVR(kernel='rbf')\n\nkfold = model_selection.KFold(n_splits=5) \n\nstk = StackingRegressor(estimators=[('Lasso', lsr), ('Linear', lr), ('ElasticNet', elst)], final_estimator=svr_rbf, cv=kfold)\n\n# Training the stacking classifier\n\nstk.fit(X_train, y_train)\n\n# Evaluate and visualize the fit\n\nprint(\"RMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f\" % (metrics.root_mean_squared_error(y_test,stk.predict(X_test)), metrics.r2_score(y_test,stk.predict(X_test)), metrics.mean_absolute_percentage_error(y_test,stk.predict(X_test)),metrics.median_absolute_error(y_test,stk.predict(X_test)),metrics.mean_absolute_error(y_test,stk.predict(X_test)) ))\n\nwith plt.style.context(('seaborn-whitegrid')):\n    plt.scatter(X_train, y_train, c='lightgray')\n    plt.plot(X_train, stk.predict(X_train), c='darkgreen', lw=2)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T07:48:44.999951Z","iopub.status.idle":"2025-07-08T07:48:45.000296Z","shell.execute_reply.started":"2025-07-08T07:48:45.000157Z","shell.execute_reply":"2025-07-08T07:48:45.000171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Perceptron, Lasso, LinearRegression, ElasticNet\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\n\nlr = LinearRegression()\nsvr_lin = SVR(kernel='linear')\nelst = ElasticNet()\nsvr_rbf = SVR(kernel='rbf')\n\nkfold = model_selection.KFold(n_splits=5) \n\nstk2 = StackingRegressor(estimators=[('ElasticNet', elst), ('Linear', lr), ('Linear SVR', svr_lin)], final_estimator=svr_rbf, cv=kfold)\n\n# Training the stacking classifier\n\nstk2.fit(X_train, y_train)\n\n# Evaluate and visualize the fit\n\nprint(\"RMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f\" % (metrics.root_mean_squared_error(y_test,stk2.predict(X_test)), metrics.r2_score(y_test,stk2.predict(X_test)), metrics.mean_absolute_percentage_error(y_test,stk2.predict(X_test)),metrics.median_absolute_error(y_test,stk2.predict(X_test)),metrics.mean_absolute_error(y_test,stk2.predict(X_test)) ))\n\nwith plt.style.context(('seaborn-whitegrid')):\n    plt.scatter(X_train, y_train, c='lightgray')\n    plt.plot(X_train, stk.predict(X_train), c='darkgreen', lw=2)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T07:48:45.002077Z","iopub.status.idle":"2025-07-08T07:48:45.002519Z","shell.execute_reply.started":"2025-07-08T07:48:45.002315Z","shell.execute_reply":"2025-07-08T07:48:45.002338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Standardise ALL, Drop ALL","metadata":{}},{"cell_type":"code","source":"## defunc\nX = df_train.loc[:, df_train.columns != 'label']\ny = df_train.loc[:, df_train.columns == 'label']\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)\n\nscaler = StandardScaler().fit(X_train)\ntemp = scaler.transform(X_train)\nrename_dict = dict(zip(temp.columns,X_train.columns))\nX_train = pandas.DataFrame(temp).rename(columns=rename_dict)\n\nscaler = StandardScaler().fit(X_test)\ntemp = scaler.transform(X_test)\nrename_dict = dict(zip(temp.columns, X_test.columns))\nX_test = pandas.DataFrame(temp).rename(columns=rename_dict)","metadata":{"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df_train.loc[:, df_train.columns != 'label']\ny = df_train['label'].values\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:22:39.675602Z","iopub.execute_input":"2025-07-23T15:22:39.675943Z","iopub.status.idle":"2025-07-23T15:22:39.780918Z","shell.execute_reply.started":"2025-07-23T15:22:39.675919Z","shell.execute_reply":"2025-07-23T15:22:39.779624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:22:39.782126Z","iopub.execute_input":"2025-07-23T15:22:39.782519Z","iopub.status.idle":"2025-07-23T15:22:39.791545Z","shell.execute_reply.started":"2025-07-23T15:22:39.782490Z","shell.execute_reply":"2025-07-23T15:22:39.790001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Lasso, LinearRegression, ElasticNet, Ridge\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\nimport sklearn.pipeline\nimport gc\n\ngc.collect()\n#sgd = Pipeline([('std', StandardScaler()), ('sgd',SGDRegressor())])\n#par = Pipeline([('std', StandardScaler()), ('par',PassiveAggressiveRegressor())])\n#etr = Pipeline([('std', StandardScaler()), ('etr',ExtraTreesRegressor())])\nhgbr =  Pipeline([('std', StandardScaler()), ('hgbr',HistGradientBoostingRegressor())])\n#rfr = Pipeline([('std', StandardScaler()), ('rfr',RandomForestRegressor())])\nlsr = Pipeline([('std', StandardScaler()), ('lsr',Lasso())])\nlr = Pipeline([('std', StandardScaler()), ('lr',LinearRegression())])\nelst = Pipeline([('std', StandardScaler()), ('elst',ElasticNet())])\nrd = Pipeline([('std', StandardScaler()), ('rd',Ridge())])\n\nmodels = [('Hist GB Reg',hgbr), ('Lasso', lsr), ('Linear', lr), ('ElasticNet', elst), ('Ridge', rd)]\n\nfor model_name, model in models:\n    ave_r2, ave_mae, ave_mape, ave_rmse, ave_mdae, ave_time = [],[],[],[],[],[]\n    for i in range(0,5):\n        kfold = model_selection.KFold(n_splits=5, random_state=i, shuffle=True) \n\t\n    \t# execute cross val to est skill of ML model (cross_validate faster)\n        start = time.time()\n        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold, scoring = ('r2','neg_root_mean_squared_error','neg_mean_absolute_error','neg_mean_absolute_percentage_error','neg_median_absolute_error'),return_train_score=False)\n        end = time.time() \n        duration =  end-start\n        \n        ave_r2.append(cv_results['test_r2'].mean())\n        ave_mae.append(cv_results['test_neg_mean_absolute_error'].mean())\n        ave_mape.append(cv_results['test_neg_mean_absolute_percentage_error'].mean())\n        ave_rmse.append(cv_results['test_neg_root_mean_squared_error'].mean())\n        ave_mdae.append(cv_results['test_neg_median_absolute_error'].mean())\n        ave_time.append(duration)\n\n    # print results\n    import statistics\n    msg1 = \"%s: mean R2: %f \\nMAE: %f \\nMAPE: %f \\nRMSE: %f \\nMDAE: %f\" % (model_name, statistics.mean(ave_r2), statistics.mean(ave_mae), statistics.mean(ave_mape), statistics.mean(ave_rmse), statistics.mean(ave_mdae))\n    print(msg1, \"\\n Time ~\", statistics.mean(ave_time))\n    gc.collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T14:25:16.324441Z","iopub.execute_input":"2025-07-22T14:25:16.32533Z","iopub.status.idle":"2025-07-22T14:28:03.51801Z","shell.execute_reply.started":"2025-07-22T14:25:16.325299Z","shell.execute_reply":"2025-07-22T14:28:03.516727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Ridge, Lasso, LinearRegression, ElasticNet\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nimport gc\n\nlsr = Pipeline([('std', StandardScaler()), ('lsr',Lasso())])\nrd = Pipeline([('std', StandardScaler()), ('lr',Ridge())])\nelst = Pipeline([('std', StandardScaler()), ('elst',ElasticNet(l1_ratio=0.2, alpha=0.9))])\npar = Pipeline([('std', StandardScaler()), ('par',PassiveAggressiveRegressor())])\nhgbr =  HistGradientBoostingRegressor()\nsgd = Pipeline([('std', StandardScaler()), ('sgd',SGDRegressor())])\n\nkfold = model_selection.KFold(n_splits=5) \n\nstk = Pipeline([('std',StandardScaler()),('stacker',StackingRegressor(estimators=[('Lasso', lsr), ('Ridge', rd), ('ElasticNet', elst)], final_estimator=rd, cv=kfold))])\nstk2 = Pipeline([('std',StandardScaler()),('stacker',StackingRegressor(estimators=[('PAR', par), ('Ridge', rd), ('SGD', sgd)], final_estimator=hgbr, cv=kfold))])\n\n\n# Training the stacking classifier\nfor stack in [stk,stk2]:\n    start = time.time()\n    stack.fit(X_train, y_train)\n    end = time.time()\n    duration = end - start\n    \n    # Evaluate and visualize the fit\n    #with plt.style.context(('seaborn-whitegrid')):\n    #    plt.scatter(X_train[['volume']], y_train, c='darkred')\n    #    plt.scatter(X_train[['volume']], stack.predict(X_train), c='darkgreen')\n    #plt.show()\n    \n    print(\"MSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (mean_squared_error(y_test,stack.predict(X_test)), r2_score(y_test,stack.predict(X_test)), mean_absolute_percentage_error(y_test,stack.predict(X_test)),median_absolute_error(y_test,stack.predict(X_test)),mean_absolute_error(y_test,stack.predict(X_test)), duration ))    \n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T13:58:05.994212Z","iopub.execute_input":"2025-07-22T13:58:05.99517Z","iopub.status.idle":"2025-07-22T13:58:31.868216Z","shell.execute_reply.started":"2025-07-22T13:58:05.995139Z","shell.execute_reply":"2025-07-22T13:58:31.867143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Updated to above, removal of SVR, too slow @ 12 months dataset, defunc now\nfrom sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Ridge, Lasso, LinearRegression, ElasticNet\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\n\nlsr = Pipeline([('std', StandardScaler()), ('lsr',Lasso())])\nrd = Pipeline([('std', StandardScaler()), ('lr',Ridge())])\nelst = Pipeline([('std', StandardScaler()), ('elst',ElasticNet())])\nsvr_rbf = SVR(kernel='rbf')\nsvr_lin = SVR(kernel='linear')\nhgbr =  Pipeline([('std', StandardScaler()), ('hgbr',HistGradientBoostingRegressor())])\n\nkfold = model_selection.KFold(n_splits=5) \n\nstk = Pipeline([('std',StandardScaler()),('stacker',StackingRegressor(estimators=[('Lasso', lsr), ('Ridge', rd), ('ElasticNet', elst)], final_estimator=svr_rbf, cv=kfold))])\nstk2 = Pipeline([('std',StandardScaler()),('stacker',StackingRegressor(estimators=[('ElasticNet', elst), ('Histogram based GB', hgbr), ('Linear SVR', svr_lin)], final_estimator=svr_rbf, cv=kfold))])\n\n\n# Training the stacking classifier\nfor stack in [stk,stk2]:\n    #stack.fit(X_train, y_train)\n    # execute cross val to est skill of ML model (cross_validate faster)\n    start = time.time()\n    cv_results = model_selection.cross_validate(stack, X_train, y_train, cv=kfold, scoring = ('r2','neg_root_mean_squared_error','neg_mean_absolute_error','neg_mean_absolute_percentage_error','neg_median_absolute_error'),return_train_score=False)\n    end = time.time()\n    duration =  end-start\n    \n    # Evaluate and visualize the fit\n    \n    #print(\"MSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f\" % (mean_squared_error(y_test,stack.predict(X_test)), r2_score(y_test,stack.predict(X_test)), mean_absolute_percentage_error(y_test,stack.predict(X_test)),median_absolute_error(y_test,stack.predict(X_test)),mean_absolute_error(y_test,stack.predict(X_test)) ))\n    msg1 = \"%s: mean R2: %f \\nMAE: %f \\nMAPE: %f \\nRMSE: %f \\nMDAE: %f\" % (model_name, statistics.mean(cv_results['test_r2']), statistics.mean(cv_results['test_neg_mean_absolute_error']), statistics.mean(cv_results['test_neg_mean_absolute_percentage_error']), statistics.mean(cv_results['test_neg_root_mean_squared_error']), statistics.mean(cv_results['test_neg_median_absolute_error']))\n    print(msg1, \"\\n Time ~\", statistics.mean(duration))\n    \n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T03:47:19.115637Z","iopub.execute_input":"2025-07-18T03:47:19.116041Z","execution_failed":"2025-07-18T03:57:40.147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with plt.style.context(('seaborn-whitegrid')):\n        plt.scatter(X_train, y_train, c='darkred')\n        plt.scatter(X_train, stack.predict(X_train), c='darkgreen', lw=2)\n    \n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Hyper Parameter Tuning","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Perceptron, Lasso, LinearRegression, ElasticNet\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nimport gc\n\nparam_grid_sgd = {\n    'loss': ['squared_error', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive'],\n    'penalty': ['l2', 'l1', 'elasticnet']\n}\n\nparam_grid = {\n    'loss': ['squared_epsilon_insensitive', 'epsilon_insensitive'],\n    'epsilon': [0.1, 0.2, 0.5],\n    'C': [0.2,1,2]\n}\n\nmodel = PassiveAggressiveRegressor(tol=1e-4)\n\nkfold = model_selection.KFold(n_splits=5)\nstart = time.time()\n# Define the grid search we want to run. Run it with six cpus in parallel.\ngs_cv = model_selection.GridSearchCV(model, param_grid, cv=kfold, n_jobs=6, verbose=100)\n\n# Run the grid search - on only the training data\ngs_cv.fit(X_train, y_train)\n\n# Print the parameters that gave us the best result\nprint(gs_cv.best_params_)\nend = time.time()\nprint(end - start)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:40:13.353516Z","iopub.execute_input":"2025-07-23T15:40:13.353888Z","iopub.status.idle":"2025-07-23T15:40:47.102822Z","shell.execute_reply.started":"2025-07-23T15:40:13.353865Z","shell.execute_reply":"2025-07-23T15:40:47.101735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##ensemble results: {'l2_regularization': 0.9, 'learning_rate': 0.3, 'min_samples_leaf': 20}\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nimport gc\n\nparam_grid = {\n    'l2_regularization': [0,0.3,0.6,0.9],\n    'learning_rate': [0,0.3,0.6,0.9],\n    'min_samples_leaf': [20,40]\n}\n\nmodel = hgbr =  HistGradientBoostingRegressor()\n\nkfold = model_selection.KFold(n_splits=5)\nstart = time.time()\n# Define the grid search we want to run. Run it with six cpus in parallel.\ngs_cv = model_selection.GridSearchCV(model, param_grid, cv=kfold, n_jobs=6, verbose=100)\n\n# Run the grid search - on only the training data\ngs_cv.fit(X_train, y_train)\nend = time.time()\n\n# Print the parameters that gave us the best result\nprint(gs_cv.best_params_)\nprint(\"RMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,gs_cv.predict(X_test))), r2_score(y_test,gs_cv.predict(X_test)), mean_absolute_percentage_error(y_test,gs_cv.predict(X_test)),median_absolute_error(y_test,gs_cv.predict(X_test)),mean_absolute_error(y_test,gs_cv.predict(X_test)),end-start ))    \n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:40:47.104656Z","iopub.execute_input":"2025-07-23T15:40:47.105102Z","iopub.status.idle":"2025-07-23T15:42:45.999700Z","shell.execute_reply.started":"2025-07-23T15:40:47.105062Z","shell.execute_reply":"2025-07-23T15:42:45.998721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##ensemble results2: {'l2_regularization': 0.3, 'learning_rate': 0.2, 'max_leaf_nodes': 51, 'min_samples_leaf': 10}\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\n\nparam_grid = {\n    'l2_regularization': [0.3,0.9,1],\n    'learning_rate': [0.1,0.2,0.3],\n    'min_samples_leaf': [10,20],\n    'max_leaf_nodes': [21,31,51]\n}\n\nmodel = hgbr =  HistGradientBoostingRegressor()\n\nkfold = model_selection.KFold(n_splits=5)\nstart = time.time()\n# Define the grid search we want to run. Run it with six cpus in parallel.\ngs_cv = model_selection.GridSearchCV(model, param_grid, cv=kfold, n_jobs=6, verbose=100)\n\n# Run the grid search - on only the training data\ngs_cv.fit(X_train, y_train)\nend = time.time()\n\n# Print the parameters that gave us the best result\nprint(gs_cv.best_params_)\nprint(\"RMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,gs_cv.predict(X_test))), r2_score(y_test,gs_cv.predict(X_test)), mean_absolute_percentage_error(y_test,gs_cv.predict(X_test)),median_absolute_error(y_test,gs_cv.predict(X_test)),mean_absolute_error(y_test,gs_cv.predict(X_test)),end-start ))    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:42:46.000644Z","iopub.execute_input":"2025-07-23T15:42:46.000951Z","iopub.status.idle":"2025-07-23T15:48:59.944824Z","shell.execute_reply.started":"2025-07-23T15:42:46.000928Z","shell.execute_reply":"2025-07-23T15:48:59.943354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##ensemble results3: {'max_iter': 900, 'tol': 1e-11}\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\n\nparam_grid = {\n    'max_iter': [50,100,200,400,800,900,1000],\n    'tol': [1e-11,1e-12]\n}\n\nmodel = hgbr =  HistGradientBoostingRegressor(l2_regularization = 0.9, learning_rate = 0.3, min_samples_leaf = 20)\n\nkfold = model_selection.KFold(n_splits=5)\nstart = time.time()\n# Define the grid search we want to run. Run it with six cpus in parallel.\ngs_cv = model_selection.GridSearchCV(model, param_grid, cv=kfold, n_jobs=6, verbose=100)\n\n# Run the grid search - on only the training data\ngs_cv.fit(X_train, y_train)\nend = time.time()\n\n# Print the parameters that gave us the best result\nprint(gs_cv.best_params_)\nprint(\"RMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,gs_cv.predict(X_test))), r2_score(y_test,gs_cv.predict(X_test)), mean_absolute_percentage_error(y_test,gs_cv.predict(X_test)),median_absolute_error(y_test,gs_cv.predict(X_test)),mean_absolute_error(y_test,gs_cv.predict(X_test)),end-start ))    \n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:48:59.946555Z","iopub.execute_input":"2025-07-23T15:48:59.948995Z","iopub.status.idle":"2025-07-23T15:51:09.753216Z","shell.execute_reply.started":"2025-07-23T15:48:59.948952Z","shell.execute_reply":"2025-07-23T15:51:09.752343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##ensemble results4: {'l2_regularization': 0, 'learning_rate': 0.3, 'max_iter': 800, 'min_samples_leaf': 20, 'tol': 1e-12}\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\n\nparam_grid = {\n    'l2_regularization': [0,0.3,0.6,0.9],\n    'learning_rate': [0,0.3,0.6,0.9],\n    'min_samples_leaf': [20,40],\n    'max_iter': [50,100,200,400,800,900,1000],\n    'tol': [1e-11,1e-12]\n}\n\nmodel = hgbr =  HistGradientBoostingRegressor()\n\nkfold = model_selection.KFold(n_splits=5)\nstart = time.time()\n# Define the grid search we want to run. Run it with six cpus in parallel.\ngs_cv = model_selection.GridSearchCV(model, param_grid, cv=kfold, n_jobs=6, verbose=100)\n\n# Run the grid search - on only the training data\ngs_cv.fit(X_train, y_train)\nend = time.time()\n\n# Print the parameters that gave us the best result\nprint(gs_cv.best_params_)\nprint(\"RMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,gs_cv.predict(X_test))), r2_score(y_test,gs_cv.predict(X_test)), mean_absolute_percentage_error(y_test,gs_cv.predict(X_test)),median_absolute_error(y_test,gs_cv.predict(X_test)),mean_absolute_error(y_test,gs_cv.predict(X_test)),end-start ))    \n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:51:09.755113Z","iopub.execute_input":"2025-07-23T15:51:09.755448Z","iopub.status.idle":"2025-07-23T16:21:08.525686Z","shell.execute_reply.started":"2025-07-23T15:51:09.755424Z","shell.execute_reply":"2025-07-23T16:21:08.524799Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Full training","metadata":{}},{"cell_type":"code","source":"## Partial fitting (Defunc)\n#getting data analysis and ML packages\nimport pandas\nimport gc\nfrom pandas.plotting import scatter_matrix\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import median_absolute_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer #transform different types\nfrom sklearn.datasets import fetch_openml\nimport numpy\nfrom numpy import sqrt\nfrom numpy import sum\nfrom numpy import square\nimport seaborn\nimport matplotlib\nimport statsmodels\nimport time\n\nimport xgboost as xgb\nfrom mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier\nfrom mlxtend.regressor import StackingRegressor\nimport lightgbm as lgbm\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom matplotlib.ticker import LinearLocator, FormatStrFormatter\nfrom mpl_toolkits.mplot3d import Axes3D\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n%load_ext cuml.accel\n%load_ext cudf.pandas\n\n!pip install ipympl\n\n# Restart the kernel after installing for the changes to take effect.\n# After restarting, you can run the cell again.\n\n\nmodel = SGDRegressor(n_jobs=6)\n\nfor s,i in [(0,44640),(44640,87840),(87840,132480),(132480,175680),(175680,220320),(220320,264960),(264960,308160),(308160,352800),(352800,396000),(396000,440640),(440640,485280),(485280,527040)]:\n    df_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\n    df_train = df_train[s:i]\n    df_train['year']   = df_train.index.year\n    df_train['month']  = df_train.index.month\n    df_train['day']    = df_train.index.day\n    df_train['hour']   = df_train.index.hour\n    df_train['minute'] = df_train.index.minute\n    df_train['weekday'] = df_train.index.weekday\n    df_train['date'] = df_train.index.date\n    df_train['second_cont'] = (df_train.index - df_train.index[0]).total_seconds()\n    df_train['minute_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 60\n    df_train['hour_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 3600\n    gc.collect()\n\n    ## Dropping infs\n    infcol = df_train.columns[np.isinf(df_train).any(axis=0)].tolist()\n    df_train = df_train.drop(columns=infcol)\n    gc.collect()\n    ## converting index from datetime to numbers. Cannot use pipeline bc not applying to entire df, would need columnTransformer, too lazy\n    df_train.index = range(0,len(df_train.index))\n\n    \n    X = df_train.loc[:, df_train.columns != 'label']\n    y = df_train.loc[:, df_train.columns == 'label']\n    X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)\n    \n    scaler = StandardScaler().fit(X_train[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890']])\n    temp = scaler.transform(X_train[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890']])\n    temp = pandas.DataFrame(temp).rename(columns={0:'bid_qty_std',1:'ask_qty_std',2:'buy_qty_std',3:'sell_qty_std',4:'volume_std',5:'X17_std',6:'X25_std',7:'X198_std',8:'X205_std',9:'X413_std',10:'X415_std',11:'X421_std',12:'X594_std',13:'X596_std',14:'X606_std',15:'X610_std',16:'X663_std',17:'X667_std',18:'X675_std',19:'X681_std',20:'X687_std',21:'X693_std',22:'X865_std',23:'X875_std',24:'X876_std',25:'X879_std',26:'X880_std',27:'X881_std',28:'X882_std',29:'X883_std',30:'X890_std'})\n    temp.index = X_train.index\n    X_train = X_train.join(temp)\n    \n    scaler = StandardScaler().fit(X_test[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890','year','month','day']])\n    temp = scaler.transform(X_test[['bid_qty','ask_qty','buy_qty','sell_qty','volume','X17','X25','X198','X205','X413','X415','X421','X594','X596','X606','X610','X663','X667','X675','X681','X687','X693','X865','X875','X876','X879','X880','X881','X882','X883','X890','year','month','day']])\n    temp = pandas.DataFrame(temp).rename(columns={0:'bid_qty_std',1:'ask_qty_std',2:'buy_qty_std',3:'sell_qty_std',4:'volume_std',5:'X17_std',6:'X25_std',7:'X198_std',8:'X205_std',9:'X413_std',10:'X415_std',11:'X421_std',12:'X594_std',13:'X596_std',14:'X606_std',15:'X610_std',16:'X663_std',17:'X667_std',18:'X675_std',19:'X681_std',20:'X687_std',21:'X693_std',22:'X865_std',23:'X875_std',24:'X876_std',25:'X879_std',26:'X880_std',27:'X881_std',28:'X882_std',29:'X883_std',30:'X890_std',31:'year_std',32:'month_std',33:'day_std'})\n    temp.index = X_test.index\n    X_test = X_test.join(temp)\n    gc.collect()\n    for epoch in range(1000):\n        model.partial_fit(X_train, y_train)\n        print('Epoch: %f RMSE: %.4f \\nR2: %.4f \\nMAE: %.4f' % (epoch, metrics.root_mean_squared_error(y_test,model.predict(X_test)), metrics.r2_score(y_test,model.predict(X_test)),metrics.mean_absolute_error(y_test,model.predict(X_test)) ) ) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:02.179094Z","iopub.status.idle":"2025-07-08T05:54:02.179334Z","shell.execute_reply.started":"2025-07-08T05:54:02.179221Z","shell.execute_reply":"2025-07-08T05:54:02.179233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Linear\n#getting data analysis and ML packages\nimport pandas\nimport gc\nfrom pandas.plotting import scatter_matrix\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import median_absolute_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer #transform different types\nfrom sklearn.datasets import fetch_openml\nimport numpy\nfrom numpy import sqrt\nfrom numpy import sum\nfrom numpy import square\nimport seaborn\nimport matplotlib\nimport statsmodels\nimport time\nimport numpy as np\n\nimport xgboost as xgb\nfrom mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier\nfrom mlxtend.regressor import StackingRegressor\nimport lightgbm as lgbm\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom matplotlib.ticker import LinearLocator, FormatStrFormatter\nfrom mpl_toolkits.mplot3d import Axes3D\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n\n!pip install ipympl\n\n# Restart the kernel after installing for the changes to take effect.\n# After restarting, you can run the cell again.\n\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Ridge, Lasso, LinearRegression, ElasticNet\n\n# feature derive from index\ndf_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\nreplace = pandas.read_csv('/kaggle/input/x1-x780/pls-762-to-5.csv')\ndf_train = df_train.drop(df_train.loc[:,'X1':'X780'].columns, axis=1)\n\ndf_train['year']   = df_train.index.year\ndf_train['month']  = df_train.index.month\ndf_train['day']    = df_train.index.day\ndf_train['hour']   = df_train.index.hour\ndf_train['minute'] = df_train.index.minute\ndf_train['weekday'] = df_train.index.weekday\ndf_train['second_cont'] = (df_train.index - df_train.index[0]).total_seconds()\ndf_train['minute_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 60\ndf_train['hour_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 3600\ngc.collect()\n\n# lag k\ndf_train['lag_1'] = df_train['label'].shift(periods=1)\ndf_train['lag_2'] = df_train['label'].shift(periods=2)\ndf_train['lag_3'] = df_train['label'].shift(periods=3)\ndf_train['lag_4'] = df_train['label'].shift(periods=4)\ndf_train['lag_5'] = df_train['label'].shift(periods=5)\ndf_train['lag_6'] = df_train['label'].shift(periods=6)\ndf_train['lag_7'] = df_train['label'].shift(periods=7)\ndf_train['lag_8'] = df_train['label'].shift(periods=8)\ndf_train['lag_9'] = df_train['label'].shift(periods=9)\ndf_train['lag_10'] = df_train['label'].shift(periods=10)\ndf_train['lag_11'] = df_train['label'].shift(periods=11)\ndf_train['lag_12'] = df_train['label'].shift(periods=12)\ndf_train['lag_13'] = df_train['label'].shift(periods=13)\ndf_train['lag_14'] = df_train['label'].shift(periods=14)\ndf_train['lag_15'] = df_train['label'].shift(periods=15)\ndf_train['lag_16'] = df_train['label'].shift(periods=16)\ndf_train.loc[:,'lag_1':'lag_16'] = df_train.loc[:,'lag_1':'lag_16'].replace({np.nan: df_train['label'].mean()})\n\n## Dropping infs\n#infcol = df_train.columns[np.isinf(df_train).any(axis=0)].tolist()\n#df_train = df_train.drop(columns=infcol)\ngc.collect()\n## converting index from datetime to numbers. Cannot use pipeline bc not applying to entire df, would need columnTransformer, too lazy\ndf_train.index = range(0,len(df_train.index))\n\n# join\ndf_train = df_train.join(replace)\ndf_train = df_train.rename(columns={'0':'PLS_1','1':'PLS_2','2':'PLS_3','3':'PLS_4','4':'PLS_5'})\n\n# split\nX = df_train.loc[:, df_train.columns != 'label']\ny = df_train['label'].values\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)\n\ngc.collect()\n\nrd = Pipeline([('std', StandardScaler()), ('rd',Ridge())])\npar = Pipeline([('std', StandardScaler()), ('par',PassiveAggressiveRegressor())])\nsgd = Pipeline([('std', StandardScaler()), ('sgd',SGDRegressor())])\n\nstart1 = time.time()\nrd.fit(X_train, y_train)\nend1 = time.time()\n\nstart2 = time.time()\npar.fit(X_train, y_train)\nend2 = time.time()\n\nstart3 = time.time()\nsgd.fit(X_train, y_train)\nend3 = time.time()\n\nstart4 = time.time()\nvt = VotingRegressor(estimators=[('PAR', par), ('Ridge', rd), ('SGD', sgd)])\nvt.fit(X_train, y_train)\nend4 = time.time()\n\n#start5 = time.time()\n#br = BaggingRegressor(estimator=HistGradientBoostingRegressor(l2_regularization = 0.9, learning_rate = 0.3), n_estimators=5, max_samples=0.1, oob_score = True)\n#br.fit(X_train, y_train)\n#end5 = time.time()\n\nprint(\"RD \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,rd.predict(X_test))), r2_score(y_test,rd.predict(X_test)), mean_absolute_percentage_error(y_test,rd.predict(X_test)),median_absolute_error(y_test,rd.predict(X_test)),mean_absolute_error(y_test,rd.predict(X_test)),end1-start1 ))\nprint(\"PAR \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,par.predict(X_test))), r2_score(y_test,par.predict(X_test)), mean_absolute_percentage_error(y_test,par.predict(X_test)),median_absolute_error(y_test,par.predict(X_test)),mean_absolute_error(y_test,par.predict(X_test)),end2-start2 ))\nprint(\"SGD \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,sgd.predict(X_test))), r2_score(y_test,sgd.predict(X_test)), mean_absolute_percentage_error(y_test,sgd.predict(X_test)),median_absolute_error(y_test,sgd.predict(X_test)),mean_absolute_error(y_test,sgd.predict(X_test)),end3-start3 ))\nprint(\"VT \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,vt.predict(X_test))), r2_score(y_test,vt.predict(X_test)), mean_absolute_percentage_error(y_test,vt.predict(X_test)),median_absolute_error(y_test,vt.predict(X_test)),mean_absolute_error(y_test,vt.predict(X_test)),end4-start4 ))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:39:49.656874Z","iopub.status.idle":"2025-07-23T15:39:49.657277Z","shell.execute_reply.started":"2025-07-23T15:39:49.657035Z","shell.execute_reply":"2025-07-23T15:39:49.657077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pandas.DataFrame({'y-true':y_test,\n                  'rd-pred':rd.predict(X_test),\n                  'par-pred':par.predict(X_test),\n                  'sgd-pred':sgd.predict(X_test),\n                  'vt-pred':vt.predict(X_test)\n    \n}).corr()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:25:19.908329Z","iopub.status.idle":"2025-07-23T15:25:19.908639Z","shell.execute_reply.started":"2025-07-23T15:25:19.908500Z","shell.execute_reply":"2025-07-23T15:25:19.908513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ensemble\n#getting data analysis and ML packages\nimport pandas\nimport gc\nfrom pandas.plotting import scatter_matrix\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import median_absolute_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer #transform different types\nfrom sklearn.datasets import fetch_openml\nimport numpy\nfrom numpy import sqrt\nfrom numpy import sum\nfrom numpy import square\nimport seaborn\nimport matplotlib\nimport statsmodels\nimport time\nimport numpy as np\n\nimport xgboost as xgb\nfrom mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier\nfrom mlxtend.regressor import StackingRegressor\nimport lightgbm as lgbm\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom matplotlib.ticker import LinearLocator, FormatStrFormatter\nfrom mpl_toolkits.mplot3d import Axes3D\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n\n!pip install ipympl\n\n# Restart the kernel after installing for the changes to take effect.\n# After restarting, you can run the cell again.\n\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Ridge, Lasso, LinearRegression, ElasticNet\n\n# feature derive from index\ndf_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\nreplace = pandas.read_csv('/kaggle/input/x1-x780/pls-762-to-5.csv')\ndf_train = df_train.drop(df_train.loc[:,'X1':'X780'].columns, axis=1)\n\n#df_train['year']   = df_train.index.year\n#df_train['month']  = df_train.index.month\n#df_train['day']    = df_train.index.day\n#df_train['hour']   = df_train.index.hour\n#df_train['minute'] = df_train.index.minute\n#df_train['weekday'] = df_train.index.weekday\n#df_train['second_cont'] = (df_train.index - df_train.index[0]).total_seconds()\n#df_train['minute_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 60\n#df_train['hour_cont'] = (df_train.index - df_train.index[0]).total_seconds() / 3600\ngc.collect()\n\n# lag k\ndf_train['lag_1'] = df_train['label'].shift(periods=1)\ndf_train['lag_2'] = df_train['label'].shift(periods=2)\ndf_train['lag_3'] = df_train['label'].shift(periods=3)\ndf_train['lag_4'] = df_train['label'].shift(periods=4)\ndf_train['lag_5'] = df_train['label'].shift(periods=5)\ndf_train['lag_6'] = df_train['label'].shift(periods=6)\ndf_train['lag_7'] = df_train['label'].shift(periods=7)\ndf_train['lag_8'] = df_train['label'].shift(periods=8)\ndf_train['lag_9'] = df_train['label'].shift(periods=9)\ndf_train['lag_10'] = df_train['label'].shift(periods=10)\ndf_train['lag_11'] = df_train['label'].shift(periods=11)\ndf_train['lag_12'] = df_train['label'].shift(periods=12)\ndf_train['lag_13'] = df_train['label'].shift(periods=13)\ndf_train['lag_14'] = df_train['label'].shift(periods=14)\ndf_train['lag_15'] = df_train['label'].shift(periods=15)\ndf_train['lag_16'] = df_train['label'].shift(periods=16)\ndf_train.loc[:,'lag_1':'lag_16'] = df_train.loc[:,'lag_1':'lag_16'].replace({np.nan: df_train['label'].mean()})\n\n## Dropping infs\n#infcol = df_train.columns[np.isinf(df_train).any(axis=0)].tolist()\n#df_train = df_train.drop(columns=infcol)\ngc.collect()\n## converting index from datetime to numbers. Cannot use pipeline bc not applying to entire df, would need columnTransformer, too lazy\ndf_train.index = range(0,len(df_train.index))\n\n# join\ndf_train = df_train.join(replace)\ndf_train = df_train.rename(columns={'0':'PLS_1','1':'PLS_2','2':'PLS_3','3':'PLS_4','4':'PLS_5'})\n\n# split\nX = df_train.loc[:, df_train.columns != 'label']\ny = df_train['label'].values\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)\n\ngc.collect()\n\nstart1 = time.time()\nv1 = HistGradientBoostingRegressor(l2_regularization= 0.6, learning_rate= 0.3, max_iter= 200, min_samples_leaf= 20, tol= 1e-12)\nv1.fit(X_train, y_train)\nend1 = time.time()\n\nstart2 = time.time()\nv2 = HistGradientBoostingRegressor(l2_regularization = 0.9, learning_rate = 0.3)\nv2.fit(X_train, y_train)\nend2 = time.time()\n\nstart3 = time.time()\nv3 = HistGradientBoostingRegressor(l2_regularization= 0.6, learning_rate= 0.3, max_iter= 200, min_samples_leaf= 20, tol= 1e-12)\nv3.fit(X_train, y_train)\nend3 = time.time()\n\nstart4 = time.time()\nrd = Pipeline([('std', StandardScaler()), ('lr',Ridge())])\nhgbr =  HistGradientBoostingRegressor(l2_regularization= 0.6, learning_rate= 0.3, max_iter= 200, min_samples_leaf= 20, tol= 1e-12)\nvt = VotingRegressor(estimators=[('rd', rd),('histo', hgbr)])\nvt.fit(X_train, y_train)\nend4 = time.time()\n\n#start5 = time.time()\n#br = BaggingRegressor(estimator=HistGradientBoostingRegressor(l2_regularization = 0.9, learning_rate = 0.3), n_estimators=5, max_samples=0.1, oob_score = True)\n#br.fit(X_train, y_train)\n#end5 = time.time()\n\nprint(\"V1 \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,v1.predict(X_test))), r2_score(y_test,v1.predict(X_test)), mean_absolute_percentage_error(y_test,v1.predict(X_test)),median_absolute_error(y_test,v1.predict(X_test)),mean_absolute_error(y_test,v1.predict(X_test)),end1-start1 ))\nprint(\"V2 \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,v2.predict(X_test))), r2_score(y_test,v2.predict(X_test)), mean_absolute_percentage_error(y_test,v2.predict(X_test)),median_absolute_error(y_test,v2.predict(X_test)),mean_absolute_error(y_test,v2.predict(X_test)),end2-start2 ))\nprint(\"V3 \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,v3.predict(X_test))), r2_score(y_test,v3.predict(X_test)), mean_absolute_percentage_error(y_test,v3.predict(X_test)),median_absolute_error(y_test,v3.predict(X_test)),mean_absolute_error(y_test,v3.predict(X_test)),end3-start3 ))\nprint(\"VT \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,vt.predict(X_test))), r2_score(y_test,vt.predict(X_test)), mean_absolute_percentage_error(y_test,vt.predict(X_test)),median_absolute_error(y_test,vt.predict(X_test)),mean_absolute_error(y_test,vt.predict(X_test)),end4-start4 ))\n#print(\"BR \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,br.predict(X_test))), r2_score(y_test,br.predict(X_test)), mean_absolute_percentage_error(y_test,br.predict(X_test)),median_absolute_error(y_test,br.predict(X_test)),mean_absolute_error(y_test,br.predict(X_test)),end5-start5 ))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:25:19.909694Z","iopub.status.idle":"2025-07-23T15:25:19.909959Z","shell.execute_reply.started":"2025-07-23T15:25:19.909837Z","shell.execute_reply":"2025-07-23T15:25:19.909848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pandas.DataFrame({'y-true':y_test,\n                  'v1-pred':v1.predict(X_test),\n                  'v2-pred':v2.predict(X_test),\n                  'v3-pred':v3.predict(X_test),\n                  'vt-pred':vt.predict(X_test)\n    \n}).corr()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:25:19.910846Z","iopub.status.idle":"2025-07-23T15:25:19.911214Z","shell.execute_reply.started":"2025-07-23T15:25:19.911032Z","shell.execute_reply":"2025-07-23T15:25:19.911072Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Full fitting for loop","metadata":{}},{"cell_type":"code","source":"#ensemble for loop\nimport pandas\nimport gc\nfrom pandas.plotting import scatter_matrix\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import median_absolute_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer #transform different types\nfrom sklearn.datasets import fetch_openml\nimport numpy\nfrom numpy import sqrt\nfrom numpy import sum\nfrom numpy import square\nimport seaborn\nimport matplotlib\nimport statsmodels\nimport time\nimport numpy as np\n\nimport xgboost as xgb\nfrom mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier\nfrom mlxtend.regressor import StackingRegressor\nimport lightgbm as lgbm\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom matplotlib.ticker import LinearLocator, FormatStrFormatter\nfrom mpl_toolkits.mplot3d import Axes3D\n\nfrom sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Lasso, LinearRegression, ElasticNet, Ridge\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\nimport gc\n\nfor i in range(100):\n    start1 = time.time()\n    v1 = HistGradientBoostingRegressor(l2_regularization= 0.6, learning_rate= 0.3, max_iter= 200, min_samples_leaf= 20, tol= 1e-12)\n    v1.fit(X_train, y_train)\n    end1 = time.time()\n    \n    start2 = time.time()\n    v2 = HistGradientBoostingRegressor(l2_regularization= 0.6, learning_rate= 0.3, max_iter= 200, min_samples_leaf= 20, tol= 1e-12)\n    v2.fit(X_train, y_train)\n    end2 = time.time()\n    \n    start3 = time.time()\n    v3 = HistGradientBoostingRegressor(l2_regularization= 0, learning_rate= 0.3, max_iter= 800, min_samples_leaf= 20, tol = 1e-12)\n    v3.fit(X_train, y_train)\n    end3 = time.time()\n    \n    start4 = time.time()\n    rd = Pipeline([('std', StandardScaler()), ('lr',Ridge())])\n    hgbr =  HistGradientBoostingRegressor(l2_regularization= 0.6, learning_rate= 0.3, max_iter= 200, min_samples_leaf= 20, tol= 1e-12)\n    vt = VotingRegressor(estimators=[('rd', rd),('histo', hgbr)])\n    vt.fit(X_train, y_train)\n    end4 = time.time()\n\n    print(\"V1 \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,v1.predict(X_test))), r2_score(y_test,v1.predict(X_test)), mean_absolute_percentage_error(y_test,v1.predict(X_test)),median_absolute_error(y_test,v1.predict(X_test)),mean_absolute_error(y_test,v1.predict(X_test)),end1-start1 ))\n    print(\"V2 \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,v2.predict(X_test))), r2_score(y_test,v2.predict(X_test)), mean_absolute_percentage_error(y_test,v2.predict(X_test)),median_absolute_error(y_test,v2.predict(X_test)),mean_absolute_error(y_test,v2.predict(X_test)),end2-start2 ))\n    print(\"V3 \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,v3.predict(X_test))), r2_score(y_test,v3.predict(X_test)), mean_absolute_percentage_error(y_test,v3.predict(X_test)),median_absolute_error(y_test,v3.predict(X_test)),mean_absolute_error(y_test,v3.predict(X_test)),end3-start3 ))\n    print(\"VT \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,vt.predict(X_test))), r2_score(y_test,vt.predict(X_test)), mean_absolute_percentage_error(y_test,vt.predict(X_test)),median_absolute_error(y_test,vt.predict(X_test)),mean_absolute_error(y_test,vt.predict(X_test)),end4-start4 ))\n\n    ok = input(\"If Break, then Yes\")\n    if ok == 'Yes':\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T16:19:45.558219Z","iopub.execute_input":"2025-07-22T16:19:45.558579Z","iopub.status.idle":"2025-07-22T16:26:08.71294Z","shell.execute_reply.started":"2025-07-22T16:19:45.558554Z","shell.execute_reply":"2025-07-22T16:26:08.711968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Pickling","metadata":{}},{"cell_type":"code","source":"## pickling\nimport joblib\njoblib.dump(v2, '/kaggle/working/best-histo.pkl')\njoblib.dump(v3, '/kaggle/working/2nd-histo.pkl')\njoblib.dump(vt, '/kaggle/working/vt-rd-histo.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T16:26:22.555216Z","iopub.execute_input":"2025-07-22T16:26:22.555599Z","iopub.status.idle":"2025-07-22T16:26:22.618847Z","shell.execute_reply.started":"2025-07-22T16:26:22.555568Z","shell.execute_reply":"2025-07-22T16:26:22.618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## loading\ntemp = joblib.load('/kaggle/working/2nd-histo.pkl')\nprint(temp.predict(X_train.iloc[0:1,:]) )\n\ntemp = joblib.load('/kaggle/working/vt-rd-histo.pkl')\nprint(temp.predict(X_train.iloc[0:1,:]) )\n\ntemp = joblib.load('/kaggle/working/best-histo.pkl')\nprint(temp.predict(X_train.iloc[0:1,:]) )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T11:59:25.211222Z","iopub.execute_input":"2025-07-19T11:59:25.21154Z","iopub.status.idle":"2025-07-19T11:59:25.320261Z","shell.execute_reply.started":"2025-07-19T11:59:25.211517Z","shell.execute_reply":"2025-07-19T11:59:25.318596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## checking\nprint(v3.predict(X_train.iloc[0:1,:]), vt.predict(X_train.iloc[0:1,:]), v2.predict(X_train.iloc[0:1,:]) )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T11:59:58.079466Z","iopub.execute_input":"2025-07-19T11:59:58.080006Z","iopub.status.idle":"2025-07-19T11:59:58.112364Z","shell.execute_reply.started":"2025-07-19T11:59:58.07994Z","shell.execute_reply":"2025-07-19T11:59:58.111638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas\nimport gc\nfrom pandas.plotting import scatter_matrix\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import median_absolute_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.metrics import mean_absolute_error,median_absolute_error,mean_absolute_percentage_error\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer #transform different types\nfrom sklearn.datasets import fetch_openml\nimport numpy\nfrom numpy import sqrt\nfrom numpy import sum\nfrom numpy import square\nimport seaborn\nimport matplotlib\nimport statsmodels\nimport time\nimport numpy as np\n\nimport xgboost as xgb\nfrom mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier\nfrom mlxtend.regressor import StackingRegressor\nimport lightgbm as lgbm\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom matplotlib.ticker import LinearLocator, FormatStrFormatter\nfrom mpl_toolkits.mplot3d import Axes3D\n\nfrom sklearn.linear_model import SGDRegressor, PassiveAggressiveRegressor, Lasso, LinearRegression, ElasticNet, Ridge\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor, VotingRegressor, StackingRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor\nfrom sklearn import metrics\nimport gc\n\n##linear for loop\nrd = Pipeline([('std', StandardScaler()), ('rd',Ridge())])\npar = Pipeline([('std', StandardScaler()), ('par',PassiveAggressiveRegressor(C= 2,epsilon= 0.2,loss= 'squared_epsilon_insensitive'))])\nsgd = Pipeline([('std', StandardScaler()), ('sgd',SGDRegressor())])\n\nfor i in range(100):\n    start1 = time.time()\n    rd.fit(X_train, y_train)\n    end1 = time.time()\n    \n    start2 = time.time()\n    par.fit(X_train, y_train)\n    end2 = time.time()\n    \n    start3 = time.time()\n    sgd.fit(X_train, y_train)\n    end3 = time.time()\n    \n    start4 = time.time()\n    vt = VotingRegressor(estimators=[('PAR', par), ('Ridge', rd), ('SGD', sgd)])\n    vt.fit(X_train, y_train)\n    end4 = time.time()\n    \n    #start5 = time.time()\n    #br = BaggingRegressor(estimator=HistGradientBoostingRegressor(l2_regularization = 0.9, learning_rate = 0.3), n_estimators=5, max_samples=0.1, oob_score = True)\n    #br.fit(X_train, y_train)\n    #end5 = time.time()\n    \n    print(\"RD \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,rd.predict(X_test))), r2_score(y_test,v1.predict(X_test)), mean_absolute_percentage_error(y_test,rd.predict(X_test)),median_absolute_error(y_test,rd.predict(X_test)),mean_absolute_error(y_test,rd.predict(X_test)),end1-start1 ))\n    print(\"PAR \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,par.predict(X_test))), r2_score(y_test,v2.predict(X_test)), mean_absolute_percentage_error(y_test,par.predict(X_test)),median_absolute_error(y_test,par.predict(X_test)),mean_absolute_error(y_test,par.predict(X_test)),end2-start2 ))\n    print(\"SGD \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,sgd.predict(X_test))), r2_score(y_test,v3.predict(X_test)), mean_absolute_percentage_error(y_test,sgd.predict(X_test)),median_absolute_error(y_test,sgd.predict(X_test)),mean_absolute_error(y_test,sgd.predict(X_test)),end3-start3 ))\n    print(\"VT \\nRMSE: %.4f \\nR2: %.4f \\nMAPE: %.4f \\nMDAE: %.4f \\nMAE: %.4f \\n~time: %.4f\" % (np.sqrt(mean_squared_error(y_test,vt.predict(X_test))), r2_score(y_test,vt.predict(X_test)), mean_absolute_percentage_error(y_test,vt.predict(X_test)),median_absolute_error(y_test,vt.predict(X_test)),mean_absolute_error(y_test,vt.predict(X_test)),end4-start4 ))\n\n    pandas.DataFrame({'y-true':y_test,\n                  'v1-pred':v1.predict(X_test),\n                  'v2-pred':v2.predict(X_test),\n                  'v3-pred':v3.predict(X_test),\n                  'vt-pred':vt.predict(X_test)\n    \n    }).corr()\n    \n    ok = input(\"If Break, then Yes\")\n    if ok == 'Yes':\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T16:34:17.18091Z","iopub.execute_input":"2025-07-22T16:34:17.181307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Pickling","metadata":{}},{"cell_type":"code","source":"import joblib\n\njoblib.dump(sgd, '/kaggle/working/sgd.pkl')\njoblib.dump(rd, '/kaggle/working/rd.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T15:25:37.576303Z","iopub.execute_input":"2025-07-19T15:25:37.576943Z","iopub.status.idle":"2025-07-19T15:25:37.585514Z","shell.execute_reply.started":"2025-07-19T15:25:37.576918Z","shell.execute_reply":"2025-07-19T15:25:37.584624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## loading\ntemp = joblib.load('/kaggle/working/sgd.pkl')\nprint(temp.predict(X_train.iloc[0:1,:]) )\n\ntemp = joblib.load('/kaggle/working/rd.pkl')\nprint(temp.predict(X_train.iloc[0:1,:]) )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T15:26:13.675751Z","iopub.execute_input":"2025-07-19T15:26:13.676437Z","iopub.status.idle":"2025-07-19T15:26:13.688517Z","shell.execute_reply.started":"2025-07-19T15:26:13.676413Z","shell.execute_reply":"2025-07-19T15:26:13.68773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## checking\nprint(sgd.predict(X_train.iloc[0:1,:]), rd.predict(X_train.iloc[0:1,:]) )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T15:24:51.958675Z","iopub.execute_input":"2025-07-19T15:24:51.959022Z","iopub.status.idle":"2025-07-19T15:24:51.969782Z","shell.execute_reply.started":"2025-07-19T15:24:51.958995Z","shell.execute_reply":"2025-07-19T15:24:51.968664Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Visualisation","metadata":{}},{"cell_type":"code","source":"X = df_train.loc[:, df_train.columns != 'label']\ny = df_train['label']\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T08:45:28.488506Z","iopub.execute_input":"2025-07-22T08:45:28.488812Z","iopub.status.idle":"2025-07-22T08:45:29.941462Z","shell.execute_reply.started":"2025-07-22T08:45:28.48879Z","shell.execute_reply":"2025-07-22T08:45:29.940689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\nrd = joblib.load(\"/kaggle/input/all-models/scikitlearn/default/1/rd.pkl\")\nsgd = joblib.load(\"/kaggle/input/all-models/scikitlearn/default/1/sgd.pkl\")\nvt = joblib.load(\"/kaggle/input/all-models/scikitlearn/default/1/vt-rd-histo.pkl\")\nbest = joblib.load(\"/kaggle/input/all-models/scikitlearn/default/1/best-histo.pkl\")\nbest2 = joblib.load(\"/kaggle/input/all-models/scikitlearn/default/1/2nd-histo.pkl\")\n\nfor name, i in [('ridge',rd), ('SGD',sgd), ('Voter',vt), ('Histo',best), ('2nd Histo',best2)]:\n    with plt.style.context(('seaborn-whitegrid')):\n        plt.scatter(X_train[['volume']], y_train, c='darkred')\n        plt.scatter(X_train[['volume']], i.predict(X_train), c='darkgreen')\n        plt.title(f\"{name} volume vs label\")\n        plt.xlabel('volume')\n        plt.ylabel('label')\n    plt.show()\n\ncombined = (rd.predict(X_train) + sgd.predict(X_train) + vt.predict(X_train) + best.predict(X_train) + best2.predict(X_train))/5\nwith plt.style.context(('seaborn-whitegrid')):\n        plt.scatter(X_train[['volume']], y_train, c='darkred')\n        plt.scatter(X_train[['volume']], combined, c='darkgreen')\n        plt.title(\"combined volume vs label\")\n        plt.xlabel('volume')\n        plt.ylabel('label')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T08:45:34.744495Z","iopub.execute_input":"2025-07-22T08:45:34.745295Z","iopub.status.idle":"2025-07-22T08:46:14.951375Z","shell.execute_reply.started":"2025-07-22T08:45:34.74527Z","shell.execute_reply":"2025-07-22T08:46:14.950565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rr = pandas.DataFrame(rd.predict(df_train.iloc[:,df_train.columns != 'label']) )\nrsgd = pandas.DataFrame(sgd.predict(df_train.iloc[:,df_train.columns != 'label']), index = rr.index )\nrvt = pandas.DataFrame(vt.predict(df_train.iloc[:,df_train.columns != 'label']) , index = rr.index )\nrbest = pandas.DataFrame(best.predict(df_train.iloc[:,df_train.columns != 'label']) , index = rr.index )\nr2best = pandas.DataFrame(best2.predict(df_train.iloc[:,df_train.columns != 'label']) , index = rr.index )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T11:11:15.877581Z","iopub.status.idle":"2025-07-22T11:11:15.877838Z","shell.execute_reply.started":"2025-07-22T11:11:15.877696Z","shell.execute_reply":"2025-07-22T11:11:15.877706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rr = rd.predict(X_train.iloc[0:1,:])[0]\nrsgd = sgd.predict(X_train.iloc[0:1,:])[0]\nrvt = vt.predict(X_train.iloc[0:1,:])[0]\nrbest = best.predict(X_train.iloc[0:1,:])[0]\nr2best = best2.predict(X_train.iloc[0:1,:])[0]\ncombined =  (rr + rsgd + rvt + rbest + r2best) / 5\ntemp = pandas.DataFrame({'y':[rr, rsgd, rvt, rbest, r2best, combined, 0.188167]})\ntemp.plot(kind='barh')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T06:57:49.813944Z","iopub.execute_input":"2025-07-22T06:57:49.814627Z","iopub.status.idle":"2025-07-22T06:57:50.713104Z","shell.execute_reply.started":"2025-07-22T06:57:49.814605Z","shell.execute_reply":"2025-07-22T06:57:50.712549Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"import joblib\nhist = joblib.load('/kaggle/input/model-new/scikitlearn/default/1/2nd-histo.pkl')\npandas.DataFrame(X.iloc[0,:]).T\nhist.predict(pandas.DataFrame(X.iloc[0,:]).T)\nhist.predict(pandas.DataFrame(X.iloc[0,:]).T)[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T04:45:16.043201Z","iopub.execute_input":"2025-07-23T04:45:16.043614Z","iopub.status.idle":"2025-07-23T04:45:16.347065Z","shell.execute_reply.started":"2025-07-23T04:45:16.043587Z","shell.execute_reply":"2025-07-23T04:45:16.346266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = pandas.DataFrame([0,0,0,0],columns=['Predictions'])\npredictions = pandas.DataFrame({'Predictions':[0,0,0,0]})\npredictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T06:24:24.720623Z","iopub.execute_input":"2025-07-23T06:24:24.720969Z","iopub.status.idle":"2025-07-23T06:24:24.731516Z","shell.execute_reply.started":"2025-07-23T06:24:24.720946Z","shell.execute_reply":"2025-07-23T06:24:24.730495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet', engine='pyarrow')\n\n\ndf_test['lag_1'] = df_test['label'].shift(periods=1)\ndf_test['lag_2'] = df_test['label'].shift(periods=2)\ndf_test['lag_3'] = df_test['label'].shift(periods=3)\ndf_test['lag_4'] = df_test['label'].shift(periods=4)\ndf_test['lag_5'] = df_test['label'].shift(periods=5)\ndf_test['lag_6'] = df_test['label'].shift(periods=6)\ndf_test['lag_7'] = df_test['label'].shift(periods=7)\ndf_test['lag_8'] = df_test['label'].shift(periods=8)\ndf_test['lag_9'] = df_test['label'].shift(periods=9)\ndf_test['lag_10'] = df_test['label'].shift(periods=10)\ndf_test['lag_11'] = df_test['label'].shift(periods=11)\ndf_test['lag_12'] = df_test['label'].shift(periods=12)\ndf_test['lag_13'] = df_test['label'].shift(periods=13)\ndf_test['lag_14'] = df_test['label'].shift(periods=14)\ndf_test['lag_15'] = df_test['label'].shift(periods=15)\ndf_test['lag_16'] = df_test['label'].shift(periods=16)\ndf_test.loc[:,'lag_1':'lag_16'] = df_test.loc[:,'lag_1':'lag_16'].replace({np.nan: 0.03612583479917425})\n## Dropping infs\ninfcol = df_test.columns[np.isinf(df_test).any(axis=0)].tolist()\nprint(infcol)\n\ndf_test = df_test.drop(columns=infcol)\n## converting index from datetime to numbers. Cannot use pipeline bc not applying to entire df, would need columnTransformer, too lazy\ndf_test.index = range(0,len(df_test.index))\n\nimport joblib\npls = joblib.load('/kaggle/input/all-models/scikitlearn/default/1/pls.pkl')\nreplace = pls.transform(df_test.loc[:, 'X1':'X780'])\nreplace = pandas.DataFrame(replace)\ndf_test = df_test.drop(df_test.loc[:,'X1':'X780'].columns, axis=1)\ndf_test = df_test.join(replace)\ndf_test = df_test.rename(columns={0:'PLS_1',1:'PLS_2',2:'PLS_3',3:'PLS_4',4:'PLS_5'})\n\nX = df_test.loc[:, df_test.columns != 'label']\ny = df_test['label'].values\n\nhist = joblib.load('/kaggle/input/model-new/scikitlearn/default/1/2nd-histo.pkl')\nhistb = joblib.load('/kaggle/input/model-new/scikitlearn/default/1/best-histo.pkl')\nrd = joblib.load('/kaggle/input/model-new/scikitlearn/default/1/rd.pkl')\nvt = joblib.load('/kaggle/input/model-new/scikitlearn/default/1/vt-rd-histo.pkl')\n\n\npredictions = []\nfor i in range(len(X)):\n    predictions.append(hist.predict(pandas.DataFrame(X.iloc[i,:]).T)[0])\n    X.loc[i+1,'lag_1':'lag_16'] = X.loc[i,'lag_1':'lag_16'].replace({0: hist.predict(pandas.DataFrame(X.iloc[i,:]).T)[0]})\nsubmission = pandas.DataFrame(predictions, columns=['Predictions-histo2'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T08:20:53.236997Z","iopub.execute_input":"2025-07-23T08:20:53.237364Z","execution_failed":"2025-07-23T08:30:50.479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pandas.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T12:58:09.474887Z","iopub.execute_input":"2025-07-22T12:58:09.475361Z","iopub.status.idle":"2025-07-22T12:58:09.960942Z","shell.execute_reply.started":"2025-07-22T12:58:09.475338Z","shell.execute_reply":"2025-07-22T12:58:09.960186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ES model (Defunc)","metadata":{}},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, RegressorMixin\nfrom statsmodels.tsa.holtwinters import ExponentialSmoothing\n\nclass ExpSmoothingWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, trend=None, seasonal=None, seasonal_periods=None):\n        self.trend = trend\n        self.seasonal = seasonal\n        self.seasonal_periods = seasonal_periods\n\n    def fit(self, X, y):\n        self.model_ = ExponentialSmoothing(\n            y, trend=self.trend, seasonal=self.seasonal, seasonal_periods=self.seasonal_periods\n        ).fit()\n        return self\n\n    def predict(self, X):\n        n_periods = len(X)\n        last_idx = self.model_.fittedvalues.shape[0] - 1\n        forecast = self.model_.predict(start=last_idx + 1, end=last_idx + n_periods)\n        return forecast","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T00:26:08.583209Z","iopub.execute_input":"2025-07-21T00:26:08.583506Z","iopub.status.idle":"2025-07-21T00:26:08.874641Z","shell.execute_reply.started":"2025-07-21T00:26:08.583485Z","shell.execute_reply":"2025-07-21T00:26:08.873698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp = ExpSmoothingWrapper(trend='add', seasonal='add', seasonal_periods=23903)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T00:26:28.778551Z","iopub.execute_input":"2025-07-21T00:26:28.778964Z","iopub.status.idle":"2025-07-21T00:26:28.783954Z","shell.execute_reply.started":"2025-07-21T00:26:28.778942Z","shell.execute_reply":"2025-07-21T00:26:28.782834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# feature derive from index\ndf_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\ndf_test = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet', engine='pyarrow')\ndf_test['label'] = range(0,538150,1)\ndf_test.index = pandas.date_range(start = \"2024/03/01\", freq= 'min', periods = 538150)\ndf_train = df_train.drop(df_train.loc[:,df_train.columns != 'label'], axis=1)\ndf_test = df_test.drop(df_test.loc[:,df_test.columns != 'label'], axis=1)\n## split\ntrain, test = df_train.iloc[:484149], df_train.iloc[484149:]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T03:48:14.385745Z","iopub.execute_input":"2025-07-21T03:48:14.386096Z","iopub.status.idle":"2025-07-21T03:49:20.098895Z","shell.execute_reply.started":"2025-07-21T03:48:14.38607Z","shell.execute_reply":"2025-07-21T03:49:20.098077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check\ndf_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T02:59:22.995202Z","iopub.execute_input":"2025-07-21T02:59:22.995501Z","iopub.status.idle":"2025-07-21T02:59:23.005718Z","shell.execute_reply.started":"2025-07-21T02:59:22.995478Z","shell.execute_reply":"2025-07-21T02:59:23.004943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check\ndf_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T02:59:19.703291Z","iopub.execute_input":"2025-07-21T02:59:19.703598Z","iopub.status.idle":"2025-07-21T02:59:19.727555Z","shell.execute_reply.started":"2025-07-21T02:59:19.703576Z","shell.execute_reply":"2025-07-21T02:59:19.726725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# defunc\n# when .loc dataframe and single column, will make dataframe into matrix, cannot .loc anymore.\n# if .loc[:,'label'].values will make 2D dataframe (n,1) into 2D matrix (n,1) into 1D array (n,) , no column name or index\ny = df_train.loc[:,'label']\ny.loc[:,'label']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp.fit(train.index,train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T00:28:04.02056Z","iopub.execute_input":"2025-07-21T00:28:04.020838Z","execution_failed":"2025-07-21T02:47:42.859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['pred'] = exp.predict(test).values\nfig, ax = plt.subplots()\nplt.plot(test.index, test['pred'])\nplt.plot(test.index, test['label'])\nplt.title(\"time series\")\nplt.tight_layout()\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-21T00:24:28.99Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\n\njoblib.dump(exp, '/kaggle/working/exp.pkl')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-21T00:24:28.99Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ES (non-wrapper)","metadata":{}},{"cell_type":"markdown","source":"#### modify into parquet","metadata":{}},{"cell_type":"code","source":"# feature derive from index\ndf_test = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet', engine='pyarrow')\ndf_test['label'] = range(0,538150,1)\ndf_test.index = pandas.date_range(start = \"2024/03/01\", freq= 'min', periods = 538150)\ndf_test = df_test.drop(df_test.loc[:,df_test.columns != 'label'], axis=1)\n#df_test.to_parquet('/kaggle/working/df_test.parquet')\ndf_train = pandas.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet', engine='pyarrow')\ndf_train = df_train.drop(df_train.loc[:,df_train.columns != 'label'], axis=1)\n## split\ndf_train = df_train.iloc[0:114912,:]\ntrain, test = df_train.iloc[0:71712,:], df_train.iloc[71712:]\n#train, test = df_train.iloc[:484149], df_train.iloc[484149:]\n#df_train.to_parquet('/kaggle/working/df_train.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T08:38:55.051622Z","iopub.execute_input":"2025-07-21T08:38:55.05192Z","iopub.status.idle":"2025-07-21T08:39:06.421904Z","shell.execute_reply.started":"2025-07-21T08:38:55.051901Z","shell.execute_reply":"2025-07-21T08:39:06.421098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### load parquet","metadata":{}},{"cell_type":"code","source":"## defunc\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport statsmodels.formula.api as smf\n\nclass DataSet(dict):\n    def __init__(self, path):\n        self.parquet = pq.ParquetFile(path)\n\n    def __getitem__(self, key):\n        try:\n            return self.parquet.read([key]).to_pandas()[key]\n        except:\n            raise KeyError\n\ndf_train = DataSet('/kaggle/input/removed-drw/df_train.parquet')\ndf_test = DataSet('/kaggle/input/removed-drw/df_test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T04:13:05.198783Z","iopub.execute_input":"2025-07-21T04:13:05.199153Z","iopub.status.idle":"2025-07-21T04:13:05.223035Z","shell.execute_reply.started":"2025-07-21T04:13:05.19913Z","shell.execute_reply":"2025-07-21T04:13:05.222345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from statsmodels.tsa.holtwinters import ExponentialSmoothing\n\nexp = statsmodels.tsa.holtwinters.ExponentialSmoothing(endog=train, trend='add', seasonal='add', seasonal_periods=23904).fit(optimized=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T08:39:27.423931Z","iopub.execute_input":"2025-07-21T08:39:27.424173Z","execution_failed":"2025-07-21T20:34:58.218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp.forecast(steps=30)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-21T20:34:58.218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp.save('/kaggle/working/exp.pkl')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-21T20:34:58.219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fexp.predict(params, start='2024-02-01 00:00:00', end='2024-02-29 23:59:00')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-21T03:12:42.787Z"}},"outputs":[],"execution_count":null}]}