{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:03:28.887274Z","iopub.execute_input":"2025-06-27T11:03:28.887789Z","iopub.status.idle":"2025-06-27T11:03:30.424307Z","shell.execute_reply.started":"2025-06-27T11:03:28.887758Z","shell.execute_reply":"2025-06-27T11:03:30.423637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport polars as pl\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:03:30.425476Z","iopub.execute_input":"2025-06-27T11:03:30.425809Z","iopub.status.idle":"2025-06-27T11:03:33.180923Z","shell.execute_reply.started":"2025-06-27T11:03:30.425790Z","shell.execute_reply":"2025-06-27T11:03:33.180308Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Read Parquet Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:03:33.181664Z","iopub.execute_input":"2025-06-27T11:03:33.182059Z","iopub.status.idle":"2025-06-27T11:04:22.879785Z","shell.execute_reply.started":"2025-06-27T11:03:33.182018Z","shell.execute_reply":"2025-06-27T11:04:22.879163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = pl.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\n# test = pl.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\n# sample_submission = pl.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:22.880617Z","iopub.execute_input":"2025-06-27T11:04:22.880843Z","iopub.status.idle":"2025-06-27T11:04:22.884797Z","shell.execute_reply.started":"2025-06-27T11:04:22.880824Z","shell.execute_reply":"2025-06-27T11:04:22.883956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:22.886770Z","iopub.execute_input":"2025-06-27T11:04:22.886989Z","iopub.status.idle":"2025-06-27T11:04:22.931809Z","shell.execute_reply.started":"2025-06-27T11:04:22.886972Z","shell.execute_reply":"2025-06-27T11:04:22.931098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:22.933181Z","iopub.execute_input":"2025-06-27T11:04:22.933365Z","iopub.status.idle":"2025-06-27T11:04:22.949752Z","shell.execute_reply.started":"2025-06-27T11:04:22.933350Z","shell.execute_reply":"2025-06-27T11:04:22.949002Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Null value handling","metadata":{}},{"cell_type":"code","source":"train.replace([np.inf, -np.inf], np.nan, inplace=True) # for Pandas\n\n# # Replace inf and -inf with null (NaN equivalent)\n# train = train.drop(['timestamp'])\n\n# train = train.with_columns([\n# pl.when(pl.col(col).is_infinite()).then(None).otherwise(pl.col(col)).alias(col)\n# for col in train.columns\n# ])\n\n# train = train.fill_null(strategy=\"zero\") # or \"zero\", \"forward\", etc.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:22.950424Z","iopub.execute_input":"2025-06-27T11:04:22.950608Z","iopub.status.idle":"2025-06-27T11:04:26.071724Z","shell.execute_reply.started":"2025-06-27T11:04:22.950593Z","shell.execute_reply":"2025-06-27T11:04:26.071011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum().sort_values(ascending=False)\n\n#train.select(pl.all().null_count())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:26.072592Z","iopub.execute_input":"2025-06-27T11:04:26.072922Z","iopub.status.idle":"2025-06-27T11:04:27.622155Z","shell.execute_reply.started":"2025-06-27T11:04:26.072887Z","shell.execute_reply":"2025-06-27T11:04:27.621367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_cols = train.isnull().sum().sort_values(ascending=False)[lambda x: x >0].index\nnull_cols\n\ntrain.drop(columns=null_cols, inplace=True)\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:27.622777Z","iopub.execute_input":"2025-06-27T11:04:27.623053Z","iopub.status.idle":"2025-06-27T11:04:30.150768Z","shell.execute_reply.started":"2025-06-27T11:04:27.623017Z","shell.execute_reply":"2025-06-27T11:04:30.150095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = train[:300000]\n# train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:30.151477Z","iopub.execute_input":"2025-06-27T11:04:30.151659Z","iopub.status.idle":"2025-06-27T11:04:30.155092Z","shell.execute_reply.started":"2025-06-27T11:04:30.151646Z","shell.execute_reply":"2025-06-27T11:04:30.154329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Split target column and Select best columns","metadata":{}},{"cell_type":"code","source":"X = train.drop(columns=['label']) #for pandas\n# X = train.drop('label')\ny = train['label']\nX.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:30.155846Z","iopub.execute_input":"2025-06-27T11:04:30.156077Z","iopub.status.idle":"2025-06-27T11:04:31.199808Z","shell.execute_reply.started":"2025-06-27T11:04:30.156030Z","shell.execute_reply":"2025-06-27T11:04:31.199171Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

SelectKBest is a feature selection method from scikit-learn\nIt selects the top K features based on a scoring function.\n✅ Purpose:\nTo automatically pick the most relevant features from your dataset, which are most correlated with the target variable.

\n

f_regression is a scoring function used with SelectKBest for regression tasks.\n\n✅ What it does:\n\nCalculates the F-statistic for each feature.\n\nMeasures the linear relationship between each input feature and the target (continuous variable).\n\nHigher the F-value, the more important the feature.

","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest, f_regression\n\nselector = SelectKBest(score_func=f_regression, k=200)\nX_selected = selector.fit_transform(X, y)\n\nmask = selector.get_support()\n\n# Get names of selected features\nselected_features = X.columns[mask]\nprint(selected_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:31.200503Z","iopub.execute_input":"2025-06-27T11:04:31.200701Z","iopub.status.idle":"2025-06-27T11:04:33.704683Z","shell.execute_reply.started":"2025-06-27T11:04:31.200685Z","shell.execute_reply":"2025-06-27T11:04:33.703972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_keep = ['bid_qty','ask_qty','buy_qty','sell_qty','volume'] + list(selected_features)\n\nX = X[cols_to_keep]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:33.705400Z","iopub.execute_input":"2025-06-27T11:04:33.705623Z","iopub.status.idle":"2025-06-27T11:04:33.987082Z","shell.execute_reply.started":"2025-06-27T11:04:33.705599Z","shell.execute_reply":"2025-06-27T11:04:33.986206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = X[:200000]\n\ny = y[:200000]\n\nX.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:33.990601Z","iopub.execute_input":"2025-06-27T11:04:33.990835Z","iopub.status.idle":"2025-06-27T11:04:33.996063Z","shell.execute_reply.started":"2025-06-27T11:04:33.990818Z","shell.execute_reply":"2025-06-27T11:04:33.995262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:33.996840Z","iopub.execute_input":"2025-06-27T11:04:33.997108Z","iopub.status.idle":"2025-06-27T11:04:34.011680Z","shell.execute_reply.started":"2025-06-27T11:04:33.997085Z","shell.execute_reply":"2025-06-27T11:04:34.010966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"type(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.012433Z","iopub.execute_input":"2025-06-27T11:04:34.012618Z","iopub.status.idle":"2025-06-27T11:04:34.028634Z","shell.execute_reply.started":"2025-06-27T11:04:34.012604Z","shell.execute_reply":"2025-06-27T11:04:34.027843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.029457Z","iopub.execute_input":"2025-06-27T11:04:34.030173Z","iopub.status.idle":"2025-06-27T11:04:34.043990Z","shell.execute_reply.started":"2025-06-27T11:04:34.030150Z","shell.execute_reply":"2025-06-27T11:04:34.043506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3,random_state=42)\n\nprint(\"Shape of X_train:\",X_train.shape)\nprint(\"Shape of X_test:\",X_test.shape)\n\nprint(\"Shape of y_train:\",y_train.shape)\nprint(\"Shape of y_test:\",y_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.044900Z","iopub.execute_input":"2025-06-27T11:04:34.045144Z","iopub.status.idle":"2025-06-27T11:04:34.366957Z","shell.execute_reply.started":"2025-06-27T11:04:34.045128Z","shell.execute_reply":"2025-06-27T11:04:34.365987Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Scalling the Feature and Column Transformer","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.compose import ColumnTransformer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.367897Z","iopub.execute_input":"2025-06-27T11:04:34.368647Z","iopub.status.idle":"2025-06-27T11:04:34.380022Z","shell.execute_reply.started":"2025-06-27T11:04:34.368626Z","shell.execute_reply":"2025-06-27T11:04:34.379419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_cols = [fea for fea in X.columns if X[fea].dtype != '0']\nlen(numeric_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.381097Z","iopub.execute_input":"2025-06-27T11:04:34.381398Z","iopub.status.idle":"2025-06-27T11:04:34.483589Z","shell.execute_reply.started":"2025-06-27T11:04:34.381375Z","shell.execute_reply":"2025-06-27T11:04:34.482579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = MinMaxScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.484546Z","iopub.execute_input":"2025-06-27T11:04:34.484823Z","iopub.status.idle":"2025-06-27T11:04:34.499660Z","shell.execute_reply.started":"2025-06-27T11:04:34.484796Z","shell.execute_reply":"2025-06-27T11:04:34.499095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformer = ColumnTransformer(\n transformers =[\n ('standard_scalling' , scaler, numeric_cols),\n ], remainder = 'passthrough'\n)\n\nX_train_tr = transformer.fit_transform(X_train)\nX_test_tr = transformer.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:34.500504Z","iopub.execute_input":"2025-06-27T11:04:34.500688Z","iopub.status.idle":"2025-06-27T11:04:35.049669Z","shell.execute_reply.started":"2025-06-27T11:04:34.500673Z","shell.execute_reply":"2025-06-27T11:04:35.049089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"type(X_train_tr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:35.050542Z","iopub.execute_input":"2025-06-27T11:04:35.051150Z","iopub.status.idle":"2025-06-27T11:04:35.055688Z","shell.execute_reply.started":"2025-06-27T11:04:35.051123Z","shell.execute_reply":"2025-06-27T11:04:35.055102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#test=test.drop(columns=['label']) ## dropping target feature from test dataframe\ntest = test[cols_to_keep]\ntest.head(5)\ntest_trf = transformer.transform(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:35.057014Z","iopub.execute_input":"2025-06-27T11:04:35.057368Z","iopub.status.idle":"2025-06-27T11:04:36.459351Z","shell.execute_reply.started":"2025-06-27T11:04:35.057351Z","shell.execute_reply":"2025-06-27T11:04:36.458332Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"## Model Training and Model Selection\nfrom sklearn.metrics import r2_score,mean_squared_error,mean_absolute_error,mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:36.460304Z","iopub.execute_input":"2025-06-27T11:04:36.460532Z","iopub.status.idle":"2025-06-27T11:04:36.464057Z","shell.execute_reply.started":"2025-06-27T11:04:36.460513Z","shell.execute_reply":"2025-06-27T11:04:36.463458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Creating a function to evaluat model\ndef evaluation(true, predicted):\n mae = mean_absolute_error(true, predicted)\n me = mean_squared_error(true, predicted)\n mse = np.sqrt(me)\n r2 = r2_score(true, predicted)\n\n r = np.corrcoef(true, predicted)[0,1]\n\n print(f\"Pearson Correlation Coefficient: {r}\")\n print(\"R2 Score:{:.4f}\".format(r2))\n print(\"MAE:{:.4f}\".format(mae))\n print(\"MSE:{:.4f}\".format(mse))\n #print(\"RMSE:{:.4f}\".format(rmse))\n \n\n return 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:36.464817Z","iopub.execute_input":"2025-06-27T11:04:36.465021Z","iopub.status.idle":"2025-06-27T11:04:36.480623Z","shell.execute_reply.started":"2025-06-27T11:04:36.465006Z","shell.execute_reply":"2025-06-27T11:04:36.480089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Model Training and Model Selection\n\nfrom sklearn.linear_model import LinearRegression,Ridge,Lasso,ElasticNet\nfrom sklearn.neighbors import KNeighborsRegressor\n\nfrom sklearn.svm import SVR\n\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\n\nimport xgboost \nfrom xgboost import XGBRegressor\n\nfrom lightgbm import LGBMRegressor\n\nfrom catboost import CatBoostRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:36.481402Z","iopub.execute_input":"2025-06-27T11:04:36.481654Z","iopub.status.idle":"2025-06-27T11:04:40.879369Z","shell.execute_reply.started":"2025-06-27T11:04:36.481632Z","shell.execute_reply":"2025-06-27T11:04:40.878733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Model training\nmodels={\n # \"Linear_Regression\":LinearRegression(),\n\n # \"Linear_Regression_with_params\": LinearRegression(\n # fit_intercept=True, \n # copy_X=True, \n # n_jobs=-1, \n # positive=False \n # ),\n \n # \"Lasso\":Lasso(),\n \n # \"Ridge\":Ridge(),\n \n # \"ElasticNet\":ElasticNet(),\n \n # \"DecisionTreeRegressor\":DecisionTreeRegressor(),\n \n # \"DecisionTreeRegressor_with_params\":DecisionTreeRegressor(\n # criterion='squared_error', \n # splitter='best', \n # max_depth=10, \n # min_samples_split=10, \n # min_samples_leaf=4, \n # max_features='sqrt', \n # random_state=42 \n # ),\n \n # \"AdaBoost\":AdaBoostRegressor(),\n \n # \"GradientBoost\":GradientBoostingRegressor(),\n \n \"XGBRegressor\":XGBRegressor(\n max_depth=20,\n colsample_bytree=0.75,\n subsample=0.9,\n n_estimators=1000,\n learning_rate=0.001,\n gamma=0.01,\n max_delta_step=2,\n eval_metric=\"rmse\",\n enable_categorical=True,\n device = 'cuda'),\n \n # \"LGBMRegressor\":LGBMRegressor(\n # n_estimators=1000,\n # learning_rate=0.005,\n # max_depth=7,\n # num_leaves=31,\n # min_child_samples=20,\n # subsample=0.8,\n # colsample_bytree=0.8,\n # random_state=42,\n # n_jobs=-1\n # ),\n \n # \"CatBoostRegressor\":CatBoostRegressor(\n # iterations= 3500,\n # depth= 12,\n # loss_function= 'RMSE',\n # l2_leaf_reg= 3,\n # random_seed= 42,\n # eval_metric= 'RMSE',\n # silent=True\n # ),\n \n # \"RandomForest\":RandomForestRegressor(\n # n_estimators=300,\n # max_depth=10,\n # min_samples_split=5,\n # min_samples_leaf=2,\n # max_features='sqrt',\n # random_state=42,\n # n_jobs=-1\n # ),\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:40.880086Z","iopub.execute_input":"2025-06-27T11:04:40.880676Z","iopub.status.idle":"2025-06-27T11:04:40.886070Z","shell.execute_reply.started":"2025-06-27T11:04:40.880647Z","shell.execute_reply":"2025-06-27T11:04:40.885374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")\n#sample_submission = sample_submission[:30000]\nid_column = sample_submission['ID']\nlen(id_column)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:40.887112Z","iopub.execute_input":"2025-06-27T11:04:40.887354Z","iopub.status.idle":"2025-06-27T11:04:41.213206Z","shell.execute_reply.started":"2025-06-27T11:04:40.887330Z","shell.execute_reply":"2025-06-27T11:04:41.212435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_name_list = []\n# corrcoef_list = []\n\n# for i in range(len(list(models))):\n# model_name = list(models.keys())[i]\n# model = list(models.values())[i]\n\n# print(\".............\",model_name, \"........................\\n\")\n\n# model.fit(X_train_tr, y_train) ## Train Model on X_train\n\n# ## Make Predictions.............................\n# y_train_pred=model.predict(X_train_tr)\n# y_test_pred=model.predict(X_test_tr)\n\n# print()\n# print(\"Evaluating Train Dataset\")\n# evaluation(y_train,y_train_pred)\n\n# print(f\"\\n{'-'*50}\\n\")\n \n# print(\"Evaluating Test Dataset\")\n# evaluation(y_test,y_test_pred)\n# print(\"=\"*60)\n# print(\"\\n\")\n\n \n# ### appending the vlaues in list \n# model_name_list.append(model_name)\n# corrcoef_list.append(np.corrcoef(y_test, y_test_pred)[0, 1])\n\n# ## prediction\n# prediction = model.predict(test_trf)\n\n# result = pd.DataFrame(\n# {\n# 'ID':id_column,\n# 'prediction':prediction\n# }\n# )\n \n# result.to_csv('submission.csv',index=False)\n# print(\"File saved as '{}_prediction.csv'....\".format(model_name))\n# print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:41.214212Z","iopub.execute_input":"2025-06-27T11:04:41.214978Z","iopub.status.idle":"2025-06-27T11:04:41.218658Z","shell.execute_reply.started":"2025-06-27T11:04:41.214951Z","shell.execute_reply":"2025-06-27T11:04:41.217880Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ## creating dataframe contains model name and their performance on X_test \n# performance_df = pd.DataFrame({\n# 'ML Algo Name': model_name_list,\n# 'Pearson Correlation Coefficient': corrcoef_list\n# })\n\n# performance_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:41.219476Z","iopub.execute_input":"2025-06-27T11:04:41.220241Z","iopub.status.idle":"2025-06-27T11:04:41.240427Z","shell.execute_reply.started":"2025-06-27T11:04:41.220221Z","shell.execute_reply":"2025-06-27T11:04:41.239879Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7 Neural Network model with pytorch","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchsummary import summary\nfrom torch.utils.data import DataLoader, TensorDataset\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Using device:', device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:41.241032Z","iopub.execute_input":"2025-06-27T11:04:41.241247Z","iopub.status.idle":"2025-06-27T11:04:48.732089Z","shell.execute_reply.started":"2025-06-27T11:04:41.241231Z","shell.execute_reply":"2025-06-27T11:04:48.731262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:48.733014Z","iopub.execute_input":"2025-06-27T11:04:48.734104Z","iopub.status.idle":"2025-06-27T11:04:48.737446Z","shell.execute_reply.started":"2025-06-27T11:04:48.734080Z","shell.execute_reply":"2025-06-27T11:04:48.736704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sparse matrix কে dense numpy array তে রূপান্তর\n# X_dense = X.toarray()\n# X_test_dense = test.toarray()\n\nX_tensor = torch.tensor(X_train_tr, dtype=torch.float32)\ny_tensor = torch.tensor(y_train.values, dtype=torch.float32).view(-1,1)\n\nX_test_tensor = torch.tensor(X_test_tr, dtype=torch.float32)\ny_test_tensor = torch.tensor(y_test.values, dtype=torch.float32).view(-1,1)\n\n#X_train, X_valid, y_train, y_valid = train_test_split(X_tensor, y_tensor, test_size=0.3, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:07:03.955547Z","iopub.execute_input":"2025-06-27T11:07:03.955928Z","iopub.status.idle":"2025-06-27T11:07:04.064429Z","shell.execute_reply.started":"2025-06-27T11:07:03.955904Z","shell.execute_reply":"2025-06-27T11:07:04.063556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create tensor dataset..............\ntrain_ds = TensorDataset(X_tensor,y_tensor)\nvalid_ds = TensorDataset(X_test_tensor,y_test_tensor)\n\ntrain_loader = DataLoader(train_ds, batch_size=128, shuffle=True)\nvalid_loader = DataLoader(valid_ds, batch_size=128, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:07:07.364573Z","iopub.execute_input":"2025-06-27T11:07:07.365128Z","iopub.status.idle":"2025-06-27T11:07:07.386820Z","shell.execute_reply.started":"2025-06-27T11:07:07.365101Z","shell.execute_reply":"2025-06-27T11:07:07.385997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NN_model(nn.Module):\n def __init__(self, input_size):\n super(NN_model, self).__init__()\n self.fc1 = nn.Linear(input_size, 1024)\n self.fc2 = nn.Linear(1024,512)\n self.fc3 = nn.Linear(512,256)\n self.fc4 = nn.Linear(256,128)\n self.fc5 = nn.Linear(128,1)\n\n\n self.dp = nn.Dropout(p=0.35) # Regularization........\n self.fc_drop = nn.Dropout(p=0.2)\n\n def forward(self, x):\n x = F.relu(self.fc1(x))\n x = self.dp(x)\n x = F.relu(self.fc2(x))\n x = self.dp(x)\n x = F.relu(self.fc3(x))\n x = self.fc_drop(x)\n x = F.relu(self.fc4(x))\n \n return self.fc5(x)\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:07:10.738371Z","iopub.execute_input":"2025-06-27T11:07:10.738693Z","iopub.status.idle":"2025-06-27T11:07:10.745721Z","shell.execute_reply.started":"2025-06-27T11:07:10.738668Z","shell.execute_reply":"2025-06-27T11:07:10.744914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = NN_model(input_size=X_tensor.shape[1])\nmodel = model.to(device)\n\n#summary(model)\nprint(model.parameters())\ncriterian = nn.MSELoss()\n#criterian = nn.CrossEntropyLoss()\n\noptimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=0.0005)\n\n\n# training the model..............................\ntrain_losses =[]\nval_losses = []\nbest_val_loss = float('inf')\nepochs = 30\n\nfor epoch in range(epochs):\n model.train()\n running_train_loss = 0.0\n for x, y in train_loader:\n x , y = x.to(device), y.to(device)\n pred = model(x)\n loss = criterian(pred, y)\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n running_train_loss+= loss\n \n train_loss = running_train_loss / len(train_loader)\n train_losses.append(train_loss)\n\n\n model.eval()\n running_val_loss = 0.0\n\n with torch.no_grad():\n for xv, yv in valid_loader:\n xv, yv = xv.to(device), yv.to(device)\n\n val_pred = model(xv)\n loss_val = criterian(val_pred, yv)\n running_val_loss+= loss_val\n\n valid_loss = running_val_loss / len(valid_loader)\n val_losses.append(valid_loss)\n\n print(f\"Epoch [{epoch + 1}/{epochs}], Training Loss: {train_loss:.4f}, Validation Loss: {valid_loss:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:07:13.826356Z","iopub.execute_input":"2025-06-27T11:07:13.826658Z","iopub.status.idle":"2025-06-27T11:09:49.720219Z","shell.execute_reply.started":"2025-06-27T11:07:13.826632Z","shell.execute_reply":"2025-06-27T11:09:49.719394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#len(id_column)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:48.904222Z","iopub.status.idle":"2025-06-27T11:04:48.904559Z","shell.execute_reply.started":"2025-06-27T11:04:48.904388Z","shell.execute_reply":"2025-06-27T11:04:48.904405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_trf_tensor = torch.tensor(test_trf, dtype=torch.float32)\ntype(test_trf_tensor)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:10:44.306948Z","iopub.execute_input":"2025-06-27T11:10:44.308310Z","iopub.status.idle":"2025-06-27T11:10:44.618853Z","shell.execute_reply.started":"2025-06-27T11:10:44.308280Z","shell.execute_reply":"2025-06-27T11:10:44.618077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(test_trf_tensor)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:10:47.815737Z","iopub.execute_input":"2025-06-27T11:10:47.816356Z","iopub.status.idle":"2025-06-27T11:10:47.821175Z","shell.execute_reply.started":"2025-06-27T11:10:47.816328Z","shell.execute_reply":"2025-06-27T11:10:47.820395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n X_test_tensor = test_trf_tensor.to(device)\n test_pred = model(X_test_tensor).squeeze()\n\n\n\ntest_preds = test_pred.cpu().numpy()\n\nsubmission = pd.DataFrame({\n 'ID' : id_column,\n 'prediction' : test_preds\n})\n\nsubmission.to_csv('submission.csv', index=False)\n\nsubmission.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:18:34.526714Z","iopub.execute_input":"2025-06-27T11:18:34.527401Z","iopub.status.idle":"2025-06-27T11:18:35.737000Z","shell.execute_reply.started":"2025-06-27T11:18:34.527374Z","shell.execute_reply":"2025-06-27T11:18:35.736326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#test_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:48.909559Z","iopub.status.idle":"2025-06-27T11:04:48.909773Z","shell.execute_reply.started":"2025-06-27T11:04:48.909673Z","shell.execute_reply":"2025-06-27T11:04:48.909682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#y_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:48.910931Z","iopub.status.idle":"2025-06-27T11:04:48.911296Z","shell.execute_reply.started":"2025-06-27T11:04:48.911078Z","shell.execute_reply":"2025-06-27T11:04:48.911094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# evaluation(y_test.values, test_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-27T11:04:48.912517Z","iopub.status.idle":"2025-06-27T11:04:48.912748Z","shell.execute_reply.started":"2025-06-27T11:04:48.912643Z","shell.execute_reply":"2025-06-27T11:04:48.912652Z"}},"outputs":[],"execution_count":null}]}