{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## In this notebook, I use technical indicators ro predict the omve in the price for the next day\n\n### To do that, I take the data form each column for the last 14 days from the current day using the function transform_data","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport datetime\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.preprocessing import MinMaxScaler\nimport warnings\nfrom sklearn.neighbors import KNeighborsClassifier as KNN\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import *\n\nfrom sklearn.linear_model import LogisticRegression as LR\nfrom sklearn.tree import DecisionTreeClassifier as DT\nfrom sklearn.ensemble import RandomForestClassifier as RF\nfrom sklearn.svm import SVC\n\nimport plotly.io as pio\nimport cufflinks as cf\nimport plotly.express as px\nfrom sklearn.model_selection import GridSearchCV\n\npd.options.display.max_columns=None\npd.set_option('display.precision', 2)\npd.set_option('display.float_format', lambda x: '%.3f' % x)\n\npd.options.plotting.backend = 'plotly'\n\npio.templates.default = \"plotly\"\ncf.go_offline()\n\nwarnings.filterwarnings('ignore')\nmpl.rcParams['figure.figsize'] = (8, 6)\nmpl.rcParams['axes.grid'] = False","metadata":{"execution":{"iopub.status.busy":"2023-01-14T10:25:30.901163Z","iopub.execute_input":"2023-01-14T10:25:30.901664Z","iopub.status.idle":"2023-01-14T10:25:30.940749Z","shell.execute_reply.started":"2023-01-14T10:25:30.901616Z","shell.execute_reply":"2023-01-14T10:25:30.939429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tickers = [\n'AAPL', 'MSFT', 'GOOGL', 'AMZN', 'JNJ', 'XOM', 'V', 'WMT', 'NVDA', 'PG',\n'LLY', 'CVX', 'MA', 'HD', 'PFE', 'ABBV', 'MRK', 'KO', 'PEP', 'AVGO', 'ORCL',\n'TMO', 'AZN', 'CSCO', 'DHR', 'MCD', 'ABT', 'TMUS', 'ACN', 'NEE', 'VZ', 'TTE',\n'LIN', 'DIS', 'PM', 'BMY', 'CMCSA', 'SCHW', 'UPS', 'TXN', 'RTX', 'COP'\n]","metadata":{"execution":{"iopub.status.busy":"2023-01-14T10:25:30.942440Z","iopub.execute_input":"2023-01-14T10:25:30.942856Z","iopub.status.idle":"2023-01-14T10:25:30.952717Z","shell.execute_reply.started":"2023-01-14T10:25:30.942819Z","shell.execute_reply":"2023-01-14T10:25:30.951467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transform data","metadata":{}},{"cell_type":"code","source":"def transform_data(df, ticker):\n    df = df.copy()\n    window = 14\n    cols = df.columns\n    for column in cols[:-1]:\n        for i in range(1, window):\n            df[f'{column}_t-{i}'] = df[column].shift(i)\n\n    df = df.dropna(how='any')\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-01-14T10:25:30.954852Z","iopub.execute_input":"2023-01-14T10:25:30.955197Z","iopub.status.idle":"2023-01-14T10:25:30.970987Z","shell.execute_reply.started":"2023-01-14T10:25:30.955168Z","shell.execute_reply":"2023-01-14T10:25:30.969520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data input","metadata":{}},{"cell_type":"code","source":"def read_data(tickers, transform=True):\n    train_dfs = []\n    test_dfs = {}\n\n    \n    for ticker in tickers[:]:\n        path = f'/kaggle/input/us-stocks-price-prediction/Data/train/Technicals/{ticker}_technicals_train.csv'\n        df = pd.read_csv(path).drop('Date', axis=1)\n\n        df['label'] = df['close'].diff().shift(-1).map(lambda x: np.sign(x)).map({-1:0, 1:1})\n        df.drop(['PSAR', 'EP', 'rsi_oversold'], axis=1, inplace=True)\n\n        try:\n            df.drop(['rsi_overbought'], axis=1, inplace=True)\n        except Exception as e:\n            print(ticker)\n\n        close = df.pop('close')\n\n        if transform:\n            df = transform_data(df, ticker)\n        \n        df['close'] = close\n        train_size = int(len(df)*0.8)\n\n        train_df = df[:train_size]\n        test_df = df[train_size:]\n\n        train_dfs.append(train_df)\n        test_dfs[ticker] = test_df\n    \n    train_df = pd.concat(train_dfs)\n    \n    return train_df, test_dfs","metadata":{"execution":{"iopub.status.busy":"2023-01-14T10:25:31.952530Z","iopub.execute_input":"2023-01-14T10:25:31.952981Z","iopub.status.idle":"2023-01-14T10:25:31.964422Z","shell.execute_reply.started":"2023-01-14T10:25:31.952949Z","shell.execute_reply":"2023-01-14T10:25:31.962988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df, test_dfs = read_data(tickers[:])\n\nx = df.drop('label', axis=1)\ny = df[['label']]\n\nx.isna().sum().to_frame().sort_values(by=0, ascending=False).sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T10:25:32.451942Z","iopub.execute_input":"2023-01-14T10:25:32.452347Z","iopub.status.idle":"2023-01-14T10:25:32.984245Z","shell.execute_reply.started":"2023-01-14T10:25:32.452316Z","shell.execute_reply":"2023-01-14T10:25:32.983100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split data and prepare models and metrics","metadata":{}},{"cell_type":"code","source":"shuffle = True\ntrain_size = int(len(x)*0.8)\n\nif shuffle:\n    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\nelse:\n    x_train, x_test, y_train, y_test = x[:train_size], x[train_size:], y[:train_size], y[train_size:]\n\nlr = LR()\ndt = DT()\nrf = RF()\nsvr = SVC(probability=True)\nknn = KNN(10)\n\nmodels = [lr, dt, rf, svr, knn]\nmodels_names = ['Linear Model', 'Descision Tree', 'Random Forest', 'SVM', 'KNN']\nmetrics = ['accuracy_score', 'precision_score', 'recall_score', 'confusion_matrix']\ntrain_scores = {}\ntest_scores = {}\nmodel_preds = {}\ntest_df = y_test.copy()\ntrain_df = y_train.copy()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Uncomment to perform hyperparameter tunning","metadata":{}},{"cell_type":"code","source":"models_names = ['Random Forest']\nmodels = [rf]\nmetrics = ['accuracy_score', 'precision_score', 'recall_score']\n\n\nparam_grid_svm = {'C': [0.1, 1, 10, 100, 1000],\n              'gamma': [1, 0.1, 0.01, 0.001, 0.0001],\n              'kernel': ['rbf']}\n\nparam_grid_dt = {'max_features': ['auto', 'sqrt', 'log2'],\n              'ccp_alpha': [0.1, .01, .001],\n              'max_depth' : [5, 6, 7, 8, 9],\n              'criterion' :['gini', 'entropy']\n             }\n\nparam_grid_rf = { \n    'n_estimators': [200, 500],\n    'max_features': ['auto', 'sqrt', 'log2'],\n    'max_depth' : [4,5,6,7,8],\n    'criterion' :['gini', 'entropy']\n}\n\ngrid_params_knn = { 'n_neighbors' : [5,7,9,11,13,15],\n               'weights' : ['uniform','distance'],\n               'metric' : ['minkowski','euclidean','manhattan']}\n\n\n# grid_params = {'SVM':param_grid_svm, 'Descision Tree':param_grid_dt, 'Random Forest':param_grid_rf, 'KNN':grid_params_knn}\ngrid_params = {'Random Forest':param_grid_rf}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_estimators = {}\nbest_score = {}\nbest_params = {}\nfor model, model_name in zip(models, models_names):\n    grid = GridSearchCV(model, grid_params[model_name], refit = True, verbose = 2, n_jobs=8)\n    grid.fit(x_train, y_train)\n    \n    best_estimators[model_name] = grid.best_estimator_\n    best_score[model_name] = grid.best_score_\n    best_params[model_name] = grid.best_params_","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nf = open('tuned_random_forest_fitted_on_all_stocks.pickle', 'wb')\npickle.dump(best_estimators, f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = y_test.copy()\ntrain_df = y_train.copy()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_scores = {}\ntest_scores = {}\nmodel_preds = {}\n\nfor model_name, model in best_estimators.items():\n\n    test_df[model_name] = model.predict(x_test).flatten()\n    train_df[model_name] = model.predict(x_train).flatten()  \n    \n    train_scores[model_name] = []\n    test_scores[model_name] = []\n    \n    for metric in metrics:\n        train_scores[model_name].append(eval(metric)(train_df[model_name], y_train))\n        test_scores[model_name].append(eval(metric)(test_df[model_name], y_test))\n        \ntest_df.sort_index(inplace=True)\ntrain_df.sort_index(inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training","metadata":{}},{"cell_type":"code","source":"# for model, model_name in zip(models[2:3], models_names[2:3]):\n#     model.fit(x_train, y_train)\n    \n#     test_df[model_name] = model.predict(x_test).flatten()\n#     train_df[model_name] = model.predict(x_train).flatten()\n    \n#     train_scores[model_name] = []\n#     test_scores[model_name] = []\n    \n#     for metric in metrics:\n#         train_scores[model_name].append(eval(metric)(train_df[model_name], y_train))\n#         test_scores[model_name].append(eval(metric)(test_df[model_name], y_test))\n        \n# test_df.sort_index(inplace=True)\n# train_df.sort_index(inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_scores = pd.DataFrame(test_scores, index = metrics)\ntrain_scores = pd.DataFrame(train_scores, index = metrics)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_scores","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_scores","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame([test_df[c].value_counts() for c in test_df]).T","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance using Random Forest","metadata":{}},{"cell_type":"code","source":"feature_names = x.columns\nrf = best_estimators['Random Forest']\n\nimportances = rf.feature_importances_\nstd = np.std([tree.feature_importances_ for tree in rf.estimators_], axis=0)\nforest_importances = pd.DataFrame(importances, index=feature_names, columns=['importance']).sort_values(ascending=False, by='importance')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(forest_importances, x=forest_importances.index, y='importance')\nfig.update_layout(width=1000, height=800)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# temp = test_df['label'] - test_df['Descision Tree']\n# px.scatter(temp, x=temp.index, y=temp)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculating PnL","metadata":{}},{"cell_type":"code","source":"p = 10000 # $10000\nreturns = {}\n\nfor ticker, test_df in test_dfs.items():\n    \n    preds = rf.predict(test_df.drop('label', axis=1))\n    signals = test_df[['close', 'label']]\n    signals['pred'] = preds\n    signals = signals.replace(0, -1)\n    signals['daily_return'] = signals.close.pct_change()*100\n    signals['diff'] = signals.close.diff().shift(-1)\n    signals['shares'] = p/signals.close\n    signals['PnL'] = signals['shares'] * signals['diff']\n        \n    returns[ticker] = [(signals.PnL * signals.label).sum(), (signals.PnL * signals.pred).sum(), (signals.close.iloc[-1] - signals.close.iloc[0]) * p/signals.close.iloc[0]]\n    \nreturns = pd.DataFrame(returns).T\nreturns.columns = ['actual_returns', 'predicted_returns', 'buy_and_hold_returns']\nreturns = returns.astype(int)\nreturns['difference'] = returns.predicted_returns - returns.buy_and_hold_returns\n# returns.drop('TSLA', inplace=True)\nreturns.sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"returns","metadata":{},"execution_count":null,"outputs":[]}]}