{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-08T05:06:13.281771Z","iopub.execute_input":"2022-07-08T05:06:13.282428Z","iopub.status.idle":"2022-07-08T05:06:13.314498Z","shell.execute_reply.started":"2022-07-08T05:06:13.282328Z","shell.execute_reply":"2022-07-08T05:06:13.313661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget https://artiya4u.keybase.pub/TA-lib/ta-lib-0.4.0-src.tar.gz &&tar -xvf ta-lib-0.4.0-src.tar.gz &&cd ta-lib/ &&./configure --prefix=/usr &&make &&sudo make install","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-08T05:06:13.316196Z","iopub.execute_input":"2022-07-08T05:06:13.316537Z","iopub.status.idle":"2022-07-08T05:08:00.553549Z","shell.execute_reply.started":"2022-07-08T05:06:13.316508Z","shell.execute_reply":"2022-07-08T05:08:00.552211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade TA-Lib==0.4.19 tables","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:08:00.555212Z","iopub.execute_input":"2022-07-08T05:08:00.555567Z","iopub.status.idle":"2022-07-08T05:09:01.66122Z","shell.execute_reply.started":"2022-07-08T05:08:00.55553Z","shell.execute_reply":"2022-07-08T05:09:01.658677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport talib\nfrom talib import RSI, BBANDS, MACD, ATR","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:01.667945Z","iopub.execute_input":"2022-07-08T05:09:01.668514Z","iopub.status.idle":"2022-07-08T05:09:02.613869Z","shell.execute_reply.started":"2022-07-08T05:09:01.668445Z","shell.execute_reply":"2022-07-08T05:09:02.612033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이터 불러오기","metadata":{}},{"cell_type":"code","source":"prices = (\n    pd.read_csv('/kaggle/input/ai-finance-korea/train.csv', parse_dates=['date'])\n    .set_index(['date', 'ticker'])\n    .swaplevel()\n    .sort_index()\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:02.617329Z","iopub.execute_input":"2022-07-08T05:09:02.617724Z","iopub.status.idle":"2022-07-08T05:09:16.909951Z","shell.execute_reply.started":"2022-07-08T05:09:02.61768Z","shell.execute_reply":"2022-07-08T05:09:16.908678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = pd.read_csv('/kaggle/input/ai-finance-korea/stocks.csv').set_index('ticker').loc[:, ['marketcap', 'sector']]","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:16.915896Z","iopub.execute_input":"2022-07-08T05:09:16.917137Z","iopub.status.idle":"2022-07-08T05:09:16.950046Z","shell.execute_reply.started":"2022-07-08T05:09:16.917087Z","shell.execute_reply":"2022-07-08T05:09:16.948261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.volume /= 1e3 # Volume 크기를 작게 만들기 \nprices.index.names = ['symbol', 'date']\nmetadata.index.name = 'symbol'","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:16.953262Z","iopub.execute_input":"2022-07-08T05:09:16.954157Z","iopub.status.idle":"2022-07-08T05:09:17.037398Z","shell.execute_reply.started":"2022-07-08T05:09:16.95412Z","shell.execute_reply":"2022-07-08T05:09:17.036334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.038733Z","iopub.execute_input":"2022-07-08T05:09:17.039195Z","iopub.status.idle":"2022-07-08T05:09:17.062375Z","shell.execute_reply.started":"2022-07-08T05:09:17.039161Z","shell.execute_reply":"2022-07-08T05:09:17.061215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.063682Z","iopub.execute_input":"2022-07-08T05:09:17.064028Z","iopub.status.idle":"2022-07-08T05:09:17.08675Z","shell.execute_reply.started":"2022-07-08T05:09:17.063961Z","shell.execute_reply":"2022-07-08T05:09:17.086038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('darkgrid')\nidx = pd.IndexSlice","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.092244Z","iopub.execute_input":"2022-07-08T05:09:17.092564Z","iopub.status.idle":"2022-07-08T05:09:17.09807Z","shell.execute_reply.started":"2022-07-08T05:09:17.092535Z","shell.execute_reply":"2022-07-08T05:09:17.096823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이터가 7년이 안되는 종목들 제거","metadata":{}},{"cell_type":"code","source":"MONTH = 21\nYEAR = 12 * MONTH","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.099639Z","iopub.execute_input":"2022-07-08T05:09:17.100264Z","iopub.status.idle":"2022-07-08T05:09:17.109275Z","shell.execute_reply.started":"2022-07-08T05:09:17.100218Z","shell.execute_reply":"2022-07-08T05:09:17.108402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_obs = 7 * YEAR\nnobs = prices.groupby(level='symbol').size()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.110788Z","iopub.execute_input":"2022-07-08T05:09:17.111479Z","iopub.status.idle":"2022-07-08T05:09:17.701006Z","shell.execute_reply.started":"2022-07-08T05:09:17.111425Z","shell.execute_reply":"2022-07-08T05:09:17.699906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nobs","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.702205Z","iopub.execute_input":"2022-07-08T05:09:17.702523Z","iopub.status.idle":"2022-07-08T05:09:17.711884Z","shell.execute_reply.started":"2022-07-08T05:09:17.702495Z","shell.execute_reply":"2022-07-08T05:09:17.710512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Min: {min_obs}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.712966Z","iopub.execute_input":"2022-07-08T05:09:17.713249Z","iopub.status.idle":"2022-07-08T05:09:17.720294Z","shell.execute_reply.started":"2022-07-08T05:09:17.713224Z","shell.execute_reply":"2022-07-08T05:09:17.719567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keep = nobs[nobs > min_obs].index\nprices = prices.loc[idx[keep, :], :]","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:09:17.721487Z","iopub.execute_input":"2022-07-08T05:09:17.721773Z","iopub.status.idle":"2022-07-08T05:15:14.05042Z","shell.execute_reply.started":"2022-07-08T05:09:17.721746Z","shell.execute_reply":"2022-07-08T05:15:14.048768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 가격 데이터와 Meta 데이터 정렬","metadata":{}},{"cell_type":"code","source":"metadata = metadata[~metadata.index.duplicated() & metadata.sector.notnull()]\nmetadata.sector = metadata.sector.str.lower().str.replace(' ', '_')","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:15:14.052578Z","iopub.execute_input":"2022-07-08T05:15:14.052956Z","iopub.status.idle":"2022-07-08T05:15:14.071695Z","shell.execute_reply.started":"2022-07-08T05:15:14.052919Z","shell.execute_reply":"2022-07-08T05:15:14.070306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MetaData에 있는 종목들만 price 데이터에서 뽑아준다. \nshared = (prices.index.get_level_values('symbol').unique()\n          .intersection(metadata.index))\nmetadata = metadata.loc[shared, :]\nprices = prices.loc[idx[shared, :], :]","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:15:14.073338Z","iopub.execute_input":"2022-07-08T05:15:14.073671Z","iopub.status.idle":"2022-07-08T05:18:58.463455Z","shell.execute_reply.started":"2022-07-08T05:15:14.073641Z","shell.execute_reply":"2022-07-08T05:18:58.461711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 시가총액 상위 1000종목만 사용","metadata":{}},{"cell_type":"code","source":"metadata.marketcap.nlargest(1000).index","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:18:58.465095Z","iopub.execute_input":"2022-07-08T05:18:58.46545Z","iopub.status.idle":"2022-07-08T05:18:58.482028Z","shell.execute_reply.started":"2022-07-08T05:18:58.465418Z","shell.execute_reply":"2022-07-08T05:18:58.48076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"universe = metadata.marketcap.nlargest(1000).index\nprices = prices.loc[idx[universe, :], :]\nmetadata = metadata.loc[universe]","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:18:58.483737Z","iopub.execute_input":"2022-07-08T05:18:58.485292Z","iopub.status.idle":"2022-07-08T05:20:45.158042Z","shell.execute_reply.started":"2022-07-08T05:18:58.485241Z","shell.execute_reply":"2022-07-08T05:20:45.156881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 종목들의 sector 분포\nsector_dist = metadata.sector.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:20:45.159934Z","iopub.execute_input":"2022-07-08T05:20:45.16039Z","iopub.status.idle":"2022-07-08T05:20:45.167884Z","shell.execute_reply.started":"2022-07-08T05:20:45.160334Z","shell.execute_reply":"2022-07-08T05:20:45.167124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 분포 출력 \nsector_dist.plot.bar()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:20:45.16887Z","iopub.execute_input":"2022-07-08T05:20:45.169438Z","iopub.status.idle":"2022-07-08T05:20:45.495753Z","shell.execute_reply.started":"2022-07-08T05:20:45.169404Z","shell.execute_reply":"2022-07-08T05:20:45.494553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 학습에 사용할 Feature 만들기","metadata":{}},{"cell_type":"markdown","source":"## 거래대금 이동평균 순위 데이터 생성","metadata":{}},{"cell_type":"code","source":"# 거래대금 생성\nprices['dollar_vol'] = prices[['close', 'volume']].prod(1).div(1e3)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:20:45.497719Z","iopub.execute_input":"2022-07-08T05:20:45.498438Z","iopub.status.idle":"2022-07-08T05:20:45.609732Z","shell.execute_reply.started":"2022-07-08T05:20:45.498392Z","shell.execute_reply":"2022-07-08T05:20:45.608493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 21일 이동평균\ndollar_vol_ma = (prices\n                 .dollar_vol\n                 .unstack('symbol')\n                 .rolling(window=21, min_periods=1) # 1 trading month\n                 .mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:20:45.611277Z","iopub.execute_input":"2022-07-08T05:20:45.611589Z","iopub.status.idle":"2022-07-08T05:20:46.569391Z","shell.execute_reply.started":"2022-07-08T05:20:45.611559Z","shell.execute_reply":"2022-07-08T05:20:46.568163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 거래대금 이동평균 순위 만들기\nprices['dollar_vol_rank'] = (dollar_vol_ma\n                            .rank(axis=1, ascending=False)\n                            .stack('symbol')\n                            .swaplevel())","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:20:46.570863Z","iopub.execute_input":"2022-07-08T05:20:46.571578Z","iopub.status.idle":"2022-07-08T05:21:07.366946Z","shell.execute_reply.started":"2022-07-08T05:20:46.571539Z","shell.execute_reply":"2022-07-08T05:21:07.365513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 거의 항상 거래대금 1등인 것을 알 수 있다. \nprices.loc['AAPL']['dollar_vol_rank'].plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:21:07.368638Z","iopub.execute_input":"2022-07-08T05:21:07.370694Z","iopub.status.idle":"2022-07-08T05:21:07.897428Z","shell.execute_reply.started":"2022-07-08T05:21:07.370654Z","shell.execute_reply":"2022-07-08T05:21:07.896436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## RSI 지표 만들기","metadata":{}},{"cell_type":"code","source":"prices['rsi'] = prices.groupby(level='symbol').close.apply(RSI)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:21:07.89889Z","iopub.execute_input":"2022-07-08T05:21:07.899229Z","iopub.status.idle":"2022-07-08T05:21:29.602584Z","shell.execute_reply.started":"2022-07-08T05:21:07.899198Z","shell.execute_reply":"2022-07-08T05:21:29.601493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.distplot(prices.rsi.dropna())\nax.axvline(30, ls='--', lw=1, c='k')\nax.axvline(70, ls='--', lw=1, c='k')\nax.set_title('RSI Distribution with Signal Threshold')\nsns.despine()\nplt.tight_layout();","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:21:29.604193Z","iopub.execute_input":"2022-07-08T05:21:29.604635Z","iopub.status.idle":"2022-07-08T05:21:38.292094Z","shell.execute_reply.started":"2022-07-08T05:21:29.60459Z","shell.execute_reply":"2022-07-08T05:21:38.290942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 볼린저밴드 만들기","metadata":{}},{"cell_type":"code","source":"def compute_bb(close):\n    high, mid, low = BBANDS(close, timeperiod=20)\n    return pd.DataFrame({'bb_high': high, 'bb_low': low}, index=close.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:21:38.297384Z","iopub.execute_input":"2022-07-08T05:21:38.297721Z","iopub.status.idle":"2022-07-08T05:21:38.303785Z","shell.execute_reply.started":"2022-07-08T05:21:38.297689Z","shell.execute_reply":"2022-07-08T05:21:38.30241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices = (prices.join(prices\n                      .groupby(level='symbol')\n                      .close\n                      .apply(compute_bb)))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:21:38.305294Z","iopub.execute_input":"2022-07-08T05:21:38.306055Z","iopub.status.idle":"2022-07-08T05:22:11.21467Z","shell.execute_reply.started":"2022-07-08T05:21:38.306013Z","shell.execute_reply":"2022-07-08T05:22:11.213401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices['bb_high'] = prices.bb_high.sub(prices.close).div(prices.bb_high).apply(np.log1p)\nprices['bb_low'] = prices.close.sub(prices.bb_low).div(prices.close).apply(np.log1p)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:22:11.216254Z","iopub.execute_input":"2022-07-08T05:22:11.216687Z","iopub.status.idle":"2022-07-08T05:22:11.291107Z","shell.execute_reply.started":"2022-07-08T05:22:11.216644Z","shell.execute_reply":"2022-07-08T05:22:11.289989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(ncols=2, figsize=(15, 5))\nsns.distplot(prices.loc[prices.dollar_vol_rank<100, 'bb_low'].dropna(), ax=axes[0])\nsns.distplot(prices.loc[prices.dollar_vol_rank<100, 'bb_high'].dropna(), ax=axes[1])\nsns.despine()\nplt.tight_layout();","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:22:11.29261Z","iopub.execute_input":"2022-07-08T05:22:11.293259Z","iopub.status.idle":"2022-07-08T05:22:13.645505Z","shell.execute_reply.started":"2022-07-08T05:22:11.293214Z","shell.execute_reply":"2022-07-08T05:22:13.644205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ATR 지표 만들기","metadata":{}},{"cell_type":"code","source":"prices['NATR'] = prices.groupby(level='symbol', \n                                group_keys=False).apply(lambda x: \n                                                        talib.NATR(x.high, x.low, x.close))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:22:13.647356Z","iopub.execute_input":"2022-07-08T05:22:13.648214Z","iopub.status.idle":"2022-07-08T05:22:27.287188Z","shell.execute_reply.started":"2022-07-08T05:22:13.648166Z","shell.execute_reply":"2022-07-08T05:22:27.285604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_atr(stock_data):\n    df = ATR(stock_data.high, stock_data.low, \n             stock_data.close, timeperiod=14)\n    return df.sub(df.mean()).div(df.std())","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:22:27.289312Z","iopub.execute_input":"2022-07-08T05:22:27.289664Z","iopub.status.idle":"2022-07-08T05:22:27.296592Z","shell.execute_reply.started":"2022-07-08T05:22:27.289632Z","shell.execute_reply":"2022-07-08T05:22:27.29532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices['ATR'] = (prices.groupby('symbol', group_keys=False)\n                 .apply(compute_atr))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:22:27.299178Z","iopub.execute_input":"2022-07-08T05:22:27.299566Z","iopub.status.idle":"2022-07-08T05:22:41.650375Z","shell.execute_reply.started":"2022-07-08T05:22:27.299534Z","shell.execute_reply":"2022-07-08T05:22:41.648557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MACD 지표 만들기","metadata":{}},{"cell_type":"code","source":"prices['PPO'] = prices.groupby(level='symbol').close.apply(talib.PPO)\ndef compute_macd(close):\n    macd = MACD(close)[0]\n    return (macd - np.mean(macd))/np.std(macd)\nprices['MACD'] = (prices\n                  .groupby('symbol', group_keys=False)\n                  .close\n                  .apply(compute_macd))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T05:22:41.652561Z","iopub.execute_input":"2022-07-08T05:22:41.65306Z","iopub.status.idle":"2022-07-08T05:23:24.195341Z","shell.execute_reply.started":"2022-07-08T05:22:41.653005Z","shell.execute_reply":"2022-07-08T05:23:24.194244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 가격데이터와 Meta 데이터 결합","metadata":{}},{"cell_type":"code","source":"metadata.sector = pd.factorize(metadata.sector)[0].astype(int)\nprices = prices.join(metadata[['sector']])","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:00:39.28146Z","iopub.execute_input":"2022-07-08T06:00:39.28183Z","iopub.status.idle":"2022-07-08T06:00:39.473948Z","shell.execute_reply.started":"2022-07-08T06:00:39.2818Z","shell.execute_reply":"2022-07-08T06:00:39.472909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 기간별 수익률 구하기","metadata":{}},{"cell_type":"code","source":"T = [1, 5, 10, 21, 42, 63]","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:00:35.55327Z","iopub.execute_input":"2022-07-08T06:00:35.55379Z","iopub.status.idle":"2022-07-08T06:00:35.559907Z","shell.execute_reply.started":"2022-07-08T06:00:35.553753Z","shell.execute_reply":"2022-07-08T06:00:35.558675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"by_sym = prices.groupby(level='symbol').close\nfor t in T:\n    prices[f'r{t:02}'] = by_sym.pct_change(t)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:00:35.982598Z","iopub.execute_input":"2022-07-08T06:00:35.983542Z","iopub.status.idle":"2022-07-08T06:00:36.869243Z","shell.execute_reply.started":"2022-07-08T06:00:35.983505Z","shell.execute_reply":"2022-07-08T06:00:36.868224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 기간별 수익률의 Quantile 구하기","metadata":{}},{"cell_type":"code","source":"for t in T:\n    prices[f'r{t:02}dec'] = (prices[f'r{t:02}']\n                             .groupby(level='date')\n                             .apply(lambda x: pd.qcut(x, \n                                                      q=10, \n                                                      labels=False, \n                                                      duplicates='drop')))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:00:41.812706Z","iopub.execute_input":"2022-07-08T06:00:41.813144Z","iopub.status.idle":"2022-07-08T06:03:00.603342Z","shell.execute_reply.started":"2022-07-08T06:00:41.813108Z","shell.execute_reply":"2022-07-08T06:03:00.602079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 기간별 섹터 Quantile 구하기","metadata":{}},{"cell_type":"code","source":"for t in T:\n    prices[f'r{t:02}q_sector'] = (prices\n                                  .groupby(['date', 'sector'])[f'r{t:02}']\n                                  .transform(lambda x: pd.qcut(x, \n                                                               q=5, \n                                                               labels=False, \n                                                               duplicates='drop')))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:03:00.605521Z","iopub.execute_input":"2022-07-08T06:03:00.605888Z","iopub.status.idle":"2022-07-08T06:05:28.155483Z","shell.execute_reply.started":"2022-07-08T06:03:00.605846Z","shell.execute_reply":"2022-07-08T06:05:28.154517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 학습을 위한 미래 수익률 가져오기","metadata":{}},{"cell_type":"code","source":"for t in [1, 5, 21]:\n    prices[f'r{t:02}_fwd'] = prices.groupby(level='symbol')[f'r{t:02}'].shift(-t)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:05:28.157259Z","iopub.execute_input":"2022-07-08T06:05:28.157683Z","iopub.status.idle":"2022-07-08T06:05:28.753231Z","shell.execute_reply.started":"2022-07-08T06:05:28.157639Z","shell.execute_reply":"2022-07-08T06:05:28.752163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Outlier 값들 제거","metadata":{}},{"cell_type":"code","source":"prices[[f'r{t:02}' for t in T]].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:05:28.75526Z","iopub.execute_input":"2022-07-08T06:05:28.755596Z","iopub.status.idle":"2022-07-08T06:05:30.123241Z","shell.execute_reply.started":"2022-07-08T06:05:28.755566Z","shell.execute_reply":"2022-07-08T06:05:30.121815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outliers = prices[prices.r01 > 1].index.get_level_values('symbol').unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:05:30.124874Z","iopub.execute_input":"2022-07-08T06:05:30.125884Z","iopub.status.idle":"2022-07-08T06:05:30.137716Z","shell.execute_reply.started":"2022-07-08T06:05:30.125836Z","shell.execute_reply":"2022-07-08T06:05:30.136386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices = prices.drop(outliers, level='symbol')","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:05:30.140046Z","iopub.execute_input":"2022-07-08T06:05:30.143248Z","iopub.status.idle":"2022-07-08T06:05:50.263495Z","shell.execute_reply.started":"2022-07-08T06:05:30.143201Z","shell.execute_reply":"2022-07-08T06:05:50.262282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sector와 시간 변수 추가","metadata":{}},{"cell_type":"code","source":"prices['year'] = prices.index.get_level_values('date').year\nprices['month'] = prices.index.get_level_values('date').month\nprices['weekday'] = prices.index.get_level_values('date').weekday","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:05:50.264946Z","iopub.execute_input":"2022-07-08T06:05:50.265459Z","iopub.status.idle":"2022-07-08T06:05:50.871569Z","shell.execute_reply.started":"2022-07-08T06:05:50.265418Z","shell.execute_reply":"2022-07-08T06:05:50.870332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature가 추가된 데이터 저장 ","metadata":{}},{"cell_type":"code","source":"prices","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:05:50.873077Z","iopub.execute_input":"2022-07-08T06:05:50.873425Z","iopub.status.idle":"2022-07-08T06:05:52.266729Z","shell.execute_reply.started":"2022-07-08T06:05:50.873393Z","shell.execute_reply":"2022-07-08T06:05:52.2654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.to_csv(\"model_data.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:09:26.451327Z","iopub.execute_input":"2022-07-08T06:09:26.451684Z","iopub.status.idle":"2022-07-08T06:10:52.713488Z","shell.execute_reply.started":"2022-07-08T06:09:26.451653Z","shell.execute_reply":"2022-07-08T06:10:52.71227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T06:10:53.121156Z","iopub.execute_input":"2022-07-08T06:10:53.121587Z","iopub.status.idle":"2022-07-08T06:10:53.188716Z","shell.execute_reply.started":"2022-07-08T06:10:53.121552Z","shell.execute_reply":"2022-07-08T06:10:53.187473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}