{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"'''\n# download TA-Lib \n!wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz \n!tar xvzf ta-lib-0.4.0-src.tar.gz\nimport os\nos.chdir('ta-lib')\n!./configure --prefix=/usr\n!make\n!make install\n!pip install TA-Lib\nimport talib as ta\nos.chdir('../')\n'''\n\n!cp ../input/talibinstall/ta-lib-0.4.0-src.tar.gzh  ./ta-lib-0.4.0-src.tar.gz\n!tar -xzvf ta-lib-0.4.0-src.tar.gz > null\n!cd ta-lib && ./configure --prefix=/usr > null && make  > null && make install > null\n!cp ../input/talibinstall/TA-Lib-0.4.21.tar.gzh TA-Lib-0.4.21.tar.gz\n!pip install TA-Lib-0.4.21.tar.gz > null\n!pip install ../input/talibinstall/numpy-1.21.4-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl >null\n\n!rm -rf ./ta-lib\n!rm  ./TA-Lib-0.4.21.tar.gz\n!rm  ./ta-lib-0.4.0-src.tar.gz\n!rm  ./null\n\nimport talib as ta","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.041689,"end_time":"2022-08-06T07:09:07.166752","exception":false,"start_time":"2022-08-06T07:09:07.125063","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:42:22.779044Z","iopub.execute_input":"2022-08-06T07:42:22.779368Z","iopub.status.idle":"2022-08-06T07:45:50.930054Z","shell.execute_reply.started":"2022-08-06T07:42:22.779282Z","shell.execute_reply":"2022-08-06T07:45:50.929112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, warnings, random, datetime, traceback, joblib\nwarnings.filterwarnings('ignore')\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport json\nfrom scipy import stats\n\nimport numpy as np\nimport pandas as pd\nfrom tensorflow.python.keras import backend as K\n\nfrom tqdm.notebook import tqdm\nfrom tensorflow.keras import layers\nfrom tensorflow import keras\nfrom tensorflow.keras import regularizers\nfrom sklearn.preprocessing import RobustScaler","metadata":{"papermill":{"duration":7.582351,"end_time":"2022-08-06T07:12:42.351567","exception":false,"start_time":"2022-08-06T07:12:34.769216","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:45:50.932295Z","iopub.execute_input":"2022-08-06T07:45:50.932576Z","iopub.status.idle":"2022-08-06T07:45:56.812464Z","shell.execute_reply.started":"2022-08-06T07:45:50.932521Z","shell.execute_reply":"2022-08-06T07:45:56.811600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_s = \"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/\"\npath_t = \"../input/jpx-tokyo-stock-exchange-prediction/train_files/\"\npath_e = \"../input/jpx-tokyo-stock-exchange-prediction/example_test_files/\"","metadata":{"papermill":{"duration":0.031897,"end_time":"2022-08-06T07:12:42.545563","exception":false,"start_time":"2022-08-06T07:12:42.513666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:45:56.813788Z","iopub.execute_input":"2022-08-06T07:45:56.814937Z","iopub.status.idle":"2022-08-06T07:45:56.819927Z","shell.execute_reply.started":"2022-08-06T07:45:56.814895Z","shell.execute_reply":"2022-08-06T07:45:56.818749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices1 = pd.read_csv(f\"{path_s}stock_prices.csv\") #2021\nprices2 = pd.read_csv(f\"{path_t}stock_prices.csv\") #2017\nprices3 = pd.read_csv(f\"{path_t}secondary_stock_prices.csv\")\n\nfinance1 = pd.read_csv(f\"{path_s}financials.csv\")\nfinance2 = pd.read_csv(f\"{path_t}financials.csv\")\n\nprices1.shape , prices2.shape, prices3.shape","metadata":{"papermill":{"duration":12.082821,"end_time":"2022-08-06T07:12:54.654265","exception":false,"start_time":"2022-08-06T07:12:42.571444","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:45:56.824332Z","iopub.execute_input":"2022-08-06T07:45:56.825581Z","iopub.status.idle":"2022-08-06T07:46:08.511193Z","shell.execute_reply.started":"2022-08-06T07:45:56.825487Z","shell.execute_reply":"2022-08-06T07:46:08.510397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices2['Target'].isnull().sum(), prices1['Target'].isnull().sum(), prices3['Target'].isnull().sum()","metadata":{"papermill":{"duration":0.049892,"end_time":"2022-08-06T07:12:54.731875","exception":false,"start_time":"2022-08-06T07:12:54.681983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:08.512419Z","iopub.execute_input":"2022-08-06T07:46:08.512896Z","iopub.status.idle":"2022-08-06T07:46:08.536977Z","shell.execute_reply.started":"2022-08-06T07:46:08.512858Z","shell.execute_reply":"2022-08-06T07:46:08.536259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train2 = prices2.dropna(subset=['Target'])\n#train1 = prices1.dropna(subset=['Target'])\n\nprices2, prices1 = prices2.fillna(0), prices1.fillna(0)\nprices2['Target'].isnull().sum(), prices1['Target'].isnull().sum()","metadata":{"papermill":{"duration":0.422499,"end_time":"2022-08-06T07:12:55.182130","exception":false,"start_time":"2022-08-06T07:12:54.759631","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:08.538284Z","iopub.execute_input":"2022-08-06T07:46:08.538648Z","iopub.status.idle":"2022-08-06T07:46:08.938263Z","shell.execute_reply.started":"2022-08-06T07:46:08.538613Z","shell.execute_reply":"2022-08-06T07:46:08.937478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices  = pd.concat([prices2,prices1])\nfinance = pd.concat([finance2.dropna(subset=['SecuritiesCode']), finance1])","metadata":{"papermill":{"duration":0.44036,"end_time":"2022-08-06T07:12:55.651295","exception":false,"start_time":"2022-08-06T07:12:55.210935","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:08.939555Z","iopub.execute_input":"2022-08-06T07:46:08.939901Z","iopub.status.idle":"2022-08-06T07:46:09.414230Z","shell.execute_reply.started":"2022-08-06T07:46:08.939864Z","shell.execute_reply":"2022-08-06T07:46:09.413434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del prices1\ndel prices2\ndel prices3","metadata":{"papermill":{"duration":0.104206,"end_time":"2022-08-06T07:12:55.785016","exception":false,"start_time":"2022-08-06T07:12:55.680810","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.415865Z","iopub.execute_input":"2022-08-06T07:46:09.416153Z","iopub.status.idle":"2022-08-06T07:46:09.493278Z","shell.execute_reply.started":"2022-08-06T07:46:09.416107Z","shell.execute_reply":"2022-08-06T07:46:09.491622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"GPU\" ","metadata":{"papermill":{"duration":0.034618,"end_time":"2022-08-06T07:12:55.848508","exception":false,"start_time":"2022-08-06T07:12:55.813890","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.494594Z","iopub.execute_input":"2022-08-06T07:46:09.495056Z","iopub.status.idle":"2022-08-06T07:46:09.500009Z","shell.execute_reply.started":"2022-08-06T07:46:09.495016Z","shell.execute_reply":"2022-08-06T07:46:09.499202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if device == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except: print(\"failed to initialize TPU\")\n    else: device = \"GPU\"\n\nif device != \"TPU\": strategy = tf.distribute.get_strategy()\nif device == \"GPU\": print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"papermill":{"duration":0.225555,"end_time":"2022-08-06T07:12:56.102883","exception":false,"start_time":"2022-08-06T07:12:55.877328","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.503468Z","iopub.execute_input":"2022-08-06T07:46:09.504362Z","iopub.status.idle":"2022-08-06T07:46:09.675181Z","shell.execute_reply.started":"2022-08-06T07:46:09.504294Z","shell.execute_reply":"2022-08-06T07:46:09.674440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)","metadata":{"papermill":{"duration":0.036839,"end_time":"2022-08-06T07:12:56.169926","exception":false,"start_time":"2022-08-06T07:12:56.133087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.676408Z","iopub.execute_input":"2022-08-06T07:46:09.676709Z","iopub.status.idle":"2022-08-06T07:46:09.681521Z","shell.execute_reply.started":"2022-08-06T07:46:09.676661Z","shell.execute_reply":"2022-08-06T07:46:09.680848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 2022\nset_all_seeds(SEED)","metadata":{"papermill":{"duration":0.036972,"end_time":"2022-08-06T07:12:56.236176","exception":false,"start_time":"2022-08-06T07:12:56.199204","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.682725Z","iopub.execute_input":"2022-08-06T07:46:09.683688Z","iopub.status.idle":"2022-08-06T07:46:09.690478Z","shell.execute_reply.started":"2022-08-06T07:46:09.683643Z","shell.execute_reply":"2022-08-06T07:46:09.689719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    is_training = False\n    output_dataset_path = \"../input/models-lstm-gru-jpx-32tf/\" #update 118F\nconfig = Config()","metadata":{"papermill":{"duration":0.035932,"end_time":"2022-08-06T07:12:56.302527","exception":false,"start_time":"2022-08-06T07:12:56.266595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.691861Z","iopub.execute_input":"2022-08-06T07:46:09.692204Z","iopub.status.idle":"2022-08-06T07:46:09.699038Z","shell.execute_reply.started":"2022-08-06T07:46:09.692158Z","shell.execute_reply":"2022-08-06T07:46:09.698287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsecurities_codes = pd.read_csv(\"../input/jpxs-cgkf-10f-118f-tf/jpx_tf/securities_codes.csv\")\nsecurities_codes_size = len(securities_codes) + 1\nwith tf.device(\"cpu\"):\n    securities_codes_lookup_layer = layers.IntegerLookup(max_tokens=securities_codes_size)\n    securities_codes_lookup_layer.adapt(securities_codes)","metadata":{"papermill":{"duration":2.735785,"end_time":"2022-08-06T07:12:59.067874","exception":false,"start_time":"2022-08-06T07:12:56.332089","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:09.700374Z","iopub.execute_input":"2022-08-06T07:46:09.700649Z","iopub.status.idle":"2022-08-06T07:46:12.006897Z","shell.execute_reply.started":"2022-08-06T07:46:09.700608Z","shell.execute_reply":"2022-08-06T07:46:12.005477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prep_prices(prices):\n    prices = prices.fillna(method=\"pad\").fillna(0)\n    return prices*1\n\ndef finance_feat(df):\n    finance_feat = df.replace(df.value_counts().index,pd.factorize(df.value_counts().index)[0])\n    df = finance_feat\n    return df\n\ndef get_finance(df, tr=True):\n    df_finance = (df.replace([r\"－\"], np.nan, regex=True).fillna(method=\"pad\").fillna(0))\n    \n    df_finance['TypeOfDocument']      = finance_feat(df_finance.TypeOfDocument)\n    df_finance['TypeOfCurrentPeriod'] = finance_feat(df_finance.TypeOfCurrentPeriod)\n    \n    if tr:\n        df_finance = df_finance.drop(['DisclosureNumber',\n                      'DisclosedDate', \n                      'CurrentPeriodEndDate',\n                      'DisclosedUnixTime',\n                      'CurrentFiscalYearStartDate', \n                      'DisclosedTime', \n                      'CurrentFiscalYearEndDate', \n                      'DateCode'],axis=1)\n    \n    return df_finance","metadata":{"papermill":{"duration":0.040948,"end_time":"2022-08-06T07:12:59.140729","exception":false,"start_time":"2022-08-06T07:12:59.099781","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.008172Z","iopub.execute_input":"2022-08-06T07:46:12.008415Z","iopub.status.idle":"2022-08-06T07:46:12.017141Z","shell.execute_reply.started":"2022-08-06T07:46:12.008381Z","shell.execute_reply":"2022-08-06T07:46:12.016090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from decimal import *\ndef adjust_price(price):\n    \"\"\"\n    Args:\n        price (pd.DataFrame)  : pd.DataFrame include stock_price\n    Returns:\n        price DataFrame (pd.DataFrame): stock_price with generated AdjustedClose\n    \"\"\"\n    # transform Date column into datetime\n    price.loc[: ,\"Date\"] = pd.to_datetime(price.loc[: ,\"Date\"], format=\"%Y-%m-%d\")\n\n    def generate_adjusted_close(df):\n        \"\"\"\n        Args:\n            df (pd.DataFrame)  : stock_price for a single SecuritiesCode\n        Returns:\n            df (pd.DataFrame): stock_price with AdjustedClose for a single SecuritiesCode\n        \"\"\"\n        # sort data to generate CumulativeAdjustmentFactor\n        df = df.sort_values(\"Date\", ascending=False)\n        # generate CumulativeAdjustmentFactor\n        df.loc[:, \"CumulativeAdjustmentFactor\"] = df[\"AdjustmentFactor\"].cumprod()\n        # generate AdjustedClose\n        df.loc[:, \"AdjustedClose\"] = (\n            df[\"CumulativeAdjustmentFactor\"] * df[\"Close\"]\n        ).map(lambda x: float(\n            Decimal(str(x)).quantize(Decimal('0.1'), rounding=ROUND_HALF_UP)\n        ))\n        # reverse order\n        df = df.sort_values(\"Date\")\n        # to fill AdjustedClose, replace 0 into np.nan\n        df.loc[df[\"AdjustedClose\"] == 0, \"AdjustedClose\"] = np.nan\n        # forward fill AdjustedClose\n        df.loc[:, \"AdjustedClose\"] = df.loc[:, \"AdjustedClose\"].ffill()\n        return df\n\n    # generate AdjustedClose\n    price = price.sort_values([\"SecuritiesCode\", \"Date\"])\n    price = price.groupby(\"SecuritiesCode\").apply(generate_adjusted_close).reset_index(drop=True)\n\n    price.set_index(\"Date\", inplace=True)\n    return price","metadata":{"papermill":{"duration":0.041174,"end_time":"2022-08-06T07:12:59.212998","exception":false,"start_time":"2022-08-06T07:12:59.171824","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.018758Z","iopub.execute_input":"2022-08-06T07:46:12.019017Z","iopub.status.idle":"2022-08-06T07:46:12.030131Z","shell.execute_reply.started":"2022-08-06T07:46:12.018983Z","shell.execute_reply":"2022-08-06T07:46:12.029260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df_prices, df_finance, tr=True):\n    df_finance['SecuritiesCode'] = df_finance['SecuritiesCode'].astype('int32').fillna(-1)\n    df_finance = get_finance(df_finance)\n    df  = pd.concat([df_prices, df_finance])\n    \n    if tr:\n        scale_features = df.columns.drop(['SecuritiesCode','Date','Target'])    \n        df[scale_features] = RobustScaler().fit_transform(df[scale_features]) \n    if not tr:\n        scale_features = df.columns.drop(['SecuritiesCode','Date',])    \n        df[scale_features] = RobustScaler().fit_transform(df[scale_features]) \n\n    df  = adjust_price(df)\n    df  = df.copy()\n\n    col = 'AdjustedClose'\n    periods=[3,5,8,12,15,26,30,35,40,45,50,60, 100,200]\n    for period in tqdm(periods):\n        df.loc[:,\"RT_{}\".format(period)] = df.groupby(\"SecuritiesCode\")[col].pct_change(period)\n        df.loc[:,\"MA_{}\".format(period)] = df.groupby(\"SecuritiesCode\")[col].rolling(window=period).mean().values\n        df.loc[:,\"EMA_{}\".format(period)] = df.groupby(\"SecuritiesCode\")[col].ewm(span=period,adjust=False).mean().values\n        df.loc[:,\"V_{}\".format(period)] = np.log(df[col]).groupby(df[\"SecuritiesCode\"]).diff().rolling(period).std()\n        \n    df = prep_prices(df)\n    \n    df['upper_Shadow']   = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['lower_Shadow']   = np.minimum(df['Close'], df['Open']) - df['Low'] \n\n    # The Golden Ratio Multiplier \n    df['GRM_0']    = (ta.MA(df['Close'], timeperiod=350, matype=0)) \n    df['GRM_1']    = (ta.MA(df['Close'], timeperiod=350, matype=0))*1.6  \n    df['GRM_2']    = (ta.MA(df['Close'], timeperiod=350, matype=0))*2\n    df['GRM_3']    = (ta.MA(df['Close'], timeperiod=350, matype=0))*3\n\n    df['Pi_Cycle'] = ta.MA(df['Close'], timeperiod=111, matype=0) \n    \n    # Momentum\n    df['RSI_14'] = ta.RSI(df['Close'], timeperiod=14)\n    df['RSI_24'] = ta.RSI(df['Close'], timeperiod=24)\n    df['RSI_42'] = ta.RSI(df['Close'], timeperiod=42)\n    \n    df['RSI1']   = df['RSI_14'].shift(-1) \n    df['RSI4']   = df['RSI_14'].shift(-4) \n    df['RSI7']   = df['RSI_14'].shift(-7) \n    df['RSI10']  = df['RSI_14'].shift(-10) \n    df['RSI13']  = df['RSI_14'].shift(-13) \n    df['RSI16']  = df['RSI_14'].shift(-16) \n    \n    df['MACD_12'], df['macdsignal_12'], df['MACD_HIST_12'] = ta.MACD(df['Close'], fastperiod=12, slowperiod=26, signalperiod=9) \n    df['MACD_48'], df['macdsignal_48'], df['MACD_HIST_48'] = ta.MACD(df['Close'], fastperiod=48, slowperiod=104, signalperiod=36)\n    \n    df['macdsignal1'] = df['macdsignal_12'].shift(-1)\n    df['macdsignal4'] = df['macdsignal_12'].shift(-4)\n    df['macdsignal7'] = df['macdsignal_12'].shift(-7)\n    df['MACD_HIST1']  = df['MACD_HIST_12'].shift(-1) \n    df['MACD_HIST4']  = df['MACD_HIST_12'].shift(-4) \n    df['MACD_HIST7']  = df['MACD_HIST_12'].shift(-7) \n    df['ROCP']     = ta.ROCP(df['Open'])\n    df['momentam'] = ta.MOM(df['Open'])\n    df['CMO']      = ta.CMO(df['Open']) \n    df['PPO']      = ta.PPO(df['Open'])\n    df['SAR']       = ta.SAR(df['High'], df['Low'], acceleration=0, maximum=0) \n    df['DI_minus']  = ta.MINUS_DI(df['High'], df['Low'],np.array(df.loc[:, 'Close']), timeperiod=14) \n    df['DI_minus1'] = df['DI_minus'].shift(-1) \n    df['DI_minus4'] = df['DI_minus'].shift(-4) \n    df['DI_minus7'] = df['DI_minus'].shift(-7)  \n    df['adx']    = ta.ADX(df['High'], df['Low'],np.array(df.loc[:, 'Close']),timeperiod=14) \n    df['adx1']   = df['adx'].shift(-1) \n    df['adx4']   = df['adx'].shift(-4) \n    df['adx+1']  = df['adx'].shift(1) \n    df['adx7']   = df['adx'].shift(-7)\n    df['DI_plus']   = ta.PLUS_DI(df['High'], df['Low'],np.array(df.loc[:, 'Close']), timeperiod=14) \n    df['DI_plus1']  = df['DI_plus'].shift(-1) \n    df['DI_plus4']  = df['DI_plus'].shift(-4) \n    df['DI_plus7']  = df['DI_plus'].shift(-7) \n    df['DI_plus10'] = df['DI_plus'].shift(-10)\n    df['APO']      = ta.APO(df['Open'])\n    df['APO1']     = df['APO'].shift(-1)\n    df['APO4']     = df['APO'].shift(-4)\n    df['APO7']     = df['APO'].shift(-7)\n    df['ROCR100']  = ta.AD(df['High'], df['Low'], df['Close'], df['Volume'])\n    df['OBV']      = ta.OBV(df['Close'], df['Volume'])\n    df['ADOSC']    = ta.ADOSC(df['High'], df['Low'], df['Close'], df['Volume'], fastperiod=3, slowperiod=10)\n    df['TSF']      = ta.TSF(df['Close'], timeperiod=14) \n    df['TSF-14']   = ta.TSF(df['Close'], timeperiod=14).shift(-14)\n    df['TSF-7']    = ta.TSF(df['Close'], timeperiod=14).shift(-7)\n    df['ATR']    = ta.ATR(df['High'], df['Low'], df['Close'], timeperiod=14)\n    df['NATR']   = ta.NATR(df['High'], df['Low'], df['Close'], timeperiod=14)\n    \n    ema_set = [3,5,8,12,15,26,30,35,40,45,50,60, 100,200]\n    # EMA\n    for i in tqdm(range(len(ema_set))):        \n        sma = df['Close'].rolling(ema_set[i]).mean()\n        ema = sma.ewm(span=ema_set[i], adjust=False).mean()\n        df[\"EMA_C_%d\"%(ema_set[i])] = ema                                    \n        df = prep_prices(df)\n        \n    if tr:\n        df = df.drop(['RowId'],axis=1)\n    return df","metadata":{"papermill":{"duration":0.062897,"end_time":"2022-08-06T07:12:59.306904","exception":false,"start_time":"2022-08-06T07:12:59.244007","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.031506Z","iopub.execute_input":"2022-08-06T07:46:12.032034Z","iopub.status.idle":"2022-08-06T07:46:12.062576Z","shell.execute_reply.started":"2022-08-06T07:46:12.031999Z","shell.execute_reply":"2022-08-06T07:46:12.061751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = list(pd.read_csv(f\"../input/jpxs-cgkf-10f-118f-tf/jpx_tf/feats.csv\")['0'])\nlen(feats)","metadata":{"papermill":{"duration":0.044762,"end_time":"2022-08-06T07:12:59.382717","exception":false,"start_time":"2022-08-06T07:12:59.337955","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.063837Z","iopub.execute_input":"2022-08-06T07:46:12.064085Z","iopub.status.idle":"2022-08-06T07:46:12.084447Z","shell.execute_reply.started":"2022-08-06T07:46:12.064052Z","shell.execute_reply":"2022-08-06T07:46:12.083592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats","metadata":{"papermill":{"duration":0.040559,"end_time":"2022-08-06T07:12:59.454616","exception":false,"start_time":"2022-08-06T07:12:59.414057","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.085768Z","iopub.execute_input":"2022-08-06T07:46:12.086015Z","iopub.status.idle":"2022-08-06T07:46:12.093212Z","shell.execute_reply.started":"2022-08-06T07:46:12.085981Z","shell.execute_reply":"2022-08-06T07:46:12.092360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_function(record_bytes):\n      return tf.io.parse_single_example(\n      record_bytes,\n      {\n          \"features\": tf.io.FixedLenFeature([len(feats)], dtype=tf.float32),\n          \"securities_code\": tf.io.FixedLenFeature([], dtype=tf.int64),\n          \"Target\": tf.io.FixedLenFeature([], dtype=tf.float32)\n      }\n  )\ndef preprocess(item):\n    return (item[\"securities_code\"], item[\"features\"]), item[\"Target\"]\ndef make_dataset(file_paths, batch_size=4096, mode=\"train\"):\n    ds = tf.data.TFRecordDataset(file_paths)\n    ds = ds.map(decode_function)\n    ds = ds.map(preprocess)\n    if mode == \"train\":\n        ds = ds.shuffle(batch_size*4*REPLICAS)\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.AUTOTUNE)\n    return ds","metadata":{"papermill":{"duration":0.04092,"end_time":"2022-08-06T07:12:59.527598","exception":false,"start_time":"2022-08-06T07:12:59.486678","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.095762Z","iopub.execute_input":"2022-08-06T07:46:12.096631Z","iopub.status.idle":"2022-08-06T07:46:12.104723Z","shell.execute_reply.started":"2022-08-06T07:46:12.096596Z","shell.execute_reply":"2022-08-06T07:46:12.103953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correlationLoss(x,y, axis=-2):\n    \n    \"\"\"Loss function that maximizes the pearson correlation coefficient between the predicted values and the labels,\n    while trying to have the same mean and variance\"\"\"\n    x = tf.convert_to_tensor(x)\n    y = math_ops.cast(y, x.dtype)\n    n = tf.cast(tf.shape(x)[axis], x.dtype)\n    xsum = tf.reduce_sum(x, axis=axis)\n    ysum = tf.reduce_sum(y, axis=axis)\n    xmean = xsum / n\n    ymean = ysum / n\n    xsqsum = tf.reduce_sum( tf.math.squared_difference(x, xmean), axis=axis)\n    ysqsum = tf.reduce_sum( tf.math.squared_difference(y, ymean), axis=axis)\n    cov = tf.reduce_sum( (x - xmean) * (y - ymean), axis=axis)\n    corr = cov / tf.sqrt(xsqsum * ysqsum)\n    return tf.convert_to_tensor( K.mean(tf.constant(1.0, dtype=x.dtype) - corr ) , dtype=tf.float32 )\n\ndef correlation(x, y, axis=-2):\n    \"\"\"Metric returning the Pearson correlation coefficient of two tensors over some axis, default -2.\"\"\"\n    x = tf.convert_to_tensor(x)\n    y = math_ops.cast(y, x.dtype)\n    n = tf.cast(tf.shape(x)[axis], x.dtype)\n    xsum = tf.reduce_sum(x, axis=axis)\n    ysum = tf.reduce_sum(y, axis=axis)\n    xmean = xsum / n\n    ymean = ysum / n\n    xvar = tf.reduce_sum( tf.math.squared_difference(x, xmean), axis=axis)\n    yvar = tf.reduce_sum( tf.math.squared_difference(y, ymean), axis=axis)\n    cov = tf.reduce_sum( (x - xmean) * (y - ymean), axis=axis)\n    corr = cov / tf.sqrt(xvar * yvar)\n    return tf.constant(1.0, dtype=x.dtype) - corr\n\ndef sharpe_loss(X_train,y_pred):\n    y_pred = tf.Variable(y_pred,dtype=tf.float64)\n    port_ret = tf.reduce_sum(tf.multiply(X_train,y_pred),axis=1)\n    s_ratio = K.mean(port_ret)/K.std(port_ret)\n    \n    return tf.math.exp(-s_ratio,  name='sharpe_loss')","metadata":{"papermill":{"duration":0.04574,"end_time":"2022-08-06T07:12:59.605277","exception":false,"start_time":"2022-08-06T07:12:59.559537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.106000Z","iopub.execute_input":"2022-08-06T07:46:12.107100Z","iopub.status.idle":"2022-08-06T07:46:12.120299Z","shell.execute_reply.started":"2022-08-06T07:46:12.107056Z","shell.execute_reply":"2022-08-06T07:46:12.119324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(hp, dim=118,fold=0):\n\n    features_inputs = tf.keras.layers.Input(shape = [dim])\n    securities_code_inputs = tf.keras.layers.Input(shape = [1])\n    \n    securities_code_x = securities_codes_lookup_layer(securities_code_inputs)\n    securities_code_x = layers.Embedding(securities_codes_size, hp.Int(f'em_f{fold}',16, 1024), input_length=1)(securities_code_x)\n    securities_code_x = layers.Reshape((-1, ))(securities_code_x)\n    securities_code_x = layers.Dense(hp.Int(f'dn_f{fold}',32, 1024), activation='swish')(securities_code_x)\n        \n    x_rnn = tf.keras.layers.BatchNormalization()(features_inputs)\n    x_rnn = tf.keras.layers.Dense(hp.Int(f'x_rnn_0_f{fold}',32, 1024, step=32), activation='swish')(x_rnn)\n    x_rnn = tf.keras.layers.Dropout(hp.Float(f'dropout_lstm_0_f{fold}',0.001, 0.99))(x_rnn)\n    x_rnn = tf.keras.layers.Reshape((1, -1))(x_rnn)\n    x_rnn = tf.keras.layers.BatchNormalization()(x_rnn)\n    x_rnn = tf.keras.layers.LSTM(hp.Int(f'x_rnn_1_f{fold}',32, 1024, step=32), dropout=hp.Float(f'dropout_lstm_1_f{fold}',0.1, 0.99), recurrent_dropout=hp.Float(f'rdropout_lstm_0_f{fold}',0.1, 0.99), return_sequences=True, activation='relu')(x_rnn)\n    x_rnn = tf.keras.layers.LSTM(hp.Int(f'x_rnn_2_f{fold}',32, 1024, step=32), dropout=hp.Float(f'dropout_lstm_2_f{fold}',0.1, 0.99), return_sequences=False, activation='relu')(x_rnn)\n    \n    x_gru = tf.keras.layers.Dense(hp.Int(f'x_gru_0_f{fold}',32, 1024, step=32), activation = \"swish\")(features_inputs)\n    x_gru = tf.keras.layers.Dense(hp.Int(f'x_gru_1_f{fold}',32, 1024, step=32), activation = \"swish\")(x_gru)\n    x_gru = tf.keras.layers.Dense(hp.Int(f'x_gru_2_f{fold}',32, 1024, step=32), activation = \"swish\")(x_gru)\n    x_gru = tf.keras.layers.BatchNormalization()(x_gru)\n    x_gru = tf.keras.layers.Reshape((1, -1))(x_gru)\n    x_gru = tf.keras.layers.GRU(hp.Int(f'x_gru_3_f{fold}',16, 1024, step=32), recurrent_dropout = hp.Float(f'rdropout_gru_0_f{fold}',0.1, 0.99), dropout = hp.Float(f'dropout_gru_0_f{fold}',0.001, 0.99), return_sequences = True)(x_gru)\n    x_gru = tf.keras.layers.GRU(hp.Int(f'x_gru_4_f{fold}',16, 1024, step=32), recurrent_dropout = hp.Float(f'rdropout_gru_1_f{fold}',0.1, 0.99), dropout = hp.Float(f'dropout_gru_1_f{fold}',0.001, 0.99), return_sequences = True)(x_gru)\n    x_gru = tf.keras.layers.GRU(hp.Int(f'x_gru_5_f{fold}',16, 1024, step=32), recurrent_dropout = hp.Float(f'rdropout_gru_2_f{fold}',0.1, 0.99), dropout = hp.Float(f'dropout_gru_2_f{fold}',0.001, 0.99), return_sequences = False)(x_gru)\n    x_gru = tf.keras.layers.Concatenate(axis=1)([securities_code_x, x_gru])\n    x_gru = tf.keras.layers.Dense(hp.Int(f'x_gru_6_f{fold}',8, 1024, step=32), activation = \"swish\")(x_gru)\n    x_gru = tf.keras.layers.Dense(hp.Int(f'x_gru_7_f{fold}',8, 1024, step=32), activation = \"swish\")(x_gru)\n    x_gru = tf.keras.layers.Dense(hp.Int(f'x_gru_8_f{fold}',8, 1024, step=32), activation = \"swish\")(x_gru)\n                                                                                                       \n    output_lstm = tf.keras.layers.Dense(1, name='output_lstm')(x_rnn)  \n    output_gru  = tf.keras.layers.Dense(1, name=\"output_gru\")(x_gru)                                              \n    output = tf.keras.layers.Average(name='output')([output_lstm, output_gru])\n\n    model  = tf.keras.Model(inputs=[securities_code_inputs, features_inputs], outputs=[output_lstm, output_gru, output])   \n    \n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=3e-2),\n                  loss = {'output_lstm': [tf.keras.losses.MeanSquaredError()],                          \n                          'output_gru' : [tf.keras.losses.MeanSquaredError()],\n                          'output'     : [tf.keras.losses.MeanSquaredError(), correlationLoss, sharpe_loss],\n                         },\n                  metrics = {'output_lstm': [tf.keras.metrics.RootMeanSquaredError(name='rmse'), correlation],                            \n                             'output_gru' : [tf.keras.metrics.RootMeanSquaredError(name='rmse'), correlation],\n                             'output'     : [tf.keras.metrics.RootMeanSquaredError(name='rmse'), correlation],\n                            },\n                 ) \n    return model","metadata":{"papermill":{"duration":0.058511,"end_time":"2022-08-06T07:12:59.696004","exception":false,"start_time":"2022-08-06T07:12:59.637493","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.122784Z","iopub.execute_input":"2022-08-06T07:46:12.123311Z","iopub.status.idle":"2022-08-06T07:46:12.144359Z","shell.execute_reply.started":"2022-08-06T07:46:12.123252Z","shell.execute_reply":"2022-08-06T07:46:12.143412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hp = pd.read_pickle('../input/hp-jpx-lstmgru-118f-10f-tf/hp_lastmgru_tf_118f_10f_tf.pkl')\ntf.keras.utils.plot_model(build_model(hp), show_shapes=True, expand_nested=True, show_dtype=True)","metadata":{"papermill":{"duration":3.494799,"end_time":"2022-08-06T07:13:03.224496","exception":false,"start_time":"2022-08-06T07:12:59.729697","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:12.146556Z","iopub.execute_input":"2022-08-06T07:46:12.147057Z","iopub.status.idle":"2022-08-06T07:46:15.447003Z","shell.execute_reply.started":"2022-08-06T07:46:12.146997Z","shell.execute_reply":"2022-08-06T07:46:15.446245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ngc.collect()\nmodels = []\nfold = [0,1,2,3,4,5,6,7,8,9]\nmodel_list = -1 #0 = lstm ,1  = gru, -1 = avg(lstm+gru) \nfor i in range(len(fold)):\n    train_path = f\"../input/jpxs-cgkf-10f-118f-tf/jpx_tf/fold_{fold[i]}_train.tfrecords\"\n    valid_path = f\"../input/jpxs-cgkf-10f-118f-tf/jpx_tf/fold_{fold[i]}_test.tfrecords\"\n    valid_ds   = make_dataset([valid_path], mode=\"valid\")\n    checkpoint = keras.callbacks.ModelCheckpoint(f\"model_{fold[i]}.tf\", monitor=\"val_output_rmse\", mode=\"min\", save_best_only=True, save_weights_only=True)\n    early_stop = keras.callbacks.EarlyStopping(patience=7, monitor=\"val_output_rmse\", mode=\"min\")\n    K.clear_session()\n    with strategy.scope(): model = build_model(hp, dim=118, fold=fold[i])\n    if config.is_training:\n        train_ds = make_dataset([train_path])\n        lr = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate = 0.003,decay_steps = 9700, decay_rate = 0.98)\n        history = model.fit(train_ds, epochs=200, validation_data=valid_ds, callbacks=[checkpoint, early_stop, lr])\n        model.load_weights(f\"model_{fold[i]}.tf\")\n        for metric in [\"output_lstm_rmse\", \"output_gru_rmse\", \"output_rmse\", \"output_correlation\"]:\n            pd.DataFrame(history.history, columns=[metric, f\"val_{metric}\"]).plot()\n            plt.title(metric.upper())\n            plt.show()\n            gc.collect()\n    else:\n        model.load_weights(f\"{config.output_dataset_path}model_{fold[i]}.tf\")\n    y_vals = []\n    for _, y in valid_ds:\n        y_vals += list(y.numpy().reshape(-1))\n    y_val = np.array(y_vals)\n    pearson_score = stats.pearsonr((model.predict(valid_ds)[model_list].reshape(-1)), y_val)[0] \n    models.append(model)\n    print(f\"Pearson Score: {pearson_score}\")\n    gc.collect()","metadata":{"_kg_hide-output":true,"papermill":{"duration":495.33986,"end_time":"2022-08-06T07:21:18.687050","exception":false,"start_time":"2022-08-06T07:13:03.347190","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:46:15.448506Z","iopub.execute_input":"2022-08-06T07:46:15.448838Z","iopub.status.idle":"2022-08-06T07:54:39.115428Z","shell.execute_reply.started":"2022-08-06T07:46:15.448799Z","shell.execute_reply":"2022-08-06T07:54:39.114322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_test(securities_code, feature):\n    return (securities_code, feature), 0\n\ndef make_test_dataset(feature, securities_code, batch_size=4096):\n    ds = tf.data.Dataset.from_tensor_slices(((securities_code, feature)))\n    ds = ds.map(preprocess_test)\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.AUTOTUNE)\n    return ds\n\ndef inference(models, ds, model_list=0):\n    y_preds = []\n    for model in models:\n        y_pred = model.predict(ds)[model_list].reshape(-1)\n        y_preds.append(y_pred)\n    return np.mean(y_preds, axis=0)","metadata":{"papermill":{"duration":0.049393,"end_time":"2022-08-06T07:21:18.777607","exception":false,"start_time":"2022-08-06T07:21:18.728214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:54:39.121047Z","iopub.execute_input":"2022-08-06T07:54:39.121256Z","iopub.status.idle":"2022-08-06T07:54:39.127623Z","shell.execute_reply.started":"2022-08-06T07:54:39.121224Z","shell.execute_reply":"2022-08-06T07:54:39.126748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jpx_tokyo_market_prediction\nenv = jpx_tokyo_market_prediction.make_env()   # initialize the environment\niter_test = env.iter_test()                    # an iterator which loops over the test files\nfor (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    \n    financials                       = financials.dropna(subset=['SecuritiesCode'])\n    prices_df                        = get_features(prices, financials, tr=False)\n    prices_df[\"Prediction\"]          = inference(models, make_test_dataset(prices_df[feats].astype(float), prices_df['SecuritiesCode']), model_list) \n    \n    prices_df_                       = pd.merge(prices, prices_df, on=[\"SecuritiesCode\"])\n    sample_prediction['Prediction']  = prices_df_['Prediction']\n    \n    sample_prediction[\"rate\"]        = sample_prediction[\"Prediction\"]\n    sample_prediction.sort_values(by = \"rate\", ascending=False, inplace=True)\n    sample_prediction.Rank           = np.arange(0,len(sample_prediction))\n    sample_prediction.sort_values(by = \"SecuritiesCode\", ascending=True, inplace=True)\n    submission                       = sample_prediction[[\"Date\",\"SecuritiesCode\",\"Rank\"]]\n    \n    cal_target = sample_prediction['Prediction']\n    first200   = pd.DataFrame(-np.sort(-cal_target.values)).loc[:199]\n    last200    = pd.DataFrame(np.sort(cal_target.values)).loc[:199]    \n    daily_spread_returns = first200 - last200\n    sharpe_ratio = daily_spread_returns.mean()/daily_spread_returns.std()\n    print(f'sharpe_ratio: {sharpe_ratio[0]}')    \n    \n    display(submission)\n    env.predict(submission) ","metadata":{"papermill":{"duration":38.85787,"end_time":"2022-08-06T07:21:57.862006","exception":false,"start_time":"2022-08-06T07:21:19.004136","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-06T07:54:39.129012Z","iopub.execute_input":"2022-08-06T07:54:39.129514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREVIOUS LEVEL\n* [JPX(1/2):  HP-AEMLP-TF: Natapong Nitarach](https://www.kaggle.com/natnitarach/jpx-1-2-hp-aemlp-tf/)","metadata":{"papermill":{"duration":0.068084,"end_time":"2022-08-06T07:21:58.152870","exception":false,"start_time":"2022-08-06T07:21:58.084786","status":"completed"},"tags":[]}}]}