{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-04T22:35:00.388517Z","iopub.execute_input":"2022-07-04T22:35:00.389393Z","iopub.status.idle":"2022-07-04T22:35:00.439201Z","shell.execute_reply.started":"2022-07-04T22:35:00.389271Z","shell.execute_reply":"2022-07-04T22:35:00.438268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import necessary libraries\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd \nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport seaborn as sns\n%matplotlib inline\nfrom matplotlib.pylab import rcParams\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt\nfrom scipy import stats\nimport random\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.datasets import make_regression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.datasets import make_regression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import RandomizedSearchCV\n\nimport jpx_tokyo_market_prediction","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:28.827794Z","iopub.execute_input":"2022-07-04T22:37:28.828152Z","iopub.status.idle":"2022-07-04T22:37:29.910544Z","shell.execute_reply.started":"2022-07-04T22:37:28.828117Z","shell.execute_reply":"2022-07-04T22:37:29.909802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#financials = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/train_files/financials.csv\")\n#stock_list = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/stock_list.csv\")\nprices = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/train_files/stock_prices.csv\")\n#financials = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/train_files/financials.csv\")\n#options = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/train_files/options.csv\")\n#sprices = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/train_files/secondary_stock_prices.csv\")\nsupplemental_prices = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv\")\n#supplemental_sprices = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/secondary_stock_prices.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:29.912403Z","iopub.execute_input":"2022-07-04T22:37:29.913164Z","iopub.status.idle":"2022-07-04T22:37:37.412961Z","shell.execute_reply.started":"2022-07-04T22:37:29.913129Z","shell.execute_reply":"2022-07-04T22:37:37.412087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices_add = prices.append(supplemental_prices,ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:37.414197Z","iopub.execute_input":"2022-07-04T22:37:37.414438Z","iopub.status.idle":"2022-07-04T22:37:37.578842Z","shell.execute_reply.started":"2022-07-04T22:37:37.414407Z","shell.execute_reply":"2022-07-04T22:37:37.578044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices_add['DateValue']=prices_add['Date'].str.replace('-','')\nsmall_prices = prices_add[prices_add['DateValue']>'20200601']\nsmall_prices=small_prices.drop(['DateValue'],axis=1)\nsmall_prices.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:37.580794Z","iopub.execute_input":"2022-07-04T22:37:37.581098Z","iopub.status.idle":"2022-07-04T22:37:40.47922Z","shell.execute_reply.started":"2022-07-04T22:37:37.581068Z","shell.execute_reply":"2022-07-04T22:37:40.47837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_prices","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:40.4806Z","iopub.execute_input":"2022-07-04T22:37:40.480824Z","iopub.status.idle":"2022-07-04T22:37:40.515969Z","shell.execute_reply.started":"2022-07-04T22:37:40.480794Z","shell.execute_reply":"2022-07-04T22:37:40.51497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering Functions","metadata":{}},{"cell_type":"code","source":"def feature_train(train):\n    \n    #train = train.groupby(\"SecuritiesCode\")\n    \n    # Add Lag Features\n    lag_features = [\"High\", \"Low\", \"Volume\", \"Close\", \"Open\"]\n    df_rolled_7d = train[lag_features].rolling(window=4, min_periods=0)\n    df_mean_7d = df_rolled_7d.mean().shift(1).reset_index().astype(np.float32)\n    df_mean_7d = df_mean_7d.drop('index', axis=1)\n    df_mean_7d = df_mean_7d.fillna(0)\n    df_mean_7d = df_mean_7d.round(2)\n    df_mean_7d['High_lag_1'] = df_mean_7d['High']\n    df_mean_7d['Open_lag_1'] = df_mean_7d['Open']\n    df_mean_7d['Close_lag_1'] = df_mean_7d['Close']\n    df_mean_7d['Volume_lag_1'] = df_mean_7d['Volume']\n    df_mean_7d['Low_lag_1'] = df_mean_7d['Low']\n    df_mean_7d = df_mean_7d.drop([\"High\", \"Low\", \"Volume\", \"Close\", \"Open\"], axis=1)\n    train = train.reset_index(drop=True)\n    train = pd.concat([train, df_mean_7d], axis=1)\n\n    # Convert Date to float\n    train['Date_Float'] = train['Date'].str.replace('-', '')\n    train['Date_Float'] = train['Date_Float'].astype(float) \n    \n    # Drop irrelevant columns for training\n    train = train.drop(\n                        ['RowId', \n                         'SupervisionFlag',\n                         'AdjustmentFactor'], axis=1)\n    \n    # Bool to int for SupervisionFlag\n    #train[\"SupervisionFlag\"] = train[\"SupervisionFlag\"].astype(int)\n    \n    # Backward, then forward fill missing values in cols\n    cols = ['Target','Open', 'High', 'Low', 'Close']\n    train.loc[:,cols] = train.loc[:,cols].bfill()\n    train.loc[:,cols] = train.loc[:,cols].ffill()\n    \n    # Add Spread Features\n    train['Daily_Spread'] = train['Close'] - train['Open']\n    train['Daily_Max_Min'] = train['High'] - train['Low']\n    train['1_Day_Spread_Close'] = train['Close'].diff()\n    train['2_Day_Spread_Close'] = train['Close'].diff(periods=2)\n    train['1_Day_Spread_Open'] = train['Open'].diff()\n    train['2_Day_Spread_Open'] = train['Open'].diff(periods=2)\n    train['1_Week_Spread'] = train['Close'].diff(periods=5)\n    \n    # Fill NaN's \n    train = train.fillna(0)\n    \n    # Add Return Features\n    train['Return_Lag_1_Close'] = (train['Close'] - train['1_Day_Spread_Close'])/train['Close']\n    train['Return_Lag_2_Close'] = (train['Close'] - train['2_Day_Spread_Close'])/train['Close']\n    \n    train['Return_Lag_1_Open'] = (train['Open'] - train['1_Day_Spread_Open'])/train['Open']\n    train['Return_Lag_2_Open'] = (train['Open'] - train['2_Day_Spread_Open'])/train['Open'] \n\n    # Fill missing and inf/-inf values with 0\n    train.replace([np.inf, -np.inf], 0, inplace=True)\n    train = train.fillna(0)\n        \n    # Add rolling ratio of mean/std of forward 1 day return\n    indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=5)\n    train['ExPost_SR_Close'] = (train['Return_Lag_1_Close'].rolling(\n        window=indexer, min_periods=1).mean())/(\n        train['Return_Lag_1_Close'].std())\n    \n    train['ExPost_SR_2_Close'] = (train['Return_Lag_2_Close'].rolling(\n        window=indexer, min_periods=1).mean())/(\n        train['Return_Lag_2_Close'].std())\n    \n    train['ExPost_SR_Open'] = (train['Return_Lag_1_Open'].rolling(\n        window=indexer, min_periods=1).mean())/(\n        train['Return_Lag_1_Open'].std())\n    \n    train['ExPost_SR_2_Open'] = (train['Return_Lag_2_Open'].rolling(\n        window=indexer, min_periods=1).mean())/(\n        train['Return_Lag_2_Open'].std())\n    \n    # Fill missing and inf/-inf values with 0\n    train.replace([np.inf, -np.inf], 0, inplace=True)\n    train = train.fillna(0)\n    \n    # Fill missing values with 0\n    #train = train.fillna(0)\n    \n    return train","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:40.517502Z","iopub.execute_input":"2022-07-04T22:37:40.517721Z","iopub.status.idle":"2022-07-04T22:37:40.545147Z","shell.execute_reply.started":"2022-07-04T22:37:40.517695Z","shell.execute_reply":"2022-07-04T22:37:40.544028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Min Max Scaler Function","metadata":{}},{"cell_type":"code","source":"def min_max(df):\n    # MinMax Scale columns (-1, 1 scale)   \n    scaler = MinMaxScaler(feature_range=(-1, 1))\n    scaled = scaler.fit_transform(df)\n    train_cols = df.columns.values.tolist()\n    trained = pd.DataFrame(data=scaled, columns=train_cols, index=df.index)\n    return trained","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:40.547089Z","iopub.execute_input":"2022-07-04T22:37:40.547413Z","iopub.status.idle":"2022-07-04T22:37:40.560646Z","shell.execute_reply.started":"2022-07-04T22:37:40.547368Z","shell.execute_reply":"2022-07-04T22:37:40.559946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Training DataFrame Function","metadata":{}},{"cell_type":"code","source":"def build_train(df):\n    # Groupby for feature engineering by securities code\n    stock_list_df = df.groupby(\"SecuritiesCode\").apply(feature_train)\n\n    # reset index\n    stock_list_df = stock_list_df.reset_index(drop=True)\n\n    # Create Securities Code df\n    sec_code_df = stock_list_df[['SecuritiesCode', 'Date']]\n    \n    # Drop Date column\n    stock_list_df = stock_list_df.drop('Date', axis=1)\n\n    # min max scale prices df\n    stock_list_df = min_max(stock_list_df)\n\n    # drop SecuritiesCode column before adding it back\n    stock_list_df = stock_list_df.drop(['SecuritiesCode'], axis=1)\n\n    # Concat dfs together\n    stock_list_df = pd.concat([sec_code_df, stock_list_df], axis=1)\n\n    # Sort df by date\n    stock_list_df = stock_list_df.sort_values(by='Date')\n\n    return stock_list_df","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:37:40.5622Z","iopub.execute_input":"2022-07-04T22:37:40.562452Z","iopub.status.idle":"2022-07-04T22:37:40.573402Z","shell.execute_reply.started":"2022-07-04T22:37:40.562423Z","shell.execute_reply":"2022-07-04T22:37:40.572446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_df = build_train(small_prices)\n\n# Drop Date column from multi_model\nsingle_model = training_df.drop('Date', axis=1)\n    \n# create Train X and y df's\nX_train = single_model.drop('Target', axis=1)\ny_train = single_model['Target']","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:38:31.885798Z","iopub.execute_input":"2022-07-04T22:38:31.886101Z","iopub.status.idle":"2022-07-04T22:39:27.938483Z","shell.execute_reply.started":"2022-07-04T22:38:31.88607Z","shell.execute_reply":"2022-07-04T22:39:27.937505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate and Fit Model\nrf = RandomForestRegressor(random_state = 42)\n\n# Train the model on training data\nrf.fit(X_train, y_train);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"env = jpx_tokyo_market_prediction.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:15:02.998635Z","iopub.execute_input":"2022-07-04T00:15:02.999609Z","iopub.status.idle":"2022-07-04T00:15:03.013253Z","shell.execute_reply.started":"2022-07-04T00:15:02.999549Z","shell.execute_reply":"2022-07-04T00:15:03.012174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    date_df = prices[['Date', 'SecuritiesCode']]\n\n    # Append small_prices and prices\n    concat_df = small_prices.append(prices)\n    train_df = build_train(concat_df) \n\n    # Create test_df from prices - date_df\n    df_testing = pd.merge(date_df, train_df, how='left', on=[\"Date\",\"SecuritiesCode\"])\n\n    # Drop duplicate rows from merge\n    df_testing_ = df_testing.drop_duplicates(subset=['SecuritiesCode'])\n\n    # Reset index \n    df_testing_ = df_testing_.reset_index(drop=True)\n\n    # Drop Date column from df_test\n    df_testing_3 = df_testing_.drop(['Target','Date'], axis=1)\n\n    # create Test X and y df's\n    X_test = df_testing_3.copy()\n\n    # df of preds by sec code\n    predictions = rf.predict(X_test)\n\n    # Create predictions df\n    sample_prediction['Target'] = predictions\n    \n    # Sort predictions df by Target\n    sample_prediction = sample_prediction.sort_values(by = \"Target\", ascending = False)\n    \n    # Rank predictions df by Target, create Rank column\n    sample_prediction['Rank'] = np.arange(len(sample_prediction.index))\n    \n    # Sort predictions column by Securities Code\n    sample_prediction = sample_prediction.sort_values(by = \"SecuritiesCode\", ascending = True)\n    \n    # Drop Target feature\n    sample_prediction.drop([\"Target\"], axis = 1)\n    \n    # Create Submission df\n    submission = sample_prediction[[\"Date\", \"SecuritiesCode\", \"Rank\"]]    \n    \n    env.predict(submission)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T00:15:03.014708Z","iopub.execute_input":"2022-07-04T00:15:03.015722Z","iopub.status.idle":"2022-07-04T00:59:00.794211Z","shell.execute_reply.started":"2022-07-04T00:15:03.015661Z","shell.execute_reply":"2022-07-04T00:59:00.792936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_test","metadata":{"execution":{"iopub.status.busy":"2022-07-03T20:09:57.721299Z","iopub.execute_input":"2022-07-03T20:09:57.722399Z","iopub.status.idle":"2022-07-03T20:09:57.757201Z","shell.execute_reply.started":"2022-07-03T20:09:57.722317Z","shell.execute_reply":"2022-07-03T20:09:57.756319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_prediction","metadata":{"execution":{"iopub.status.busy":"2022-07-03T20:09:45.397888Z","iopub.execute_input":"2022-07-03T20:09:45.398249Z","iopub.status.idle":"2022-07-03T20:09:45.41463Z","shell.execute_reply.started":"2022-07-03T20:09:45.39821Z","shell.execute_reply":"2022-07-03T20:09:45.413236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission","metadata":{"execution":{"iopub.status.busy":"2022-07-03T20:09:18.639939Z","iopub.execute_input":"2022-07-03T20:09:18.64187Z","iopub.status.idle":"2022-07-03T20:09:18.666366Z","shell.execute_reply.started":"2022-07-03T20:09:18.641798Z","shell.execute_reply":"2022-07-03T20:09:18.665708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}