{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":792900,"sourceType":"datasetVersion","datasetId":129}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"1. Виконати описовий аналіз набору даних https://www.kaggle.com/datasets/aaron7sun/stocknews\n\n2. Проаналізувати підходи до побудови гібридних моделей прогнозування ринку на основі новин.\n\n3. Дослідити використання моделі FinBERT (https://huggingface.co/ProsusAI/finbert)","metadata":{}},{"cell_type":"code","source":"!pip install --upgrade \"protobuf<4\"\n\nimport os\nimport gc\nimport math\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\n# Hugging Face transformers (FinBERT)\nfrom transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification\n\n# sklearn / xgboost / keras\nfrom sklearn.model_selection import train_test_split, TimeSeriesSplit\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score, mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\nimport xgboost as xgb\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, LSTM, Dropout, Bidirectional\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n\nDATA_DIR = Path('/kaggle/input/stocknews')\nassert DATA_DIR.exists(), \"Dataset not found at /kaggle/input/stocknews — add the dataset to your Kaggle notebook\"\n\ncsv_files = list(DATA_DIR.glob('**/Combined_News_DJIA.csv'))\nprint('Found CSV files:', csv_files)\n\nmain_csv = csv_files[0]\n\ndf = pd.read_csv(main_csv)\nprint('Initial shape:', df.shape)\nprint('Columns:', df.columns.tolist())\n\nprint(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T09:18:09.332919Z","iopub.execute_input":"2025-12-04T09:18:09.333474Z","iopub.status.idle":"2025-12-04T09:18:40.426422Z","shell.execute_reply.started":"2025-12-04T09:18:09.333453Z","shell.execute_reply":"2025-12-04T09:18:40.425663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('\\nMissing values per column:\\n', df.isna().sum())\n\nprint('\\nDtypes:\\n', df.dtypes)\nfor c in df.columns:\n    if df[c].nunique() < 20:\n        print(f\"Unique values for {c}:\\n\", df[c].value_counts(dropna=False).head(20))\n\ndate_cols = [c for c in df.columns if 'date' in c.lower() or 'time' in c.lower() or 'day' in c.lower()]\nprint('Candidate date columns:', date_cols)\n\nparsed_date = None\nfor c in date_cols:\n    try:\n        df[c] = pd.to_datetime(df[c])\n        parsed_date = c\n        print(f'Parsed {c} as datetime')\n        break\n    except Exception as e:\n        print('Failed parsing', c, e)\n\nif parsed_date is None:\n    if 'Date' in df.columns:\n        df['Date'] = pd.to_datetime(df['Date'], errors='coerce')\n        parsed_date = 'Date' if df['Date'].notna().sum() > 0 else None\n\nprint('Using date column:', parsed_date)\n\nif parsed_date is None:\n    df.index = range(len(df))\nelse:\n    df = df.sort_values(parsed_date).reset_index(drop=True)\n\nprice_cols = [c for c in df.columns if any(x in c.lower() for x in ['open','close','adj close','adj_close','price','target','label','trend'])]\nprint('Candidate price/target columns:', price_cols)\n\nprint(df.head())\n\ntext_cols = [c for c in df.columns if df[c].dtype == 'object' and df[c].str.len().mean() > 5]\nprint('Candidate text columns:', text_cols)\n\ntext_col = None\nfor name in ['News','news','Headline','headline','Title','title','Text','text']:\n    if name in df.columns:\n        text_col = name\n        break\nif text_col is None and len(text_cols)>0:\n    text_col = text_cols[0]\n\nprint('Using text column:', text_col)\n\nif parsed_date is not None and 'Close' in df.columns:\n    plt.figure(figsize=(12,4))\n    plt.plot(df[parsed_date], df['Close'])\n    plt.title('Close over time')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T09:18:50.517349Z","iopub.execute_input":"2025-12-04T09:18:50.517939Z","iopub.status.idle":"2025-12-04T09:18:50.594166Z","shell.execute_reply.started":"2025-12-04T09:18:50.517910Z","shell.execute_reply":"2025-12-04T09:18:50.593412Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"headline_cols = [c for c in df.columns if 'top' in c.lower()]\n\nif len(headline_cols) == 0:\n    raise RuntimeError(\"Не знайшов колонок з новинами (Top1...Top25). Перевір датасет.\")\n\ndf['combined_text'] = df[headline_cols].fillna('').agg(' '.join, axis=1)\ntext_col = 'combined_text'\n\nprint(\"Text column selected:\", text_col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T09:19:07.745393Z","iopub.execute_input":"2025-12-04T09:19:07.746243Z","iopub.status.idle":"2025-12-04T09:19:07.778738Z","shell.execute_reply.started":"2025-12-04T09:19:07.746215Z","shell.execute_reply":"2025-12-04T09:19:07.778072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline\nprint('\\nLoading FinBERT model pipeline...')\nmodel_name = 'ProsusAI/finbert'\n\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = AutoModelForSequenceClassification.from_pretrained(model_name)\n\nsent_pipe = pipeline(\n    'sentiment-analysis',\n    model=model,\n    tokenizer=tokenizer,\n    truncation=True,\n    max_length=512,\n)\n\nBATCH = 64\ntexts = df[text_col].fillna('').astype(str).tolist()\n\nsentiments = []\nfor i in tqdm(range(0, len(texts), BATCH)):\n    batch = texts[i:i+BATCH]\n    res = sent_pipe(batch)\n    for r in res:\n        lbl = r['label'].lower()\n        score = float(r['score'])\n        sentiments.append((lbl, score))\n\nsent_df = pd.DataFrame(sentiments, columns=[f'{text_col}_sent_label', f'{text_col}_sent_score'])\ndf = pd.concat([df.reset_index(drop=True), sent_df], axis=1)\n\nprint(df[[text_col, f'{text_col}_sent_label', f'{text_col}_sent_score']].head())\n\ndef label_to_signed_score(r):\n    lbl = r[0]\n    sc = r[1]\n    if 'neg' in lbl or 'negative' in lbl:\n        return -sc\n    if 'pos' in lbl or 'positive' in lbl:\n        return sc\n    return 0.0\n\nsigned_scores = [label_to_signed_score(t) for t in df[[f'{text_col}_sent_label', f'{text_col}_sent_score']].to_numpy()]\ndf[f'{text_col}_sent_signed'] = signed_scores\n\nif parsed_date is not None:\n    df['date_only'] = df[parsed_date].dt.date\n    agg = df.groupby('date_only')[f'{text_col}_sent_signed'].agg(['mean','median','count']).rename(columns={'mean':'sent_mean','median':'sent_median','count':'sent_count'})\n    agg = agg.reset_index()\n    print('Aggregated daily sentiment sample:\\n', agg.head())\n    df = df.merge(agg, on='date_only', how='left')\n\nif parsed_date is not None:\n    df = df.sort_values(parsed_date).reset_index(drop=True)\n    df['text_len'] = df[text_col].fillna('').str.len()\n    df['sent_roll_3'] = df[f'{text_col}_sent_signed'].rolling(3, min_periods=1).mean()\n    df['sent_roll_7'] = df[f'{text_col}_sent_signed'].rolling(7, min_periods=1).mean()\n    df['text_len_roll_7'] = df['text_len'].rolling(7, min_periods=1).mean()\n\nTARGET = None\nif any('label' in c.lower() for c in df.columns):\n    for c in df.columns:\n        if 'label' in c.lower():\n            TARGET = c\n            break\nelif any('trend' in c.lower() for c in df.columns):\n    for c in df.columns:\n        if 'trend' in c.lower():\n            TARGET = c\n            break\nelif 'Close' in df.columns or 'close' in [c.lower() for c in df.columns]:\n    for c in df.columns:\n        if c.lower()=='close':\n            TARGET = c\n            break\n\nprint('Detected target column (if any):', TARGET)\n\nif TARGET is None and 'date_only' in df.columns and 'sent_mean' in df.columns:\n    df['future_sent'] = df[f'{text_col}_sent_signed'].shift(-1)\n    df['target_direction'] = (df['future_sent'] > 0).astype(int)\n    TARGET = 'target_direction'\n    print('Created fallback target:', TARGET)\n\nfeature_cols = [c for c in df.columns if any(x in c.lower() for x in ['sent','text_len','roll'])]\nprint('Feature cols used:', feature_cols)\n\nmodel_df = df.dropna(subset=feature_cols + [TARGET]) if TARGET else df\n\nX = model_df[feature_cols].fillna(0).values\nif TARGET:\n    y = model_df[TARGET].values\nelse:\n    raise RuntimeError('No target found or created. Inspect dataset and choose a target column.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T09:19:13.367461Z","iopub.execute_input":"2025-12-04T09:19:13.367983Z","iopub.status.idle":"2025-12-04T09:20:34.918201Z","shell.execute_reply.started":"2025-12-04T09:19:13.367957Z","shell.execute_reply":"2025-12-04T09:20:34.917598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\n\n# Визначаємо категоріальні та числові колонки\ncat_cols = [c for c in df.columns if 'label' in c.lower() or \n            (df[c].dtype == object and 'sent' in c.lower())]\n\nnum_cols = [c for c in df.columns \n            if any(x in c.lower() for x in ['sent', 'score', 'roll', 'text_len']) \n            and c not in cat_cols]\n\nprint('Numeric columns:', num_cols)\nprint('Categorical columns:', cat_cols)\n\n# One-hot encoding для категоріальних колонок\nencoder = OneHotEncoder(sparse=False, handle_unknown='ignore')\ncat_encoded = encoder.fit_transform(df[cat_cols].fillna('unknown'))\n\nX_all = np.hstack([df[num_cols].fillna(0).values, cat_encoded])\n\nmodel_df = df.dropna(subset=[TARGET])\ny = model_df[TARGET].values\n\nX = X_all[model_df.index]\n\nprint('Feature shape:', X.shape)\n\nif 'date_only' in model_df.columns:\n    split_idx = int(len(model_df)*0.8)\n    X_train, X_test = X[:split_idx], X[split_idx:]\n    y_train, y_test = y[:split_idx], y[split_idx:]\nelse:\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=False)\n\nprint('\\nTraining XGBoost...')\nif len(np.unique(y_train)) <= 2:\n    model_xgb = xgb.XGBClassifier(n_estimators=200, learning_rate=0.05, max_depth=4,\n                                  use_label_encoder=False, eval_metric='logloss')\n    model_xgb.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)\n    preds = model_xgb.predict(X_test)\n    print('Accuracy:', accuracy_score(y_test, preds))\n    print('F1:', f1_score(y_test, preds, average='binary'))\nelse:\n    model_xgb = xgb.XGBRegressor(n_estimators=200, learning_rate=0.05, max_depth=4)\n    model_xgb.fit(X_train, y_train)\n    preds = model_xgb.predict(X_test)\n    print('RMSE:', math.sqrt(mean_squared_error(y_test, preds)))\n\nSEQ_LEN = 7\n\ndef make_sequences(X_array, seq_len=SEQ_LEN):\n    Xs = []\n    for i in range(seq_len, len(X_array)):\n        Xs.append(X_array[i-seq_len:i])\n    return np.array(Xs)\n\nif len(X_train) > SEQ_LEN:\n    X_seq = make_sequences(np.vstack([X_train, X_test]))\n    y_seq = y[SEQ_LEN:SEQ_LEN + len(X_seq)]\n    \n    train_len = int(len(X_seq) * 0.8)\n    Xs_train, Xs_test = X_seq[:train_len], X_seq[train_len:]\n    ys_train, ys_test = y_seq[:train_len], y_seq[train_len:]\n\n    print('Sequence shapes:', Xs_train.shape, Xs_test.shape)\n\n    model_lstm = Sequential()\n    model_lstm.add(Bidirectional(LSTM(64, return_sequences=False), input_shape=(Xs_train.shape[1], Xs_train.shape[2])))\n    model_lstm.add(Dropout(0.2))\n    model_lstm.add(Dense(32, activation='relu'))\n\n    if len(np.unique(y_train)) <= 2:\n        model_lstm.add(Dense(1, activation='sigmoid'))\n        model_lstm.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n    else:\n        model_lstm.add(Dense(1, activation='linear'))\n        model_lstm.compile(optimizer='adam', loss='mse', metrics=[tf.keras.metrics.RootMeanSquaredError()])\n\n    es = EarlyStopping(patience=10, restore_best_weights=True)\n\n    model_lstm.fit(Xs_train, ys_train, epochs=50, batch_size=32,\n                   validation_data=(Xs_test, ys_test), callbacks=[es], verbose=2)\n\n    preds_seq = model_lstm.predict(Xs_test)\n    if len(np.unique(y_train)) <= 2:\n        preds_bin = (preds_seq.flatten() > 0.5).astype(int)\n        print('LSTM Accuracy:', accuracy_score(ys_test, preds_bin))\n    else:\n        print('LSTM RMSE:', math.sqrt(mean_squared_error(ys_test, preds_seq.flatten())))\nelse:\n    print('Not enough data to build sequences for LSTM. Try increasing dataset or decreasing SEQ_LEN.')\n\n# === Збереження оброблених фіч ===\nout_path = '/kaggle/working/stocknews_processed.csv'\ndf.to_csv(out_path, index=False)\nprint('Saved processed dataframe to', out_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T09:27:41.163242Z","iopub.execute_input":"2025-12-04T09:27:41.163904Z","iopub.status.idle":"2025-12-04T09:27:58.472005Z","shell.execute_reply.started":"2025-12-04T09:27:41.163876Z","shell.execute_reply":"2025-12-04T09:27:58.471353Z"}},"outputs":[],"execution_count":null}]}