{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\nfrom IPython.display import clear_output\nfrom tqdm import tqdm\nfrom tqdm.notebook import tqdm_notebook\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nnp.random.seed(0)\ntqdm_notebook.pandas()\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-18T18:34:23.944887Z","iopub.execute_input":"2022-07-18T18:34:23.945325Z","iopub.status.idle":"2022-07-18T18:34:25.085685Z","shell.execute_reply.started":"2022-07-18T18:34:23.945233Z","shell.execute_reply":"2022-07-18T18:34:25.084421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## First look on data","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip\", delimiter=\"\\t\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:25.088370Z","iopub.execute_input":"2022-07-18T18:34:25.089220Z","iopub.status.idle":"2022-07-18T18:34:26.255003Z","shell.execute_reply.started":"2022-07-18T18:34:25.089173Z","shell.execute_reply":"2022-07-18T18:34:26.253825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.256772Z","iopub.execute_input":"2022-07-18T18:34:26.257471Z","iopub.status.idle":"2022-07-18T18:34:26.278178Z","shell.execute_reply.started":"2022-07-18T18:34:26.257419Z","shell.execute_reply":"2022-07-18T18:34:26.277391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.280304Z","iopub.execute_input":"2022-07-18T18:34:26.280752Z","iopub.status.idle":"2022-07-18T18:34:26.315178Z","shell.execute_reply.started":"2022-07-18T18:34:26.280713Z","shell.execute_reply":"2022-07-18T18:34:26.314163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.316596Z","iopub.execute_input":"2022-07-18T18:34:26.316972Z","iopub.status.idle":"2022-07-18T18:34:26.336131Z","shell.execute_reply.started":"2022-07-18T18:34:26.316938Z","shell.execute_reply":"2022-07-18T18:34:26.335286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.337349Z","iopub.execute_input":"2022-07-18T18:34:26.337848Z","iopub.status.idle":"2022-07-18T18:34:26.453590Z","shell.execute_reply.started":"2022-07-18T18:34:26.337817Z","shell.execute_reply":"2022-07-18T18:34:26.452463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.455421Z","iopub.execute_input":"2022-07-18T18:34:26.456133Z","iopub.status.idle":"2022-07-18T18:34:26.473204Z","shell.execute_reply.started":"2022-07-18T18:34:26.456089Z","shell.execute_reply":"2022-07-18T18:34:26.471975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Distribution of classes","metadata":{}},{"cell_type":"code","source":"sns.countplot(data[\"sentiment\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.474832Z","iopub.execute_input":"2022-07-18T18:34:26.475436Z","iopub.status.idle":"2022-07-18T18:34:26.682576Z","shell.execute_reply.started":"2022-07-18T18:34:26.475401Z","shell.execute_reply":"2022-07-18T18:34:26.681344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Length of sentences and word number distributions","metadata":{}},{"cell_type":"code","source":"sentences_length = data[\"review\"].apply(len).rename(\"review length\")\nword_count = data[\"review\"].apply(lambda row: len(str(row).split(\" \"))).rename(\"word count\")\n\nfig=plt.figure(figsize=(14,8))\nfig.add_subplot(1, 2, 1)\nsns.distplot(sentences_length,color='red')\nfig.add_subplot(1, 2, 2)\nsns.distplot(word_count,color='blue')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:26.684108Z","iopub.execute_input":"2022-07-18T18:34:26.684512Z","iopub.status.idle":"2022-07-18T18:34:28.006696Z","shell.execute_reply.started":"2022-07-18T18:34:26.684477Z","shell.execute_reply":"2022-07-18T18:34:28.005342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences_length.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:28.012828Z","iopub.execute_input":"2022-07-18T18:34:28.013292Z","iopub.status.idle":"2022-07-18T18:34:28.028593Z","shell.execute_reply.started":"2022-07-18T18:34:28.013178Z","shell.execute_reply":"2022-07-18T18:34:28.027751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_count.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:28.029812Z","iopub.execute_input":"2022-07-18T18:34:28.030275Z","iopub.status.idle":"2022-07-18T18:34:28.042960Z","shell.execute_reply.started":"2022-07-18T18:34:28.030245Z","shell.execute_reply":"2022-07-18T18:34:28.040993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Word cloud of uncleaned data","metadata":{}},{"cell_type":"code","source":"cloud = WordCloud(width=800, height=600, stopwords=set(STOPWORDS)).generate(\" \".join(data[\"review\"]))\nplt.figure(figsize=(16,10))\nplt.imshow(cloud)\nplt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:28.044456Z","iopub.execute_input":"2022-07-18T18:34:28.045393Z","iopub.status.idle":"2022-07-18T18:34:54.283867Z","shell.execute_reply.started":"2022-07-18T18:34:28.045353Z","shell.execute_reply":"2022-07-18T18:34:54.282761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cleaning data","metadata":{}},{"cell_type":"code","source":"import re\nfrom bs4 import BeautifulSoup\nimport nltk\nnltk.download('stopwords')\n\nfrom nltk.corpus import stopwords\n\n\nstop_words=set(stopwords.words('english'))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:54.285742Z","iopub.execute_input":"2022-07-18T18:34:54.286284Z","iopub.status.idle":"2022-07-18T18:34:55.436416Z","shell.execute_reply.started":"2022-07-18T18:34:54.286246Z","shell.execute_reply":"2022-07-18T18:34:55.435508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_data(string: str) -> str:\n    # delete HTML-tags\n    text = BeautifulSoup(string, \"lxml\").get_text()\n    # delete no text data\n    letters_only = re.sub(\"[^a-zA-Z]\", \" \", text)\n    # process into lower case\n    lowercase_text = letters_only.lower()\n    # delete stop-words\n    no_stop_words = [word for word in lowercase_text.split() if word not in stop_words]\n    return \" \".join(no_stop_words)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:55.437838Z","iopub.execute_input":"2022-07-18T18:34:55.438149Z","iopub.status.idle":"2022-07-18T18:34:55.445054Z","shell.execute_reply.started":"2022-07-18T18:34:55.438120Z","shell.execute_reply":"2022-07-18T18:34:55.443769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_data = data.copy()\ncleaned_data[\"review\"] = cleaned_data[\"review\"].progress_apply(clean_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:34:55.447079Z","iopub.execute_input":"2022-07-18T18:34:55.447808Z","iopub.status.idle":"2022-07-18T18:35:11.867010Z","shell.execute_reply.started":"2022-07-18T18:34:55.447762Z","shell.execute_reply":"2022-07-18T18:35:11.865580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:11.869186Z","iopub.execute_input":"2022-07-18T18:35:11.869676Z","iopub.status.idle":"2022-07-18T18:35:11.882784Z","shell.execute_reply.started":"2022-07-18T18:35:11.869630Z","shell.execute_reply":"2022-07-18T18:35:11.881826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cloud = WordCloud(width=800, height=600, stopwords=set(STOPWORDS)).generate(\" \".join(cleaned_data[\"review\"]))\nplt.figure(figsize=(16,10))\nplt.imshow(cloud)\nplt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:11.884511Z","iopub.execute_input":"2022-07-18T18:35:11.884916Z","iopub.status.idle":"2022-07-18T18:35:47.709712Z","shell.execute_reply.started":"2022-07-18T18:35:11.884871Z","shell.execute_reply":"2022-07-18T18:35:47.708065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Deep text processing with lemmatization","metadata":{}},{"cell_type":"code","source":"!{sys.executable} -m pip install spacy\n!{sys.executable} -m spacy download en_core_web_sm\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:47.712076Z","iopub.execute_input":"2022-07-18T18:35:47.712481Z","iopub.status.idle":"2022-07-18T18:35:49.330766Z","shell.execute_reply.started":"2022-07-18T18:35:47.712443Z","shell.execute_reply":"2022-07-18T18:35:49.329580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\n\nnlp = spacy.load(\"en_core_web_sm\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:49.333026Z","iopub.execute_input":"2022-07-18T18:35:49.334955Z","iopub.status.idle":"2022-07-18T18:35:51.645353Z","shell.execute_reply.started":"2022-07-18T18:35:49.334878Z","shell.execute_reply":"2022-07-18T18:35:51.644272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_data[\"review\"].apply(len).max() < nlp.max_length","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:51.648041Z","iopub.execute_input":"2022-07-18T18:35:51.648435Z","iopub.status.idle":"2022-07-18T18:35:51.680093Z","shell.execute_reply.started":"2022-07-18T18:35:51.648399Z","shell.execute_reply":"2022-07-18T18:35:51.679217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lemmatization example","metadata":{}},{"cell_type":"code","source":"print(\"Original text:\")\nprint(cleaned_data.iloc[0][\"review\"])\nprint(\"=\" * 40)\nprint(\"Text after lemmatization:\")\ndoc = nlp(cleaned_data.iloc[0][\"review\"])\nprint(\" \".join([token.lemma_ for token in doc]))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:51.681198Z","iopub.execute_input":"2022-07-18T18:35:51.682438Z","iopub.status.idle":"2022-07-18T18:35:51.782882Z","shell.execute_reply.started":"2022-07-18T18:35:51.682397Z","shell.execute_reply":"2022-07-18T18:35:51.781915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_lemmatization(row: str) -> str:\n    doc = nlp(row)\n    return \" \".join([token.lemma_ for token in doc])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:51.784348Z","iopub.execute_input":"2022-07-18T18:35:51.785313Z","iopub.status.idle":"2022-07-18T18:35:51.791239Z","shell.execute_reply.started":"2022-07-18T18:35:51.785263Z","shell.execute_reply":"2022-07-18T18:35:51.790300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lemmatization_data = cleaned_data.copy()\nlemmatization_data[\"review\"] = lemmatization_data[\"review\"].progress_apply(text_lemmatization)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:35:51.792767Z","iopub.execute_input":"2022-07-18T18:35:51.793431Z","iopub.status.idle":"2022-07-18T18:47:40.621697Z","shell.execute_reply.started":"2022-07-18T18:35:51.793390Z","shell.execute_reply":"2022-07-18T18:47:40.620070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lemmatization_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:47:40.623459Z","iopub.execute_input":"2022-07-18T18:47:40.623868Z","iopub.status.idle":"2022-07-18T18:47:40.640399Z","shell.execute_reply.started":"2022-07-18T18:47:40.623831Z","shell.execute_reply":"2022-07-18T18:47:40.638248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create word vectors","metadata":{}},{"cell_type":"code","source":"from gensim.models import Word2Vec\nfrom multiprocessing import cpu_count\nfrom nltk.tokenize import TreebankWordTokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:47:40.641951Z","iopub.execute_input":"2022-07-18T18:47:40.642799Z","iopub.status.idle":"2022-07-18T18:47:41.010115Z","shell.execute_reply.started":"2022-07-18T18:47:40.642762Z","shell.execute_reply":"2022-07-18T18:47:41.008832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_features = 400\nembedding_dims = 128","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:47:41.013021Z","iopub.execute_input":"2022-07-18T18:47:41.013510Z","iopub.status.idle":"2022-07-18T18:47:41.018913Z","shell.execute_reply.started":"2022-07-18T18:47:41.013459Z","shell.execute_reply":"2022-07-18T18:47:41.017460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = []\ntokenizer = TreebankWordTokenizer()\n\nfor review in tqdm(lemmatization_data[\"review\"].to_list()):\n    sentence = []\n    raw_sentences = tokenizer.tokenize(str(review).rstrip())\n    for raw_sentence in raw_sentences:\n        if len(raw_sentence) > 0:\n            sentence.append(raw_sentence)\n    sentences.append(sentence)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:47:41.021138Z","iopub.execute_input":"2022-07-18T18:47:41.021615Z","iopub.status.idle":"2022-07-18T18:48:09.123428Z","shell.execute_reply.started":"2022-07-18T18:47:41.021564Z","shell.execute_reply":"2022-07-18T18:48:09.122198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(sentences))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:48:09.132306Z","iopub.execute_input":"2022-07-18T18:48:09.132749Z","iopub.status.idle":"2022-07-18T18:48:09.138909Z","shell.execute_reply.started":"2022-07-18T18:48:09.132712Z","shell.execute_reply":"2022-07-18T18:48:09.137927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nword_2_vec_model = Word2Vec(vector_size=num_features, window=5, min_count=1, workers=cpu_count())\nword_2_vec_model.build_vocab(sentences)\nword_2_vec_model.train(sentences, total_examples=word_2_vec_model.corpus_count, epochs=20)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:48:09.140079Z","iopub.execute_input":"2022-07-18T18:48:09.140380Z","iopub.status.idle":"2022-07-18T18:49:57.186988Z","shell.execute_reply.started":"2022-07-18T18:48:09.140353Z","shell.execute_reply":"2022-07-18T18:49:57.185682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_feature_vector(words: list, model: Word2Vec, size: int) -> np.array:\n    feature_vector = np.zeros((size,), dtype=\"float32\")\n    for word in words:\n        try:\n            feature_vector = np.add(feature_vector, model.wv[str(word)])\n        except KeyError:\n            pass\n    feature_vector = np.divide(feature_vector, len(words))\n    return feature_vector\n\n\ndef tokenize_and_vectorize_text(rows: list, model: Word2Vec, size: int) -> np.array:\n    _tokenizer = TreebankWordTokenizer()\n    vectorized_data = np.zeros((len(rows), size), dtype=\"float32\")\n    for i, sample in tqdm(enumerate(rows)):\n        tokens = _tokenizer.tokenize(sample)\n        vectorized_data[i] = make_feature_vector(tokens, model, size)\n    return vectorized_data","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:49:57.189222Z","iopub.execute_input":"2022-07-18T18:49:57.190249Z","iopub.status.idle":"2022-07-18T18:49:57.207040Z","shell.execute_reply.started":"2022-07-18T18:49:57.190197Z","shell.execute_reply":"2022-07-18T18:49:57.205779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorize_data = tokenize_and_vectorize_text(lemmatization_data[\"review\"].to_list(), word_2_vec_model, size=num_features)\nlabels = np.array(lemmatization_data[\"sentiment\"].to_list())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:49:57.209027Z","iopub.execute_input":"2022-07-18T18:49:57.209943Z","iopub.status.idle":"2022-07-18T18:50:41.209257Z","shell.execute_reply.started":"2022-07-18T18:49:57.209893Z","shell.execute_reply":"2022-07-18T18:50:41.207399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(vectorize_data.shape)\nprint(labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:50:41.210838Z","iopub.execute_input":"2022-07-18T18:50:41.211663Z","iopub.status.idle":"2022-07-18T18:50:41.219040Z","shell.execute_reply.started":"2022-07-18T18:50:41.211622Z","shell.execute_reply":"2022-07-18T18:50:41.217500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare data for learning","metadata":{}},{"cell_type":"markdown","source":"### CatBoost","metadata":{}},{"cell_type":"code","source":"X = vectorize_data\ny = labels","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:50:41.221357Z","iopub.execute_input":"2022-07-18T18:50:41.222244Z","iopub.status.idle":"2022-07-18T18:50:41.237029Z","shell.execute_reply.started":"2022-07-18T18:50:41.222194Z","shell.execute_reply":"2022-07-18T18:50:41.235574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n\nX_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.33, random_state=0)\nX_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=0.33, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:50:41.238265Z","iopub.execute_input":"2022-07-18T18:50:41.238979Z","iopub.status.idle":"2022-07-18T18:50:41.298714Z","shell.execute_reply.started":"2022-07-18T18:50:41.238940Z","shell.execute_reply":"2022-07-18T18:50:41.297753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, confusion_matrix, roc_curve\nfrom sklearn.model_selection import cross_val_score, KFold\nfrom catboost import CatBoostClassifier\n\n\ndef cross_validation(model, x_, y_, score=\"roc_auc\") -> tuple:\n    cv = KFold(n_splits=10, shuffle=True, random_state=0)\n    scores = cross_val_score(model, x_, y_, scoring=score, cv=cv, n_jobs=-1, error_score=\"raise\")\n    return np.mean(scores), np.std(scores)\n\n\ndef plot_learning_history_nn(history) -> None:\n    plt.plot(history.history[\"accuracy\"], label=\"Accuracy on training data\")\n    plt.plot(history.history[\"val_accuracy\"], label=\"Accuracy on validation data\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Accuracy\")\n    plt.legend()\n    plt.show()\n\ndef plot_roc_curve(y_pred: list, y_true: list) -> None:\n    fpr, tpr, _ = roc_curve(y_true, y_pred)\n    plt.plot(fpr, tpr)\n    plt.axis([0,1,0,1])\n    plt.title(\"ROC-Curve\")\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.show()\n    print(f\"ROC-AUC Score: {roc_auc_score(y_true, y_pred)}\")\n\n\ndef plot_confusion_matrix(y_true: list, y_pred: list) -> None:\n    cf_matrix = confusion_matrix(y_true, y_pred)\n    ax = sns.heatmap(cf_matrix, annot=True, cmap=\"Blues\")\n    ax.set_title(\"Confusion matrix\")\n    ax.set_xlabel(\"Prediction\")\n    ax.set_ylabel(\"True\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:50:41.300118Z","iopub.execute_input":"2022-07-18T18:50:41.300698Z","iopub.status.idle":"2022-07-18T18:50:41.741385Z","shell.execute_reply.started":"2022-07-18T18:50:41.300661Z","shell.execute_reply":"2022-07-18T18:50:41.740222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost = CatBoostClassifier(\n    iterations=250,\n    eval_metric=\"AUC\",\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:50:41.742924Z","iopub.execute_input":"2022-07-18T18:50:41.743672Z","iopub.status.idle":"2022-07-18T18:50:41.754090Z","shell.execute_reply.started":"2022-07-18T18:50:41.743625Z","shell.execute_reply":"2022-07-18T18:50:41.752566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_val_accuracy, cross_val_std = cross_validation(catboost, X_train_val, y_train_val)\nclear_output()\nprint(f\"Accuracy on cross-validation: {cross_val_accuracy} ({cross_val_std})\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:50:41.755657Z","iopub.execute_input":"2022-07-18T18:50:41.756268Z","iopub.status.idle":"2022-07-18T18:53:42.583472Z","shell.execute_reply.started":"2022-07-18T18:50:41.756223Z","shell.execute_reply":"2022-07-18T18:53:42.581827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost.fit(X_train, y_train, eval_set=(X_val, y_val), plot=True, use_best_model=True, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:53:42.585265Z","iopub.execute_input":"2022-07-18T18:53:42.586414Z","iopub.status.idle":"2022-07-18T18:53:59.092792Z","shell.execute_reply.started":"2022-07-18T18:53:42.586361Z","shell.execute_reply":"2022-07-18T18:53:59.091434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_roc_curve(catboost.predict(X_test), y_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:53:59.095038Z","iopub.execute_input":"2022-07-18T18:53:59.095571Z","iopub.status.idle":"2022-07-18T18:53:59.455692Z","shell.execute_reply.started":"2022-07-18T18:53:59.095502Z","shell.execute_reply":"2022-07-18T18:53:59.454156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confusion_matrix(y_test, catboost.predict(X_test))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:53:59.457065Z","iopub.execute_input":"2022-07-18T18:53:59.458320Z","iopub.status.idle":"2022-07-18T18:53:59.855038Z","shell.execute_reply.started":"2022-07-18T18:53:59.458280Z","shell.execute_reply":"2022-07-18T18:53:59.853805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost.fit(X_train_val, y_train_val, eval_set=(X_test, y_test), plot=True, use_best_model=True, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:53:59.856666Z","iopub.execute_input":"2022-07-18T18:53:59.857558Z","iopub.status.idle":"2022-07-18T18:54:17.324163Z","shell.execute_reply.started":"2022-07-18T18:53:59.857506Z","shell.execute_reply":"2022-07-18T18:54:17.323250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Neural network","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras_preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense , LSTM , Embedding, Dropout\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Sequential\n\n\nmax_features = 6000\nkeras_tokenizer = Tokenizer(num_words=max_features)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:17.325621Z","iopub.execute_input":"2022-07-18T18:54:17.326489Z","iopub.status.idle":"2022-07-18T18:54:27.773625Z","shell.execute_reply.started":"2022-07-18T18:54:17.326449Z","shell.execute_reply":"2022-07-18T18:54:27.772197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lemmatization_data[\"review\"].apply(lambda x: len(x.split(\" \"))).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:27.775375Z","iopub.execute_input":"2022-07-18T18:54:27.776495Z","iopub.status.idle":"2022-07-18T18:54:27.999453Z","shell.execute_reply.started":"2022-07-18T18:54:27.776438Z","shell.execute_reply":"2022-07-18T18:54:27.998105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocess data with keras tools\nX_nn = lemmatization_data[\"review\"].to_list()\nkeras_tokenizer.fit_on_texts(X_nn)\nX_nn = pad_sequences(keras_tokenizer.texts_to_sequences(X_nn), maxlen=120)\ny_nn = np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:28.001034Z","iopub.execute_input":"2022-07-18T18:54:28.001412Z","iopub.status.idle":"2022-07-18T18:54:33.444014Z","shell.execute_reply.started":"2022-07-18T18:54:28.001370Z","shell.execute_reply":"2022-07-18T18:54:33.442766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_nn.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:33.445453Z","iopub.execute_input":"2022-07-18T18:54:33.445830Z","iopub.status.idle":"2022-07-18T18:54:33.453675Z","shell.execute_reply.started":"2022-07-18T18:54:33.445789Z","shell.execute_reply":"2022-07-18T18:54:33.452127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_val_nn, X_test_nn, y_train_val_nn, y_test_nn = train_test_split(X_nn, y_nn, test_size=0.33, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:33.455486Z","iopub.execute_input":"2022-07-18T18:54:33.456849Z","iopub.status.idle":"2022-07-18T18:54:33.480489Z","shell.execute_reply.started":"2022-07-18T18:54:33.456796Z","shell.execute_reply":"2022-07-18T18:54:33.479270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_val_nn.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:33.483445Z","iopub.execute_input":"2022-07-18T18:54:33.484166Z","iopub.status.idle":"2022-07-18T18:54:33.493268Z","shell.execute_reply.started":"2022-07-18T18:54:33.484111Z","shell.execute_reply":"2022-07-18T18:54:33.491844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_model = Sequential()\nnn_model.add(Embedding(max_features, 128))\nnn_model.add(Bidirectional(LSTM(32, return_sequences=True)))\nnn_model.add(GlobalMaxPool1D())\nnn_model.add(Dense(20, activation=\"relu\"))\nnn_model.add(Dropout(0.05))\nnn_model.add(Dense(1, activation=\"sigmoid\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:33.495126Z","iopub.execute_input":"2022-07-18T18:54:33.496283Z","iopub.status.idle":"2022-07-18T18:54:34.192008Z","shell.execute_reply.started":"2022-07-18T18:54:33.496229Z","shell.execute_reply":"2022-07-18T18:54:34.190933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:34.193666Z","iopub.execute_input":"2022-07-18T18:54:34.194050Z","iopub.status.idle":"2022-07-18T18:54:34.212425Z","shell.execute_reply.started":"2022-07-18T18:54:34.194016Z","shell.execute_reply":"2022-07-18T18:54:34.211032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:34.216451Z","iopub.execute_input":"2022-07-18T18:54:34.217627Z","iopub.status.idle":"2022-07-18T18:54:34.225897Z","shell.execute_reply.started":"2022-07-18T18:54:34.217569Z","shell.execute_reply":"2022-07-18T18:54:34.224764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10\nbatch_size = 100","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:34.227316Z","iopub.execute_input":"2022-07-18T18:54:34.228269Z","iopub.status.idle":"2022-07-18T18:54:34.236225Z","shell.execute_reply.started":"2022-07-18T18:54:34.228222Z","shell.execute_reply":"2022-07-18T18:54:34.235079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_history = nn_model.fit(X_train_val_nn, y_train_val_nn, batch_size=batch_size, epochs=epochs, validation_split=0.333)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:54:34.237872Z","iopub.execute_input":"2022-07-18T18:54:34.238295Z","iopub.status.idle":"2022-07-18T18:58:27.263741Z","shell.execute_reply.started":"2022-07-18T18:54:34.238259Z","shell.execute_reply":"2022-07-18T18:58:27.262816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_history_nn(nn_history)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:58:27.265372Z","iopub.execute_input":"2022-07-18T18:58:27.265734Z","iopub.status.idle":"2022-07-18T18:58:27.422241Z","shell.execute_reply.started":"2022-07-18T18:58:27.265701Z","shell.execute_reply":"2022-07-18T18:58:27.420969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_predictions =  nn_model.predict(X_test_nn)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:58:27.423441Z","iopub.execute_input":"2022-07-18T18:58:27.423764Z","iopub.status.idle":"2022-07-18T18:58:36.649828Z","shell.execute_reply.started":"2022-07-18T18:58:27.423736Z","shell.execute_reply":"2022-07-18T18:58:36.648926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_roc_curve(nn_predictions, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:58:36.651395Z","iopub.execute_input":"2022-07-18T18:58:36.651822Z","iopub.status.idle":"2022-07-18T18:58:36.860019Z","shell.execute_reply.started":"2022-07-18T18:58:36.651776Z","shell.execute_reply":"2022-07-18T18:58:36.858667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_predictions_int = [1 if pred[0] > 0.5 else 0 for pred in nn_predictions]\nplot_confusion_matrix(nn_predictions_int, y_test_nn)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:58:36.862276Z","iopub.execute_input":"2022-07-18T18:58:36.862933Z","iopub.status.idle":"2022-07-18T18:58:37.078581Z","shell.execute_reply.started":"2022-07-18T18:58:36.862892Z","shell.execute_reply":"2022-07-18T18:58:37.077294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}