{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gensim\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = \"/kaggle/input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\nembeddings= gensim.models.KeyedVectors.load_word2vec_format(path, binary =True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"url = \"https://raw.githubusercontent.com/skathirmani/datasets/master/imdb_sentiment.csv\"\nimdb= pd.read_csv(url)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imdb.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\ndoc = imdb.loc[0,\"review\"]\nwords = nltk.word_tokenize(doc.lower())\ntemp= pd.DataFrame()\nfor word in words:\n    try:\n        print(word,embeddings[word][:5])\n        temp=temp.append(pd.Series(embeddings[word]),ignore_index=True)\n    except:\n        print(word,\"is not there\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs = imdb[\"review\"].str.replace(\"-\",\" \").str.lower().str.replace(\"[^a-z ]\",\"\")\ndocs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords=nltk.corpus.stopwords.words(\"english\")\ndef clean_sentance(doc):\n    words = doc.split(\" \")\n    words_clean = [word for word in nltk.word_tokenize(doc) if word  not in stopwords]\n    docs_clean= \" \".join(words_clean)\n    return docs_clean\ndocs_clean=docs.apply(clean_sentance)\ndocs_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs_vectors = pd.DataFrame()\nfor doc in docs_clean:\n    words = nltk.word_tokenize(doc)\n    temp = pd.DataFrame()\n    for word in words:\n        try:\n            word_vec = embeddings[word]\n            temp = temp.append(pd.Series(word_vec),ignore_index=True)\n        except:\n                pass\n    docs_vectors= docs_vectors.append(temp.mean(),ignore_index=True)\ndocs_vectors.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs_vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.isnull(docs_vectors).sum(axis=1).sort_values(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = docs_vectors.drop([64,590])\ny = imdb[\"sentiment\"].drop([64,590])\nfrom sklearn.model_selection import train_test_split\ntrain_x,test_x,train_y,test_y = train_test_split(x,y,test_size=0.2,random_state=100)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier\nfrom sklearn.metrics import accuracy_score\nmodel_rf = RandomForestClassifier(n_estimators=300).fit(train_x,train_y)\ntest_pred = model_rf.predict(test_x)\nprint(accuracy_score(test_y,test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_Ad = AdaBoostClassifier(n_estimators=300).fit(train_x,train_y)\ntest_pred = model_Ad.predict(test_x)\nprint(accuracy_score(test_y,test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}