{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk\nfrom nltk.corpus import stopwords                # stopwords are removed from text to keep just useful info\nfrom nltk import word_tokenize, sent_tokenize\n\nimport keras\nfrom keras.preprocessing.text import one_hot, Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, Embedding, LSTM, SpatialDropout1D, Input, Bidirectional,Dropout\nfrom sklearn.model_selection import train_test_split\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.regularizers import l2\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"raw_train_data_labeled = pd.read_csv(\"../input/word2vec-nlp-tutorial/labeledTrainData.tsv\", header=0, delimiter=\"\\t\", quoting=3)\nraw_train_data_unlabeled = pd.read_csv(\"../input/word2vec-nlp-tutorial/unlabeledTrainData.tsv\", header=0, delimiter=\"\\t\", quoting=3)\nraw_test_data = pd.read_csv(\"../input/word2vec-nlp-tutorial/testData.tsv\", header=0, delimiter=\"\\t\", quoting=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6efec74779805b9151de3f1fca05b09f3658b45a"},"cell_type":"code","source":"# importing IMDB dataset from another source....this is done to increase our training dataset\n# w/o this the max accuracy was around 88%, but using this validation set acc. increased to around 93%\nimdb_data = pd.read_csv('../input/imdb-review-dataset/imdb_master.csv',encoding=\"latin-1\")\nimdb_data.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"412c843421ad8cb8aaac78abdf0b2841fa4c4515"},"cell_type":"code","source":"imdb_data = imdb_data.drop([\"Unnamed: 0\",\"type\",\"file\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf4aba61a506be1a7a16c080f0347631d12cbdb8"},"cell_type":"code","source":"imdb_data['sentiment'] = imdb_data['label'].map({\"neg\":0, \"pos\":1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b55dd431023e6e8c2d0406781127aefa1c2c272"},"cell_type":"code","source":"imdb_data = imdb_data.drop([\"label\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b4e07e3f201eda3138354e13b28f8b0a84dbc8b"},"cell_type":"code","source":"imdb_data = imdb_data.dropna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b9788071e52e8cdd98af923184f7447c4bab376"},"cell_type":"code","source":"imdb_data = imdb_data[['sentiment','review']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cff5f137d8138c68daa63ff387f392152975ff10"},"cell_type":"code","source":"raw_train_data_labeled.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ae981562a1dc0c8ff6c655fcd5a04510e1d5198"},"cell_type":"code","source":"raw_train_data_unlabeled.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69e646e0883accb0a15afcc4232cecaf0e502aba"},"cell_type":"code","source":"X = raw_train_data_labeled['review']\ny = raw_train_data_labeled['sentiment']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae2f8fa01ddf609e4dd0877a1cd8e22fa787d7e0"},"cell_type":"code","source":"review_data = X.append(raw_test_data['review'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca02977d61ee228eefb9658c5187ea061dfeaa64"},"cell_type":"code","source":"review_data = review_data.append(imdb_data['review'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd3cb84028cca739b7b14133460644eebb5c6800"},"cell_type":"code","source":"review_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c505c97e839b2ebeb2f8b1505660496cb73d727"},"cell_type":"code","source":"ntrain = X.shape[0]\ndf = raw_train_data_labeled.append(raw_test_data, sort=False)\ndf = df.drop(['sentiment'], axis=1)\ndf_review = df['review']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dd50a20edb8f578a8ab8f9d533c2d52df93183c"},"cell_type":"code","source":"'''\nHere we will do preprocessing\n1. Removing punctuations\n2. Lowering all words\n3. removing non-alphabet things\n4. removing stop words\n5. Tokenizing the sentence\n'''\nimport string\n\nreview_lines = list()\nlines = review_data.values.tolist()\n\nfor line in lines:\n    \n    '''\n    breaks line into it's sub parts like each word and comma etc,\n    https://pythonspot.com/tokenizing-words-and-sentences-with-nltk/\n    '''\n    tokens = word_tokenize(line)   \n    \n     #convert to lower case\n    tokens = [w.lower() for w in tokens]\n    \n    #remove punctuation from each word\n    # brief detail: https://pythonadventures.wordpress.com/2017/02/05/remove-punctuations-from-a-text/\n    table = str.maketrans('','', string.punctuation)\n    stripped = [w.translate(table) for w in tokens]\n     \n    # remove remaining tokens that are not alphabetic\n    words = [w for w in stripped if w.isalpha()]\n    \n    # filter out stop words\n    stop_words = set(stopwords.words('english'))\n    words = [w for w in words if w not in stop_words]\n    \n    review_lines.append(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae04942364945ebfde23943ad80dc622f6d8791d"},"cell_type":"code","source":"'''\ngensim is python library for training word embeddings in given data\nfor more information visit: \n1. https://machinelearningmastery.com/develop-word-embeddings-python-gensim/\n2. http://kavita-ganesan.com/gensim-word2vec-tutorial-starter-code/#.XEoWKVwzbIV\n'''\nimport gensim\n\nembedding_vector_size = 150\n# now training embeddings for each word \nmodel_1 = gensim.models.Word2Vec(sentences = review_lines, size=embedding_vector_size, min_count=1, window=5, workers=4 )\n\n# to get total number of unique words\nwords = list(model_1.wv.vocab)\n\nprint(\"vocab size:\", len(words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b3de65731787dfcc4fc01cec9a344a66e6bd384"},"cell_type":"code","source":"#len(sequence)\nleng=0\nlength = [(leng + len(x)) for x in review_lines]\nplt.hist(length)\nplt.xlabel('length of words')\nplt.ylabel('frequency')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"921711312ca9cad19ed8c926b973654e6abd3ceb"},"cell_type":"code","source":"import math\navg_length = sum(length)/len(review_lines)\n\n# if words are more than max_length then they are skipped, if less than padding with 0 is done\nprint(avg_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"507aa8c0fb3e266d51edb692ad71bd50aad84a10"},"cell_type":"code","source":"#max_len = math.ceil(avg_length)             # this is used to decide how man words in seq to keep\nmax_len = math.ceil(avg_length) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe27d22f6d3fb3b3a269aae6d7cb28e208e514a1"},"cell_type":"code","source":"'''\nNow we have trained the embeddings, we now have embedding vector for each word. We will\nconvert our text training data to numeric using theseword embeddings.\nFirst, we need to make length of each input same, therefore we'll do padding. But padding happends \non numeric data, therefore we'll convert texts to sequences using tokenize() function. Then add padding\nThen we'll replace each non-zero numeric resulted from texts to sequences to its corresponding word\nembedding.\n'''\nmax_features = 6000\ntokenizer = Tokenizer(num_words=max_features)       #keeps 6000 most common words\ntrain_test_data = review_lines                       # contains word tokens extracted from lines\ntokenizer.fit_on_texts(train_test_data)\nsequence = tokenizer.texts_to_sequences(train_test_data)\ntrain_test_data = pad_sequences(sequence, maxlen = max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b6cab0b83856550162999db22715d7fe0886612"},"cell_type":"code","source":"# Preparing embedding matrix\nvocab_size = len(tokenizer.word_index)+1\nembedding_matrix = np.zeros((vocab_size, embedding_vector_size))\n# +1 is done because i starts from 1 instead of 0, and goes till len(vocab)\nfor  word, i in tokenizer.word_index.items():\n    embedding_vector = model_1.wv[word]\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdad67b3996ca6c1690f5e3b35aff85ce0672c2e"},"cell_type":"code","source":"X = train_test_data[:ntrain,:]\nX = np.append(X,train_test_data[ntrain+25000: ,:])\nX = X.reshape(-1,123)\ny1 = y.append(imdb_data['sentiment'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0ffcc8e80601ef653034506b5d22db5c6cad1ad"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X , y1, test_size=0.2, random_state=42, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5d8d48d3220e50093e230bb92927acf8000b78b"},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Embedding(input_dim = vocab_size, output_dim = embedding_vector_size, \n                    input_length = max_len, weights = [embedding_matrix]))\nmodel.add(Bidirectional(LSTM(64, dropout=0.25, recurrent_dropout=0.1)))\nmodel.add(Dense(10))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation='sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62ad68e3ab52c2b5ebdead844eb17d836cf1f140"},"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"077194cdb8628183b530c3edb9b8cc0d12ffe53a"},"cell_type":"code","source":"model.compile(optimizer='RMSprop', loss='binary_crossentropy', metrics=['acc'])\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train, epochs = 30, batch_size = 700, validation_data=(X_test, y_test),callbacks = [learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31fe7c7d99f99378cc0108c83cb30633c7e2cf37","scrolled":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f68c7a2e7579a1f35587cad84d8e1fc0611c96e"},"cell_type":"code","source":"y_test_pred = model.predict(X_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d94bec332f4fecbb3cbbab89371b3400c234af71"},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nroc_auc_score(y_test, y_test_pred, average = 'weighted')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"481cceaadcca94200f4389ff2b7efc39ffff920d"},"cell_type":"code","source":"#predicting test_data\ny_pred = model.predict(train_test_data[ntrain:ntrain+25000 , :])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5565948b52b441af8e8e92eb36fda62b8f463e64"},"cell_type":"code","source":"predictions = [1 if (x>0.5) else 0 for x in y_pred ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d2dc365cd355752a868b8564cc22e91791d736f"},"cell_type":"code","source":"predictions = pd.Series(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9b37c7d630c5eb2acb5e3430c6fd47938c7d8db"},"cell_type":"code","source":"ids = raw_test_data['id'].str.replace('\"', '')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32f45cb8fe8afa9e6e23d58c9a1c3d5463e2a521"},"cell_type":"code","source":"submission = pd.DataFrame({'id': ids, 'sentiment':predictions})\nsubmission.to_csv('word2vec.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9827b43c6e22b2800dd417c4297c4b93152ef2f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}