{"cells":[{"metadata":{"_uuid":"5f20c3d53c1ca42328974211ee9fda652619008e"},"cell_type":"markdown","source":"# Discover the dataset with these intuitive visual insights !"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_colwidth', -1)\nimport os\nprint(os.listdir(\"../input\"))\nimport re\n\n#Plot\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport cufflinks as cf\ncf.go_offline()\n\n# Sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\nfrom sklearn.manifold import TSNE\n\n# Keras NLP\nfrom keras.preprocessing import sequence, text\n\n# Keras training\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\n\n# RNN\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\n\n# print\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25ad7118ce26ac2d504735662bb9cb37bd426a2f","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train[train['target']==0].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da6216fe5d42f28627821552be638d822ad1b41a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train[train['target']==1].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b45242717352e3600889686bd11b983b0661d572","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# remove punc\ndef remove_punc(question):\n    return question.replace(\"?\", \"\").replace(\"!\", \"\").replace(\",\", \"\").replace(\";\", \"\").replace(\".\", \"\").replace(\"\\\\\", \"\").replace(\"(\", \"\").replace(\")\", \"\")\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4d58f904ea37d10d26e5deef27e481105f419b4"},"cell_type":"markdown","source":"# Some EDA"},{"metadata":{"_uuid":"414aa29ac464a45e805c60525f647981805ed561"},"cell_type":"markdown","source":"## Datasets size"},{"metadata":{"trusted":true,"_uuid":"1144cb4fcc2418b2dd4b40dcb446a572c10b4ac0","_kg_hide-input":true},"cell_type":"code","source":"nb_train = train.shape[0]\n\nprint(\"There are %s questions in the training set.\" % nb_train)\nprint(\"There are %s questions in the testing set.\\n\" % test.shape[0])\n\nnb_insincere = train[train['target']==1].shape[0]\nprint(\"There are {} insincere questions in the training set. ({}% total)\".format(nb_insincere, round(100*nb_insincere/nb_train,2)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b71601d6f6e593d3abe6c348422241f7fd1fa86"},"cell_type":"markdown","source":"Note: Imbalanced dataset"},{"metadata":{"_uuid":"5a88da36f54813ab263007d3773944ced90a6d36"},"cell_type":"markdown","source":"## Questions size"},{"metadata":{"trusted":true,"_uuid":"17e6c3038eb4037dc4613e674c01d5842d8cfead","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def question_size(question):\n    return len(question.split(\" \"))\n\ntrain['question_size'] = train[\"question_text\"].apply(question_size)\ntest['question_size'] = test[\"question_text\"].apply(question_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6866f8fa9e4feec69b9a6ca60939fdac7052844c","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train': train['question_size'].sample(frac=0.01),\n                   'test': test['question_size'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram',\n              histnorm='probability',\n              title='Train/test questions size distribution (from samples)',\n              filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9d82976dbde0bdc9d32658eeebc2f3bcfa4b59d","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train_insincere': train[train['target']==1]['question_size'].sample(frac=0.05),\n                   'train_sincere': train[train['target']==0]['question_size'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram',\n              histnorm='probability',\n              title='Insincere/sincere questions size distribution (from samples)',\n              filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90f9cb35fdf0b67061e843a41927c47560ef4491"},"cell_type":"markdown","source":"Note: insincere questions are longer on average !"},{"metadata":{"_uuid":"ae3f235aea03ff4494fdab4e8fd18c60ecd2a582"},"cell_type":"markdown","source":"## Number of sentences/questions in each question"},{"metadata":{"trusted":true,"_uuid":"607cecccba577cae6c73c8a1c6174cb14fe6c093","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#do better here\ntrain['number_questions'] = train[\"question_text\"].apply(lambda x:x.count('? '))\ntrain['number_statements'] = train[\"question_text\"].apply(lambda x:x.count('. '))\n#train['number_statements'] = train[\"question_text\"].apply(lambda x:x.count(\"[^.]{15,}.\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fc8df4fbeee223f53d3c408f584fb26e2d35c38","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train_insincere': train[train['target']==1]['number_statements'].sample(frac=0.05),\n                   'train_sincere': train[train['target']==0]['number_statements'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram',\n              histnorm='probability',\n              title='Train/test, number of statements in questions distribution (from samples)',\n              filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b9b7f6566db5115002f3f28d8bf2e1077e8c5b7d"},"cell_type":"markdown","source":"## Average word length"},{"metadata":{"trusted":true,"_uuid":"c81fae713d03f9ca8eec4bb9f73b81badeef2eb2","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"punc_list = ['\\\\', '?', '.', ';', ',', '-']\n\ndef average_word_length(question):\n    words = re.sub(\"|\".join(punc_list), \"\", question).split(\" \")\n    return np.mean([len(w) for w in words])\n\ntrain['average_word_length'] = train[\"question_text\"].apply(average_word_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"931951ab20d29df91d5ded43ed3dff337f37d219","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train_insincere': train[train['target']==1]['average_word_length'].sample(frac=0.01),\n                   'train_sincere': train[train['target']==0]['average_word_length'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram', histnorm='probability',\n              title='Train/test, average word length in questions distribution (from samples)', filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a7d6d9c0e129deaf6ff7c3ebf89a010c0d69b14"},"cell_type":"markdown","source":"No real diff here"},{"metadata":{"_uuid":"56a861de611bd99b016f04e8037155f88cabef6b"},"cell_type":"markdown","source":"## Number of stopwords"},{"metadata":{"trusted":true,"_uuid":"ff6f40b19450dfdc82f9159a78cc24e183a9b84e","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\neng_stopwords = set(stopwords.words(\"english\"))\n\ndef nb_stop_words(question):\n    words = re.sub(\"|\".join(punc_list), \"\", question).split(\" \")\n    return len([w for w in words if w in eng_stopwords])\n\ntrain['nb_stop_words'] = train[\"question_text\"].apply(nb_stop_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"422b287be582797b5182778f0056735a091f40c4","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train_insincere': train[train['target']==1]['nb_stop_words'].sample(frac=0.01),\n                   'train_sincere': train[train['target']==0]['nb_stop_words'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram', histnorm='probability',\n              title='Train/test, number of stopwords in questions distribution (from samples)', filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36f2fe01f29466b072b1455d77838e9fd223db81"},"cell_type":"markdown","source":"We still see the difference but its less interesting than the sentence size"},{"metadata":{"_uuid":"26ef5f37b574aa41f96f3c6678d9358bc4df3239"},"cell_type":"markdown","source":"## First names"},{"metadata":{"trusted":true,"_uuid":"c508863fdfe7a7bf4534a58a3ea4cb8b3a83342f","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def nb_up_case_word(question):\n    return question.count()\n\ntrain['nb_up_case_word'] = train[\"question_text\"].apply(lambda x:len(re.findall('[A-Z][a-z]+ ',x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"948d09a48f132f9f367c44663480560ccf2578bb","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train_positive': train[train['target']==1]['nb_up_case_word'].sample(frac=0.01),\n                   'train_negative': train[train['target']==0]['nb_up_case_word'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram', histnorm='probability',\n              title='Train/test, number of First names in questions distribution (from samples)', filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"965f81c9431f11e5e791e67a2fec8756156eb7a6"},"cell_type":"markdown","source":"Seems like a nice feature"},{"metadata":{"_uuid":"ae1a2fea4c8391834b4c1b8ca44fc152af1e7d29"},"cell_type":"markdown","source":"## Question types"},{"metadata":{"trusted":true,"_uuid":"d742a105ed452af71ee234e4aacd09575a44cc6c","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def get_question_type(question):\n    question = question.lower()\n    question_types = [\"why\", \"who\", \"what\", \"when\", \"how\", \"can\", \"do\"]\n    \n    for qt in question_types:\n        if question.startswith(qt):\n            return qt\n    \n    return \"other\"\n\ntrain[\"question_type\"] = train[\"question_text\"].apply(get_question_type)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3566a05ae0c970bdb66dd9ece305de9aff0d9fe","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train[\"sum\"] = train.groupby([\"target\"]).qid.count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e48a931c2dfe7571243d3cf20062e8d47cfb61e","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train_grouped = train.groupby([\"question_type\", \"target\"],as_index=False).qid.count()\n\ntrain_grouped[\"sum\"] = train_grouped.groupby(\"target\").qid.transform(np.sum)\ntrain_grouped[\"qid\"] = train_grouped[\"qid\"] / train_grouped[\"sum\"]\n\ntrain_pivoted = train_grouped.pivot(index=\"question_type\", columns=\"target\", values=\"qid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ea32cfeeb3207b6ad9923796e3d6ca2b5a66287","_kg_hide-input":true},"cell_type":"code","source":"train_pivoted.iplot(kind='bar',\n              title='Train/test, type of questions distribution (from samples)', filename='cufflinks/bar-chart-row')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82d44bcabba49fe91c59b466131af686721926ee"},"cell_type":"markdown","source":"A genuine question involves more 'what' (and 'how') and less 'why', this makes sense as 'why' is usually followed by a statement. "},{"metadata":{"_uuid":"32108a5c1312f983a2bcbbc6a384685dd739f630"},"cell_type":"markdown","source":"## Term frequency comparison training positive/negative"},{"metadata":{"trusted":true,"_uuid":"af0d4e047ed3a357d5158bcd3e9ec5a3d1e17c70","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#Build vocabulay\nctv = CountVectorizer(analyzer='word',token_pattern=r\"\\w{1,}'?[t]?\", ngram_range=(1, 1), stop_words = 'english')\n\n# Get vocabulary\ntrain['question_text_no_punc'] = train['question_text'].apply(remove_punc)\nctv.fit(train['question_text_no_punc'])\n\ntrain_pos = ctv.transform(train[train['target']==1]['question_text_no_punc'])\ntrain_neg = ctv.transform(train[train['target']==0]['question_text_no_punc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19e8f100b31ca99ee32ce0cd502273b658b30094","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train_pos = train_pos.sum(axis=0) / train_pos.sum()\ntrain_neg = train_neg.sum(axis=0) / train_neg.sum()\n\ntrain_diff = train_pos - train_neg\n\ntrain_diff = train_diff.tolist()[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"412faafcf468f47b70edde251c170eaf3db02ffe","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"inv_voc = {v:k for k,v in ctv.vocabulary_.items()}\ntup_list = [(value, inv_voc[ind]) for (ind, value) in enumerate(train_diff)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20c83c2af23c6b979617eebc2e4d2fe39d7dd567","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"most_sincere_words = sorted(tup_list)\nmost_insincere_words = sorted(tup_list, reverse=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"121de6ad8f73c7e7fad3878cc278508bdb77822d","_kg_hide-input":true},"cell_type":"code","source":"[pos_scores, pos_words] = zip(*most_insincere_words)\n[neg_scores, neg_words] = zip(*most_sincere_words)\n\ntrace1 = go.Bar(\n    y=list(reversed(pos_words[:300])),\n    x=list(reversed(pos_scores[:300])),\n    orientation = 'h',\n    marker=dict(\n        color='rgba(244, 80, 65, 0.6)'\n    ),\n)\ntrace2 = go.Bar(\n    y=list(reversed(neg_words[:300])),\n    x=list(reversed(neg_scores[:300])),\n    orientation = 'h',\n    marker=dict(\n        color='rgba(134, 244, 66, 0.6)'\n    ),\n)\n\nfig = tools.make_subplots(rows=1, cols=2)\n\nfig.append_trace(trace2, 1, 1)\nfig.append_trace(trace1, 1, 2)\n\nfig['layout'].update(height=8000, title='Words that are more used in sincere(left)/insincere(right) questions')\npy.iplot(fig, filename='simple-subplot-with-annotations')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8644f35e1dcb6b5d392ac40e019971dbf73a6169"},"cell_type":"markdown","source":"## Number of typos"},{"metadata":{"trusted":true,"_uuid":"1b087fd8cfae5487c185d446a758f953667ff8d3","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# user embeddings to find words that don't exist\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt' # https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afefdf89efbce875b2fd0d9ced27d3f41bf04f7f","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def nb_typos(question):\n    real_words = embeddings_index.keys()\n    typos=[w for w in remove_punc(question.lower()).split(\" \") if w not in real_words]\n    \n    return len(typos)\n    \ntrain[\"nb_typos\"] = train[\"question_text\"].apply(nb_typos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81759ca807542853dc88843685e416fbb7be9e8f","_kg_hide-input":true},"cell_type":"code","source":"# Sampled histograms\nto_plot = pd.DataFrame({'train_positive': train[train['target']==1]['nb_typos'].sample(frac=0.01),\n                   'train_negative': train[train['target']==0]['nb_typos'].sample(frac=0.01)})\n\nto_plot.iplot(kind='histogram', histnorm='probability',\n              title='Train/test, number of typos in questions distribution (from samples)', filename='cufflinks/basic-histogram')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4901918146b8534ec78ba6adb816cadcfcc700d"},"cell_type":"markdown","source":"## Embedding visualisations"},{"metadata":{"trusted":true,"_uuid":"95f492391618058475077ce38b71bebdd17a697e","_kg_hide-input":true},"cell_type":"code","source":"# match and highlight in 2D\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9910e2eaed1dd599fab6b0fbd0648e3b21de88bc"},"cell_type":"code","source":"top_sincere_words = most_sincere_words[:1000]\ntop_insincere_words = most_insincere_words[:1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"152f14704bdeb4670582675c4026bcc5b82568cc"},"cell_type":"code","source":"words_to_keep = top_sincere_words + top_insincere_words\nemb = np.array([list(embeddings_index[word[1]]) + [abs(word[0])/word[0], word[1]] for word in words_to_keep if word[1] in embeddings_index ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1885f2d9bd69895869387307df9f0e033e98a919"},"cell_type":"code","source":"emb.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb0d29278a358f220765073c801ae4a35636d034"},"cell_type":"code","source":"X = emb\nX_embedded = TSNE(n_components=2).fit_transform(X[:,:300])\nX_embedded.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d13661639cc1fab629743d1e9af71609a5e83f20"},"cell_type":"code","source":"# Create a trace\ntrace = go.Scatter(\n    x = X_embedded[:,0],\n    y = X_embedded[:,1],\n    mode = 'markers',\n    marker=dict(\n        color= ['rgb(51, 206, 111)' if val=='-1.0' else 'rgb(244, 86, 66)'  for val in list(X[:,300])]\n    ),\n    text = X[:,301]\n)\n\ndata = [trace]\nlayout = go.Layout(title='top sincere/insincere words according to their embeddings (projected on 2D by T-SNE)')\nfig = go.Figure(data=data, layout=layout)\n# Plot and embed in ipython notebook!\npy.iplot(fig, filename='basic-scatter')\n\n# or plot with: plot_url = py.plot(data, filename='basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b5dfe33e0aa10126c17c91dbd9e0447f7acc962"},"cell_type":"markdown","source":"## TFIDF and Logistic Regression"},{"metadata":{"_uuid":"e807abf6365cabb96e448cc20139fc7af8b700b1"},"cell_type":"markdown","source":"# Train/valid"},{"metadata":{"trusted":true,"_uuid":"59fb08d43c8adba7f362e61c4b30272e871f3f72"},"cell_type":"code","source":"train_count = train.groupby('target').count()\ntrain_count.qid[0] / (train_count.qid[0] + train_count.qid[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"280d7d8e535e3523937b5202b1b63327b219ad7e"},"cell_type":"code","source":"xtrain, xvalid, ytrain, yvalid = train_test_split(train.question_text, train.target, \n                                                  stratify=train.target, \n                                                  random_state=42, \n                                                  test_size=0.1, shuffle=True)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d3c6b4526d1f1a344d403d17d129b58f0a6d971"},"cell_type":"markdown","source":"# TFIDF + Logistic regression"},{"metadata":{"trusted":true,"_uuid":"7b33860d6c57c6ea52be374d1ea85a28df37bfd5"},"cell_type":"code","source":"tfv = TfidfVectorizer(min_df=3,  max_features=None, \n            strip_accents='unicode', analyzer='word',token_pattern=r'\\w{1,}',\n            ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,\n            stop_words = 'english')\n\n# Fitting TF-IDF to both training and test sets (semi-supervised learning)\ntfv.fit(list(xvalid) + list(xtrain))\n\ntrain_tfv =  tfv.transform(xtrain)\nvalid_tfv =  tfv.transform(xvalid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc892bf9759eabe068463213ecd54fd291bf8e56"},"cell_type":"code","source":"train_tfv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5c41bc323248eb0bf3edcb3f5c337ef0facdff5"},"cell_type":"code","source":"valid_tfv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d758d23058b418a00cf761c630fcf9d89dce95d"},"cell_type":"code","source":"# Fitting a simple Logistic Regression on TFIDF\nclf = LogisticRegression(C=1.0)\nclf.fit(train_tfv, ytrain)\npredictions = clf.predict_proba(valid_tfv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"564020f9a9493e396526d104bac776e35377e16a"},"cell_type":"code","source":"# Plot different F1 scores according to threshold\nx = np.linspace(0,1, num=25)\ndef pred(threshold):\n    return [0 if (y<threshold) else 1 for y in predictions[:,1]]\n\nscores = [f1_score(yvalid, pred(xx)) for xx in x]\n\n# Create a trace\ntrace = go.Scatter(\n    x = x,\n    y = scores\n)\n\ndata = [trace]\n\npy.iplot(data, filename='basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4e34b1bcf9beec05e8aede622ea85ad47f8ca61"},"cell_type":"markdown","source":"# CountVectorizer"},{"metadata":{"trusted":true,"_uuid":"9a08e8da8b380db18223d61576f615142245bccf"},"cell_type":"code","source":"ctv = CountVectorizer(analyzer='word',token_pattern=r'\\w{1,}',\n            ngram_range=(1, 3), stop_words = 'english')\n\n# Fitting Count Vectorizer to both training and test sets (semi-supervised learning)\nctv.fit(list(xtrain) + list(xvalid))\nxtrain_ctv =  ctv.transform(xtrain) \nxvalid_ctv = ctv.transform(xvalid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58fa53a3a835fca5bbc9ccf5a558b4107e40aa26"},"cell_type":"code","source":"# Fitting a simple Logistic Regression on TFIDF\nclf = LogisticRegression(C=1.0)\nclf.fit(xtrain_ctv, ytrain)\npredictions = clf.predict_proba(xvalid_ctv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88c88ff9258874eda2a84032c0d884ddc7b23ab1"},"cell_type":"code","source":"# Plot different F1 scores according to threshold\nx = np.linspace(0,1, num=35)\ndef pred(threshold):\n    return [0 if (y<threshold) else 1 for y in predictions[:,1]]\n\nscores = [f1_score(yvalid, pred(xx)) for xx in x]\n\n# Create a trace\ntrace = go.Scatter(\n    x = x,\n    y = scores\n)\n\ndata = [trace]\n\npy.iplot(data, filename='basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e81f36fc8794b93abc0f83df64aa8e880f58e54"},"cell_type":"markdown","source":"## LSTM\n"},{"metadata":{"_uuid":"942c7bd667a13ee3fccc85080ff3c43ff0ac6cb8"},"cell_type":"markdown","source":"### Read Embeddings"},{"metadata":{"trusted":true,"_uuid":"3877a5665bf8c1aad2613c3790c830a1d1c0e8f6"},"cell_type":"code","source":"# Reload embeddings even if already loaded (for each section to be independant)\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5f03402d62a010ac041054496858b274a4b7d43"},"cell_type":"markdown","source":"### Split validation/train"},{"metadata":{"trusted":true,"_uuid":"e2448b500b4f04fb55eeabc78743d00a9f0015e2"},"cell_type":"code","source":"xtrain, xvalid, ytrain, yvalid = train_test_split(train.question_text, train.target, \n                                                  stratify=train.target, \n                                                  random_state=42, \n                                                  test_size=0.1, shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8322727e52dc786d5833c3a5bef19ffd5a49d3d3"},"cell_type":"markdown","source":"### Sentences preprocessing: sentences to seq of token vector"},{"metadata":{"trusted":true,"_uuid":"ae443df0fa35d88ecd4d5f2e634971ee7e77de3d"},"cell_type":"code","source":"# using keras tokenizer here\ntoken = text.Tokenizer(num_words=None)\nmax_len = 70\n\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\n# zero pad the sequences\nxtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)\n\nword_index = token.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1e2f343d112b387b812004410528fbbbce92976"},"cell_type":"code","source":"# we need to binarize the labels for the neural net\nytrain_enc = np_utils.to_categorical(ytrain)\nyvalid_enc = np_utils.to_categorical(yvalid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c802da06e767a65afc2c328ca044f8018f07464","scrolled":true},"cell_type":"code","source":"# create an embedding matrix for the words we have in the dataset\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fcbe00d1504c0905bbcefc933bce8221e9fc5a2"},"cell_type":"code","source":"# A simple LSTM with glove embeddings and two dense layers\nmodel = Sequential()\nmodel.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\nmodel.add(SpatialDropout1D(0.3))\nmodel.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3))\n\nmodel.add(Dense(1024, activation='relu'))\nmodel.add(Dropout(0.8))\n\nmodel.add(Dense(1024, activation='relu'))\nmodel.add(Dropout(0.8))\n\nmodel.add(Dense(2))\nmodel.add(Activation('softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d47ac3c51d669d9c3453cdad9faf88f6d153e74b"},"cell_type":"code","source":"model.fit(xtrain_pad, y=ytrain_enc, batch_size=512, epochs=10, verbose=1, validation_data=(xvalid_pad, yvalid_enc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e6c73f684c229fe77d809edc07efae7c0c2532b"},"cell_type":"code","source":"predictions = model.predict_proba(xvalid_pad)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89b57f370801295b4524d87479eaad44622254eb"},"cell_type":"code","source":"# Plot different F1 scores according to threshold\nx = np.linspace(0,1, num=100)\ndef pred(threshold):\n    return [0 if (y<threshold) else 1 for y in predictions[:,1]]\n\nscores = [f1_score(yvalid, pred(xx)) for xx in x]\n\n# Create a trace\ntrace = go.Scatter(\n    x = x,\n    y = scores\n)\n\ndata = [trace]\n\npy.iplot(data, filename='basic-line')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21b77b6dbbcd328cc722858d90aa76860c507de3"},"cell_type":"markdown","source":"## CNN"},{"metadata":{"trusted":true,"_uuid":"c37370e8059e0ed2570b2ecc206e3f7f22782718"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}