{"cells":[{"metadata":{"_uuid":"0c1216e788a53209d3e967c0e1fa1f5d5b806864"},"cell_type":"markdown","source":"## QUORA INSINCERE PROJECT "},{"metadata":{"trusted":true,"_uuid":"b3da620543c3236c5d1f11b3a68d14011b711476"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport random\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1b9fd2501238879f3b4642e838c24555b70eaaf"},"cell_type":"markdown","source":"## Data extraction and load "},{"metadata":{"trusted":true,"_uuid":"3d00a0b08b8686552a6608aa71bc4c3578cbbce7"},"cell_type":"code","source":"df = pd.read_csv(os.path.join('../input', 'train.csv'))\ndf_test = pd.read_csv(os.path.join('../input', 'test.csv'))\nX_test = df_test['question_text']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a37da5674c709e22b68ab122dd7556cc36dca6b5"},"cell_type":"code","source":"df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11e65ab0fb0babffbfeedf83e4346e03f01ea7e6"},"cell_type":"code","source":"df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a664cc9eb2559dee53e517475bab0b13658a1dff"},"cell_type":"code","source":"df['target'].unique()\ndf[df['question_text'] == ''].sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3a4e790fb3d5e123fe1cc74b3951a92af61049e"},"cell_type":"markdown","source":"** Datas don't have NaN values and '', datas don't need to be cleaned **"},{"metadata":{"trusted":true,"_uuid":"7183dee0f098ca5c445b0b131e28ad1e504b9599"},"cell_type":"code","source":"sincere_q = (df['target'] == 0).sum()\ninsincere_q = (df['target'] == 1).sum()\n\nsincere_q, insincere_q","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fedc644d91d94ceceb3a37f121f1c6dd2268e002"},"cell_type":"code","source":"rate_sincere_q = (sincere_q/len(df['target']))*100\nrate_insincere_q = (insincere_q/len(df['target']))*100\nrate_sincere_q, rate_insincere_q\nprint( '{}% of questions are sincere and {}% are insincere'.format(rate_sincere_q, rate_insincere_q))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba812214be535a94bb6732b986b62991533fb1b8"},"cell_type":"markdown","source":"** Proportion very unbalanced, we have to try to scale datas ** "},{"metadata":{"trusted":true,"_uuid":"f9fe69c0213cc4003fd9c6c4d0b9c61b6205a95c"},"cell_type":"code","source":"index_insincere_q = np.array(df[df['target'] == 1].index) # len = 80810 \nindex_sincere_q = np.array(df[df['target'] == 0].index)\nindex_sincere_q_reduc = random.sample(list(index_sincere_q), int(1.8*len(index_insincere_q)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74c88eb982aab91ccd31c2905f8d09ef6d1b7878"},"cell_type":"code","source":"X = pd.concat([df['question_text'][index_insincere_q], df['question_text'][index_sincere_q_reduc]])\ny = pd.concat([df['target'][index_insincere_q], df['target'][index_sincere_q_reduc]])\n#X = df['question_text']\n#y = df['target']\nX.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5438f33d708f59b9fa8f019241d208322205c42"},"cell_type":"markdown","source":"** The proportion of insincere questions and sincere questions is the same ** [](http://)"},{"metadata":{"_uuid":"b3f4f24c372f2be3e40d39715f9a1458d5763b61"},"cell_type":"markdown","source":"** Split datas in train and test set **"},{"metadata":{"trusted":true,"_uuid":"87b482213571d0f62b9d24c636998705b82071df"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=.2, random_state=42, stratify=y)\nX_train.shape, y_train.shape, X_valid.shape, y_valid.shape\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14b065433a77d026720774073d6a46793343ca4b"},"cell_type":"markdown","source":"** Preprocessing with nltk ** "},{"metadata":{"trusted":true,"_uuid":"cddc6e7a62d13791c65025a721f75483c1fd4770"},"cell_type":"code","source":"import nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import sent_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.stem import PorterStemmer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"003e03d8a5b8781c5bc016943e9acb9dfb3aebe4"},"cell_type":"markdown","source":"** Tokenizing **"},{"metadata":{"_uuid":"576b8e291eb4b32e7e484c8512aa9414577a5cd6"},"cell_type":"markdown","source":"** Creation of a tokenize's function permitting to automatically tokenize our train and test set ** "},{"metadata":{"trusted":true,"_uuid":"2c6d959fbb936c8bb0d5b74d0c49fd6e094a0006"},"cell_type":"code","source":"def tokenize(data):\n    tokenized_docs = [word_tokenize(doc.lower()) for doc in data]\n    alpha_tokens = [[t for t in doc if t.isalpha() == True] for doc in tokenized_docs]\n    stemmer = PorterStemmer ()\n    stemmed_tokens = [[stemmer.stem(alpha) for alpha in doc] for doc in alpha_tokens]\n    X_stem_as_string = [\" \".join(x_t) for x_t in stemmed_tokens]\n    return X_stem_as_string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62b546be60a8cc525a978c1b1225fd104cf7f7aa"},"cell_type":"code","source":"X_train_pr = tokenize(X_train)\nX_valid_pr = tokenize(X_valid)\nX_test_pr = tokenize(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27ccc657891ef414b77eb00970369f67c58c7658"},"cell_type":"markdown","source":"** Preprocessing pipeline **"},{"metadata":{"trusted":true,"_uuid":"3c674f01cead4e4aa2cd2c436444af033c028ec9"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.pipeline import Pipeline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd0bfd22e1652af6ec287dfdb0ec49a163b7892b"},"cell_type":"code","source":"vct = CountVectorizer(stop_words='english', ngram_range=(2, 3), lowercase=False)\nsvd = TruncatedSVD(n_components=100, random_state=42)\ntfvec = TfidfVectorizer(stop_words='english', lowercase=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7928e8f88037689a1315c9d9446eaa7ca0e861b"},"cell_type":"code","source":"preprocessing_pipe = Pipeline([\n    ('vectorizer', tfvec),\n    ('svd', svd),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e70688a54571e4577ef15421e1edda424e2be67"},"cell_type":"code","source":"lsa_train = preprocessing_pipe.fit_transform(X_train_pr)\nlsa_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06ff9cb51efea39b7fed7716b01f5385227b5062"},"cell_type":"markdown","source":"**Visualizing the most informative features ** "},{"metadata":{"trusted":true,"_uuid":"6f46c6981835430316dc69c26f3126ad9d4dd8d0"},"cell_type":"code","source":"components = pd.DataFrame(data=svd.components_, columns=preprocessing_pipe.named_steps['vectorizer'].get_feature_names())\ncomponents","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"623581b1cc8c8b0ba4c7b5c845a4098b6f611402"},"cell_type":"code","source":"fig, axes = plt.subplots(10, 2, figsize=(18, 30))\nfor i, ax in enumerate(axes.flat):\n    components.iloc[i].sort_values(ascending=False)[:10].sort_values().plot.barh(ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"831c7fca23b43acf84230a4396cd483bfe7e2b4a"},"cell_type":"markdown","source":"## MACHINE LEARNING MODELS"},{"metadata":{"_uuid":"a7ceb4825da9ead568199361c82f2f943d98cb5b"},"cell_type":"markdown","source":"** Creation of a pipeline with prepocessing pipeline ** "},{"metadata":{"trusted":true,"_uuid":"c26521f8eff8129cf46082dedbbe76d9e8779e03"},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.naive_bayes import MultinomialNB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"497dbcbfadc508396346ae8eeba205d0eeef368e"},"cell_type":"code","source":"rf = RandomForestClassifier (class_weight='balanced_subsample')\nxgbc = xgb.XGBClassifier() \nmb = MultinomialNB()\npipe = Pipeline([\n    ('vectorizer', tfvec),\n    ('mb', mb)\n])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94a9d9874e76e4c54600e5e50beab9a079e97caa"},"cell_type":"code","source":"X_train_pr = tokenize(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d8c27ec815158ea73c47c475fbe7648b7a54689"},"cell_type":"code","source":"pipe.fit(X_train_pr, y_train)\ny_pred = pipe.predict(X_valid_pr)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"85afcc67f8b197edb01df64adc1d798424ca1bbd"},"cell_type":"markdown","source":"** Confusion Matrix ** "},{"metadata":{"trusted":true,"_uuid":"90dbae29623d41043a285a613a1a2df3c35c22fc"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b099ca630dcdd72e09a708ec8fc01e0fbc43443"},"cell_type":"code","source":"cm = confusion_matrix(y_valid, y_pred)\ncm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13c3acc8b8f7bb7171eb414cbe4c4a0aa25304ee"},"cell_type":"code","source":"labels = ['sincere', 'unsincere']\ndf_cm = pd.DataFrame(cm, columns=labels, index=labels)\ndf_cm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"841acf812ffdf9f284e866de9e7569bfd0aa9ffe"},"cell_type":"markdown","source":"** VALIDATION **"},{"metadata":{"trusted":true,"_uuid":"e3a73b4ff0e19f5ecd87fb1c64ecff7060ec84f9"},"cell_type":"code","source":" from sklearn.model_selection import cross_val_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af45500bcaf2e38971e82dd0837fd453a47530b7"},"cell_type":"code","source":"score = cross_val_score(pipe, X_valid_pr, y=y_valid, cv=5, scoring='f1_macro')\nscore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c720546e5b0b71ecaf845bfd9dda61cb91e1f39"},"cell_type":"code","source":"print(classification_report(y_valid, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e247b2fd9b0d51d2b8ed8594dfb2b711a98de670"},"cell_type":"code","source":"y_test_true = pipe.predict(X_test_pr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dded228e29d318ad16fd7c86474cfad49f77a37d"},"cell_type":"code","source":"#df_sample_submission = pd.DataFrame({'qid' : df_test['qid'], 'y_pred' : y_test_true})\n#index_insin = np.array(df_sample_submission[df_sample_submission['y_pred'] == 1].index) \n#df_sample_submission['qid'][index_insin]\nsub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = y_test_true\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}