{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Goal\n\nOur goal here is to create a baseline model.","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport seaborn as sns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_sub = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')\ndf_train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ndf_test = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_sub.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for null values\ndf_train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking if the problem is balanced or unbalanced --> unbalanced\ndf_train.target.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=df_train, x='target')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nimport string\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\n\nnltk.download('stopwords')\nnltk_stopwords = stopwords.words('english')\n\nwordnet_lemmatizer = WordNetLemmatizer()\n\ndef lemSentence(sentence):\n    token_words = word_tokenize(sentence)\n    lem_sentence = []\n    for word in token_words:\n        lem_sentence.append(wordnet_lemmatizer.lemmatize(word, pos=\"v\"))\n        lem_sentence.append(\" \")\n    return \"\".join(lem_sentence)\n\ndef clean(message, lem=True):\n    # Remove ponctuation\n    message = message.translate(str.maketrans('', '', string.punctuation))\n    \n    # Remove numbers\n    message = message.translate(str.maketrans('', '', string.digits))\n    \n    # Remove stop words\n    message = [word for word in word_tokenize(message) if not word.lower() in nltk_stopwords]\n    message = ' '.join(message)\n    \n    # Lemmatization (root of the word)\n    if lem:\n        message = lemSentence(message)\n    \n    return message","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning the questions (Really slow process, to be changed !)\ndf_train['question_text_cleaned'] = df_train.question_text.apply(lambda x: clean(x, True))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\n\ncount_vectorizer = CountVectorizer()\nmodel = LogisticRegression(C=1, random_state=0)\n\nvectorize_model_pipeline = Pipeline([\n    ('count_vectorizer', count_vectorizer),\n    ('model', model)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(df_train['question_text_cleaned'], df_train['target'], test_size=0.3)\nvectorize_model_pipeline.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = vectorize_model_pipeline.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Accuracy :', accuracy_score(y_test, predictions))\nprint('F1 score :', accuracy_score(y_test, predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nprint(classification_report(y_test, predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning the questions (Really slow process, to be changed !)\ndf_test['question_text_cleaned'] = df_test.question_text.apply(lambda x: clean(x, True))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['prediction'] = vectorize_model_pipeline.predict(df_test['question_text_cleaned'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final = df_test[['qid','prediction']]\ndf_final.set_index('qid', inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}