{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-06T18:36:46.237518Z","iopub.execute_input":"2023-06-06T18:36:46.237965Z","iopub.status.idle":"2023-06-06T18:36:46.270179Z","shell.execute_reply.started":"2023-06-06T18:36:46.237931Z","shell.execute_reply":"2023-06-06T18:36:46.269052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport string\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn import linear_model\n\n# Compute text statistics\ndef compute_text_statistics(data):\n    data[\"num_words\"] = data[\"question_text\"].apply(lambda x: len(str(x).split()))\n    data[\"num_chars\"] = data[\"question_text\"].apply(lambda x: len(str(x)))\n    data[\"num_stopwords\"] = data[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))\n    data[\"num_punctuations\"] = data[\"question_text\"].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n    data[\"num_upper\"] = data[\"question_text\"].apply(lambda x: len([u for u in str(x) if u.isupper()]))\n    data[\"mean_word_len\"] = data[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n\n# Load the data\ntrain_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\n\n# Compute text statistics\ncompute_text_statistics(train_df)\n\n# Visualize text statistics\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\nsns.boxplot(x='target', y='num_words', data=train_df, ax=axes[0, 0])\naxes[0, 0].set_title('Number of Words')\nsns.boxplot(x='target', y='num_chars', data=train_df, ax=axes[0, 1])\naxes[0, 1].set_title('Number of Characters')\nsns.boxplot(x='target', y='num_stopwords', data=train_df, ax=axes[0, 2])\naxes[0, 2].set_title('Number of Stopwords')\nsns.boxplot(x='target', y='num_punctuations', data=train_df, ax=axes[1, 0])\naxes[1, 0].set_title('Number of Punctuations')\nsns.boxplot(x='target', y='num_upper', data=train_df, ax=axes[1, 1])\naxes[1, 1].set_title('Number of Uppercase Characters')\nsns.boxplot(x='target', y='mean_word_len', data=train_df, ax=axes[1, 2])\naxes[1, 2].set_title('Mean Word Length')\nplt.show()\n\n# TF-IDF features\ntrain_x = train_df[\"question_text\"]\ntest_x = test_df[\"question_text\"]\ntrain_y = train_df[\"target\"]\n\ntfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1, 3))\ntrain_tfidf = tfidf_vec.fit_transform(train_x)\ntest_tfidf = tfidf_vec.transform(test_x)\n\n# SVD decomposition\nsvd = TruncatedSVD(n_components=100, random_state=42)\ntrain_svd = svd.fit_transform(train_tfidf)\ntest_svd = svd.transform(test_tfidf)\n\n# Model training\nclf = linear_model.LogisticRegression()\nclf.fit(train_svd, train_y)\n\n# Model evaluation\npredictions = clf.predict_proba(train_svd)[:, 1]\ntrain_df[\"pred\"] = predictions\n\nplt.figure(figsize=(8, 6))\nsns.distplot(train_df[train_df['target'] == 0][\"pred\"], color='b', label='Not Insincere')\nsns.distplot(train_df[train_df['target'] == 1][\"pred\"], color='r', label='Insincere')\nplt.title('Distribution of Predictions', fontsize=18)\nplt.xlabel('Prediction', fontsize=12)\nplt.ylabel('Density', fontsize=12)\nplt.legend()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-06T19:29:41.985035Z","iopub.execute_input":"2023-06-06T19:29:41.985476Z","iopub.status.idle":"2023-06-06T19:44:03.474972Z","shell.execute_reply.started":"2023-06-06T19:29:41.985437Z","shell.execute_reply":"2023-06-06T19:44:03.473685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_word_frequency(data):\n    sincere_words = ' '.join(data[data['target'] == 0]['question_text']).lower()\n    insincere_words = ' '.join(data[data['target'] == 1]['question_text']).lower()\n\n    sincere_word_freq = pd.Series(sincere_words.split()).value_counts()\n    insincere_word_freq = pd.Series(insincere_words.split()).value_counts()\n\n    plt.figure(figsize=(12, 6))\n    plt.subplot(1, 2, 1)\n    sns.barplot(x=sincere_word_freq[:10].values, y=sincere_word_freq[:10].index, color='b')\n    plt.title('Top 10 Most Frequent Words in Sincere Questions')\n    plt.xlabel('Frequency')\n    plt.ylabel('Words')\n\n    plt.subplot(1, 2, 2)\n    sns.barplot(x=insincere_word_freq[:10].values, y=insincere_word_freq[:10].index, color='r')\n    plt.title('Top 10 Most Frequent Words in Insincere Questions')\n    plt.xlabel('Frequency')\n    plt.ylabel('Words')\n\n    plt.tight_layout()\n    plt.show()\nplot_word_frequency(data)","metadata":{"execution":{"iopub.status.busy":"2023-06-06T19:46:03.183089Z","iopub.execute_input":"2023-06-06T19:46:03.183416Z","iopub.status.idle":"2023-06-06T19:46:12.934373Z","shell.execute_reply.started":"2023-06-06T19:46:03.183389Z","shell.execute_reply":"2023-06-06T19:46:12.933169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport string\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn import linear_model\nfrom nltk.corpus import stopwords\n\nSTOPWORDS = set(stopwords.words('english'))\n\n# Compute text statistics\ndef compute_text_statistics(data):\n    data[\"num_words\"] = data[\"question_text\"].apply(lambda x: len(str(x).split()))\n    data[\"num_chars\"] = data[\"question_text\"].apply(lambda x: len(str(x)))\n    data[\"num_stopwords\"] = data[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))\n    data[\"num_punctuations\"] = data[\"question_text\"].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n    data[\"num_upper\"] = data[\"question_text\"].apply(lambda x: len([u for u in str(x) if u.isupper()]))\n    data[\"mean_word_len\"] = data[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n\n# Preprocess text\ndef preprocess(text):\n    text = text.lower()\n    text = text.translate(str.maketrans(\"\", \"\", string.punctuation))\n    text = text.strip()\n    return text\n\n# Load the data\ntrain_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\n\n# Compute text statistics\ncompute_text_statistics(train_df)\n\n# Visualize text statistics\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\nsns.boxplot(x='target', y='num_words', data=train_df, ax=axes[0, 0])\naxes[0, 0].set_title('Number of Words')\nsns.boxplot(x='target', y='num_chars', data=train_df, ax=axes[0, 1])\naxes[0, 1].set_title('Number of Characters')\nsns.boxplot(x='target', y='num_stopwords', data=train_df, ax=axes[0, 2])\naxes[0, 2].set_title('Number of Stopwords')\nsns.boxplot(x='target', y='num_punctuations', data=train_df, ax=axes[1, 0])\naxes[1, 0].set_title('Number of Punctuations')\nsns.boxplot(x='target', y='num_upper', data=train_df, ax=axes[1, 1])\naxes[1, 1].set_title('Number of Uppercase Letters')\nsns.boxplot(x='target', y='mean_word_len', data=train_df, ax=axes[1, 2])\naxes[1, 2].set_title('Mean Word Length')\nplt.show()\n\n# TF-IDF features\ntrain_x = train_df[\"question_text\"].apply(preprocess)\ntest_x = test_df[\"question_text\"].apply(preprocess)\ntrain_y = train_df[\"target\"]\n\ntfidf_vec = TfidfVectorizer(stop_words=list(STOPWORDS), ngram_range=(1, 3))\ntrain_tfidf = tfidf_vec.fit_transform(train_x)\ntest_tfidf = tfidf_vec.transform(test_x)\n\n# SVD decomposition\nsvd = TruncatedSVD(n_components=100, random_state=42)\ntrain_svd = svd.fit_transform(train_tfidf)\ntest_svd = svd.transform(test_tfidf)\n\n# Model training\nclf = linear_model.LogisticRegression()\nclf.fit(train_svd, train_y)\n\n# Model evaluation\npredictions = clf.predict_proba(train_svd)[:, 1]\ntrain_df[\"pred\"] = predictions\n\nplt.figure(figsize=(8, 6))\nsns.distplot(train_df[train_df['target'] == 0][\"pred\"], color='b', label='Not Insincere')\nsns.distplot(train_df[train_df['target'] == 1][\"pred\"], color='r', label='Insincere')\nplt.title('Distribution of Predictions', fontsize=18)\nplt.xlabel('Prediction', fontsize=12)\nplt.ylabel('Density', fontsize=12)\nplt.legend()\nplt.show()\n\ndef plot_word_frequency(data):\n    sincere_words = ' '.join(data[data['target'] == 0]['question_text']).lower()\n    insincere_words = ' '.join(data[data['target'] == 1]['question_text']).lower()\n\n    sincere_word_freq = pd.Series(sincere_words.split()).value_counts()\n    insincere_word_freq = pd.Series(insincere_words.split()).value_counts()\n\n    plt.figure(figsize=(12, 6))\n    plt.subplot(1, 2, 1)\n    sns.barplot(x=sincere_word_freq[:10].values, y=sincere_word_freq[:10].index, color='b')\n    plt.title('Top 10 Most Frequent Words in Sincere Questions')\n    plt.xlabel('Frequency')\n    plt.ylabel('Words')\n\n    plt.subplot(1, 2, 2)\n    sns.barplot(x=insincere_word_freq[:10].values, y=insincere_word_freq[:10].index, color='r')\n    plt.title('Top 10 Most Frequent Words in Insincere Questions')\n    plt.xlabel('Frequency')\n    plt.ylabel('Words')\n\n    plt.tight_layout()\n    plt.show()\n\nplot_word_frequency(train_df)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-07T11:35:25.590361Z","iopub.execute_input":"2023-06-07T11:35:25.590968Z","iopub.status.idle":"2023-06-07T11:57:47.234873Z","shell.execute_reply.started":"2023-06-07T11:35:25.590915Z","shell.execute_reply":"2023-06-07T11:57:47.233191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.probability import FreqDist\nfrom nltk.util import ngrams\nfrom wordcloud import WordCloud\nimport re\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nstopwords = set(stopwords.words('english'))\n\ntrain_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\n\ntrain_df.head()\n\n## target count ##\ncnt_srs = train_df['target'].value_counts()\nplt.figure(figsize=(10, 6))\nsns.barplot(x=cnt_srs.index, y=cnt_srs.values, palette=\"deep\", alpha=0.8)\nplt.title('Target Count', fontsize=18)\nplt.show()\n\n## target distribution ##\nplt.figure(figsize=(6, 6))\nlabels = train_df['target'].value_counts().index\nsizes = train_df['target'].value_counts().values\nplt.pie(sizes, labels=labels, autopct='%1.1f%%', startangle=90)\nplt.title('Target Distribution', fontsize=18)\nplt.show()\n\n# def plot_wordcloud(text, title=None):\n#     wordcloud = WordCloud(stopwords=stopwords, background_color='black', max_words=200, max_font_size=100, random_state=42)\n#     wordcloud.generate(text)\n    \n#     plt.figure(figsize=(12, 8))\n#     plt.imshow(wordcloud, interpolation='bilinear')\n#     plt.title(title, fontsize=18)\n#     plt.axis('off')\n#     plt.show()\n\n# plot_wordcloud(' '.join(train_df[train_df['target'] == 1]['question_text']), 'Word Cloud of Insincere Questions')\n\ndef clean_text(text):\n    text = text.lower()\n    text = re.sub(r\"[^a-zA-Z]\", \" \", text)\n    text = word_tokenize(text)\n    text = [word for word in text if word not in stopwords]\n    text = ' '.join(text)\n    return text\n\ntrain_df['cleaned_text'] = train_df['question_text'].apply(clean_text)\n\ndef plot_freq_dist(text):\n    tokens = word_tokenize(text)\n    fdist = FreqDist(tokens)\n    \n    plt.figure(figsize=(12, 6))\n    fdist.plot(30)\n    plt.title('Frequency Distribution', fontsize=18)\n    plt.show()\n\nplot_freq_dist(' '.join(train_df[train_df['target'] == 1]['cleaned_text']))\n\n\ndef generate_ngrams(text, n):\n    tokens = word_tokenize(text)\n    ngrams_list = list(ngrams(tokens, n))\n    return [' '.join(gram) for gram in ngrams_list]\n\ndef plot_ngrams_freq_dist(text, n, title=None):\n    ngrams_list = generate_ngrams(text, n)\n    fdist = FreqDist(ngrams_list)\n    \n    plt.figure(figsize=(12, 6))\n    fdist.plot(30)\n    plt.title(title, fontsize=18)\n    plt.show()\n\nplot_ngrams_freq_dist(' '.join(train_df[train_df['target'] == 1]['cleaned_text']), 2, 'Frequency Distribution of Bigrams')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-07T12:30:31.388186Z","iopub.execute_input":"2023-06-07T12:30:31.38861Z","iopub.status.idle":"2023-06-07T12:35:34.286973Z","shell.execute_reply.started":"2023-06-07T12:30:31.388564Z","shell.execute_reply":"2023-06-07T12:35:34.28561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport string\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.probability import FreqDist\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import train_test_split\n\n# Load the data\ntrain_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\nprint(\"Test shape:\", test_df.shape)\nprint(\"Train shape:\", train_df.shape)\nprint(\"\\n\\n\\n\\n\\n\")\n\n\n# Distribution of the target variable\nplt.figure(figsize=(6, 6))\ntrain_df['target'].value_counts().plot(kind='pie', autopct='%1.1f%%', startangle=90)\nplt.title('Target Distribution')\nplt.show()\n\n# Word Frequency plot of sincere & insincere questions\nsincere_text = ' '.join(train_df[train_df['target'] == 0]['question_text'])\ninsincere_text = ' '.join(train_df[train_df['target'] == 1]['question_text'])\n\ndef plot_word_frequency(text, title):\n    tokens = word_tokenize(text)\n    fdist = FreqDist(tokens)\n    \n    plt.figure(figsize=(12, 6))\n    fdist.plot(30)\n    plt.title(title)\n    plt.show()\n\nplot_word_frequency(sincere_text, 'Word Frequency - Sincere Questions')\nplot_word_frequency(insincere_text, 'Word Frequency - Insincere Questions')\n\n# Bigram frequency plots\ndef generate_ngrams(text, n):\n    tokens = word_tokenize(text)\n    ngrams_list = list(ngrams(tokens, n))\n    return [' '.join(gram) for gram in ngrams_list]\n\ndef plot_ngrams_frequency(text, n, title):\n    ngrams_list = generate_ngrams(text, n)\n    fdist = FreqDist(ngrams_list)\n    \n    plt.figure(figsize=(12, 6))\n    fdist.plot(30)\n    plt.title(title)\n    plt.show()\n\nplot_ngrams_frequency(sincere_text, 2, 'Bigram Frequency - Sincere Questions')\nplot_ngrams_frequency(insincere_text, 2, 'Bigram Frequency - Insincere Questions')\n\n# Meta Features\nstopwords = set(stopwords.words('english'))\n\ndef get_meta_features(text):\n    words = word_tokenize(text)\n    word_count = len(words)\n    unique_word_count = len(set(words))\n    char_count = len(text)\n    stopwords_count = len([word for word in words if word.lower() in stopwords])\n    punctuation_count = len([char for char in text if char in string.punctuation])\n    uppercase_count = len([word for word in words if word.isupper()])\n    titlecase_count = len([word for word in words if word.istitle()])\n    avg_word_length = np.mean([len(word) for word in words])\n    \n    return word_count, unique_word_count, char_count, stopwords_count, punctuation_count, uppercase_count, titlecase_count, avg_word_length\n\ntrain_df['word_count'], train_df['unique_word_count'], train_df['char_count'], train_df['stopwords_count'], train_df['punctuation_count'], train_df['uppercase_count'], train_df['titlecase_count'], train_df['avg_word_length'] = zip(*train_df['question_text'].apply(get_meta_features))\n\nmeta_features = ['word_count', 'unique_word_count', 'char_count', 'stopwords_count', 'punctuation_count', 'uppercase_count', 'titlecase_count', 'avg_word_length']\nplt.figure(figsize=(12, 6))\nfor feature in meta_features:\n    sns.kdeplot(train_df[train_df['target'] == 0][feature], label='Sincere')\n    sns.kdeplot(train_df[train_df['target'] == 1][feature], label='Insincere')\nplt.title('Meta Features Distribution')\nplt.legend()\nplt.show()\n\n# Baseline Model: Logistic Regression with TF-IDF vectors\nX = train_df['question_text']\ny = train_df['target']\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\ntfidf = TfidfVectorizer()\nX_train_tfidf = tfidf.fit_transform(X_train)\nX_val_tfidf = tfidf.transform(X_val)\n\nmodel = LogisticRegression()\nmodel.fit(X_train_tfidf, y_train)\n\ny_pred = model.predict(X_val_tfidf)\nprint(classification_report(y_val, y_pred))\n\n# Getting the best threshold based on validation sample\nthresholds = np.arange(0.1, 0.9, 0.1)\nbest_threshold = None\nbest_f1_score = 0.0\n\nfor threshold in thresholds:\n    y_pred_thresh = (model.predict_proba(X_val_tfidf)[:, 1] > threshold).astype(int)\n    f1_score = classification_report(y_val, y_pred_thresh, output_dict=True)['1']['f1-score']\n    \n    if f1_score > best_f1_score:\n        best_f1_score = f1_score\n        best_threshold = threshold\n\nprint('Best Threshold:', best_threshold)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-08T15:29:33.607884Z","iopub.execute_input":"2023-06-08T15:29:33.609205Z","iopub.status.idle":"2023-06-08T15:40:07.142912Z","shell.execute_reply.started":"2023-06-08T15:29:33.609152Z","shell.execute_reply":"2023-06-08T15:40:07.141796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}