{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30588,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

Quora Insincere Questions Classification 🕵️‍🤔

\n
","metadata":{}},{"cell_type":"markdown","source":"
\n \"Animated\n
","metadata":{}},{"cell_type":"markdown","source":"
\n

IMPORT DEPENDENCIES

\n
","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport nltk\nfrom sklearn.pipeline import Pipeline\nfrom nltk.corpus import stopwords\nfrom string import punctuation \nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\nfrom sklearn import model_selection\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.corpus import stopwords \nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nimport string\nfrom sklearn.model_selection import train_test_split\nimport nltk\nfrom nltk.corpus import stopwords\nimport pandas as pd\nfrom imblearn.under_sampling import RandomUnderSampler\nimport pandas as pd\nfrom sklearn.metrics import confusion_matrix, classification_report\nnltk.download('stopwords')\nnltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:47:51.080804Z","iopub.execute_input":"2023-12-05T19:47:51.081170Z","iopub.status.idle":"2023-12-05T19:47:51.097435Z","shell.execute_reply.started":"2023-12-05T19:47:51.081139Z","shell.execute_reply":"2023-12-05T19:47:51.096208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:30.552112Z","iopub.execute_input":"2023-12-05T19:39:30.552852Z","iopub.status.idle":"2023-12-05T19:39:35.530748Z","shell.execute_reply.started":"2023-12-05T19:39:30.552819Z","shell.execute_reply":"2023-12-05T19:39:35.529651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:39.164749Z","iopub.execute_input":"2023-12-05T19:39:39.165133Z","iopub.status.idle":"2023-12-05T19:39:39.178886Z","shell.execute_reply.started":"2023-12-05T19:39:39.165098Z","shell.execute_reply":"2023-12-05T19:39:39.177747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:43.984047Z","iopub.execute_input":"2023-12-05T19:39:43.984724Z","iopub.status.idle":"2023-12-05T19:39:43.994227Z","shell.execute_reply.started":"2023-12-05T19:39:43.984693Z","shell.execute_reply":"2023-12-05T19:39:43.993082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

DATA PREPROCESSING AND EDA

\n
","metadata":{}},{"cell_type":"code","source":"df_train.shape,df_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:48.739713Z","iopub.execute_input":"2023-12-05T19:39:48.740569Z","iopub.status.idle":"2023-12-05T19:39:48.747090Z","shell.execute_reply.started":"2023-12-05T19:39:48.740534Z","shell.execute_reply":"2023-12-05T19:39:48.746128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:50.782554Z","iopub.execute_input":"2023-12-05T19:39:50.782955Z","iopub.status.idle":"2023-12-05T19:39:51.077419Z","shell.execute_reply.started":"2023-12-05T19:39:50.782924Z","shell.execute_reply":"2023-12-05T19:39:51.076388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:54.071327Z","iopub.execute_input":"2023-12-05T19:39:54.071660Z","iopub.status.idle":"2023-12-05T19:39:54.148387Z","shell.execute_reply.started":"2023-12-05T19:39:54.071628Z","shell.execute_reply":"2023-12-05T19:39:54.147404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"target\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:56.451227Z","iopub.execute_input":"2023-12-05T19:39:56.451592Z","iopub.status.idle":"2023-12-05T19:39:56.472303Z","shell.execute_reply.started":"2023-12-05T19:39:56.451563Z","shell.execute_reply":"2023-12-05T19:39:56.471253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere = df_train[df_train[\"target\"] == 1]\nsincere = df_train[df_train[\"target\"] == 0]","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:39:59.174156Z","iopub.execute_input":"2023-12-05T19:39:59.174555Z","iopub.status.idle":"2023-12-05T19:39:59.265919Z","shell.execute_reply.started":"2023-12-05T19:39:59.174523Z","shell.execute_reply":"2023-12-05T19:39:59.265095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:04.850741Z","iopub.execute_input":"2023-12-05T19:40:04.851111Z","iopub.status.idle":"2023-12-05T19:40:04.861081Z","shell.execute_reply.started":"2023-12-05T19:40:04.851084Z","shell.execute_reply":"2023-12-05T19:40:04.860168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:08.034827Z","iopub.execute_input":"2023-12-05T19:40:08.035667Z","iopub.status.idle":"2023-12-05T19:40:08.045723Z","shell.execute_reply.started":"2023-12-05T19:40:08.035626Z","shell.execute_reply":"2023-12-05T19:40:08.044601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question_class = df_train[\"target\"].value_counts()\ncolors = [\"#B92B27\", \"dodgerblue\"]\nquestion_class.plot(kind=\"bar\", color=colors, edgecolor=\"black\")\nplt.xlabel(\"Question Class\")\nplt.ylabel(\"Number of Questions\")\nplt.title(\"Distribution of Question Classes in Training Data\")\nfor i, count in enumerate(question_class):\n plt.text(i, count + 0.1, str(count), ha=\"center\", va=\"bottom\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:10.619033Z","iopub.execute_input":"2023-12-05T19:40:10.620027Z","iopub.status.idle":"2023-12-05T19:40:10.942046Z","shell.execute_reply.started":"2023-12-05T19:40:10.619994Z","shell.execute_reply":"2023-12-05T19:40:10.941129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train[\"target\"].value_counts())\nprint(sum(df_train[\"target\"] == 1) / sum(df_train[\"target\"] == 0) * 100, \"percent of questions are insincere.\")\nprint(100 - sum(df_train[\"target\"] == 1) / sum(df_train[\"target\"] == 0) * 100, \"percent of questions are sincere\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:14.517666Z","iopub.execute_input":"2023-12-05T19:40:14.518594Z","iopub.status.idle":"2023-12-05T19:40:15.147836Z","shell.execute_reply.started":"2023-12-05T19:40:14.518558Z","shell.execute_reply":"2023-12-05T19:40:15.146911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Original Class Distribution:\")\nprint(df_train[\"target\"].value_counts())\n\nrus = RandomUnderSampler(random_state=42)\nX_resampled, y_resampled = rus.fit_resample(df_train.drop(\"target\", axis=1), df_train[\"target\"])\n\ndf_balanced = pd.concat([X_resampled, y_resampled], axis=1)\n\nprint(\"\\nBalanced Class Distribution:\")\nprint(df_balanced[\"target\"].value_counts())\n\ninsincere_percent = sum(df_balanced[\"target\"] == 1) / len(df_balanced) * 100\nsincere_percent = 100 - insincere_percent\n\nprint(f\"\\n{insincere_percent:.2f}% of questions are insincere.\")\nprint(f\"{sincere_percent:.2f}% of questions are sincere.\")\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:17.847747Z","iopub.execute_input":"2023-12-05T19:40:17.848616Z","iopub.status.idle":"2023-12-05T19:40:18.328292Z","shell.execute_reply.started":"2023-12-05T19:40:17.848582Z","shell.execute_reply":"2023-12-05T19:40:18.327366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_balanced.copy()\ndf_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:22.387655Z","iopub.execute_input":"2023-12-05T19:40:22.388041Z","iopub.status.idle":"2023-12-05T19:40:22.402593Z","shell.execute_reply.started":"2023-12-05T19:40:22.388012Z","shell.execute_reply":"2023-12-05T19:40:22.401534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords = set(STOPWORDS)\nsincere_wordcloud = WordCloud(width=600, height=400).generate(str(sincere[\"question_text\"]))\nplt.figure(figsize=(10,8), facecolor=\"black\")\nplt.imshow(sincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:33.202507Z","iopub.execute_input":"2023-12-05T19:40:33.202889Z","iopub.status.idle":"2023-12-05T19:40:33.784192Z","shell.execute_reply.started":"2023-12-05T19:40:33.202859Z","shell.execute_reply":"2023-12-05T19:40:33.783209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords = set(STOPWORDS)\ninsincere_wordcloud = WordCloud(width=600, height=400).generate(str(insincere[\"question_text\"]))\nplt.figure(figsize=(10,8), facecolor=\"black\")\nplt.imshow(insincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:36.946705Z","iopub.execute_input":"2023-12-05T19:40:36.947559Z","iopub.status.idle":"2023-12-05T19:40:37.467885Z","shell.execute_reply.started":"2023-12-05T19:40:36.947528Z","shell.execute_reply":"2023-12-05T19:40:37.466931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"number_words\"] = df_train[\"question_text\"].apply(lambda x: len(x.split()))\ndf_test[\"number_words\"] = df_test[\"question_text\"].apply(lambda x: len(x.split()))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:41.705088Z","iopub.execute_input":"2023-12-05T19:40:41.705741Z","iopub.status.idle":"2023-12-05T19:40:42.548988Z","shell.execute_reply.started":"2023-12-05T19:40:41.705708Z","shell.execute_reply":"2023-12-05T19:40:42.547999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"num_unique_words\"] = df_train[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ndf_test[\"num_unique_words\"] = df_test[\"question_text\"].apply(lambda x: len(set(str(x).split())))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:45.083275Z","iopub.execute_input":"2023-12-05T19:40:45.083920Z","iopub.status.idle":"2023-12-05T19:40:46.755257Z","shell.execute_reply.started":"2023-12-05T19:40:45.083885Z","shell.execute_reply":"2023-12-05T19:40:46.754068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"num_chars\"] = df_train[\"question_text\"].apply(lambda x: len(str(x)))\ndf_test[\"num_chars\"] = df_test[\"question_text\"].apply(lambda x: len(str(x)))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:40:47.924258Z","iopub.execute_input":"2023-12-05T19:40:47.925128Z","iopub.status.idle":"2023-12-05T19:40:48.339390Z","shell.execute_reply.started":"2023-12-05T19:40:47.925094Z","shell.execute_reply":"2023-12-05T19:40:48.338389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\nimport pandas as pd\nnltk.download('stopwords')\nstop_words = set(stopwords.words(\"english\"))\ndf_train[\"num_stopwords\"] = df_train[\"question_text\"].apply(lambda x : len([nw for nw in str(x).split() if nw.lower() in stop_words]))\ndf_test[\"num_stopwords\"] = df_test[\"question_text\"].apply(lambda x : len([nw for nw in str(x).split() if nw.lower() in stop_words]))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:11.583601Z","iopub.execute_input":"2023-12-05T19:41:11.584557Z","iopub.status.idle":"2023-12-05T19:41:14.117967Z","shell.execute_reply.started":"2023-12-05T19:41:11.584522Z","shell.execute_reply":"2023-12-05T19:41:14.116933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"num_punctuation\"] = df_train[\"question_text\"].apply(lambda x : len([np for np in str(x) if np in punctuation]))\ndf_test[\"num_punctuation\"] = df_test[\"question_text\"].apply(lambda x : len([np for np in str(x) if np in punctuation]))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:17.577366Z","iopub.execute_input":"2023-12-05T19:41:17.577759Z","iopub.status.idle":"2023-12-05T19:41:20.657560Z","shell.execute_reply.started":"2023-12-05T19:41:17.577730Z","shell.execute_reply":"2023-12-05T19:41:20.656663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"num_uppercase\"] = df_train[\"question_text\"].apply(lambda x : len([nu for nu in str(x).split() if nu.isupper()]))\ndf_test[\"num_uppercase\"] = df_test[\"question_text\"].apply(lambda x : len([nu for nu in str(x).split() if nu.isupper()]))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:20.659159Z","iopub.execute_input":"2023-12-05T19:41:20.659454Z","iopub.status.idle":"2023-12-05T19:41:22.336072Z","shell.execute_reply.started":"2023-12-05T19:41:20.659429Z","shell.execute_reply":"2023-12-05T19:41:22.335287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"num_lowercase\"] = df_train[\"question_text\"].apply(lambda x : len([nl for nl in str(x).split() if nl.islower()]))\ndf_test[\"num_lowercase\"] = df_test[\"question_text\"].apply(lambda x : len([nl for nl in str(x).split() if nl.islower()]))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:22.337636Z","iopub.execute_input":"2023-12-05T19:41:22.337992Z","iopub.status.idle":"2023-12-05T19:41:24.555206Z","shell.execute_reply.started":"2023-12-05T19:41:22.337966Z","shell.execute_reply":"2023-12-05T19:41:24.554169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"num_title\"] = df_train[\"question_text\"].apply(lambda x : len([nl for nl in str(x).split() if nl.istitle()]))\ndf_test[\"num_title\"] = df_test[\"question_text\"].apply(lambda x : len([nl for nl in str(x).split() if nl.istitle()]))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:24.560613Z","iopub.execute_input":"2023-12-05T19:41:24.561372Z","iopub.status.idle":"2023-12-05T19:41:26.390329Z","shell.execute_reply.started":"2023-12-05T19:41:24.561336Z","shell.execute_reply":"2023-12-05T19:41:26.389534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train[\"target\"] == 1].describe()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:26.612453Z","iopub.execute_input":"2023-12-05T19:41:26.613469Z","iopub.status.idle":"2023-12-05T19:41:26.684385Z","shell.execute_reply.started":"2023-12-05T19:41:26.613439Z","shell.execute_reply":"2023-12-05T19:41:26.683325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train[\"target\"] == 0].describe()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:30.083434Z","iopub.execute_input":"2023-12-05T19:41:30.083786Z","iopub.status.idle":"2023-12-05T19:41:30.147209Z","shell.execute_reply.started":"2023-12-05T19:41:30.083740Z","shell.execute_reply":"2023-12-05T19:41:30.146210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 10))\ncolors = [\"#B92B27\", \"dodgerblue\"]\nsns.set_palette(sns.color_palette(colors))\nsns.boxplot(data=df_train, y=\"number_words\", x=\"target\", orient=\"v\", ax=ax)\nax.set(xlabel=\"Target\", ylabel=\"Number of Words\", title=\"Box Plot of Number of Words According to the Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:34.246547Z","iopub.execute_input":"2023-12-05T19:41:34.247286Z","iopub.status.idle":"2023-12-05T19:41:34.512508Z","shell.execute_reply.started":"2023-12-05T19:41:34.247253Z","shell.execute_reply":"2023-12-05T19:41:34.511603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 10))\ncolors = [\"#B92B27\", \"dodgerblue\"]\nsns.set_palette(sns.color_palette(colors))\nsns.boxplot(data=df_train, y=\"num_unique_words\", x=\"target\", orient=\"v\", ax=ax)\nax.set(xlabel=\"Target\", ylabel=\"Number of Unique Words\", title=\"Box Plot of Number of Unique Words According to the Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:37.361824Z","iopub.execute_input":"2023-12-05T19:41:37.362665Z","iopub.status.idle":"2023-12-05T19:41:37.843765Z","shell.execute_reply.started":"2023-12-05T19:41:37.362627Z","shell.execute_reply":"2023-12-05T19:41:37.842789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 10))\ncolors = [\"#B92B27\", \"dodgerblue\"]\nsns.set_palette(sns.color_palette(colors))\nsns.boxplot(data=df_train, y=\"num_chars\", x=\"target\", orient=\"v\", ax=ax)\nax.set(xlabel=\"Target\", ylabel=\"Number of Chars\", title=\"Box Plot of Number of Chars According to the Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:40.547104Z","iopub.execute_input":"2023-12-05T19:41:40.547956Z","iopub.status.idle":"2023-12-05T19:41:40.822840Z","shell.execute_reply.started":"2023-12-05T19:41:40.547922Z","shell.execute_reply":"2023-12-05T19:41:40.821936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 10))\ncolors = [\"#B92B27\", \"dodgerblue\"]\nsns.set_palette(sns.color_palette(colors))\nsns.boxplot(data=df_train, y=\"num_stopwords\", x=\"target\", orient=\"v\", ax=ax)\nax.set(xlabel=\"Target\", ylabel=\"Number of Stop Words\", title=\"Box Plot of Number of Stop Words According to the Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:43.370435Z","iopub.execute_input":"2023-12-05T19:41:43.370830Z","iopub.status.idle":"2023-12-05T19:41:43.648067Z","shell.execute_reply.started":"2023-12-05T19:41:43.370799Z","shell.execute_reply":"2023-12-05T19:41:43.647000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 10))\ncolors = [\"#B92B27\", \"dodgerblue\"]\nsns.set_palette(sns.color_palette(colors))\nsns.boxplot(data=df_train, y=\"num_lowercase\", x=\"target\", orient=\"v\", ax=ax)\nax.set(xlabel=\"Target\", ylabel=\"Number of LowerCase Words\", title=\"Box Plot of Number of LowerCase Words According to the Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:46.476698Z","iopub.execute_input":"2023-12-05T19:41:46.477424Z","iopub.status.idle":"2023-12-05T19:41:46.747273Z","shell.execute_reply.started":"2023-12-05T19:41:46.477391Z","shell.execute_reply":"2023-12-05T19:41:46.746339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 10))\ncolors = [\"#B92B27\", \"dodgerblue\"]\nsns.set_palette(sns.color_palette(colors))\nsns.boxplot(data=df_train, y=\"num_title\", x=\"target\", orient=\"v\", ax=ax)\nax.set(xlabel=\"Target\", ylabel=\"Number of Titles\", title=\"Box Plot of Number of Titles According to the Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:49.566619Z","iopub.execute_input":"2023-12-05T19:41:49.566995Z","iopub.status.idle":"2023-12-05T19:41:49.834542Z","shell.execute_reply.started":"2023-12-05T19:41:49.566968Z","shell.execute_reply":"2023-12-05T19:41:49.833561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_process(text):\n text = text.lower()\n text = text.translate(str.maketrans(\"\", \"\", string.punctuation))\n tokens = word_tokenize(text)\n stop_words = set(stopwords.words('english'))\n tokens = [word for word in tokens if word not in stop_words]\n processed_text = ' '.join(tokens)\n return processed_text","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:41:54.371234Z","iopub.execute_input":"2023-12-05T19:41:54.371588Z","iopub.status.idle":"2023-12-05T19:41:54.377197Z","shell.execute_reply.started":"2023-12-05T19:41:54.371558Z","shell.execute_reply":"2023-12-05T19:41:54.376322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['clean_train'] = df_train[\"question_text\"].apply(text_process)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:42:20.655509Z","iopub.execute_input":"2023-12-05T19:42:20.656438Z","iopub.status.idle":"2023-12-05T19:43:20.475849Z","shell.execute_reply.started":"2023-12-05T19:42:20.656403Z","shell.execute_reply":"2023-12-05T19:43:20.474815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['clean_test'] = df_test[\"question_text\"].apply(text_process)\ndf_test","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:43:25.936287Z","iopub.execute_input":"2023-12-05T19:43:25.937236Z","iopub.status.idle":"2023-12-05T19:45:38.041309Z","shell.execute_reply.started":"2023-12-05T19:43:25.937198Z","shell.execute_reply":"2023-12-05T19:45:38.040338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,X_val,Y_train,Y_val = train_test_split(df_train['clean_train'],df_train['target'],test_size=0.2)\nX_train.shape,X_val.shape,Y_train.shape,Y_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:46:01.870419Z","iopub.execute_input":"2023-12-05T19:46:01.870802Z","iopub.status.idle":"2023-12-05T19:46:01.897438Z","shell.execute_reply.started":"2023-12-05T19:46:01.870761Z","shell.execute_reply":"2023-12-05T19:46:01.896567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

MODEL ENGINEERING

\n
","metadata":{}},{"cell_type":"code","source":"pipeline = Pipeline(\n [\n (\"cv\",CountVectorizer(analyzer=\"word\",ngram_range=(1,4),max_df=0.9)),\n (\"clf\",LogisticRegression(solver=\"saga\", class_weight=\"balanced\", C=0.45, max_iter=250, verbose=1))\n ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:46:32.247513Z","iopub.execute_input":"2023-12-05T19:46:32.247908Z","iopub.status.idle":"2023-12-05T19:46:32.253723Z","shell.execute_reply.started":"2023-12-05T19:46:32.247877Z","shell.execute_reply":"2023-12-05T19:46:32.252718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_model = pipeline.fit(X_train,Y_train)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:46:44.715138Z","iopub.execute_input":"2023-12-05T19:46:44.715836Z","iopub.status.idle":"2023-12-05T19:47:26.752762Z","shell.execute_reply.started":"2023-12-05T19:46:44.715803Z","shell.execute_reply":"2023-12-05T19:47:26.751805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = lr_model.predict(X_val)\nprint(classification_report(Y_val,Y_pred))\ncm = confusion_matrix(Y_val,Y_pred)\nsns.heatmap(cm, cmap=\"Blues\", annot=True, square=True, fmt=\".0f\");","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:48:37.629075Z","iopub.execute_input":"2023-12-05T19:48:37.629462Z","iopub.status.idle":"2023-12-05T19:48:39.064249Z","shell.execute_reply.started":"2023-12-05T19:48:37.629434Z","shell.execute_reply":"2023-12-05T19:48:39.063313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_final = pipeline.predict(df_test['clean_test'])\ny_pred_final","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:49:26.671324Z","iopub.execute_input":"2023-12-05T19:49:26.672218Z","iopub.status.idle":"2023-12-05T19:49:37.420139Z","shell.execute_reply.started":"2023-12-05T19:49:26.672185Z","shell.execute_reply":"2023-12-05T19:49:37.419086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.DataFrame({\"qid\":df_test[\"qid\"], \"prediction\":y_pred_final})\ndf_sub.to_csv('submission.csv', index=False)\ndf_sub.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T19:49:59.475149Z","iopub.execute_input":"2023-12-05T19:49:59.475550Z","iopub.status.idle":"2023-12-05T19:50:00.432655Z","shell.execute_reply.started":"2023-12-05T19:49:59.475521Z","shell.execute_reply":"2023-12-05T19:50:00.431672Z"},"trusted":true},"execution_count":null,"outputs":[]}]}