{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nimport string\nfrom textblob import Word","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-13T14:36:01.995571Z","iopub.execute_input":"2024-07-13T14:36:01.995960Z","iopub.status.idle":"2024-07-13T14:36:03.179666Z","shell.execute_reply.started":"2024-07-13T14:36:01.995920Z","shell.execute_reply":"2024-07-13T14:36:03.178555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:03.181534Z","iopub.execute_input":"2024-07-13T14:36:03.181987Z","iopub.status.idle":"2024-07-13T14:36:06.463422Z","shell.execute_reply.started":"2024-07-13T14:36:03.181956Z","shell.execute_reply":"2024-07-13T14:36:06.462330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"memory used by train dataset : {train.memory_usage(deep=True).sum()/1024**2} mb\")","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:06.465413Z","iopub.execute_input":"2024-07-13T14:36:06.465718Z","iopub.status.idle":"2024-07-13T14:36:07.026776Z","shell.execute_reply.started":"2024-07-13T14:36:06.465693Z","shell.execute_reply":"2024-07-13T14:36:07.025645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:07.028299Z","iopub.execute_input":"2024-07-13T14:36:07.028771Z","iopub.status.idle":"2024-07-13T14:36:07.037768Z","shell.execute_reply.started":"2024-07-13T14:36:07.028733Z","shell.execute_reply":"2024-07-13T14:36:07.036838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'].value_counts(normalize = True , dropna = False)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:07.076031Z","iopub.execute_input":"2024-07-13T14:36:07.076404Z","iopub.status.idle":"2024-07-13T14:36:07.098017Z","shell.execute_reply.started":"2024-07-13T14:36:07.076374Z","shell.execute_reply":"2024-07-13T14:36:07.096968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dataset is heavinly imbalanced.\ntrain[train['target']==1]['question_text'].iloc[7605]\n","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:08.271040Z","iopub.execute_input":"2024-07-13T14:36:08.271427Z","iopub.status.idle":"2024-07-13T14:36:08.300221Z","shell.execute_reply.started":"2024-07-13T14:36:08.271397Z","shell.execute_reply":"2024-07-13T14:36:08.299152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop = set(stopwords.words('english') + list(string.punctuation))\nstop","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:09.480808Z","iopub.execute_input":"2024-07-13T14:36:09.481194Z","iopub.status.idle":"2024-07-13T14:36:09.493537Z","shell.execute_reply.started":"2024-07-13T14:36:09.481161Z","shell.execute_reply":"2024-07-13T14:36:09.492384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer\n \nlemmatizer = WordNetLemmatizer()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:44.315538Z","iopub.execute_input":"2024-07-13T14:36:44.315925Z","iopub.status.idle":"2024-07-13T14:36:44.320717Z","shell.execute_reply.started":"2024-07-13T14:36:44.315893Z","shell.execute_reply":"2024-07-13T14:36:44.319506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('wordnet')","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:36:39.771294Z","iopub.execute_input":"2024-07-13T14:36:39.771675Z","iopub.status.idle":"2024-07-13T14:36:39.890019Z","shell.execute_reply.started":"2024-07-13T14:36:39.771640Z","shell.execute_reply":"2024-07-13T14:36:39.888779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip /usr/share/nltk_data/corpora/wordnet.zip -d /usr/share/nltk_data/corpora/","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:37:30.281254Z","iopub.execute_input":"2024-07-13T14:37:30.281747Z","iopub.status.idle":"2024-07-13T14:37:31.697092Z","shell.execute_reply.started":"2024-07-13T14:37:30.281710Z","shell.execute_reply":"2024-07-13T14:37:31.695828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nltk.data.path.append('/kaggle/working/nltk_data')\nlemmatizer.lemmatize(\"heelo\")","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:37:44.156812Z","iopub.execute_input":"2024-07-13T14:37:44.157220Z","iopub.status.idle":"2024-07-13T14:37:45.875516Z","shell.execute_reply.started":"2024-07-13T14:37:44.157189Z","shell.execute_reply":"2024-07-13T14:37:45.874491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -V","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:38:17.886132Z","iopub.execute_input":"2024-07-13T14:38:17.886904Z","iopub.status.idle":"2024-07-13T14:38:18.958063Z","shell.execute_reply.started":"2024-07-13T14:38:17.886864Z","shell.execute_reply":"2024-07-13T14:38:18.956444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cleaning_text(text):\n    text = text.lower()\n    text  = [i for i in word_tokenize(text) if i not in stop] \n    for i in range(len(text)):\n        text[i] = lemmatizer.lemmatize(text[i])\n    text = \" \".join(text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:41:49.445594Z","iopub.execute_input":"2024-07-13T14:41:49.446547Z","iopub.status.idle":"2024-07-13T14:41:49.452524Z","shell.execute_reply.started":"2024-07-13T14:41:49.446510Z","shell.execute_reply":"2024-07-13T14:41:49.451414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train['question_text'].isnull()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:41:51.215651Z","iopub.execute_input":"2024-07-13T14:41:51.216050Z","iopub.status.idle":"2024-07-13T14:41:51.220781Z","shell.execute_reply.started":"2024-07-13T14:41:51.216020Z","shell.execute_reply":"2024-07-13T14:41:51.219662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['cleaned_text'] = train['question_text'].apply(cleaning_text)\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:41:52.441383Z","iopub.execute_input":"2024-07-13T14:41:52.441765Z","iopub.status.idle":"2024-07-13T14:45:53.764261Z","shell.execute_reply.started":"2024-07-13T14:41:52.441735Z","shell.execute_reply":"2024-07-13T14:45:53.763251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_len(text):\n    text = text.split(\" \")\n    return len(text)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:10.002717Z","iopub.execute_input":"2024-07-13T14:46:10.003688Z","iopub.status.idle":"2024-07-13T14:46:10.009328Z","shell.execute_reply.started":"2024-07-13T14:46:10.003640Z","shell.execute_reply":"2024-07-13T14:46:10.008043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['length'] = train['cleaned_text'].apply(split_len)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:11.290689Z","iopub.execute_input":"2024-07-13T14:46:11.291120Z","iopub.status.idle":"2024-07-13T14:46:12.510390Z","shell.execute_reply.started":"2024-07-13T14:46:11.291088Z","shell.execute_reply":"2024-07-13T14:46:12.509212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmax(train['length']) , min(train['length'])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:14.356006Z","iopub.execute_input":"2024-07-13T14:46:14.356389Z","iopub.status.idle":"2024-07-13T14:46:14.603989Z","shell.execute_reply.started":"2024-07-13T14:46:14.356360Z","shell.execute_reply":"2024-07-13T14:46:14.602902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['length']==0]","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:17.520527Z","iopub.execute_input":"2024-07-13T14:46:17.520901Z","iopub.status.idle":"2024-07-13T14:46:17.534987Z","shell.execute_reply.started":"2024-07-13T14:46:17.520873Z","shell.execute_reply":"2024-07-13T14:46:17.533773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets see if our dataset is seperable by length.\nimport matplotlib.pyplot as plt\nplt.figure(figsize = (10,6))\ntrain[train['target']==0]['length'].plot(kind = 'hist' , alpha=0.5 , bins=20 , label = 'class-0')\ntrain[train['target']==1]['length'].plot(kind = 'hist' , alpha=0.5 , bins=20 , label = 'class-1')\nplt.legend()\nplt.title(\"Distribution based on length\")\nplt.plot()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:37.645420Z","iopub.execute_input":"2024-07-13T14:46:37.645801Z","iopub.status.idle":"2024-07-13T14:46:38.647347Z","shell.execute_reply.started":"2024-07-13T14:46:37.645774Z","shell.execute_reply":"2024-07-13T14:46:38.646190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#least frequent used word\npd.Series(\" \".join(train['cleaned_text']).split()).value_counts()[-10:]","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:43.315989Z","iopub.execute_input":"2024-07-13T14:46:43.316418Z","iopub.status.idle":"2024-07-13T14:46:47.049656Z","shell.execute_reply.started":"2024-07-13T14:46:43.316384Z","shell.execute_reply":"2024-07-13T14:46:47.048532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#most frequent used word\npd.Series(\" \".join(train['cleaned_text']).split()).value_counts()[:10]","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:46:52.480660Z","iopub.execute_input":"2024-07-13T14:46:52.481047Z","iopub.status.idle":"2024-07-13T14:46:55.809338Z","shell.execute_reply.started":"2024-07-13T14:46:52.481017Z","shell.execute_reply":"2024-07-13T14:46:55.808181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#total unique words\npd.Series(\" \".join(train['cleaned_text']).split()).nunique()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:47:23.717790Z","iopub.execute_input":"2024-07-13T14:47:23.718183Z","iopub.status.idle":"2024-07-13T14:47:26.300855Z","shell.execute_reply.started":"2024-07-13T14:47:23.718152Z","shell.execute_reply":"2024-07-13T14:47:26.299561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#so out dataset is not seperable by length.\n#lets use embeddings to embed it .\nfrom sklearn.feature_extraction.text import TfidfVectorizer\ntfidf = TfidfVectorizer(max_features=10000 , lowercase=True, analyzer='word',\n stop_words= 'english',ngram_range=(1,1))\ntrain_tfidf = tfidf.fit_transform(train['cleaned_text'])\n","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:48:07.710405Z","iopub.execute_input":"2024-07-13T14:48:07.710778Z","iopub.status.idle":"2024-07-13T14:48:21.839908Z","shell.execute_reply.started":"2024-07-13T14:48:07.710750Z","shell.execute_reply":"2024-07-13T14:48:21.838631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train , X_test , y_train  , y_test = train_test_split(train_tfidf , train['target'] , test_size = 0.3 , random_state = 43 ,stratify = train['target'])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:51:36.241972Z","iopub.execute_input":"2024-07-13T14:51:36.242783Z","iopub.status.idle":"2024-07-13T14:51:36.878609Z","shell.execute_reply.started":"2024-07-13T14:51:36.242743Z","shell.execute_reply":"2024-07-13T14:51:36.877527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nlr = LogisticRegression()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:52:47.426463Z","iopub.execute_input":"2024-07-13T14:52:47.426851Z","iopub.status.idle":"2024-07-13T14:52:47.432114Z","shell.execute_reply.started":"2024-07-13T14:52:47.426823Z","shell.execute_reply":"2024-07-13T14:52:47.430804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr.fit(X_train , y_train)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:53:00.866187Z","iopub.execute_input":"2024-07-13T14:53:00.866606Z","iopub.status.idle":"2024-07-13T14:53:12.707460Z","shell.execute_reply.started":"2024-07-13T14:53:00.866574Z","shell.execute_reply":"2024-07-13T14:53:12.706401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = lr.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:53:31.136410Z","iopub.execute_input":"2024-07-13T14:53:31.137379Z","iopub.status.idle":"2024-07-13T14:53:31.151875Z","shell.execute_reply.started":"2024-07-13T14:53:31.137344Z","shell.execute_reply":"2024-07-13T14:53:31.150639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report , f1_score","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:55:20.457516Z","iopub.execute_input":"2024-07-13T14:55:20.457907Z","iopub.status.idle":"2024-07-13T14:55:20.462570Z","shell.execute_reply.started":"2024-07-13T14:55:20.457875Z","shell.execute_reply":"2024-07-13T14:55:20.461519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(y_test , y_pred)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:55:49.885948Z","iopub.execute_input":"2024-07-13T14:55:49.886472Z","iopub.status.idle":"2024-07-13T14:55:50.070405Z","shell.execute_reply.started":"2024-07-13T14:55:49.886432Z","shell.execute_reply":"2024-07-13T14:55:50.069269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\nsample_sub = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:56:57.040960Z","iopub.execute_input":"2024-07-13T14:56:57.041494Z","iopub.status.idle":"2024-07-13T14:56:58.757366Z","shell.execute_reply.started":"2024-07-13T14:56:57.041460Z","shell.execute_reply":"2024-07-13T14:56:58.756289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:57:03.450372Z","iopub.execute_input":"2024-07-13T14:57:03.450729Z","iopub.status.idle":"2024-07-13T14:57:03.461058Z","shell.execute_reply.started":"2024-07-13T14:57:03.450703Z","shell.execute_reply":"2024-07-13T14:57:03.459858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['cleaned_text'] = test['question_text'].apply(cleaning_text)\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:57:47.400911Z","iopub.execute_input":"2024-07-13T14:57:47.401329Z","iopub.status.idle":"2024-07-13T14:58:57.243297Z","shell.execute_reply.started":"2024-07-13T14:57:47.401295Z","shell.execute_reply":"2024-07-13T14:58:57.242230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tf = tfidf.transform(test['cleaned_text'])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T14:59:40.202575Z","iopub.execute_input":"2024-07-13T14:59:40.203538Z","iopub.status.idle":"2024-07-13T14:59:44.082890Z","shell.execute_reply.started":"2024-07-13T14:59:40.203504Z","shell.execute_reply":"2024-07-13T14:59:44.081897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = lr.predict(test_tf)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T15:00:07.001126Z","iopub.execute_input":"2024-07-13T15:00:07.001552Z","iopub.status.idle":"2024-07-13T15:00:07.016349Z","shell.execute_reply.started":"2024-07-13T15:00:07.001519Z","shell.execute_reply":"2024-07-13T15:00:07.015155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['prediction'] = pred","metadata":{"execution":{"iopub.status.busy":"2024-07-13T15:00:28.976226Z","iopub.execute_input":"2024-07-13T15:00:28.976647Z","iopub.status.idle":"2024-07-13T15:00:28.982639Z","shell.execute_reply.started":"2024-07-13T15:00:28.976614Z","shell.execute_reply":"2024-07-13T15:00:28.981299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T15:00:41.277093Z","iopub.execute_input":"2024-07-13T15:00:41.278225Z","iopub.status.idle":"2024-07-13T15:00:41.288066Z","shell.execute_reply.started":"2024-07-13T15:00:41.278166Z","shell.execute_reply":"2024-07-13T15:00:41.286794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv(\"submission.csv\" , index = False)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T15:01:25.280639Z","iopub.execute_input":"2024-07-13T15:01:25.281070Z","iopub.status.idle":"2024-07-13T15:01:25.698812Z","shell.execute_reply.started":"2024-07-13T15:01:25.281034Z","shell.execute_reply":"2024-07-13T15:01:25.697777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}