{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:23.953913Z","iopub.execute_input":"2024-10-08T21:43:23.954313Z","iopub.status.idle":"2024-10-08T21:43:24.858397Z","shell.execute_reply.started":"2024-10-08T21:43:23.954268Z","shell.execute_reply":"2024-10-08T21:43:24.857506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/quora-insincere-questions-classification'\ntrain_fname= data_dir + '/train.csv'\ntest_fname= data_dir + '/test.csv'\nsub_fname= data_dir + '/sample_submission.csv'","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:24.859906Z","iopub.execute_input":"2024-10-08T21:43:24.860366Z","iopub.status.idle":"2024-10-08T21:43:24.867185Z","shell.execute_reply.started":"2024-10-08T21:43:24.860330Z","shell.execute_reply":"2024-10-08T21:43:24.866269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis","metadata":{}},{"cell_type":"code","source":"df=pd.read_csv(train_fname)\ndf.sample(10)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:24.868435Z","iopub.execute_input":"2024-10-08T21:43:24.868805Z","iopub.status.idle":"2024-10-08T21:43:29.519826Z","shell.execute_reply.started":"2024-10-08T21:43:24.868759Z","shell.execute_reply":"2024-10-08T21:43:29.518730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df=df[df['target']==0]\ninsincere_df=df[df['target']==1]","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:29.522508Z","iopub.execute_input":"2024-10-08T21:43:29.523169Z","iopub.status.idle":"2024-10-08T21:43:29.641751Z","shell.execute_reply.started":"2024-10-08T21:43:29.523132Z","shell.execute_reply":"2024-10-08T21:43:29.640572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sincere_df.sample(10).question_text[:5].values)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:29.643447Z","iopub.execute_input":"2024-10-08T21:43:29.643789Z","iopub.status.idle":"2024-10-08T21:43:29.701386Z","shell.execute_reply.started":"2024-10-08T21:43:29.643745Z","shell.execute_reply":"2024-10-08T21:43:29.700356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(insincere_df.sample(10).question_text[:5].values)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:29.702925Z","iopub.execute_input":"2024-10-08T21:43:29.705310Z","iopub.status.idle":"2024-10-08T21:43:29.714173Z","shell.execute_reply.started":"2024-10-08T21:43:29.705252Z","shell.execute_reply":"2024-10-08T21:43:29.713055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['target'].value_counts(normalize=True)*100","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:29.715516Z","iopub.execute_input":"2024-10-08T21:43:29.715888Z","iopub.status.idle":"2024-10-08T21:43:29.745061Z","shell.execute_reply.started":"2024-10-08T21:43:29.715845Z","shell.execute_reply":"2024-10-08T21:43:29.744041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=pd.read_csv(test_fname)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:29.746421Z","iopub.execute_input":"2024-10-08T21:43:29.746769Z","iopub.status.idle":"2024-10-08T21:43:30.911343Z","shell.execute_reply.started":"2024-10-08T21:43:29.746736Z","shell.execute_reply":"2024-10-08T21:43:30.910325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.sample(10).question_text[0:5].values","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:30.912687Z","iopub.execute_input":"2024-10-08T21:43:30.913104Z","iopub.status.idle":"2024-10-08T21:43:30.934449Z","shell.execute_reply.started":"2024-10-08T21:43:30.913060Z","shell.execute_reply":"2024-10-08T21:43:30.933223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sampling","metadata":{}},{"cell_type":"code","source":"sample_size= len(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:59:36.432200Z","iopub.execute_input":"2024-10-08T21:59:36.433089Z","iopub.status.idle":"2024-10-08T21:59:36.437059Z","shell.execute_reply.started":"2024-10-08T21:59:36.433051Z","shell.execute_reply":"2024-10-08T21:59:36.436082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:59:45.338729Z","iopub.execute_input":"2024-10-08T21:59:45.339165Z","iopub.status.idle":"2024-10-08T21:59:45.345555Z","shell.execute_reply.started":"2024-10-08T21:59:45.339125Z","shell.execute_reply":"2024-10-08T21:59:45.344571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df=df.sample(sample_size, random_state=43)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:30.946091Z","iopub.execute_input":"2024-10-08T21:43:30.946523Z","iopub.status.idle":"2024-10-08T21:43:31.017915Z","shell.execute_reply.started":"2024-10-08T21:43:30.946479Z","shell.execute_reply":"2024-10-08T21:43:31.016947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:31.019146Z","iopub.execute_input":"2024-10-08T21:43:31.019519Z","iopub.status.idle":"2024-10-08T21:43:31.030951Z","shell.execute_reply.started":"2024-10-08T21:43:31.019477Z","shell.execute_reply":"2024-10-08T21:43:31.030065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Tokenization","metadata":{}},{"cell_type":"code","source":"q0= sincere_df.question_text.values[1]\nq0","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:31.032245Z","iopub.execute_input":"2024-10-08T21:43:31.032563Z","iopub.status.idle":"2024-10-08T21:43:31.039313Z","shell.execute_reply.started":"2024-10-08T21:43:31.032531Z","shell.execute_reply":"2024-10-08T21:43:31.038466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1= insincere_df.question_text.values[1]\nq1","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:31.040326Z","iopub.execute_input":"2024-10-08T21:43:31.040601Z","iopub.status.idle":"2024-10-08T21:43:31.049509Z","shell.execute_reply.started":"2024-10-08T21:43:31.040570Z","shell.execute_reply":"2024-10-08T21:43:31.048502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom nltk.tokenize import word_tokenize","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:31.050824Z","iopub.execute_input":"2024-10-08T21:43:31.051252Z","iopub.status.idle":"2024-10-08T21:43:32.138557Z","shell.execute_reply.started":"2024-10-08T21:43:31.051208Z","shell.execute_reply":"2024-10-08T21:43:32.137678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(q0)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.139802Z","iopub.execute_input":"2024-10-08T21:43:32.140289Z","iopub.status.idle":"2024-10-08T21:43:32.161919Z","shell.execute_reply.started":"2024-10-08T21:43:32.140244Z","shell.execute_reply":"2024-10-08T21:43:32.160933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(q1)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.163153Z","iopub.execute_input":"2024-10-08T21:43:32.163470Z","iopub.status.idle":"2024-10-08T21:43:32.171514Z","shell.execute_reply.started":"2024-10-08T21:43:32.163438Z","shell.execute_reply":"2024-10-08T21:43:32.170520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok = word_tokenize(q0)\nq1_tok=word_tokenize(q1)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.172730Z","iopub.execute_input":"2024-10-08T21:43:32.173496Z","iopub.status.idle":"2024-10-08T21:43:32.179338Z","shell.execute_reply.started":"2024-10-08T21:43:32.173461Z","shell.execute_reply":"2024-10-08T21:43:32.178531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stop Word Removal","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.180602Z","iopub.execute_input":"2024-10-08T21:43:32.180932Z","iopub.status.idle":"2024-10-08T21:43:32.189229Z","shell.execute_reply.started":"2024-10-08T21:43:32.180899Z","shell.execute_reply":"2024-10-08T21:43:32.188206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_stopwords= stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.190690Z","iopub.execute_input":"2024-10-08T21:43:32.191627Z","iopub.status.idle":"2024-10-08T21:43:32.200477Z","shell.execute_reply.started":"2024-10-08T21:43:32.191533Z","shell.execute_reply":"2024-10-08T21:43:32.199538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\", \".join(english_stopwords))","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.201775Z","iopub.execute_input":"2024-10-08T21:43:32.202207Z","iopub.status.idle":"2024-10-08T21:43:32.207473Z","shell.execute_reply.started":"2024-10-08T21:43:32.202171Z","shell.execute_reply":"2024-10-08T21:43:32.206538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def removd_stopwords(tokens):\n    return [word for word in tokens if word.lower() not in english_stopwords]","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.208907Z","iopub.execute_input":"2024-10-08T21:43:32.209733Z","iopub.status.idle":"2024-10-08T21:43:32.215849Z","shell.execute_reply.started":"2024-10-08T21:43:32.209687Z","shell.execute_reply":"2024-10-08T21:43:32.214699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0, q0_tok","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.217112Z","iopub.execute_input":"2024-10-08T21:43:32.217416Z","iopub.status.idle":"2024-10-08T21:43:32.226196Z","shell.execute_reply.started":"2024-10-08T21:43:32.217382Z","shell.execute_reply":"2024-10-08T21:43:32.225240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp= removd_stopwords(q0_tok)\nq0_stp","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.227498Z","iopub.execute_input":"2024-10-08T21:43:32.228165Z","iopub.status.idle":"2024-10-08T21:43:32.236675Z","shell.execute_reply.started":"2024-10-08T21:43:32.228129Z","shell.execute_reply":"2024-10-08T21:43:32.235663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp= removd_stopwords(q1_tok)\nq1_stp","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.238105Z","iopub.execute_input":"2024-10-08T21:43:32.238767Z","iopub.status.idle":"2024-10-08T21:43:32.245936Z","shell.execute_reply.started":"2024-10-08T21:43:32.238722Z","shell.execute_reply":"2024-10-08T21:43:32.245058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stemming","metadata":{}},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.247370Z","iopub.execute_input":"2024-10-08T21:43:32.248265Z","iopub.status.idle":"2024-10-08T21:43:32.253611Z","shell.execute_reply.started":"2024-10-08T21:43:32.248219Z","shell.execute_reply":"2024-10-08T21:43:32.252774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer= SnowballStemmer(language='english')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.254845Z","iopub.execute_input":"2024-10-08T21:43:32.255641Z","iopub.status.idle":"2024-10-08T21:43:32.262619Z","shell.execute_reply.started":"2024-10-08T21:43:32.255570Z","shell.execute_reply":"2024-10-08T21:43:32.261700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer.stem('going'), stemmer.stem('supposedly'), stemmer.stem('done'), stemmer.stem('doing'), stemmer.stem('do')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.269908Z","iopub.execute_input":"2024-10-08T21:43:32.270354Z","iopub.status.idle":"2024-10-08T21:43:32.277351Z","shell.execute_reply.started":"2024-10-08T21:43:32.270319Z","shell.execute_reply":"2024-10-08T21:43:32.276225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stem= [stemmer.stem(word) for word in q0_stp]\nprint(q0_stp)\nq0_stem","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.278420Z","iopub.execute_input":"2024-10-08T21:43:32.278755Z","iopub.status.idle":"2024-10-08T21:43:32.289200Z","shell.execute_reply.started":"2024-10-08T21:43:32.278723Z","shell.execute_reply":"2024-10-08T21:43:32.288129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stem= [stemmer.stem(word) for word in q1_stp]\nprint(q1_stp)\nq1_stem","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.290350Z","iopub.execute_input":"2024-10-08T21:43:32.292640Z","iopub.status.idle":"2024-10-08T21:43:32.300226Z","shell.execute_reply.started":"2024-10-08T21:43:32.292603Z","shell.execute_reply":"2024-10-08T21:43:32.299286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lemmitization","metadata":{}},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer\nfrom nltk.corpus import wordnet","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.301596Z","iopub.execute_input":"2024-10-08T21:43:32.301926Z","iopub.status.idle":"2024-10-08T21:43:32.307353Z","shell.execute_reply.started":"2024-10-08T21:43:32.301884Z","shell.execute_reply":"2024-10-08T21:43:32.306489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lemmatizer=WordNetLemmatizer()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:43:32.308460Z","iopub.execute_input":"2024-10-08T21:43:32.308788Z","iopub.status.idle":"2024-10-08T21:43:32.316134Z","shell.execute_reply.started":"2024-10-08T21:43:32.308755Z","shell.execute_reply":"2024-10-08T21:43:32.315139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# q0_lem= [lemmatizer.lemmatize(word) for word in q0_stp]\n# print(\"Ques: \", q0, \"\\n\", \"Stopword Removed: \", q0_stp, \"\\n\" , \"Ques Stemmed: \", q0_stem, \"\\n\" , \"Ques Lemmatized: \", q0_lem)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:24.769921Z","iopub.execute_input":"2024-10-08T21:45:24.770729Z","iopub.status.idle":"2024-10-08T21:45:24.774810Z","shell.execute_reply.started":"2024-10-08T21:45:24.770690Z","shell.execute_reply":"2024-10-08T21:45:24.773731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# q1_lem= [lemmatizer.lemmatize(word) for word in q1_stp]\n# print(\"Ques: \", q1, \"\\n\", \"Stopword Removed: \", q1_stp, \"\\n\" , \"Ques Stemmed: \", q1_stem, \"\\n\" , \"Ques Lemmatized: \", q1_lem)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:29.475681Z","iopub.execute_input":"2024-10-08T21:45:29.476577Z","iopub.status.idle":"2024-10-08T21:45:29.480568Z","shell.execute_reply.started":"2024-10-08T21:45:29.476537Z","shell.execute_reply":"2024-10-08T21:45:29.479468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bag of Words","metadata":{}},{"cell_type":"markdown","source":"## Create a Vocabolary","metadata":{}},{"cell_type":"code","source":"small_df= sample_df[:5]\nsmall_df.question_text.values","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:36.769283Z","iopub.execute_input":"2024-10-08T21:45:36.770016Z","iopub.status.idle":"2024-10-08T21:45:36.776901Z","shell.execute_reply.started":"2024-10-08T21:45:36.769959Z","shell.execute_reply":"2024-10-08T21:45:36.775929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:37.773926Z","iopub.execute_input":"2024-10-08T21:45:37.774879Z","iopub.status.idle":"2024-10-08T21:45:37.778925Z","shell.execute_reply.started":"2024-10-08T21:45:37.774836Z","shell.execute_reply":"2024-10-08T21:45:37.777905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vec=CountVectorizer()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:38.497211Z","iopub.execute_input":"2024-10-08T21:45:38.498128Z","iopub.status.idle":"2024-10-08T21:45:38.502117Z","shell.execute_reply.started":"2024-10-08T21:45:38.498086Z","shell.execute_reply":"2024-10-08T21:45:38.501190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vec.fit(small_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:39.019897Z","iopub.execute_input":"2024-10-08T21:45:39.020795Z","iopub.status.idle":"2024-10-08T21:45:39.035777Z","shell.execute_reply.started":"2024-10-08T21:45:39.020754Z","shell.execute_reply":"2024-10-08T21:45:39.034773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vec.vocabulary_","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:39.513909Z","iopub.execute_input":"2024-10-08T21:45:39.514808Z","iopub.status.idle":"2024-10-08T21:45:39.522244Z","shell.execute_reply.started":"2024-10-08T21:45:39.514768Z","shell.execute_reply":"2024-10-08T21:45:39.521128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vec.get_feature_names_out()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:40.435976Z","iopub.execute_input":"2024-10-08T21:45:40.436712Z","iopub.status.idle":"2024-10-08T21:45:40.442943Z","shell.execute_reply.started":"2024-10-08T21:45:40.436671Z","shell.execute_reply":"2024-10-08T21:45:40.441956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformation","metadata":{}},{"cell_type":"code","source":"vectors= small_vec.transform(small_df.question_text)\nvectors.toarray(), vectors.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:41.661824Z","iopub.execute_input":"2024-10-08T21:45:41.662217Z","iopub.status.idle":"2024-10-08T21:45:41.670396Z","shell.execute_reply.started":"2024-10-08T21:45:41.662181Z","shell.execute_reply":"2024-10-08T21:45:41.669357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuring Count Vectorizer Parameters","metadata":{}},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:42.813127Z","iopub.execute_input":"2024-10-08T21:45:42.813530Z","iopub.status.idle":"2024-10-08T21:45:42.818388Z","shell.execute_reply.started":"2024-10-08T21:45:42.813494Z","shell.execute_reply":"2024-10-08T21:45:42.817220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n    return [stemmer.stem(word) for word in word_tokenize(text) if word.lower() not in english_stopwords]","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:43.402535Z","iopub.execute_input":"2024-10-08T21:45:43.403462Z","iopub.status.idle":"2024-10-08T21:45:43.407973Z","shell.execute_reply.started":"2024-10-08T21:45:43.403420Z","shell.execute_reply":"2024-10-08T21:45:43.407049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenize('this just, a texting lot feeling doing of ; punc and often.. stop words')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:44.123065Z","iopub.execute_input":"2024-10-08T21:45:44.123455Z","iopub.status.idle":"2024-10-08T21:45:44.130609Z","shell.execute_reply.started":"2024-10-08T21:45:44.123419Z","shell.execute_reply":"2024-10-08T21:45:44.129506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer(lowercase=True, tokenizer=tokenize, max_features=1000)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:44.630706Z","iopub.execute_input":"2024-10-08T21:45:44.631628Z","iopub.status.idle":"2024-10-08T21:45:44.635926Z","shell.execute_reply.started":"2024-10-08T21:45:44.631584Z","shell.execute_reply":"2024-10-08T21:45:44.634914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:45:45.062772Z","iopub.execute_input":"2024-10-08T21:45:45.063660Z","iopub.status.idle":"2024-10-08T21:46:13.833731Z","shell.execute_reply.started":"2024-10-08T21:45:45.063619Z","shell.execute_reply":"2024-10-08T21:46:13.832769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vectorizer.vocabulary_)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:46:13.835722Z","iopub.execute_input":"2024-10-08T21:46:13.836238Z","iopub.status.idle":"2024-10-08T21:46:13.842691Z","shell.execute_reply.started":"2024-10-08T21:46:13.836194Z","shell.execute_reply":"2024-10-08T21:46:13.841685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[0:100]","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:46:18.326041Z","iopub.execute_input":"2024-10-08T21:46:18.326903Z","iopub.status.idle":"2024-10-08T21:46:18.334152Z","shell.execute_reply.started":"2024-10-08T21:46:18.326866Z","shell.execute_reply":"2024-10-08T21:46:18.333007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninputs=vectorizer.transform(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:46:20.577396Z","iopub.execute_input":"2024-10-08T21:46:20.577788Z","iopub.status.idle":"2024-10-08T21:46:49.673154Z","shell.execute_reply.started":"2024-10-08T21:46:20.577751Z","shell.execute_reply":"2024-10-08T21:46:49.672208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:46:49.674791Z","iopub.execute_input":"2024-10-08T21:46:49.675125Z","iopub.status.idle":"2024-10-08T21:46:49.681307Z","shell.execute_reply.started":"2024-10-08T21:46:49.675090Z","shell.execute_reply":"2024-10-08T21:46:49.680304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.question_text.values[0]","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:46:49.682731Z","iopub.execute_input":"2024-10-08T21:46:49.683060Z","iopub.status.idle":"2024-10-08T21:46:49.692226Z","shell.execute_reply.started":"2024-10-08T21:46:49.683017Z","shell.execute_reply":"2024-10-08T21:46:49.691324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_inputs=vectorizer.transform(test_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:46:49.694142Z","iopub.execute_input":"2024-10-08T21:46:49.694529Z","iopub.status.idle":"2024-10-08T21:48:37.489202Z","shell.execute_reply.started":"2024-10-08T21:46:49.694493Z","shell.execute_reply":"2024-10-08T21:48:37.488219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spliting","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.490526Z","iopub.execute_input":"2024-10-08T21:48:37.490838Z","iopub.status.idle":"2024-10-08T21:48:37.497637Z","shell.execute_reply.started":"2024-10-08T21:48:37.490804Z","shell.execute_reply":"2024-10-08T21:48:37.496723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df, train_target, val_target= train_test_split(inputs, sample_df['target'], test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.498775Z","iopub.execute_input":"2024-10-08T21:48:37.499151Z","iopub.status.idle":"2024-10-08T21:48:37.519348Z","shell.execute_reply.started":"2024-10-08T21:48:37.499110Z","shell.execute_reply":"2024-10-08T21:48:37.518504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape, train_target.shape, val_df.shape, val_target.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.520538Z","iopub.execute_input":"2024-10-08T21:48:37.520854Z","iopub.status.idle":"2024-10-08T21:48:37.527442Z","shell.execute_reply.started":"2024-10-08T21:48:37.520822Z","shell.execute_reply":"2024-10-08T21:48:37.526442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.529232Z","iopub.execute_input":"2024-10-08T21:48:37.529689Z","iopub.status.idle":"2024-10-08T21:48:37.534486Z","shell.execute_reply.started":"2024-10-08T21:48:37.529643Z","shell.execute_reply":"2024-10-08T21:48:37.533535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_itterations=1000","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.535547Z","iopub.execute_input":"2024-10-08T21:48:37.535846Z","iopub.status.idle":"2024-10-08T21:48:37.542899Z","shell.execute_reply.started":"2024-10-08T21:48:37.535814Z","shell.execute_reply":"2024-10-08T21:48:37.542041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=LogisticRegression(max_iter=max_itterations, solver='sag')","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.546177Z","iopub.execute_input":"2024-10-08T21:48:37.546703Z","iopub.status.idle":"2024-10-08T21:48:37.551070Z","shell.execute_reply.started":"2024-10-08T21:48:37.546668Z","shell.execute_reply":"2024-10-08T21:48:37.550187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.fit(train_df, train_target)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:37.552179Z","iopub.execute_input":"2024-10-08T21:48:37.552478Z","iopub.status.idle":"2024-10-08T21:48:43.970896Z","shell.execute_reply.started":"2024-10-08T21:48:37.552447Z","shell.execute_reply":"2024-10-08T21:48:43.969888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Predictions Using Model","metadata":{}},{"cell_type":"code","source":"train_pred= model.predict(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:43.972437Z","iopub.execute_input":"2024-10-08T21:48:43.972781Z","iopub.status.idle":"2024-10-08T21:48:43.980181Z","shell.execute_reply.started":"2024-10-08T21:48:43.972745Z","shell.execute_reply":"2024-10-08T21:48:43.979283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_pred).value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:43.981325Z","iopub.execute_input":"2024-10-08T21:48:43.981615Z","iopub.status.idle":"2024-10-08T21:48:43.990465Z","shell.execute_reply.started":"2024-10-08T21:48:43.981577Z","shell.execute_reply":"2024-10-08T21:48:43.989585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_target).value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:43.991580Z","iopub.execute_input":"2024-10-08T21:48:43.991896Z","iopub.status.idle":"2024-10-08T21:48:44.001468Z","shell.execute_reply.started":"2024-10-08T21:48:43.991864Z","shell.execute_reply":"2024-10-08T21:48:44.000491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, f1_score","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.002566Z","iopub.execute_input":"2024-10-08T21:48:44.002867Z","iopub.status.idle":"2024-10-08T21:48:44.011776Z","shell.execute_reply.started":"2024-10-08T21:48:44.002836Z","shell.execute_reply":"2024-10-08T21:48:44.010872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(train_target,train_pred)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.013138Z","iopub.execute_input":"2024-10-08T21:48:44.013439Z","iopub.status.idle":"2024-10-08T21:48:44.028274Z","shell.execute_reply.started":"2024-10-08T21:48:44.013408Z","shell.execute_reply":"2024-10-08T21:48:44.027261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_pred= np.random.choice((0,1), len(train_target))\nf1_score(train_target,train_pred), f1_score(train_target,random_pred)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.029517Z","iopub.execute_input":"2024-10-08T21:48:44.030151Z","iopub.status.idle":"2024-10-08T21:48:44.096012Z","shell.execute_reply.started":"2024-10-08T21:48:44.030105Z","shell.execute_reply":"2024-10-08T21:48:44.095051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred= model.predict(val_df)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.097123Z","iopub.execute_input":"2024-10-08T21:48:44.097434Z","iopub.status.idle":"2024-10-08T21:48:44.102434Z","shell.execute_reply.started":"2024-10-08T21:48:44.097402Z","shell.execute_reply":"2024-10-08T21:48:44.101431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(val_target,val_pred)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.103670Z","iopub.execute_input":"2024-10-08T21:48:44.104010Z","iopub.status.idle":"2024-10-08T21:48:44.114326Z","shell.execute_reply.started":"2024-10-08T21:48:44.103963Z","shell.execute_reply":"2024-10-08T21:48:44.113359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(val_target,val_pred)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.115754Z","iopub.execute_input":"2024-10-08T21:48:44.116821Z","iopub.status.idle":"2024-10-08T21:48:44.130605Z","shell.execute_reply.started":"2024-10-08T21:48:44.116786Z","shell.execute_reply":"2024-10-08T21:48:44.129749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(vectorizer.transform(sincere_df.question_text.values[:10]))","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.132609Z","iopub.execute_input":"2024-10-08T21:48:44.132903Z","iopub.status.idle":"2024-10-08T21:48:44.143783Z","shell.execute_reply.started":"2024-10-08T21:48:44.132871Z","shell.execute_reply":"2024-10-08T21:48:44.142624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(vectorizer.transform(insincere_df.question_text.values[:10]))","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.145190Z","iopub.execute_input":"2024-10-08T21:48:44.146174Z","iopub.status.idle":"2024-10-08T21:48:44.157660Z","shell.execute_reply.started":"2024-10-08T21:48:44.146090Z","shell.execute_reply":"2024-10-08T21:48:44.156677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting on Kaggle Test Set","metadata":{}},{"cell_type":"code","source":"test_pred = model.predict(test_inputs)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.158964Z","iopub.execute_input":"2024-10-08T21:48:44.159585Z","iopub.status.idle":"2024-10-08T21:48:44.180621Z","shell.execute_reply.started":"2024-10-08T21:48:44.159541Z","shell.execute_reply":"2024-10-08T21:48:44.179750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df= pd.read_csv(sub_fname)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.181986Z","iopub.execute_input":"2024-10-08T21:48:44.182514Z","iopub.status.idle":"2024-10-08T21:48:44.515596Z","shell.execute_reply.started":"2024-10-08T21:48:44.182467Z","shell.execute_reply":"2024-10-08T21:48:44.514716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.516740Z","iopub.execute_input":"2024-10-08T21:48:44.517064Z","iopub.status.idle":"2024-10-08T21:48:44.529333Z","shell.execute_reply.started":"2024-10-08T21:48:44.517024Z","shell.execute_reply":"2024-10-08T21:48:44.528287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction=test_pred","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.530815Z","iopub.execute_input":"2024-10-08T21:48:44.531516Z","iopub.status.idle":"2024-10-08T21:48:44.537493Z","shell.execute_reply.started":"2024-10-08T21:48:44.531479Z","shell.execute_reply":"2024-10-08T21:48:44.536584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.538554Z","iopub.execute_input":"2024-10-08T21:48:44.538832Z","iopub.status.idle":"2024-10-08T21:48:44.551396Z","shell.execute_reply.started":"2024-10-08T21:48:44.538801Z","shell.execute_reply":"2024-10-08T21:48:44.550507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T21:48:44.556131Z","iopub.execute_input":"2024-10-08T21:48:44.556456Z","iopub.status.idle":"2024-10-08T21:48:44.967987Z","shell.execute_reply.started":"2024-10-08T21:48:44.556423Z","shell.execute_reply":"2024-10-08T21:48:44.967173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}