{"metadata":{"colab":{"provenance":[],"gpuType":"T4"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU","kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Text Classification using Bag of words","metadata":{"id":"7wfHRh4Zzd52"}},{"cell_type":"markdown","source":"Outline:\n\n   Download and explore data\n\n  Apply text preprocessing techniques\n\n  Implement the bag of words model\n\n  Train ML models for text classification\n\n  Make Predicitions\n","metadata":{"id":"kzMs8rGFzoJF"}},{"cell_type":"markdown","source":"### Download and explore data","metadata":{"id":"Ja_g2jHPz7sK"}},{"cell_type":"code","source":"!ls","metadata":{"id":"I3LfGsl51AbI","outputId":"7325c070-9ec1-428d-cf87-1719c4cac076"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n","metadata":{"id":"DApDDdkrM0X2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.environ['KAGGLE_CONFIG_DIR'] = '.'","metadata":{"id":"kUlsVeAeM0Np"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data","metadata":{"id":"5B4dq8w91AYv","outputId":"cd91f821-ee60-4a08-8794-9d9bd0103373"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data\n# !kaggle competitions download -c quora-insincere-questions-classification -f sample_submission.csv -p data","metadata":{"id":"zX8wGzN01AXP","outputId":"76a85008-b2e6-4db5-e58b-90847030c037"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Explore the data","metadata":{"id":"xqKvLT6MOjQm"}},{"cell_type":"code","source":"# train_fname = 'data/train.csv.zip'\n# test_fname = 'data/test.csv.zip'\n# sample_fname = 'data/sample_submission.csv.zip'","metadata":{"id":"9_iBxVUU1ATS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/input/quora-insincere-questions-classification'\ntrain_fname = data_dir + '/train.csv'\ntest_fname = data_dir + '/test.csv'\nsample_fname = data_dir + '/sample_submission.csv'","metadata":{"execution":{"iopub.status.busy":"2024-05-11T23:13:52.538706Z","iopub.execute_input":"2024-05-11T23:13:52.539220Z","iopub.status.idle":"2024-05-11T23:13:52.544991Z","shell.execute_reply.started":"2024-05-11T23:13:52.539159Z","shell.execute_reply":"2024-05-11T23:13:52.543650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"id":"OU224tmFRkdz","execution":{"iopub.status.busy":"2024-05-11T23:13:56.584442Z","iopub.execute_input":"2024-05-11T23:13:56.584877Z","iopub.status.idle":"2024-05-11T23:13:56.590505Z","shell.execute_reply.started":"2024-05-11T23:13:56.584845Z","shell.execute_reply":"2024-05-11T23:13:56.588981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df = pd.read_csv(train_fname)","metadata":{"id":"KmlDd13GRcqH","execution":{"iopub.status.busy":"2024-05-11T23:13:57.103835Z","iopub.execute_input":"2024-05-11T23:13:57.104280Z","iopub.status.idle":"2024-05-11T23:14:02.480332Z","shell.execute_reply.started":"2024-05-11T23:13:57.104245Z","shell.execute_reply":"2024-05-11T23:14:02.478768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df","metadata":{"id":"IQRLXYI-Rqxt","outputId":"1a4a4cb0-dcf5-45a5-859b-2e55b4b78203","execution":{"iopub.status.busy":"2024-05-11T23:14:04.177853Z","iopub.execute_input":"2024-05-11T23:14:04.178301Z","iopub.status.idle":"2024-05-11T23:14:04.208845Z","shell.execute_reply.started":"2024-05-11T23:14:04.178268Z","shell.execute_reply":"2024-05-11T23:14:04.207475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts()","metadata":{"id":"kZ1mY4DmSHYd","outputId":"69a15915-b71b-422e-8a5a-32acff0fb055","execution":{"iopub.status.busy":"2024-05-11T23:14:19.191192Z","iopub.execute_input":"2024-05-11T23:14:19.191962Z","iopub.status.idle":"2024-05-11T23:14:19.227685Z","shell.execute_reply.started":"2024-05-11T23:14:19.191923Z","shell.execute_reply":"2024-05-11T23:14:19.225990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True)","metadata":{"id":"nsYpF46GTGd7","outputId":"c04c3617-cf4a-42d3-dd67-32a3141515e2","execution":{"iopub.status.busy":"2024-05-11T23:14:19.517325Z","iopub.execute_input":"2024-05-11T23:14:19.518149Z","iopub.status.idle":"2024-05-11T23:14:19.543391Z","shell.execute_reply.started":"2024-05-11T23:14:19.518095Z","shell.execute_reply":"2024-05-11T23:14:19.541830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df = raw_df[raw_df.target == 0]","metadata":{"id":"K7dla-ClR_2J","execution":{"iopub.status.busy":"2024-05-11T23:14:19.699686Z","iopub.execute_input":"2024-05-11T23:14:19.700401Z","iopub.status.idle":"2024-05-11T23:14:19.804541Z","shell.execute_reply.started":"2024-05-11T23:14:19.700351Z","shell.execute_reply":"2024-05-11T23:14:19.803571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"id":"ehXpY5IsSOxJ","outputId":"4040cb14-5df8-44a8-bd86-7a32e00ae759","execution":{"iopub.status.busy":"2024-05-11T23:14:19.864041Z","iopub.execute_input":"2024-05-11T23:14:19.864781Z","iopub.status.idle":"2024-05-11T23:14:19.873697Z","shell.execute_reply.started":"2024-05-11T23:14:19.864734Z","shell.execute_reply":"2024-05-11T23:14:19.872476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target == 1]","metadata":{"id":"FyADIbxbSS7w","execution":{"iopub.status.busy":"2024-05-11T23:14:19.999706Z","iopub.execute_input":"2024-05-11T23:14:20.000440Z","iopub.status.idle":"2024-05-11T23:14:20.025114Z","shell.execute_reply.started":"2024-05-11T23:14:20.000394Z","shell.execute_reply":"2024-05-11T23:14:20.023667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"id":"mFfO3ArqSS3V","outputId":"1cbd85f6-c08d-42e3-da86-9b423fe17e5a","execution":{"iopub.status.busy":"2024-05-11T23:14:20.157536Z","iopub.execute_input":"2024-05-11T23:14:20.157939Z","iopub.status.idle":"2024-05-11T23:14:20.165474Z","shell.execute_reply.started":"2024-05-11T23:14:20.157908Z","shell.execute_reply":"2024-05-11T23:14:20.164368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True).plot(kind = 'bar')","metadata":{"id":"rS13SRjgTY0v","outputId":"c7a78a50-6522-4021-de5d-a59b7ef8ba28","execution":{"iopub.status.busy":"2024-05-11T23:14:20.316343Z","iopub.execute_input":"2024-05-11T23:14:20.317585Z","iopub.status.idle":"2024-05-11T23:14:20.618243Z","shell.execute_reply.started":"2024-05-11T23:14:20.317534Z","shell.execute_reply":"2024-05-11T23:14:20.616407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(test_fname)","metadata":{"id":"7jscza0uTnzO","execution":{"iopub.status.busy":"2024-05-11T23:14:20.746096Z","iopub.execute_input":"2024-05-11T23:14:20.747135Z","iopub.status.idle":"2024-05-11T23:14:22.173623Z","shell.execute_reply.started":"2024-05-11T23:14:20.747092Z","shell.execute_reply":"2024-05-11T23:14:22.172305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(sample_fname)\nsub_df","metadata":{"id":"vTumCpMVTuA4","outputId":"6a60ae02-bf7a-43eb-c1ab-6407d424ddbc","execution":{"iopub.status.busy":"2024-05-11T23:14:22.176434Z","iopub.execute_input":"2024-05-11T23:14:22.177264Z","iopub.status.idle":"2024-05-11T23:14:22.663107Z","shell.execute_reply.started":"2024-05-11T23:14:22.177213Z","shell.execute_reply":"2024-05-11T23:14:22.662202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Text Preprocessing techniques","metadata":{"id":"PEPI01Y30PsF"}},{"cell_type":"markdown","source":"#### Tokenization","metadata":{"id":"Lm27Ky8rgDq6"}},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')","metadata":{"id":"mBp5nObbg72Z","outputId":"b727973d-cc96-4f24-900e-1dd83c6cb731","execution":{"iopub.status.busy":"2024-05-11T23:14:22.664456Z","iopub.execute_input":"2024-05-11T23:14:22.664951Z","iopub.status.idle":"2024-05-11T23:14:24.287019Z","shell.execute_reply.started":"2024-05-11T23:14:22.664922Z","shell.execute_reply":"2024-05-11T23:14:24.285608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize","metadata":{"id":"ujJUUSi80_z7","execution":{"iopub.status.busy":"2024-05-11T23:14:24.288764Z","iopub.execute_input":"2024-05-11T23:14:24.289332Z","iopub.status.idle":"2024-05-11T23:14:24.295710Z","shell.execute_reply.started":"2024-05-11T23:14:24.289297Z","shell.execute_reply":"2024-05-11T23:14:24.294141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0 = sincere_df.question_text.values[0]","metadata":{"id":"7_U_JsVu0_xd","execution":{"iopub.status.busy":"2024-05-11T23:14:24.300512Z","iopub.execute_input":"2024-05-11T23:14:24.301033Z","iopub.status.idle":"2024-05-11T23:14:24.307978Z","shell.execute_reply.started":"2024-05-11T23:14:24.300963Z","shell.execute_reply":"2024-05-11T23:14:24.306949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0","metadata":{"id":"UJ_S33Iz0_vl","outputId":"ee6d22b9-7a53-4c86-d375-48e4e6f7af21","execution":{"iopub.status.busy":"2024-05-11T23:14:24.310050Z","iopub.execute_input":"2024-05-11T23:14:24.310747Z","iopub.status.idle":"2024-05-11T23:14:24.323462Z","shell.execute_reply.started":"2024-05-11T23:14:24.310711Z","shell.execute_reply":"2024-05-11T23:14:24.322122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok = word_tokenize(q0)","metadata":{"id":"1gGbaOex0_sd","execution":{"iopub.status.busy":"2024-05-11T23:14:24.324684Z","iopub.execute_input":"2024-05-11T23:14:24.325036Z","iopub.status.idle":"2024-05-11T23:14:24.348447Z","shell.execute_reply.started":"2024-05-11T23:14:24.325005Z","shell.execute_reply":"2024-05-11T23:14:24.347027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok","metadata":{"id":"QRN8Jsbenj0S","outputId":"d78b4dbb-516a-42d0-c40c-0fa6996657e2","execution":{"iopub.status.busy":"2024-05-11T23:14:24.351805Z","iopub.execute_input":"2024-05-11T23:14:24.352302Z","iopub.status.idle":"2024-05-11T23:14:24.361767Z","shell.execute_reply.started":"2024-05-11T23:14:24.352267Z","shell.execute_reply":"2024-05-11T23:14:24.359909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Stopwords","metadata":{"id":"y_oI5TEbn5Gp"}},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"id":"QPS2350QmtyZ","execution":{"iopub.status.busy":"2024-05-11T23:14:24.363612Z","iopub.execute_input":"2024-05-11T23:14:24.363950Z","iopub.status.idle":"2024-05-11T23:14:24.371069Z","shell.execute_reply.started":"2024-05-11T23:14:24.363922Z","shell.execute_reply":"2024-05-11T23:14:24.369592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('stopwords')","metadata":{"id":"MY8iBNXsmtuI","outputId":"d82c7658-b4a9-451f-d8cb-b268021a3730","execution":{"iopub.status.busy":"2024-05-11T23:14:24.372750Z","iopub.execute_input":"2024-05-11T23:14:24.373148Z","iopub.status.idle":"2024-05-11T23:14:24.390686Z","shell.execute_reply.started":"2024-05-11T23:14:24.373117Z","shell.execute_reply":"2024-05-11T23:14:24.389550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_words = stopwords.words('english')","metadata":{"id":"FWNeq28UmtrD","execution":{"iopub.status.busy":"2024-05-11T23:14:24.392514Z","iopub.execute_input":"2024-05-11T23:14:24.392937Z","iopub.status.idle":"2024-05-11T23:14:24.404309Z","shell.execute_reply.started":"2024-05-11T23:14:24.392906Z","shell.execute_reply":"2024-05-11T23:14:24.403372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\", \".join(english_words)","metadata":{"id":"Bjwa-VFxm70A","outputId":"d238ffcf-6223-4a37-a14d-ecae4b728bec","execution":{"iopub.status.busy":"2024-05-11T23:14:24.405583Z","iopub.execute_input":"2024-05-11T23:14:24.406159Z","iopub.status.idle":"2024-05-11T23:14:24.415006Z","shell.execute_reply.started":"2024-05-11T23:14:24.406127Z","shell.execute_reply":"2024-05-11T23:14:24.414244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(tokens):\n  return [word for word in tokens if word.lower() not in english_words]\n","metadata":{"id":"C5Xd6GASnOCi","execution":{"iopub.status.busy":"2024-05-11T23:14:24.416091Z","iopub.execute_input":"2024-05-11T23:14:24.417529Z","iopub.status.idle":"2024-05-11T23:14:24.425403Z","shell.execute_reply.started":"2024-05-11T23:14:24.417495Z","shell.execute_reply":"2024-05-11T23:14:24.424242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok","metadata":{"id":"uOa2UAnSns0n","outputId":"abdb7729-440a-4ac3-f820-b4f4c7e61008","execution":{"iopub.status.busy":"2024-05-11T23:14:24.429138Z","iopub.execute_input":"2024-05-11T23:14:24.429653Z","iopub.status.idle":"2024-05-11T23:14:24.440224Z","shell.execute_reply.started":"2024-05-11T23:14:24.429613Z","shell.execute_reply":"2024-05-11T23:14:24.438992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp = remove_stopwords(q0_tok)","metadata":{"id":"_QDf_g1e08PH","execution":{"iopub.status.busy":"2024-05-11T23:14:24.441816Z","iopub.execute_input":"2024-05-11T23:14:24.442376Z","iopub.status.idle":"2024-05-11T23:14:24.451443Z","shell.execute_reply.started":"2024-05-11T23:14:24.442341Z","shell.execute_reply":"2024-05-11T23:14:24.450237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp","metadata":{"id":"uPUQxz0B08Mg","outputId":"9fac5cfc-ff35-474a-8bdc-ceeadeaf6600","execution":{"iopub.status.busy":"2024-05-11T23:14:24.452788Z","iopub.execute_input":"2024-05-11T23:14:24.453655Z","iopub.status.idle":"2024-05-11T23:14:24.463158Z","shell.execute_reply.started":"2024-05-11T23:14:24.453613Z","shell.execute_reply":"2024-05-11T23:14:24.462237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Stemming","metadata":{"id":"9tTszDrRn7Wr"}},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer\n\n#the stemmer requires a language parameter\nstemmer = SnowballStemmer(language='english')","metadata":{"id":"PrOi6kVin-bA","execution":{"iopub.status.busy":"2024-05-11T23:14:24.464561Z","iopub.execute_input":"2024-05-11T23:14:24.465654Z","iopub.status.idle":"2024-05-11T23:14:24.472738Z","shell.execute_reply.started":"2024-05-11T23:14:24.465618Z","shell.execute_reply":"2024-05-11T23:14:24.471795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stm = [stemmer.stem(word) for word in q0_stp]","metadata":{"id":"dRjRPVKen-WF","execution":{"iopub.status.busy":"2024-05-11T23:14:24.474239Z","iopub.execute_input":"2024-05-11T23:14:24.474614Z","iopub.status.idle":"2024-05-11T23:14:24.484711Z","shell.execute_reply.started":"2024-05-11T23:14:24.474585Z","shell.execute_reply":"2024-05-11T23:14:24.483814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stm","metadata":{"id":"YSSJkp5nn-RY","outputId":"b20681a9-a135-4653-9194-19337dd11baa","execution":{"iopub.status.busy":"2024-05-11T23:14:24.486157Z","iopub.execute_input":"2024-05-11T23:14:24.487118Z","iopub.status.idle":"2024-05-11T23:14:24.497033Z","shell.execute_reply.started":"2024-05-11T23:14:24.487087Z","shell.execute_reply":"2024-05-11T23:14:24.496225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Implement Bag of Words\n\n1. Create a vocabulary using Count Vectorizer\n2. Transform text to vectors using Count Vectorizer\n3. Configure text preprocessing in Count Vectorizer\n\n","metadata":{"id":"1WcOoRNF0P9o"}},{"cell_type":"code","source":"small_df = raw_df[:5]","metadata":{"id":"TdkX2nTY08J5","execution":{"iopub.status.busy":"2024-05-11T23:14:24.628804Z","iopub.execute_input":"2024-05-11T23:14:24.629245Z","iopub.status.idle":"2024-05-11T23:14:24.634945Z","shell.execute_reply.started":"2024-05-11T23:14:24.629212Z","shell.execute_reply":"2024-05-11T23:14:24.633964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"id":"WBXryUFW08HC","execution":{"iopub.status.busy":"2024-05-11T23:14:24.789100Z","iopub.execute_input":"2024-05-11T23:14:24.789554Z","iopub.status.idle":"2024-05-11T23:14:24.795439Z","shell.execute_reply.started":"2024-05-11T23:14:24.789522Z","shell.execute_reply":"2024-05-11T23:14:24.793973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer()","metadata":{"id":"h9S4gMQo08E7","execution":{"iopub.status.busy":"2024-05-11T23:14:24.973183Z","iopub.execute_input":"2024-05-11T23:14:24.973670Z","iopub.status.idle":"2024-05-11T23:14:24.980837Z","shell.execute_reply.started":"2024-05-11T23:14:24.973632Z","shell.execute_reply":"2024-05-11T23:14:24.979588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.fit(small_df.question_text)","metadata":{"id":"O2y_xwx208CJ","outputId":"2287271f-74b0-4224-e4f1-eae10d8399d9","execution":{"iopub.status.busy":"2024-05-11T23:14:25.132230Z","iopub.execute_input":"2024-05-11T23:14:25.132625Z","iopub.status.idle":"2024-05-11T23:14:25.154629Z","shell.execute_reply.started":"2024-05-11T23:14:25.132595Z","shell.execute_reply":"2024-05-11T23:14:25.152766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()","metadata":{"id":"RiH4OeUwtvjQ","outputId":"b93101c6-7dd0-44f9-c19f-e213378c2974","execution":{"iopub.status.busy":"2024-05-11T23:14:25.519410Z","iopub.execute_input":"2024-05-11T23:14:25.519857Z","iopub.status.idle":"2024-05-11T23:14:25.528500Z","shell.execute_reply.started":"2024-05-11T23:14:25.519818Z","shell.execute_reply":"2024-05-11T23:14:25.526946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Transform documents to vector","metadata":{"id":"W2-l6p1it08k"}},{"cell_type":"code","source":"vectors = vectorizer.transform(small_df.question_text)","metadata":{"id":"3vv_tCuRtzzY","execution":{"iopub.status.busy":"2024-05-11T23:14:26.053553Z","iopub.execute_input":"2024-05-11T23:14:26.053993Z","iopub.status.idle":"2024-05-11T23:14:26.059681Z","shell.execute_reply.started":"2024-05-11T23:14:26.053961Z","shell.execute_reply":"2024-05-11T23:14:26.058693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors","metadata":{"id":"HeVPUZu3uAFQ","outputId":"ad1cd382-94b4-472d-c7f3-a86597c374b8","execution":{"iopub.status.busy":"2024-05-11T23:14:26.259619Z","iopub.execute_input":"2024-05-11T23:14:26.260009Z","iopub.status.idle":"2024-05-11T23:14:26.269175Z","shell.execute_reply.started":"2024-05-11T23:14:26.259979Z","shell.execute_reply":"2024-05-11T23:14:26.267645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors.toarray()","metadata":{"id":"K0yx8OvjuB_J","outputId":"88b34d1e-02e9-4c5e-ec32-a997cabac332","execution":{"iopub.status.busy":"2024-05-11T23:14:26.664462Z","iopub.execute_input":"2024-05-11T23:14:26.665516Z","iopub.status.idle":"2024-05-11T23:14:26.674251Z","shell.execute_reply.started":"2024-05-11T23:14:26.665462Z","shell.execute_reply":"2024-05-11T23:14:26.673081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors.shape","metadata":{"id":"t4K9ZERKuFxE","outputId":"313a635f-45c4-42ab-f6d6-0c743980bafc","execution":{"iopub.status.busy":"2024-05-11T23:14:27.089879Z","iopub.execute_input":"2024-05-11T23:14:27.090337Z","iopub.status.idle":"2024-05-11T23:14:27.097340Z","shell.execute_reply.started":"2024-05-11T23:14:27.090304Z","shell.execute_reply":"2024-05-11T23:14:27.096152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Configure Count Vectorizer Parameters","metadata":{"id":"p_f5dFXExIl8"}},{"cell_type":"code","source":"def tokenize(text):\n    return [stemmer.stem(word) for word in word_tokenize(text)]","metadata":{"id":"ZpHqCRWexE9H","execution":{"iopub.status.busy":"2024-05-11T23:14:27.833151Z","iopub.execute_input":"2024-05-11T23:14:27.833648Z","iopub.status.idle":"2024-05-11T23:14:27.839102Z","shell.execute_reply.started":"2024-05-11T23:14:27.833612Z","shell.execute_reply":"2024-05-11T23:14:27.838029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer(lowercase=True,\n                             tokenizer=tokenize,\n                             stop_words=english_words,\n                             max_features=1000) #vocabulary will have top 1000 words in vocabuary","metadata":{"id":"ft7CE5CIxPh6","execution":{"iopub.status.busy":"2024-05-11T23:14:27.995696Z","iopub.execute_input":"2024-05-11T23:14:27.996189Z","iopub.status.idle":"2024-05-11T23:14:28.002865Z","shell.execute_reply.started":"2024-05-11T23:14:27.996137Z","shell.execute_reply":"2024-05-11T23:14:28.001296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = raw_df.sample(100000, random_state=42)","metadata":{"id":"erLXh8CqxYeU","execution":{"iopub.status.busy":"2024-05-11T23:14:28.398027Z","iopub.execute_input":"2024-05-11T23:14:28.398450Z","iopub.status.idle":"2024-05-11T23:14:28.503667Z","shell.execute_reply.started":"2024-05-11T23:14:28.398420Z","shell.execute_reply":"2024-05-11T23:14:28.502725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.fit(sample_df.question_text) #creates vocabulary","metadata":{"id":"BzI4-Qn_0MNy","outputId":"5fdbfa1c-bacb-4802-b8a3-4b6662e2e7bf","execution":{"iopub.status.busy":"2024-05-11T23:14:28.772400Z","iopub.execute_input":"2024-05-11T23:14:28.773233Z","iopub.status.idle":"2024-05-11T23:15:18.118415Z","shell.execute_reply.started":"2024-05-11T23:14:28.773157Z","shell.execute_reply":"2024-05-11T23:15:18.116870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vectorizer.vocabulary_) #len of vocabulary","metadata":{"id":"jmCJ-CWhx2GM","outputId":"d8162aa2-da85-4230-a109-26a7863af392","execution":{"iopub.status.busy":"2024-05-11T23:15:18.121062Z","iopub.execute_input":"2024-05-11T23:15:18.121818Z","iopub.status.idle":"2024-05-11T23:15:18.130151Z","shell.execute_reply.started":"2024-05-11T23:15:18.121776Z","shell.execute_reply":"2024-05-11T23:15:18.128903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100] # first 100 words in vocabulary","metadata":{"id":"Hk7OSuvT0d7X","outputId":"a5dc8f10-992e-4691-9dec-0fd4d9e50845","execution":{"iopub.status.busy":"2024-05-11T23:15:18.132326Z","iopub.execute_input":"2024-05-11T23:15:18.132756Z","iopub.status.idle":"2024-05-11T23:15:18.145521Z","shell.execute_reply.started":"2024-05-11T23:15:18.132720Z","shell.execute_reply":"2024-05-11T23:15:18.144236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = vectorizer.transform(sample_df.question_text) #convert words to vectors","metadata":{"id":"4pgZ9FwW0siJ","execution":{"iopub.status.busy":"2024-05-11T23:15:18.147931Z","iopub.execute_input":"2024-05-11T23:15:18.148428Z","iopub.status.idle":"2024-05-11T23:16:06.997126Z","shell.execute_reply.started":"2024-05-11T23:15:18.148400Z","shell.execute_reply":"2024-05-11T23:16:06.995397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"id":"zcrylI3_01N-","outputId":"324d8c1b-9b00-4f62-84ff-77e8dafe03dc","execution":{"iopub.status.busy":"2024-05-11T23:16:06.998849Z","iopub.execute_input":"2024-05-11T23:16:06.999286Z","iopub.status.idle":"2024-05-11T23:16:07.006796Z","shell.execute_reply.started":"2024-05-11T23:16:06.999253Z","shell.execute_reply":"2024-05-11T23:16:07.005661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"id":"XrCL6hiv1EvM","outputId":"6d45d95c-921c-460d-8aba-0a3b1bb237fb","execution":{"iopub.status.busy":"2024-05-11T23:16:07.008607Z","iopub.execute_input":"2024-05-11T23:16:07.008996Z","iopub.status.idle":"2024-05-11T23:16:07.022542Z","shell.execute_reply.started":"2024-05-11T23:16:07.008961Z","shell.execute_reply":"2024-05-11T23:16:07.021606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"id":"J6yfGmPY0_lQ","outputId":"03aa5dc9-a86e-4079-b1ae-02e18d871b29","execution":{"iopub.status.busy":"2024-05-11T23:16:07.024089Z","iopub.execute_input":"2024-05-11T23:16:07.025094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ML models and text classification","metadata":{"id":"Uph3IkiE1JWN"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"id":"3kIGxYoX0-5C","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, sample_df.target,\n                                                                        test_size=0.3, random_state=42)\n","metadata":{"id":"Yd4oRw3_0-1H","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs.shape","metadata":{"id":"YZhxSzD80-yo","outputId":"176fadd1-bde0-4861-b84b-d4cff64751bd","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_inputs.shape","metadata":{"id":"hapO3qAn0-wI","outputId":"b961a4af-d126-4b0a-8833-30c406f15cf4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train Logistic Regression model","metadata":{"id":"rZzlbuW01mxp"}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"id":"puaPdRjz0-sa","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_ITER = 1000","metadata":{"id":"o-mZZJxF1rA-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LogisticRegression(max_iter=MAX_ITER, solver='sag')","metadata":{"id":"u27XKdqn1tm9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.fit(train_inputs, train_targets)","metadata":{"id":"xoV67beo1tkD","outputId":"eac165ea-c2b0-40e5-c2e2-7a9ec207efbe","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predictions","metadata":{"id":"aJzjo1fJ0QTo"}},{"cell_type":"code","source":"train_preds = model.predict(train_inputs)","metadata":{"id":"l9AJJjbH17Ym","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets","metadata":{"id":"E7omuiQC05Vs","outputId":"a96ec534-a7a0-4f74-d0cb-f4b6ce9913af","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds","metadata":{"id":"SqrKKQ9g05Rx","outputId":"bdb716c8-017a-47e6-9b6e-ad60400a00da","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_preds).value_counts()","metadata":{"id":"SGsf5hU005Or","outputId":"fd90f456-47cc-4c87-a82f-f966eb3cb973","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_targets).value_counts()\n","metadata":{"id":"ppTTIBAX05Kv","outputId":"5ca82b2c-4ebf-479b-aae8-e6e0118c6a52","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"id":"6xUH3T4D05II","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(train_targets, train_preds)","metadata":{"id":"twK8Endt2rjr","outputId":"0c8029e7-212d-4235-a35c-9a0dba3e8d4b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{"id":"MGLzI4ur2tlO","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(train_targets, train_preds)","metadata":{"id":"Hnhz0f092xoZ","outputId":"ee1cf54b-4b15-40a2-80da-98cd8f609b93","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = model.predict(val_inputs)","metadata":{"id":"tDrsF9iF20M6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(val_targets, val_preds)","metadata":{"id":"UKPfA-mb26dD","outputId":"33b2d882-1251-4f12-d847-936fb3dc9c4c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(val_targets, val_preds)","metadata":{"id":"i2uL6Pr028QN","outputId":"575dad54-e285-453b-ae8f-cd548515ab4d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = model.predict(test_inputs)","metadata":{"id":"UrrxSqEY3F6H","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"id":"TNVjW_Yz3OXH","outputId":"dca0ae95-2879-4904-b287-a0c94c805c63","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"id":"sOAH94jI3F3-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"id":"3pp1MHJM3F17","outputId":"56a61fcb-9169-42b6-a7b4-fbb3f7bed943","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"id":"dAlRbSZx3Fyb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"id":"d2ShgYSz3Fvw","outputId":"7b28e59c-1b12-4bdf-e169-addda1bb3656"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"lZnBw5CI3VrX"},"execution_count":null,"outputs":[]}]}