{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text Classification with Bag of Words\n\nOutline:\n\n- Download and explore the data\n- Apply text preprocessing techniques\n- Implement the bag of words model\n- Train ML models for text classification\n- Make predictions and submit to Kaggle","metadata":{"id":"8ch5A_OOGko-"}},{"cell_type":"markdown","source":"## Download and Explore the Data\n\nOutline:\n1. Download the dataset from Kaggle to Colab\n2. Explore the data using Pandas\n3. Create a small working sample\n","metadata":{"id":"SD_uNyHRHN6e"}},{"cell_type":"markdown","source":"### Download the Data to Colab","metadata":{"id":"qa-_8C4aJu6i"}},{"cell_type":"code","source":"!ls","metadata":{"id":"kjxLzC7AHr7N","outputId":"528c1bb6-faef-40b7-f7d9-8c6a2a05d657","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:37.536956Z","iopub.execute_input":"2025-06-20T08:40:37.537268Z","iopub.status.idle":"2025-06-20T08:40:37.666999Z","shell.execute_reply.started":"2025-06-20T08:40:37.537243Z","shell.execute_reply":"2025-06-20T08:40:37.665934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!mkdir -p ~/.kaggle\n!cp kaggle.json ~/.kaggle/\n!chmod 600 ~/.kaggle/kaggle.json\n","metadata":{"id":"52CjtauSOWOr","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:37.668824Z","iopub.execute_input":"2025-06-20T08:40:37.669093Z","iopub.status.idle":"2025-06-20T08:40:38.032051Z","shell.execute_reply.started":"2025-06-20T08:40:37.669069Z","shell.execute_reply":"2025-06-20T08:40:38.031100Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport os\n\nos.listdir('/kaggle/input/')\n","metadata":{"id":"Lns0KO6QLL4V","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.033136Z","iopub.execute_input":"2025-06-20T08:40:38.033390Z","iopub.status.idle":"2025-06-20T08:40:38.041757Z","shell.execute_reply.started":"2025-06-20T08:40:38.033363Z","shell.execute_reply":"2025-06-20T08:40:38.040682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.listdir('/kaggle/input/quora-insincere-questions-classification/')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.043387Z","iopub.execute_input":"2025-06-20T08:40:38.043709Z","iopub.status.idle":"2025-06-20T08:40:38.063122Z","shell.execute_reply.started":"2025-06-20T08:40:38.043686Z","shell.execute_reply":"2025-06-20T08:40:38.062087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# os.environ['KAGGLE_CONFIG_DIR'] = ' .'","metadata":{"id":"YM-yFAOjK-b8","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.064123Z","iopub.execute_input":"2025-06-20T08:40:38.064455Z","iopub.status.idle":"2025-06-20T08:40:38.075196Z","shell.execute_reply.started":"2025-06-20T08:40:38.064423Z","shell.execute_reply":"2025-06-20T08:40:38.074171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip install -q kaggle\n","metadata":{"id":"OVRKXPUrOjrp","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.076106Z","iopub.execute_input":"2025-06-20T08:40:38.076372Z","iopub.status.idle":"2025-06-20T08:40:38.092273Z","shell.execute_reply.started":"2025-06-20T08:40:38.076350Z","shell.execute_reply":"2025-06-20T08:40:38.091209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data","metadata":{"id":"gMsUlm4VHrtf","outputId":"0ef8a501-69ad-412e-9629-842dcf68a419","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.093749Z","iopub.execute_input":"2025-06-20T08:40:38.094082Z","iopub.status.idle":"2025-06-20T08:40:38.106366Z","shell.execute_reply.started":"2025-06-20T08:40:38.094051Z","shell.execute_reply":"2025-06-20T08:40:38.105385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data\n# !kaggle competitions download -c quora-insincere-questions-classification -f sample_submission.csv -p data","metadata":{"id":"ZWrqW0yNHrbg","outputId":"080c0186-57bd-4c27-b4df-38e293a5adcf","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.107216Z","iopub.execute_input":"2025-06-20T08:40:38.107458Z","iopub.status.idle":"2025-06-20T08:40:38.116551Z","shell.execute_reply.started":"2025-06-20T08:40:38.107432Z","shell.execute_reply":"2025-06-20T08:40:38.115625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Rename .csv to .zip (only if these are actually ZIP archives)\n# !mv data/train.csv data/train.zip\n# !mv data/test.csv data/test.zip\n# !mv data/sample_submission.csv data/sample_submission.zip\n\n# # Unzip all of them to the same 'data/' folder\n# !unzip data/train.zip -d data/\n# !unzip data/test.zip -d data/\n# !unzip data/sample_submission.zip -d data/\n\n# # Optional: Remove the zip files to avoid confusion\n# !rm data/*.zip\n\n# # Check extracted files\n# !ls data\n","metadata":{"id":"9yUDb1ULqIW7","outputId":"5072f809-ea7b-41e0-bd6f-173b24ce800c","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.117601Z","iopub.execute_input":"2025-06-20T08:40:38.117958Z","iopub.status.idle":"2025-06-20T08:40:38.133965Z","shell.execute_reply.started":"2025-06-20T08:40:38.117928Z","shell.execute_reply":"2025-06-20T08:40:38.133011Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Explore the Data using Pandas","metadata":{"id":"xFIVVj9xPcTY"}},{"cell_type":"code","source":"# train_fname = 'data/train.csv'\n# test_fname = 'data/test.csv'\n# sample_fname = 'data/sample_submission.csv'","metadata":{"id":"YGb7d54FPgis","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.137034Z","iopub.execute_input":"2025-06-20T08:40:38.137345Z","iopub.status.idle":"2025-06-20T08:40:38.149826Z","shell.execute_reply.started":"2025-06-20T08:40:38.137322Z","shell.execute_reply":"2025-06-20T08:40:38.148907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = '/kaggle/input/quora-insincere-questions-classification'\n\ntrain_fname = data_dir + '/train.csv'\ntest_fname = data_dir + '/test.csv'\nsample_fname = data_dir + '/sample_submission.csv'\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.150782Z","iopub.execute_input":"2025-06-20T08:40:38.151097Z","iopub.status.idle":"2025-06-20T08:40:38.166534Z","shell.execute_reply.started":"2025-06-20T08:40:38.151071Z","shell.execute_reply":"2025-06-20T08:40:38.165592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd","metadata":{"id":"bn9knYKgld38","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.167575Z","iopub.execute_input":"2025-06-20T08:40:38.167904Z","iopub.status.idle":"2025-06-20T08:40:38.595667Z","shell.execute_reply.started":"2025-06-20T08:40:38.167878Z","shell.execute_reply":"2025-06-20T08:40:38.595020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df = pd.read_csv(train_fname)","metadata":{"id":"IGgS9uVIlduy","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:38.596581Z","iopub.execute_input":"2025-06-20T08:40:38.596975Z","iopub.status.idle":"2025-06-20T08:40:43.175746Z","shell.execute_reply.started":"2025-06-20T08:40:38.596953Z","shell.execute_reply":"2025-06-20T08:40:43.174822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df","metadata":{"id":"5K4McOZLldj7","outputId":"107b659c-3c0a-4e78-cb73-ab722bd91baf","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.176791Z","iopub.execute_input":"2025-06-20T08:40:43.177087Z","iopub.status.idle":"2025-06-20T08:40:43.205206Z","shell.execute_reply.started":"2025-06-20T08:40:43.177066Z","shell.execute_reply":"2025-06-20T08:40:43.204223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df = raw_df[raw_df.target == 0]","metadata":{"id":"TEeC6ux3swS6","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.206073Z","iopub.execute_input":"2025-06-20T08:40:43.206390Z","iopub.status.idle":"2025-06-20T08:40:43.289591Z","shell.execute_reply.started":"2025-06-20T08:40:43.206367Z","shell.execute_reply":"2025-06-20T08:40:43.288900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"id":"jEWHBXlrswJS","outputId":"b107ee21-368c-4f85-9fb7-d080c406e372","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.290522Z","iopub.execute_input":"2025-06-20T08:40:43.290784Z","iopub.status.idle":"2025-06-20T08:40:43.296995Z","shell.execute_reply.started":"2025-06-20T08:40:43.290764Z","shell.execute_reply":"2025-06-20T08:40:43.296133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target == 1]","metadata":{"id":"f-hE1Shksv83","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.297940Z","iopub.execute_input":"2025-06-20T08:40:43.298289Z","iopub.status.idle":"2025-06-20T08:40:43.331034Z","shell.execute_reply.started":"2025-06-20T08:40:43.298257Z","shell.execute_reply":"2025-06-20T08:40:43.330157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"id":"Jw2T2zc3tPky","outputId":"81b400b7-f83a-41af-d4f1-d57ecdff9e0b","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.332065Z","iopub.execute_input":"2025-06-20T08:40:43.332354Z","iopub.status.idle":"2025-06-20T08:40:43.346278Z","shell.execute_reply.started":"2025-06-20T08:40:43.332333Z","shell.execute_reply":"2025-06-20T08:40:43.345494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True).plot(kind='bar')","metadata":{"id":"Ux8EQDTNthEm","outputId":"29aa0577-411a-44b7-a35b-07291dcc568d","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.347050Z","iopub.execute_input":"2025-06-20T08:40:43.347304Z","iopub.status.idle":"2025-06-20T08:40:43.632147Z","shell.execute_reply.started":"2025-06-20T08:40:43.347279Z","shell.execute_reply":"2025-06-20T08:40:43.631349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(test_fname)","metadata":{"id":"D4WkPT6s6BnV","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:43.633230Z","iopub.execute_input":"2025-06-20T08:40:43.633876Z","iopub.status.idle":"2025-06-20T08:40:44.910210Z","shell.execute_reply.started":"2025-06-20T08:40:43.633823Z","shell.execute_reply":"2025-06-20T08:40:44.909339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"id":"nzVJ1M-Y6BaL","outputId":"249c504c-accb-4a5e-85d3-f937bdde362e","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:44.911020Z","iopub.execute_input":"2025-06-20T08:40:44.911227Z","iopub.status.idle":"2025-06-20T08:40:44.920791Z","shell.execute_reply.started":"2025-06-20T08:40:44.911209Z","shell.execute_reply":"2025-06-20T08:40:44.919898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df = pd.read_csv(sample_fname)","metadata":{"id":"_eZGBB-i6BN7","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:44.921893Z","iopub.execute_input":"2025-06-20T08:40:44.922215Z","iopub.status.idle":"2025-06-20T08:40:45.265383Z","shell.execute_reply.started":"2025-06-20T08:40:44.922188Z","shell.execute_reply":"2025-06-20T08:40:45.264605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df","metadata":{"id":"gN8Ky7QDtPUt","outputId":"26d18a2e-5832-465d-b7da-ec814e7a0da6","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.266224Z","iopub.execute_input":"2025-06-20T08:40:45.266519Z","iopub.status.idle":"2025-06-20T08:40:45.276613Z","shell.execute_reply.started":"2025-06-20T08:40:45.266481Z","shell.execute_reply":"2025-06-20T08:40:45.275600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"id":"-tQN044C64zR","outputId":"85f33730-2bbf-4d6a-fbad-548442c397b9","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.277557Z","iopub.execute_input":"2025-06-20T08:40:45.277799Z","iopub.status.idle":"2025-06-20T08:40:45.297797Z","shell.execute_reply.started":"2025-06-20T08:40:45.277780Z","shell.execute_reply":"2025-06-20T08:40:45.296824Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create a Working Sample","metadata":{"id":"y-L0IY7M7BNA"}},{"cell_type":"code","source":"SAMPLE_SIZE = 100_000","metadata":{"id":"PqKAHFg17QKG","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.298913Z","iopub.execute_input":"2025-06-20T08:40:45.299543Z","iopub.status.idle":"2025-06-20T08:40:45.317129Z","shell.execute_reply.started":"2025-06-20T08:40:45.299520Z","shell.execute_reply":"2025-06-20T08:40:45.316276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df = raw_df.sample(SAMPLE_SIZE, random_state= 42)","metadata":{"id":"_vU_Axbh64eu","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.317934Z","iopub.execute_input":"2025-06-20T08:40:45.318274Z","iopub.status.idle":"2025-06-20T08:40:45.436972Z","shell.execute_reply.started":"2025-06-20T08:40:45.318244Z","shell.execute_reply":"2025-06-20T08:40:45.435722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df","metadata":{"id":"KQqjzSmL64WU","outputId":"e6a0918d-9359-4f93-b247-f9cb61fc0eb5","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.438107Z","iopub.execute_input":"2025-06-20T08:40:45.438452Z","iopub.status.idle":"2025-06-20T08:40:45.450219Z","shell.execute_reply.started":"2025-06-20T08:40:45.438422Z","shell.execute_reply":"2025-06-20T08:40:45.449219Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" ## Text Preprocessing Tehniques\n\n Outline:\n 1. Understand the bag of words model\n 2. Tokenization\n 3. Stop word removal\n 4. Stemming","metadata":{"id":"MHAqrH2MHTog"}},{"cell_type":"markdown","source":"## Bag of Words Intuition\n 1. Create a list of all the words across all the text documents\n 2. You convert each documents into vector counts of each word\n\n Limitations:\n 1. There may be too many words in the dataset\n 2. Some words may occur too frequently\n 3. Some words may occur very rarely or only once\n 4. A single word may have many forms(go, gone, going or bird vs.birds)\n","metadata":{"id":"OZe8uoF-8onC"}},{"cell_type":"code","source":"sincere_df","metadata":{"id":"kSBH-Ws4-Ovk","outputId":"d843d1d8-6d93-4f2e-9dd4-1884db0cd22a","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.457959Z","iopub.execute_input":"2025-06-20T08:40:45.458261Z","iopub.status.idle":"2025-06-20T08:40:45.477632Z","shell.execute_reply.started":"2025-06-20T08:40:45.458238Z","shell.execute_reply":"2025-06-20T08:40:45.476092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0 = sincere_df.question_text.values[1]","metadata":{"id":"aKOa-wF3-Rrf","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.478745Z","iopub.execute_input":"2025-06-20T08:40:45.479140Z","iopub.status.idle":"2025-06-20T08:40:45.491879Z","shell.execute_reply.started":"2025-06-20T08:40:45.479104Z","shell.execute_reply":"2025-06-20T08:40:45.490649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0","metadata":{"id":"lt6v6FfR-fBz","outputId":"84d76b94-892c-4353-cc23-64788ecd384b","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.492994Z","iopub.execute_input":"2025-06-20T08:40:45.493239Z","iopub.status.idle":"2025-06-20T08:40:45.512108Z","shell.execute_reply.started":"2025-06-20T08:40:45.493219Z","shell.execute_reply":"2025-06-20T08:40:45.510926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1 = raw_df[raw_df.target == 1].question_text.values[0]","metadata":{"id":"3BpSOCQq-iMJ","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.513317Z","iopub.execute_input":"2025-06-20T08:40:45.514016Z","iopub.status.idle":"2025-06-20T08:40:45.550317Z","shell.execute_reply.started":"2025-06-20T08:40:45.513982Z","shell.execute_reply":"2025-06-20T08:40:45.549563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1","metadata":{"id":"O6hDIgJ6-zbB","outputId":"7ed001de-d933-4a4a-bd9f-070796899368","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.551352Z","iopub.execute_input":"2025-06-20T08:40:45.551595Z","iopub.status.idle":"2025-06-20T08:40:45.556683Z","shell.execute_reply.started":"2025-06-20T08:40:45.551578Z","shell.execute_reply":"2025-06-20T08:40:45.555890Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Tokenization\nsplitting a document into words and separators","metadata":{"id":"xa8jdN7U-ADX"}},{"cell_type":"code","source":"import nltk","metadata":{"id":"OvEGH499HuwC","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:45.557653Z","iopub.execute_input":"2025-06-20T08:40:45.557916Z","iopub.status.idle":"2025-06-20T08:40:46.929669Z","shell.execute_reply.started":"2025-06-20T08:40:45.557896Z","shell.execute_reply":"2025-06-20T08:40:46.928734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize","metadata":{"id":"zhYsSBQiHuko","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:46.930720Z","iopub.execute_input":"2025-06-20T08:40:46.931780Z","iopub.status.idle":"2025-06-20T08:40:46.935933Z","shell.execute_reply.started":"2025-06-20T08:40:46.931752Z","shell.execute_reply":"2025-06-20T08:40:46.934902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nltk.download('punkt')\nnltk.download('punkt_tab')","metadata":{"id":"ozUQRw5T_xiE","outputId":"acb6c230-585a-49bb-96ea-f8120d906993","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:46.936906Z","iopub.execute_input":"2025-06-20T08:40:46.937547Z","iopub.status.idle":"2025-06-20T08:40:47.126126Z","shell.execute_reply.started":"2025-06-20T08:40:46.937512Z","shell.execute_reply":"2025-06-20T08:40:47.125317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0","metadata":{"id":"_OJciAdfHuXh","outputId":"0b9a7503-5340-4bb6-810c-8443a3e544da","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.127168Z","iopub.execute_input":"2025-06-20T08:40:47.128224Z","iopub.status.idle":"2025-06-20T08:40:47.133585Z","shell.execute_reply.started":"2025-06-20T08:40:47.128191Z","shell.execute_reply":"2025-06-20T08:40:47.132726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"word_tokenize(q0)","metadata":{"id":"w2latYsv_5qL","outputId":"4290a4e4-1dd5-4a1c-940b-bdadbe80d4d7","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.134666Z","iopub.execute_input":"2025-06-20T08:40:47.135005Z","iopub.status.idle":"2025-06-20T08:40:47.181162Z","shell.execute_reply.started":"2025-06-20T08:40:47.134978Z","shell.execute_reply":"2025-06-20T08:40:47.180210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1","metadata":{"id":"t9md9tgM_5my","outputId":"9bedd83d-cca8-413a-e763-dfadc19bf7cf","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.182165Z","iopub.execute_input":"2025-06-20T08:40:47.183133Z","iopub.status.idle":"2025-06-20T08:40:47.188216Z","shell.execute_reply.started":"2025-06-20T08:40:47.183109Z","shell.execute_reply":"2025-06-20T08:40:47.187373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"word_tokenize(q1)","metadata":{"id":"zW02LT-Z_5fS","outputId":"084175a1-dcee-4d26-a32b-87c6ce3e5c03","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.189301Z","iopub.execute_input":"2025-06-20T08:40:47.189673Z","iopub.status.idle":"2025-06-20T08:40:47.203999Z","shell.execute_reply.started":"2025-06-20T08:40:47.189652Z","shell.execute_reply":"2025-06-20T08:40:47.202874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_tok = word_tokenize(q0)\nq1_tok = word_tokenize(q1)","metadata":{"id":"wKc7o3QnAVjM","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.205136Z","iopub.execute_input":"2025-06-20T08:40:47.205628Z","iopub.status.idle":"2025-06-20T08:40:47.219498Z","shell.execute_reply.started":"2025-06-20T08:40:47.205605Z","shell.execute_reply":"2025-06-20T08:40:47.218599Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Stop Word Removal\nRemoving commonly occuring words","metadata":{"id":"k8BKqCbVAj7Y"}},{"cell_type":"code","source":"q1_tok","metadata":{"id":"9cQbU5zrAs6Z","outputId":"52b27370-7b1c-4809-b43c-e0ac5fb52a54","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.220504Z","iopub.execute_input":"2025-06-20T08:40:47.221032Z","iopub.status.idle":"2025-06-20T08:40:47.237413Z","shell.execute_reply.started":"2025-06-20T08:40:47.221009Z","shell.execute_reply":"2025-06-20T08:40:47.236544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"id":"2kjScRWXAs0A","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.238366Z","iopub.execute_input":"2025-06-20T08:40:47.238623Z","iopub.status.idle":"2025-06-20T08:40:47.252825Z","shell.execute_reply.started":"2025-06-20T08:40:47.238603Z","shell.execute_reply":"2025-06-20T08:40:47.251960Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nltk.download('stopwords')\nenglish_stopwords = stopwords.words('english')","metadata":{"id":"Fq6JUHv3AVYk","outputId":"75996888-2a0a-4871-9026-82c4eefe8a55","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.253688Z","iopub.execute_input":"2025-06-20T08:40:47.254017Z","iopub.status.idle":"2025-06-20T08:40:47.277405Z","shell.execute_reply.started":"2025-06-20T08:40:47.253989Z","shell.execute_reply":"2025-06-20T08:40:47.276489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\", \".join(english_stopwords)","metadata":{"id":"FnLdemUEBL6l","outputId":"89df5a6c-9331-4efd-c582-d402c8cc00e9","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.278404Z","iopub.execute_input":"2025-06-20T08:40:47.278638Z","iopub.status.idle":"2025-06-20T08:40:47.284067Z","shell.execute_reply.started":"2025-06-20T08:40:47.278617Z","shell.execute_reply":"2025-06-20T08:40:47.283255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_stopwords(tokens):\n  return [word for word in tokens if word.lower() not in english_stopwords]","metadata":{"id":"3M0fAKDpBLxD","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.285015Z","iopub.execute_input":"2025-06-20T08:40:47.285255Z","iopub.status.idle":"2025-06-20T08:40:47.306443Z","shell.execute_reply.started":"2025-06-20T08:40:47.285236Z","shell.execute_reply":"2025-06-20T08:40:47.305702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_tok","metadata":{"id":"-v03ROufBzaf","outputId":"255c3c30-4ff7-45e5-b8c6-e435255a08a8","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.307272Z","iopub.execute_input":"2025-06-20T08:40:47.307502Z","iopub.status.idle":"2025-06-20T08:40:47.323459Z","shell.execute_reply.started":"2025-06-20T08:40:47.307482Z","shell.execute_reply":"2025-06-20T08:40:47.322417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stp = remove_stopwords(q0_tok)","metadata":{"id":"5W-bmBEABzOY","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.324347Z","iopub.execute_input":"2025-06-20T08:40:47.324628Z","iopub.status.idle":"2025-06-20T08:40:47.340576Z","shell.execute_reply.started":"2025-06-20T08:40:47.324608Z","shell.execute_reply":"2025-06-20T08:40:47.339605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stp","metadata":{"id":"fGmpkSjcBzG6","outputId":"437be4ae-abd3-4959-9194-b729bba1900a","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.341606Z","iopub.execute_input":"2025-06-20T08:40:47.341912Z","iopub.status.idle":"2025-06-20T08:40:47.359602Z","shell.execute_reply.started":"2025-06-20T08:40:47.341887Z","shell.execute_reply":"2025-06-20T08:40:47.358788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stp = remove_stopwords(q1_tok)","metadata":{"id":"Hi2HPFRPCGzQ","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.360654Z","iopub.execute_input":"2025-06-20T08:40:47.360912Z","iopub.status.idle":"2025-06-20T08:40:47.375035Z","shell.execute_reply.started":"2025-06-20T08:40:47.360888Z","shell.execute_reply":"2025-06-20T08:40:47.374179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stp","metadata":{"id":"Yec_A4lfCPTe","outputId":"10f9afea-6abb-4520-9a3a-c8f6f96bef56","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.376054Z","iopub.execute_input":"2025-06-20T08:40:47.376359Z","iopub.status.idle":"2025-06-20T08:40:47.394151Z","shell.execute_reply.started":"2025-06-20T08:40:47.376333Z","shell.execute_reply":"2025-06-20T08:40:47.393227Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Stemming\n \"go\",\"gone\",\"going\" ->\"go\" \"birds\",\"bird\"->\"bird\"","metadata":{"id":"TDK5Dz5SCXwH"}},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer","metadata":{"id":"q_RKFXydCXQj","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.394979Z","iopub.execute_input":"2025-06-20T08:40:47.395317Z","iopub.status.idle":"2025-06-20T08:40:47.410202Z","shell.execute_reply.started":"2025-06-20T08:40:47.395296Z","shell.execute_reply":"2025-06-20T08:40:47.409383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer = SnowballStemmer(language ='english')","metadata":{"id":"UhZp4Q8UCXAD","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.411343Z","iopub.execute_input":"2025-06-20T08:40:47.411781Z","iopub.status.idle":"2025-06-20T08:40:47.428501Z","shell.execute_reply.started":"2025-06-20T08:40:47.411752Z","shell.execute_reply":"2025-06-20T08:40:47.427636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer.stem('going')","metadata":{"id":"ONnobMuXCVgu","outputId":"4d63ccbd-cec3-4cad-ba1d-338fc6712a78","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.429451Z","iopub.execute_input":"2025-06-20T08:40:47.430041Z","iopub.status.idle":"2025-06-20T08:40:47.448716Z","shell.execute_reply.started":"2025-06-20T08:40:47.430012Z","shell.execute_reply":"2025-06-20T08:40:47.447748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer.stem('supposedly')","metadata":{"id":"R6W8aBL_CPHE","outputId":"f7a23aba-2678-430a-f97e-2f92bee97d47","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.449734Z","iopub.execute_input":"2025-06-20T08:40:47.450036Z","iopub.status.idle":"2025-06-20T08:40:47.465498Z","shell.execute_reply.started":"2025-06-20T08:40:47.450014Z","shell.execute_reply":"2025-06-20T08:40:47.464634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stm = [stemmer.stem(word) for word in q0_stp]","metadata":{"id":"d8UCXBBBDLcK","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.466455Z","iopub.execute_input":"2025-06-20T08:40:47.466738Z","iopub.status.idle":"2025-06-20T08:40:47.480408Z","shell.execute_reply.started":"2025-06-20T08:40:47.466707Z","shell.execute_reply":"2025-06-20T08:40:47.479606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stp","metadata":{"id":"tPBwM4goDLOA","outputId":"00592197-29db-4d56-df3b-69069b9e93a0","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.481417Z","iopub.execute_input":"2025-06-20T08:40:47.481726Z","iopub.status.idle":"2025-06-20T08:40:47.498005Z","shell.execute_reply.started":"2025-06-20T08:40:47.481705Z","shell.execute_reply":"2025-06-20T08:40:47.497121Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stm","metadata":{"id":"ndqt5uZWDTNq","outputId":"9cd3da3d-3bd6-42cf-ebf7-be26bc2ccf2e","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.498871Z","iopub.execute_input":"2025-06-20T08:40:47.499231Z","iopub.status.idle":"2025-06-20T08:40:47.513242Z","shell.execute_reply.started":"2025-06-20T08:40:47.499208Z","shell.execute_reply":"2025-06-20T08:40:47.512240Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stm = [stemmer.stem(word) for word in q1_stp]","metadata":{"id":"uvFg5m98DTKN","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.514135Z","iopub.execute_input":"2025-06-20T08:40:47.514395Z","iopub.status.idle":"2025-06-20T08:40:47.529038Z","shell.execute_reply.started":"2025-06-20T08:40:47.514375Z","shell.execute_reply":"2025-06-20T08:40:47.528230Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stp","metadata":{"id":"M08SmJEKDbWR","outputId":"7f7eebee-3873-4815-b20f-b076d438aa11","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.530056Z","iopub.execute_input":"2025-06-20T08:40:47.530629Z","iopub.status.idle":"2025-06-20T08:40:47.546049Z","shell.execute_reply.started":"2025-06-20T08:40:47.530600Z","shell.execute_reply":"2025-06-20T08:40:47.544977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stm","metadata":{"id":"r-Q2eDRPDbH1","outputId":"a8321f18-1a6a-430b-f47a-3d2ba7933082","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.546868Z","iopub.execute_input":"2025-06-20T08:40:47.547190Z","iopub.status.idle":"2025-06-20T08:40:47.565437Z","shell.execute_reply.started":"2025-06-20T08:40:47.547160Z","shell.execute_reply":"2025-06-20T08:40:47.564543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Lemmatization\n\"love\" -> \"love\"\n\"loving\" -> \" love\"\n\"lovable\" -> \"love\"","metadata":{"id":"YKDzEptSDf5Q"}},{"cell_type":"code","source":"import nltk\nnltk.download('wordnet')","metadata":{"id":"o6XmDxAwEOAB","outputId":"980c1ab3-e765-4051-93c0-c033d3659cda","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.566279Z","iopub.execute_input":"2025-06-20T08:40:47.566560Z","iopub.status.idle":"2025-06-20T08:40:47.640974Z","shell.execute_reply.started":"2025-06-20T08:40:47.566539Z","shell.execute_reply":"2025-06-20T08:40:47.640003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer as wnl\nprint(wnl().lemmatize('us', 'n'))","metadata":{"id":"hieHYWvrEAK7","outputId":"c2616296-e4ab-4df3-892a-4d8130031711","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:47.642054Z","iopub.execute_input":"2025-06-20T08:40:47.642469Z","iopub.status.idle":"2025-06-20T08:40:50.727335Z","shell.execute_reply.started":"2025-06-20T08:40:47.642444Z","shell.execute_reply":"2025-06-20T08:40:50.726349Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Implement Bag of Words\n\nOutline:\n1. Create a vocabulary using Count Vectorizer\n2. Transform text to vectors using Count Vectorizer\n3. Configure text preprocessing in Count Vectorizer","metadata":{"id":"0zZlnAjoHadT"}},{"cell_type":"markdown","source":"### Create a Vocubalry","metadata":{"id":"xc7Xr6JAFV__"}},{"cell_type":"code","source":"sample_df","metadata":{"id":"iqpHqDnk7wxX","outputId":"7f01e8bd-9e65-4447-e1f9-cecbb1f2fe9e","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.728328Z","iopub.execute_input":"2025-06-20T08:40:50.728595Z","iopub.status.idle":"2025-06-20T08:40:50.739330Z","shell.execute_reply.started":"2025-06-20T08:40:50.728575Z","shell.execute_reply":"2025-06-20T08:40:50.738293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df = sample_df[:5]","metadata":{"id":"qetFNsX67wt8","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.740759Z","iopub.execute_input":"2025-06-20T08:40:50.741127Z","iopub.status.idle":"2025-06-20T08:40:50.759550Z","shell.execute_reply.started":"2025-06-20T08:40:50.741098Z","shell.execute_reply":"2025-06-20T08:40:50.758635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df.question_text.values","metadata":{"id":"AJqqUqFh7wkO","outputId":"ef12eb66-36ab-4c1d-92b6-4745e7c1357f","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.760581Z","iopub.execute_input":"2025-06-20T08:40:50.760888Z","iopub.status.idle":"2025-06-20T08:40:50.779697Z","shell.execute_reply.started":"2025-06-20T08:40:50.760828Z","shell.execute_reply":"2025-06-20T08:40:50.778681Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ML Models for Text Classification","metadata":{"id":"EksQll7jHec5"}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"id":"HtZNaXje7zFw","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.780689Z","iopub.execute_input":"2025-06-20T08:40:50.780955Z","iopub.status.idle":"2025-06-20T08:40:50.795714Z","shell.execute_reply.started":"2025-06-20T08:40:50.780936Z","shell.execute_reply":"2025-06-20T08:40:50.794888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_vect = CountVectorizer()","metadata":{"id":"pfQyDO057zAx","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.796593Z","iopub.execute_input":"2025-06-20T08:40:50.796869Z","iopub.status.idle":"2025-06-20T08:40:50.815764Z","shell.execute_reply.started":"2025-06-20T08:40:50.796820Z","shell.execute_reply":"2025-06-20T08:40:50.814989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_vect.fit(small_df.question_text)","metadata":{"id":"mgsPRG_S7y5B","outputId":"aa8928cd-244e-41c2-acbe-39e5b24e65d9","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.816941Z","iopub.execute_input":"2025-06-20T08:40:50.817634Z","iopub.status.idle":"2025-06-20T08:40:50.847345Z","shell.execute_reply.started":"2025-06-20T08:40:50.817598Z","shell.execute_reply":"2025-06-20T08:40:50.846389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_vect.get_feature_names_out()","metadata":{"id":"qb3sjqJgGE-E","outputId":"edadc74e-56f5-45c1-b543-7adeb35d6a70","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.848315Z","iopub.execute_input":"2025-06-20T08:40:50.848610Z","iopub.status.idle":"2025-06-20T08:40:50.856851Z","shell.execute_reply.started":"2025-06-20T08:40:50.848588Z","shell.execute_reply":"2025-06-20T08:40:50.855854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Tranform documents into Vectors","metadata":{"id":"qIKAOOyFGXGT"}},{"cell_type":"code","source":"vectors = small_vect.transform(small_df.question_text)","metadata":{"id":"rdzh44ryGbtX","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.857679Z","iopub.execute_input":"2025-06-20T08:40:50.857985Z","iopub.status.idle":"2025-06-20T08:40:50.875757Z","shell.execute_reply.started":"2025-06-20T08:40:50.857964Z","shell.execute_reply":"2025-06-20T08:40:50.874866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors","metadata":{"id":"kVQe0QgUGbi2","outputId":"79500d5c-2191-4f64-8579-6a701a7d3053","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.876668Z","iopub.execute_input":"2025-06-20T08:40:50.876958Z","iopub.status.idle":"2025-06-20T08:40:50.896780Z","shell.execute_reply.started":"2025-06-20T08:40:50.876939Z","shell.execute_reply":"2025-06-20T08:40:50.895977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors.toarray()","metadata":{"id":"pfDFFk9tG1PM","outputId":"50dfdd24-63df-4ee3-ebae-bbc5fc72ff2c","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.897796Z","iopub.execute_input":"2025-06-20T08:40:50.898078Z","iopub.status.idle":"2025-06-20T08:40:50.915585Z","shell.execute_reply.started":"2025-06-20T08:40:50.898059Z","shell.execute_reply":"2025-06-20T08:40:50.914685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors[0].toarray()","metadata":{"id":"uL6EFiNsHSCM","outputId":"527aa317-c572-4061-e483-7089845918f2","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.916308Z","iopub.execute_input":"2025-06-20T08:40:50.916518Z","iopub.status.idle":"2025-06-20T08:40:50.934511Z","shell.execute_reply.started":"2025-06-20T08:40:50.916501Z","shell.execute_reply":"2025-06-20T08:40:50.933644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors.shape","metadata":{"id":"JxxdH7J5G1Dj","outputId":"9addebf8-eb83-4e49-bb4b-180da6775ad9","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.935355Z","iopub.execute_input":"2025-06-20T08:40:50.935663Z","iopub.status.idle":"2025-06-20T08:40:50.951438Z","shell.execute_reply.started":"2025-06-20T08:40:50.935642Z","shell.execute_reply":"2025-06-20T08:40:50.950438Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Configure Count Vectorizer Parameters","metadata":{"id":"Hkp6_zyIHY-f"}},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"id":"2zNoi1JOG8By","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.952555Z","iopub.execute_input":"2025-06-20T08:40:50.953180Z","iopub.status.idle":"2025-06-20T08:40:50.968979Z","shell.execute_reply.started":"2025-06-20T08:40:50.953156Z","shell.execute_reply":"2025-06-20T08:40:50.968013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tokenize(text):\n    return [stemmer.stem(word) for word in word_tokenize(text)]","metadata":{"id":"rCkAlGjXG72a","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.969991Z","iopub.execute_input":"2025-06-20T08:40:50.970218Z","iopub.status.idle":"2025-06-20T08:40:50.984677Z","shell.execute_reply.started":"2025-06-20T08:40:50.970201Z","shell.execute_reply":"2025-06-20T08:40:50.983788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenize('What is the really (dealing) here?')","metadata":{"id":"Kohu1pgiIvGZ","outputId":"6ed6df61-c6ff-435d-dfaa-baec5a930968","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:50.985658Z","iopub.execute_input":"2025-06-20T08:40:50.986300Z","iopub.status.idle":"2025-06-20T08:40:51.004035Z","shell.execute_reply.started":"2025-06-20T08:40:50.986277Z","shell.execute_reply":"2025-06-20T08:40:51.003220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer = CountVectorizer(lowercase=True,\n                             tokenizer=tokenize,\n                             stop_words=english_stopwords,\n                             max_features=1000)","metadata":{"id":"VgFduswGIu7P","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:51.004748Z","iopub.execute_input":"2025-06-20T08:40:51.004989Z","iopub.status.idle":"2025-06-20T08:40:51.020872Z","shell.execute_reply.started":"2025-06-20T08:40:51.004971Z","shell.execute_reply":"2025-06-20T08:40:51.019773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"id":"9L_d-bSOI1KR","outputId":"390d00c9-0476-49d9-ee4b-b031ef387c58","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:40:51.021791Z","iopub.execute_input":"2025-06-20T08:40:51.022103Z","iopub.status.idle":"2025-06-20T08:41:13.508785Z","shell.execute_reply.started":"2025-06-20T08:40:51.022082Z","shell.execute_reply":"2025-06-20T08:41:13.507892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(vectorizer.vocabulary_)","metadata":{"id":"R3HMBqtXI1As","outputId":"4f6b6c08-e0ca-4b71-aaf5-978f57dfe65c","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:13.509756Z","iopub.execute_input":"2025-06-20T08:41:13.510017Z","iopub.status.idle":"2025-06-20T08:41:13.515549Z","shell.execute_reply.started":"2025-06-20T08:41:13.509997Z","shell.execute_reply":"2025-06-20T08:41:13.514710Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100]","metadata":{"id":"FKbO-PgWJJUA","outputId":"c93374ae-c2f4-4afd-e6df-f50aa672a328","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:13.516577Z","iopub.execute_input":"2025-06-20T08:41:13.516929Z","iopub.status.idle":"2025-06-20T08:41:13.537696Z","shell.execute_reply.started":"2025-06-20T08:41:13.516901Z","shell.execute_reply":"2025-06-20T08:41:13.536717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{"id":"pdouyuaxJJHO","outputId":"27bc3be1-fb50-4edd-f3ae-be30cc9fcc1d","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:13.549063Z","iopub.execute_input":"2025-06-20T08:41:13.549365Z","iopub.status.idle":"2025-06-20T08:41:36.176720Z","shell.execute_reply.started":"2025-06-20T08:41:13.549344Z","shell.execute_reply":"2025-06-20T08:41:36.175899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs.shape","metadata":{"id":"NayjSlgoJI1j","outputId":"57094329-5348-4a2c-a76c-88244a0b3019","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:36.177603Z","iopub.execute_input":"2025-06-20T08:41:36.177867Z","iopub.status.idle":"2025-06-20T08:41:36.183573Z","shell.execute_reply.started":"2025-06-20T08:41:36.177823Z","shell.execute_reply":"2025-06-20T08:41:36.182735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs","metadata":{"id":"1_AKXAAoJkRZ","outputId":"5d4d386b-ab99-4d33-c6de-f9082eb21d47","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:36.184542Z","iopub.execute_input":"2025-06-20T08:41:36.184865Z","iopub.status.idle":"2025-06-20T08:41:36.203713Z","shell.execute_reply.started":"2025-06-20T08:41:36.184811Z","shell.execute_reply":"2025-06-20T08:41:36.202668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.question_text.values[0]","metadata":{"id":"f1y6R3NHJuur","outputId":"e3e5cb05-aa7b-4dcd-a4fe-ebd3708d9d02","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:36.204715Z","iopub.execute_input":"2025-06-20T08:41:36.205565Z","iopub.status.idle":"2025-06-20T08:41:36.223307Z","shell.execute_reply.started":"2025-06-20T08:41:36.205528Z","shell.execute_reply":"2025-06-20T08:41:36.222300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"id":"BQjm1UCGJl6q","outputId":"94d79a49-f5a9-4c30-9306-f85340a8ad52","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:36.224276Z","iopub.execute_input":"2025-06-20T08:41:36.224936Z","iopub.status.idle":"2025-06-20T08:41:36.247858Z","shell.execute_reply.started":"2025-06-20T08:41:36.224907Z","shell.execute_reply":"2025-06-20T08:41:36.246890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"id":"E5uEKj-3Jpw2","outputId":"01337651-8b4f-428d-8acf-864bc96d9754","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:41:36.248736Z","iopub.execute_input":"2025-06-20T08:41:36.249040Z","iopub.status.idle":"2025-06-20T08:43:00.330731Z","shell.execute_reply.started":"2025-06-20T08:41:36.249018Z","shell.execute_reply":"2025-06-20T08:43:00.329634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ML Models for Text Classification\n\nOutline:\n- Create a training & validation set\n- Train a logiatic regression model\n- Make predictions on training , validation & test data","metadata":{"id":"4NrP2e1WKbCR"}},{"cell_type":"markdown","source":"### Split into Training and validation Set","metadata":{"id":"nIgAlTJjLctx"}},{"cell_type":"code","source":"sample_df","metadata":{"id":"uCgCsIhrJpg7","outputId":"9be766be-59c8-4124-db1c-2a6c25c04eaa","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.331562Z","iopub.execute_input":"2025-06-20T08:43:00.331782Z","iopub.status.idle":"2025-06-20T08:43:00.342650Z","shell.execute_reply.started":"2025-06-20T08:43:00.331766Z","shell.execute_reply":"2025-06-20T08:43:00.341726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs.shape","metadata":{"id":"MkZd4GqkLmrL","outputId":"bac89fef-2970-43fb-a1a4-b6580fdfd287","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.343577Z","iopub.execute_input":"2025-06-20T08:43:00.343814Z","iopub.status.idle":"2025-06-20T08:43:00.361283Z","shell.execute_reply.started":"2025-06-20T08:43:00.343796Z","shell.execute_reply":"2025-06-20T08:43:00.360348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"id":"oMs4QWBbLmn0","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.362003Z","iopub.execute_input":"2025-06-20T08:43:00.362232Z","iopub.status.idle":"2025-06-20T08:43:00.408497Z","shell.execute_reply.started":"2025-06-20T08:43:00.362213Z","shell.execute_reply":"2025-06-20T08:43:00.407420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, sample_df.target, test_size=0.3, random_state=42)","metadata":{"id":"JBtJThlXLmk1","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.409382Z","iopub.execute_input":"2025-06-20T08:43:00.409634Z","iopub.status.idle":"2025-06-20T08:43:00.437168Z","shell.execute_reply.started":"2025-06-20T08:43:00.409614Z","shell.execute_reply":"2025-06-20T08:43:00.436195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_inputs.shape","metadata":{"id":"F_PyXj4CLmiH","outputId":"48b37b77-407c-4ae6-ff18-b05c9b439b3f","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.438171Z","iopub.execute_input":"2025-06-20T08:43:00.438510Z","iopub.status.idle":"2025-06-20T08:43:00.444382Z","shell.execute_reply.started":"2025-06-20T08:43:00.438483Z","shell.execute_reply":"2025-06-20T08:43:00.443476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_targets.shape","metadata":{"id":"TqhSHDM6Lmbn","outputId":"00b6f67a-7259-430d-d834-0f2675e27c1d","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.445323Z","iopub.execute_input":"2025-06-20T08:43:00.445558Z","iopub.status.idle":"2025-06-20T08:43:00.463026Z","shell.execute_reply.started":"2025-06-20T08:43:00.445541Z","shell.execute_reply":"2025-06-20T08:43:00.462195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_inputs.shape","metadata":{"id":"HiRtPAInNG3I","outputId":"b92e3f2d-e479-402c-f10a-3d146ba945b7","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.463989Z","iopub.execute_input":"2025-06-20T08:43:00.464253Z","iopub.status.idle":"2025-06-20T08:43:00.480202Z","shell.execute_reply.started":"2025-06-20T08:43:00.464234Z","shell.execute_reply":"2025-06-20T08:43:00.479151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_targets.shape","metadata":{"id":"y1sgwlrSNF6N","outputId":"18f5afb1-b5b9-4b92-b0c3-f7b8f0a1bdfb","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.481201Z","iopub.execute_input":"2025-06-20T08:43:00.481545Z","iopub.status.idle":"2025-06-20T08:43:00.496380Z","shell.execute_reply.started":"2025-06-20T08:43:00.481517Z","shell.execute_reply":"2025-06-20T08:43:00.495381Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train Logistic Regression model","metadata":{"id":"QrZiOVmwNMJx"}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"id":"SzBGJLBmNStO","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.497277Z","iopub.execute_input":"2025-06-20T08:43:00.497941Z","iopub.status.idle":"2025-06-20T08:43:00.512513Z","shell.execute_reply.started":"2025-06-20T08:43:00.497912Z","shell.execute_reply":"2025-06-20T08:43:00.511650Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_ITER = 1000","metadata":{"id":"yv6ntLyHNSiG","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.513647Z","iopub.execute_input":"2025-06-20T08:43:00.513968Z","iopub.status.idle":"2025-06-20T08:43:00.530884Z","shell.execute_reply.started":"2025-06-20T08:43:00.513940Z","shell.execute_reply":"2025-06-20T08:43:00.530032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = LogisticRegression(max_iter = 1000, solver ='sag')","metadata":{"id":"vbUyS4QtOE2e","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.532299Z","iopub.execute_input":"2025-06-20T08:43:00.532630Z","iopub.status.idle":"2025-06-20T08:43:00.548088Z","shell.execute_reply.started":"2025-06-20T08:43:00.532600Z","shell.execute_reply":"2025-06-20T08:43:00.547177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nmodel.fit(train_inputs, train_targets)","metadata":{"id":"USYGyWB_OEoe","outputId":"5c7844ee-68cb-468a-f4ce-249e8e5a5e17","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:00.549087Z","iopub.execute_input":"2025-06-20T08:43:00.549386Z","iopub.status.idle":"2025-06-20T08:43:20.901201Z","shell.execute_reply.started":"2025-06-20T08:43:00.549364Z","shell.execute_reply":"2025-06-20T08:43:20.900293Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Make predictions using the model\n","metadata":{"id":"KBShDWgvSELW"}},{"cell_type":"code","source":"train_pred = model.predict(train_inputs)","metadata":{"id":"z8-y_3wjOEbN","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:20.902261Z","iopub.execute_input":"2025-06-20T08:43:20.902883Z","iopub.status.idle":"2025-06-20T08:43:20.910485Z","shell.execute_reply.started":"2025-06-20T08:43:20.902852Z","shell.execute_reply":"2025-06-20T08:43:20.909393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_targets","metadata":{"id":"NPF0MnBPOEOJ","outputId":"0e119c16-8b38-4fdd-a45e-d883daf9f7bf","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:20.911309Z","iopub.execute_input":"2025-06-20T08:43:20.911556Z","iopub.status.idle":"2025-06-20T08:43:20.928144Z","shell.execute_reply.started":"2025-06-20T08:43:20.911538Z","shell.execute_reply":"2025-06-20T08:43:20.927269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_pred","metadata":{"id":"Dnk2x-uxSPvJ","outputId":"992010d8-f62d-4ac8-fa88-02063d0265a5","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:20.929244Z","iopub.execute_input":"2025-06-20T08:43:20.929509Z","iopub.status.idle":"2025-06-20T08:43:20.945506Z","shell.execute_reply.started":"2025-06-20T08:43:20.929483Z","shell.execute_reply":"2025-06-20T08:43:20.944577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(train_pred).value_counts()","metadata":{"id":"52XJrQaXSPro","outputId":"697f7e3f-ce62-44e6-fa0a-a8fdaef8eb7a","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:20.946298Z","iopub.execute_input":"2025-06-20T08:43:20.946536Z","iopub.status.idle":"2025-06-20T08:43:20.967752Z","shell.execute_reply.started":"2025-06-20T08:43:20.946517Z","shell.execute_reply":"2025-06-20T08:43:20.966781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(train_targets).value_counts()","metadata":{"id":"-NAgCE7RSPot","outputId":"32bf5fae-cc38-4061-84bd-6b78b03806e0","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:20.968740Z","iopub.execute_input":"2025-06-20T08:43:20.969013Z","iopub.status.idle":"2025-06-20T08:43:20.986158Z","shell.execute_reply.started":"2025-06-20T08:43:20.968994Z","shell.execute_reply":"2025-06-20T08:43:20.985168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"id":"0-5ifkbHSPlp","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:20.987242Z","iopub.execute_input":"2025-06-20T08:43:20.988246Z","iopub.status.idle":"2025-06-20T08:43:21.001036Z","shell.execute_reply.started":"2025-06-20T08:43:20.988221Z","shell.execute_reply":"2025-06-20T08:43:20.999909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(train_targets, train_pred)","metadata":{"id":"cIMdoge6SPiO","outputId":"87fe868a-73bf-479b-ba75-619880731c49","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.002044Z","iopub.execute_input":"2025-06-20T08:43:21.002472Z","iopub.status.idle":"2025-06-20T08:43:21.025367Z","shell.execute_reply.started":"2025-06-20T08:43:21.002446Z","shell.execute_reply":"2025-06-20T08:43:21.024438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np","metadata":{"id":"BtJ4hItRS2n2","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.026351Z","iopub.execute_input":"2025-06-20T08:43:21.026617Z","iopub.status.idle":"2025-06-20T08:43:21.031240Z","shell.execute_reply.started":"2025-06-20T08:43:21.026597Z","shell.execute_reply":"2025-06-20T08:43:21.030253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(train_targets, np.zeros(len(train_targets)))","metadata":{"id":"ZfLgHBTkS20A","outputId":"d3f774bd-fb65-4471-d534-2548233dbe09","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.032062Z","iopub.execute_input":"2025-06-20T08:43:21.032456Z","iopub.status.idle":"2025-06-20T08:43:21.059642Z","shell.execute_reply.started":"2025-06-20T08:43:21.032399Z","shell.execute_reply":"2025-06-20T08:43:21.058716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{"id":"asYTROgdS3Pd","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.060565Z","iopub.execute_input":"2025-06-20T08:43:21.060798Z","iopub.status.idle":"2025-06-20T08:43:21.065356Z","shell.execute_reply.started":"2025-06-20T08:43:21.060779Z","shell.execute_reply":"2025-06-20T08:43:21.064320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(train_targets, train_pred)","metadata":{"id":"zppr_YC5TGs5","outputId":"70b0febf-8add-4cb3-bd04-0c3d448011b4","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.066447Z","iopub.execute_input":"2025-06-20T08:43:21.066677Z","iopub.status.idle":"2025-06-20T08:43:21.108701Z","shell.execute_reply.started":"2025-06-20T08:43:21.066659Z","shell.execute_reply":"2025-06-20T08:43:21.107686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(train_targets, np.zeros(len(train_targets)))","metadata":{"id":"AvamCqfzTGhw","outputId":"bd7a1910-3a16-4c44-d529-5e25390624cf","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.109458Z","iopub.execute_input":"2025-06-20T08:43:21.109712Z","iopub.status.idle":"2025-06-20T08:43:21.149730Z","shell.execute_reply.started":"2025-06-20T08:43:21.109693Z","shell.execute_reply":"2025-06-20T08:43:21.148739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_preds = np.random.choice((0, 1), len(train_targets))\nf1_score(train_targets, random_preds)","metadata":{"id":"_RmrbYLNTZdr","outputId":"45862cae-5d62-48a3-ea13-436fe1d2de23","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.150751Z","iopub.execute_input":"2025-06-20T08:43:21.151059Z","iopub.status.idle":"2025-06-20T08:43:21.183400Z","shell.execute_reply.started":"2025-06-20T08:43:21.151036Z","shell.execute_reply":"2025-06-20T08:43:21.182357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds = model.predict(val_inputs)","metadata":{"id":"ZRLeyB-YTZUl","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.184322Z","iopub.execute_input":"2025-06-20T08:43:21.184583Z","iopub.status.idle":"2025-06-20T08:43:21.189801Z","shell.execute_reply.started":"2025-06-20T08:43:21.184563Z","shell.execute_reply":"2025-06-20T08:43:21.189053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(val_targets, val_preds)","metadata":{"id":"nTj_o_LWS3cm","outputId":"5556dbe8-78f1-47d4-8f8b-5046f43b8dab","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.190641Z","iopub.execute_input":"2025-06-20T08:43:21.190887Z","iopub.status.idle":"2025-06-20T08:43:21.210546Z","shell.execute_reply.started":"2025-06-20T08:43:21.190863Z","shell.execute_reply":"2025-06-20T08:43:21.209564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(val_targets, val_preds)","metadata":{"id":"JNuy4lNUUnVM","outputId":"37b45842-0988-4c2e-db9b-256f65c94ca5","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.211489Z","iopub.execute_input":"2025-06-20T08:43:21.212101Z","iopub.status.idle":"2025-06-20T08:43:21.235459Z","shell.execute_reply.started":"2025-06-20T08:43:21.212072Z","shell.execute_reply":"2025-06-20T08:43:21.234393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"id":"48XjdUhOUnRz","outputId":"3e8b8139-ae7a-49db-f4ec-0a0318d3b742","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.236366Z","iopub.execute_input":"2025-06-20T08:43:21.236690Z","iopub.status.idle":"2025-06-20T08:43:21.243665Z","shell.execute_reply.started":"2025-06-20T08:43:21.236661Z","shell.execute_reply":"2025-06-20T08:43:21.242794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.target.values[:10]","metadata":{"id":"vJC2yrZiVC5J","outputId":"bc5abdde-f433-4144-e378-7a0bebcdfa18","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.244625Z","iopub.execute_input":"2025-06-20T08:43:21.244879Z","iopub.status.idle":"2025-06-20T08:43:21.263672Z","shell.execute_reply.started":"2025-06-20T08:43:21.244822Z","shell.execute_reply":"2025-06-20T08:43:21.262660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.predict(vectorizer.transform(sincere_df.question_text.values[:10]))","metadata":{"id":"rcslGwE0VKjI","outputId":"d3f73037-98eb-4649-e15d-b886a23d9b64","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.264732Z","iopub.execute_input":"2025-06-20T08:43:21.265310Z","iopub.status.idle":"2025-06-20T08:43:21.286703Z","shell.execute_reply.started":"2025-06-20T08:43:21.265287Z","shell.execute_reply":"2025-06-20T08:43:21.285661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"id":"t6g6kur7Vobu","outputId":"014b9321-8d15-4c4e-f4d2-21c23a8c5a68","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.287564Z","iopub.execute_input":"2025-06-20T08:43:21.287804Z","iopub.status.idle":"2025-06-20T08:43:21.306141Z","shell.execute_reply.started":"2025-06-20T08:43:21.287785Z","shell.execute_reply":"2025-06-20T08:43:21.304898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.target.values[:10]","metadata":{"id":"VNzj20y1VCSk","outputId":"4c6d4bec-5430-4f04-d778-420e5f92a887","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.307212Z","iopub.execute_input":"2025-06-20T08:43:21.307480Z","iopub.status.idle":"2025-06-20T08:43:21.322397Z","shell.execute_reply.started":"2025-06-20T08:43:21.307460Z","shell.execute_reply":"2025-06-20T08:43:21.321358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.predict(vectorizer.transform(insincere_df.question_text.values[:10]))","metadata":{"id":"SjUsQO3QVm_H","outputId":"e8532618-c273-4f8d-94bd-8aae7c0720ac","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.323246Z","iopub.execute_input":"2025-06-20T08:43:21.323459Z","iopub.status.idle":"2025-06-20T08:43:21.344134Z","shell.execute_reply.started":"2025-06-20T08:43:21.323443Z","shell.execute_reply":"2025-06-20T08:43:21.343054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"OidFEwOFVmyn","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Make prediction and submit to Kaggle","metadata":{"id":"i-_u3SlpHlMU"}},{"cell_type":"code","source":"test_df","metadata":{"id":"qgKZmvWP73j7","outputId":"8cecee6c-77c3-4697-b430-fe0eee7b9465","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.344959Z","iopub.execute_input":"2025-06-20T08:43:21.345178Z","iopub.status.idle":"2025-06-20T08:43:21.363779Z","shell.execute_reply.started":"2025-06-20T08:43:21.345161Z","shell.execute_reply":"2025-06-20T08:43:21.363018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_inputs.shape","metadata":{"id":"fJS0B56B73Rp","outputId":"760b3edd-3f5c-4c17-fa2c-859a557f7f64","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.364940Z","iopub.execute_input":"2025-06-20T08:43:21.365346Z","iopub.status.idle":"2025-06-20T08:43:21.381037Z","shell.execute_reply.started":"2025-06-20T08:43:21.365316Z","shell.execute_reply":"2025-06-20T08:43:21.380042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds = model.predict(test_inputs)","metadata":{"id":"MTWlIhhvWydr","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.381824Z","iopub.execute_input":"2025-06-20T08:43:21.382171Z","iopub.status.idle":"2025-06-20T08:43:21.420061Z","shell.execute_reply.started":"2025-06-20T08:43:21.382150Z","shell.execute_reply":"2025-06-20T08:43:21.419049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"id":"tte1RwosHqMH","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.421066Z","iopub.execute_input":"2025-06-20T08:43:21.421340Z","iopub.status.idle":"2025-06-20T08:43:21.426183Z","shell.execute_reply.started":"2025-06-20T08:43:21.421319Z","shell.execute_reply":"2025-06-20T08:43:21.425194Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"id":"-8Ix2pZWXOWy","outputId":"76341a97-2ee8-4eb2-9ba0-b68817bdef74","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.427153Z","iopub.execute_input":"2025-06-20T08:43:21.427479Z","iopub.status.idle":"2025-06-20T08:43:21.452029Z","shell.execute_reply.started":"2025-06-20T08:43:21.427451Z","shell.execute_reply":"2025-06-20T08:43:21.451188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df","metadata":{"id":"JZHd2P8OWfFP","outputId":"d939417c-9074-4f18-86b1-42e3616169d3","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.453349Z","iopub.execute_input":"2025-06-20T08:43:21.453690Z","iopub.status.idle":"2025-06-20T08:43:21.472607Z","shell.execute_reply.started":"2025-06-20T08:43:21.453657Z","shell.execute_reply":"2025-06-20T08:43:21.471666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"id":"qBYqumMAXV16","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.473457Z","iopub.execute_input":"2025-06-20T08:43:21.473769Z","iopub.status.idle":"2025-06-20T08:43:21.937607Z","shell.execute_reply.started":"2025-06-20T08:43:21.473740Z","shell.execute_reply":"2025-06-20T08:43:21.936698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.csv","metadata":{"id":"n0TJGwPiXg9B","outputId":"d012fc2b-bc43-44a9-da56-1c65f44da867","trusted":true,"execution":{"iopub.status.busy":"2025-06-20T08:43:21.938560Z","iopub.execute_input":"2025-06-20T08:43:21.938802Z","iopub.status.idle":"2025-06-20T08:43:22.092059Z","shell.execute_reply.started":"2025-06-20T08:43:21.938783Z","shell.execute_reply":"2025-06-20T08:43:22.091139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"YJpo-tP4Xn2i","trusted":true},"outputs":[],"execution_count":null}]}