{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.6"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IS_KAGGLE = 'KAGGLE_KERNEL_RUN_TYPE' in os.environ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if IS_KAGGLE:\n    data_dir = '../input/quora-insincere-questions-classification'\n    train_fname = data_dir + '/train.csv'\n    test_fname = data_dir + '/test.csv'\n    sample_fname = data_dir + '/sample_submission.csv'\nelse:\n    os.environ['KAGGLE_CONFIG_DIR'] = '.'\n    !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data\n    !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data\n    !kaggle competitions download -c quora-insincere-questions-classification -f sample_submission.csv -p data\n    train_fname = 'data/train.csv.zip'\n    test_fname = 'data/test.csv.zip'\n    sample_fname = 'data/sample_submission.csv.zip' ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df = pd.read_csv(train_fname, low_memory=False)\nraw_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df = raw_df[raw_df.target == 0]\nsincere_df.question_text.values[:10]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target == 1]\ninsincere_df.question_text.values[:10]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# normalize = True gives the percentage of counts\nraw_df.target.value_counts(normalize=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True).plot(kind='bar')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Average length of each question is: {np.mean(raw_df.question_text.apply(len))}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(test_fname)\ntest_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df = pd.read_csv(sample_fname)\nsub_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if IS_KAGGLE:\n    SAMPLE_SIZE = len(raw_df)\nelse:\n    SAMPLE_SIZE = 100_000","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df = raw_df.sample(SAMPLE_SIZE, random_state=42)\nsample_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0 = sincere_df.question_text.values[1]\nq0","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1 = insincere_df.question_text.values[0]\nq1","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem.snowball import SnowballStemmer\nimport re","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tokenization\nq0_tok = word_tokenize(q0)\nq1_tok = word_tokenize(q1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\", \".join(stopwords.words('english'))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_stopwords(tokens):\n    return [word for word in tokens if word.lower() not in stopwords.words('english')]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Removing stopwords\nq0_stp = remove_stopwords(q0_tok)\nq0_stp","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Removing stopwords\nq1_stp = remove_stopwords(q1_tok)\nq1_stp","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Perform stemming\nq0_stm = [stemmer.stem(word) for word in q0_stp]\nq0_stm","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Perform stemming\nq1_stm = [stemmer.stem(word) for word in q1_stp]\nq1_stm","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df = sample_df[:5]\nsmall_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df.question_text.values","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv = CountVectorizer(binary=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv.fit(small_df.question_text)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv.vocabulary_","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv.get_feature_names_out()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df.question_text.values[0]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors = cv.transform(small_df.question_text)\nvectors[0].toarray()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tokenize(text):\n    text = re.sub('[^a-zA-Z]', ' ', text)\n    text = text.lower()\n    # Tokenize, remove stopwords, and stemming\n    return [stemmer.stem(word) for word in word_tokenize(text) if word not in stopwords.words('english')]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenize('What is the really (dealing) here?')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer = CountVectorizer(tokenizer=tokenize, max_features=1000, binary=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer.vocabulary_","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.question_text.values[0]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs[0].toarray()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.question_text.values[0]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_inputs[0].toarray()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, sample_df.target, test_size=0.3, random_state=42)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_inputs.shape, train_targets.shape","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_inputs.shape, val_targets.shape","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_ITER = 1000","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = LogisticRegression(max_iter=MAX_ITER, solver='sag')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nmodel.fit(train_inputs, train_targets)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preds = model.predict(train_inputs)\npd.Series(train_preds).value_counts()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_targets.value_counts()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.score(train_inputs, train_targets)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.score(train_inputs, np.zeros(train_targets.shape))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(train_targets, train_preds)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(train_targets, np.zeros(len(train_targets)))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_preds = np.random.choice((0, 1), len(train_targets))\nf1_score(train_targets, random_preds)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds = model.predict(val_inputs)\nval_preds","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.score(val_inputs, val_targets)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(val_targets, val_preds)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.target.values[:10]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.predict(vectorizer.transform(sincere_df.question_text.values[:10]))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.target.values[:10]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.predict(vectorizer.transform(insincere_df.question_text.values[:10]))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds = model.predict(test_inputs)\ntest_preds","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df['prediction'] = test_preds\nsub_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{},"outputs":[],"execution_count":null}]}