{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-15T12:42:31.751502Z","iopub.execute_input":"2024-04-15T12:42:31.753992Z","iopub.status.idle":"2024-04-15T12:42:31.783049Z","shell.execute_reply.started":"2024-04-15T12:42:31.753693Z","shell.execute_reply":"2024-04-15T12:42:31.779602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T12:42:32.806687Z","iopub.execute_input":"2024-04-15T12:42:32.807677Z","iopub.status.idle":"2024-04-15T12:42:42.994861Z","shell.execute_reply.started":"2024-04-15T12:42:32.807615Z","shell.execute_reply":"2024-04-15T12:42:42.99157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom imblearn.over_sampling import SMOTE\nimport pandas as pd\n\n# Assuming you have already loaded the dataset into a DataFrame named 'data'\n\n# Separate features and labels\nX = data['question_text']\ny = data['target']\n\n# Initialize TF-IDF vectorizer\ntfidf_vectorizer = TfidfVectorizer()\n\n# Transform text data into numerical features\nX_tfidf = tfidf_vectorizer.fit_transform(X)\n\n# Initialize SMOTE\nsmote = SMOTE(random_state=42)\n\n# Resample using SMOTE\nX_resampled, y_resampled = smote.fit_resample(X_tfidf, y)\n\n# Convert the resampled data back to DataFrame\nresampled_data = pd.DataFrame(X_resampled.toarray(), columns=tfidf_vectorizer.get_feature_names_out())\nresampled_data['target'] = y_resampled\n\n# Filter resampled data to get only 200 samples for each class\nresampled_data_200_samples = pd.concat([resampled_data[resampled_data['target'] == 0].head(1),\n                                        resampled_data[resampled_data['target'] == 1].head(1)])\n\n# Print the resampled data with 200 samples for each class\nprint(resampled_data_200_samples)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-15T12:42:45.466837Z","iopub.execute_input":"2024-04-15T12:42:45.467593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_zero_target = data[data['target'] == 0].head(10)\nprint(data_zero_target)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. Loading the language library\nimport spacy\nnlp = spacy.load('en_core_web_sm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"question_text\"][:100]\n# print(data.iloc[0: 5]);\ndata[\"question_text\"][0:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2. Building a Pipline Object\ndoc = nlp(data[\"question_text\"].iloc[:3].to_string(index=False))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tabulate import tabulate\n\ntoken_info = []\nfor token in doc:\n    token_info.append([token.text, token.pos_, token.dep_, token.lemma_])\n\nheaders = [\"Token\", \"POS\", \"Dependency\", \"Lemma\"]\n\nprint(tabulate(token_info, headers=headers, tablefmt=\"pretty\"))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:22:45.510152Z","iopub.execute_input":"2024-04-14T15:22:45.510584Z","iopub.status.idle":"2024-04-14T15:22:45.525402Z","shell.execute_reply.started":"2024-04-14T15:22:45.510553Z","shell.execute_reply":"2024-04-14T15:22:45.52394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for entity in doc.ents:\n    print(f\"{entity.text:-<{20}}{entity.label_:-<{20}}{str(spacy.explain(entity.label_))}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.258734Z","iopub.execute_input":"2024-04-14T15:12:23.259204Z","iopub.status.idle":"2024-04-14T15:12:23.268322Z","shell.execute_reply.started":"2024-04-14T15:12:23.259165Z","shell.execute_reply":"2024-04-14T15:12:23.266848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for chunk in doc.noun_chunks:\n    print(chunk.text)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.270193Z","iopub.execute_input":"2024-04-14T15:12:23.27114Z","iopub.status.idle":"2024-04-14T15:12:23.279417Z","shell.execute_reply.started":"2024-04-14T15:12:23.271103Z","shell.execute_reply":"2024-04-14T15:12:23.277994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"doc = nlp(data[\"question_text\"].iloc[:50].to_string(index=False))\nimport spacy\n\nprint(\"{:<30} {:<15} {:<50}\".format(\"Entity\", \"Label\", \"Description\"))\n\nfor entity in doc.ents:\n    print(\"{:<30} {:<15} {:<50}\".format(entity.text, entity.label_, spacy.explain(entity.label_)))","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:43:44.975008Z","iopub.execute_input":"2024-04-14T15:43:44.975776Z","iopub.status.idle":"2024-04-14T15:43:45.096552Z","shell.execute_reply.started":"2024-04-14T15:43:44.975739Z","shell.execute_reply":"2024-04-14T15:43:45.095599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from spacy import displacy\ndisplacy.render(doc, style='dep', jupyter=True, options={'distance':90})","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.285213Z","iopub.execute_input":"2024-04-14T15:12:23.285628Z","iopub.status.idle":"2024-04-14T15:12:23.320853Z","shell.execute_reply.started":"2024-04-14T15:12:23.285597Z","shell.execute_reply":"2024-04-14T15:12:23.319497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"displacy.render(doc, style='ent', jupyter=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.32239Z","iopub.execute_input":"2024-04-14T15:12:23.322853Z","iopub.status.idle":"2024-04-14T15:12:23.33767Z","shell.execute_reply.started":"2024-04-14T15:12:23.322816Z","shell.execute_reply":"2024-04-14T15:12:23.335631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tabulate import tabulate\n\n# Initialize an empty list to store token information\ntoken_info = []\n\n# Iterate over tokens and append token information to the list\nfor token in doc:\n    token_info.append([token.text, token.pos_, token.lemma_, token.lemma_])\n\n# Define headers for the table\nheaders = [\"Token\", \"POS\", \"Lemma ID\", \"Lemma\"]\n\n# Print the table using tabulate with borders\nprint(tabulate(token_info, headers=headers, tablefmt=\"pretty\"))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.339085Z","iopub.execute_input":"2024-04-14T15:12:23.339845Z","iopub.status.idle":"2024-04-14T15:12:23.383343Z","shell.execute_reply.started":"2024-04-14T15:12:23.339765Z","shell.execute_reply":"2024-04-14T15:12:23.381938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This shows the similarity between two questions\ndoc_1 = nlp(data[\"question_text\"][1])\ndoc_2 = nlp(data[\"question_text\"][2])\n\nprint(doc_1.similarity(doc_2))\nprint(doc_2.similarity(doc_1))","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.384794Z","iopub.execute_input":"2024-04-14T15:12:23.385173Z","iopub.status.idle":"2024-04-14T15:12:23.420237Z","shell.execute_reply.started":"2024-04-14T15:12:23.385144Z","shell.execute_reply":"2024-04-14T15:12:23.418398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\ndef calculate_similarity_matrix(questions):\n    num_questions = len(questions)\n    similarity_matrix = np.zeros((num_questions, num_questions))\n    for i in range(num_questions):\n        for j in range(num_questions):\n            similarity_matrix[i, j] = nlp(questions[i]).similarity(nlp(questions[j]))\n    return similarity_matrix\n\nquestions = data[\"question_text\"][:10]  \n\nsimilarity_matrix = calculate_similarity_matrix(questions)\n\nplt.figure(figsize=(10, 8))\nsns.set(font_scale=1.2)\nsns.heatmap(similarity_matrix, annot=True, cmap=\"YlGnBu\", xticklabels=range(1, 11), yticklabels=range(1, 11))\nplt.xlabel(\"Question Index\")\nplt.ylabel(\"Question Index\")\nplt.title(\"Similarity Matrix between Questions\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:23.422553Z","iopub.execute_input":"2024-04-14T15:12:23.423016Z","iopub.status.idle":"2024-04-14T15:12:27.146075Z","shell.execute_reply.started":"2024-04-14T15:12:23.422961Z","shell.execute_reply":"2024-04-14T15:12:27.144478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data[\"question_text\"][3])\nprint(data[\"question_text\"][7])","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:27.147644Z","iopub.execute_input":"2024-04-14T15:12:27.148109Z","iopub.status.idle":"2024-04-14T15:12:27.155729Z","shell.execute_reply.started":"2024-04-14T15:12:27.148075Z","shell.execute_reply":"2024-04-14T15:12:27.153785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport spacy\nfrom spacy import displacy\n\nnlp = spacy.load(\"en_core_web_sm\")\n\nfor text in data[\"question_text\"].iloc[:5]:\n   \n    doc = nlp(text)\n\n    displacy.render(doc, style=\"dep\", jupyter=True)\n\n    displacy.render(doc, style=\"ent\", jupyter=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:27.157823Z","iopub.execute_input":"2024-04-14T15:12:27.158275Z","iopub.status.idle":"2024-04-14T15:12:28.656231Z","shell.execute_reply.started":"2024-04-14T15:12:27.158244Z","shell.execute_reply":"2024-04-14T15:12:28.654425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Working with POS Tags**","metadata":{}},{"cell_type":"code","source":"for text in data[\"question_text\"].iloc[:5]:\n    doc = nlp(text)\n   \n    token_info = []\n    \n    for token in doc:\n        token_info.append([token.text, token.pos_, token.tag_, spacy.explain(token.tag_)])\n\n    headers = [\"Token\", \"POS\", \"Tag\", \"Explanation\"]\n\n    print(tabulate(token_info, headers=headers))\n\n    print(\"-\" * 100) ","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:28.65774Z","iopub.execute_input":"2024-04-14T15:12:28.65871Z","iopub.status.idle":"2024-04-14T15:12:28.733313Z","shell.execute_reply.started":"2024-04-14T15:12:28.658644Z","shell.execute_reply":"2024-04-14T15:12:28.73203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfor text in data[\"question_text\"].iloc[:5]:\n\n    doc = nlp(text)\n    \n    pos_count = doc.count_by(spacy.attrs.POS)\n    \n    pos_count_text = {doc.vocab[key].text: value for key, value in pos_count.items()}\n    plt.figure(figsize=(8, 6))\n    plt.bar(pos_count_text.keys(), pos_count_text.values(), color='skyblue')\n    plt.xlabel('POS Tag')\n    plt.ylabel('Count')\n    plt.title('POS Tag Counts')\n    plt.xticks(rotation=45)  \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:28.734885Z","iopub.execute_input":"2024-04-14T15:12:28.735402Z","iopub.status.idle":"2024-04-14T15:12:31.075153Z","shell.execute_reply.started":"2024-04-14T15:12:28.735368Z","shell.execute_reply":"2024-04-14T15:12:31.073555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize an empty dictionary to store POS tag counts\npos_count_total = {}\n\n# Assuming 'data' is your DataFrame containing text data\nfor text in data[\"question_text\"].iloc[:5]:\n    # Process the text with spaCy\n    doc = nlp(text)\n    \n    # Count POS tags for the current sentence\n    pos_count = doc.count_by(spacy.attrs.POS)\n    \n    # Aggregate POS tag counts across all sentences\n    for key, value in pos_count.items():\n        pos_tag = doc.vocab[key].text\n        pos_count_total[pos_tag] = pos_count_total.get(pos_tag, 0) + value\n\n# Plot the combined POS tag counts using a bar chart\nplt.figure(figsize=(8, 6))\nplt.bar(pos_count_total.keys(), pos_count_total.values(), color='green')\nplt.xlabel('POS Tag')\nplt.ylabel('Count')\nplt.title('Combined POS Tag Counts')\nplt.xticks(rotation=45)  # Rotate x-axis labels for better readability\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:31.077109Z","iopub.execute_input":"2024-04-14T15:12:31.077562Z","iopub.status.idle":"2024-04-14T15:12:31.632862Z","shell.execute_reply.started":"2024-04-14T15:12:31.077529Z","shell.execute_reply":"2024-04-14T15:12:31.630968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos = []\nlemma = []\ntext = []\n\nfor tok in doc:\n    pos.append(tok.pos_)\n    lemma.append(tok.lemma_)\n    text.append(tok.text)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:31.634092Z","iopub.execute_input":"2024-04-14T15:12:31.634499Z","iopub.status.idle":"2024-04-14T15:12:31.64215Z","shell.execute_reply.started":"2024-04-14T15:12:31.634469Z","shell.execute_reply":"2024-04-14T15:12:31.64056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlp_table =  pd.DataFrame({\"Text\": text , \"Lemma\": lemma, \"PoS\": pos})\nnlp_table","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:31.643888Z","iopub.execute_input":"2024-04-14T15:12:31.644347Z","iopub.status.idle":"2024-04-14T15:12:31.664886Z","shell.execute_reply.started":"2024-04-14T15:12:31.644314Z","shell.execute_reply":"2024-04-14T15:12:31.663805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noun = []\n\nfor review in data[\"question_text\"].iloc[:10000]:\n    doc = nlp(review)\n    for tok in doc:\n        if tok.pos_ == 'NOUN':\n            noun.append(tok.lemma_.lower())\n            \n\npd.Series(noun).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:12:31.666135Z","iopub.execute_input":"2024-04-14T15:12:31.667327Z","iopub.status.idle":"2024-04-14T15:14:16.10356Z","shell.execute_reply.started":"2024-04-14T15:12:31.667291Z","shell.execute_reply":"2024-04-14T15:14:16.100738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Assuming data[\"question_text\"] contains your list of questions and nlp is your spaCy model\n# Make sure you have imported the necessary libraries and initialized the spaCy model (nlp)\n\n# Create empty dictionary to store POS tag counts\npos_counts = {}\n\n# Iterate through each review in the dataset\nfor review in data[\"question_text\"].iloc[:10000]:\n    doc = nlp(review)\n    # Iterate through each token in the review\n    for token in doc:\n        pos_tag = token.pos_\n        # Check if POS tag is already in the dictionary, if not, add it with count 1\n        if pos_tag not in pos_counts:\n            pos_counts[pos_tag] = 1\n        else:\n            # If POS tag is already in the dictionary, increment its count by 1\n            pos_counts[pos_tag] += 1\n\n# Convert dictionary to pandas Series for easier manipulation\npos_counts_series = pd.Series(pos_counts)\n\n# Sort the Series by counts in descending order\npos_counts_series_sorted = pos_counts_series.sort_values(ascending=False)\n\n# Plot the distribution of POS tags\nplt.figure(figsize=(10, 6))\npos_counts_series_sorted.plot(kind='bar')\nplt.title('Distribution of POS Tags in Questions')\nplt.xlabel('POS Tag')\nplt.ylabel('Count')\nplt.xticks(rotation=45)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:14:16.105028Z","iopub.execute_input":"2024-04-14T15:14:16.105384Z","iopub.status.idle":"2024-04-14T15:15:54.44118Z","shell.execute_reply.started":"2024-04-14T15:14:16.105356Z","shell.execute_reply":"2024-04-14T15:15:54.439768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#We now know that people mention people, way, year etc. But we still don't know in what context they mention these keywords","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.442824Z","iopub.execute_input":"2024-04-14T15:15:54.443979Z","iopub.status.idle":"2024-04-14T15:15:54.449091Z","shell.execute_reply.started":"2024-04-14T15:15:54.443937Z","shell.execute_reply":"2024-04-14T15:15:54.447688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Extract all the prefixes and suffixes of \"people\"","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.450561Z","iopub.execute_input":"2024-04-14T15:15:54.450904Z","iopub.status.idle":"2024-04-14T15:15:54.463199Z","shell.execute_reply.started":"2024-04-14T15:15:54.450876Z","shell.execute_reply":"2024-04-14T15:15:54.46188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\npattern = re.compile(r\"\\b\\w+\\speople\\s\\w+\\b\")  \n\ntext = \" \".join(data[\"question_text\"].iloc[:1000].astype(str))\n\nprefixes_suffixes = re.findall(pattern, text)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.464769Z","iopub.execute_input":"2024-04-14T15:15:54.465257Z","iopub.status.idle":"2024-04-14T15:15:54.485959Z","shell.execute_reply.started":"2024-04-14T15:15:54.465192Z","shell.execute_reply":"2024-04-14T15:15:54.48459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tabulate import tabulate\n\noccurrences_data = []\n\nfor idx, occurrence in enumerate(prefixes_suffixes):\n    occurrences_data.append([idx + 1, occurrence])\n\nprint(tabulate(occurrences_data, headers=[\"Index\", \"Occurrence\"]))\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.487615Z","iopub.execute_input":"2024-04-14T15:15:54.487978Z","iopub.status.idle":"2024-04-14T15:15:54.500754Z","shell.execute_reply.started":"2024-04-14T15:15:54.487937Z","shell.execute_reply":"2024-04-14T15:15:54.499141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prefixes = []\nsuffixes = []\nfor p in prefixes_suffixes:\n    l = p.split(\" \")\n    prefixes.append(l[0].lower())\n    suffixes.append(l[-1].lower())","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.508398Z","iopub.execute_input":"2024-04-14T15:15:54.508834Z","iopub.status.idle":"2024-04-14T15:15:54.517506Z","shell.execute_reply.started":"2024-04-14T15:15:54.508802Z","shell.execute_reply":"2024-04-14T15:15:54.516129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(prefixes).value_counts().head(5)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.519017Z","iopub.execute_input":"2024-04-14T15:15:54.51936Z","iopub.status.idle":"2024-04-14T15:15:54.531413Z","shell.execute_reply.started":"2024-04-14T15:15:54.519333Z","shell.execute_reply":"2024-04-14T15:15:54.530503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(suffixes).value_counts().head(5)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.532765Z","iopub.execute_input":"2024-04-14T15:15:54.533381Z","iopub.status.idle":"2024-04-14T15:15:54.546841Z","shell.execute_reply.started":"2024-04-14T15:15:54.533353Z","shell.execute_reply":"2024-04-14T15:15:54.545514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prefixes=pd.Series(prefixes).value_counts().head(5).index\nsuffixes=pd.Series(suffixes).value_counts().head(5).index","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.548543Z","iopub.execute_input":"2024-04-14T15:15:54.549247Z","iopub.status.idle":"2024-04-14T15:15:54.558795Z","shell.execute_reply.started":"2024-04-14T15:15:54.549193Z","shell.execute_reply":"2024-04-14T15:15:54.557722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({'prefixes':prefixes,'keyword':['people']*len(prefixes),'suffixes':suffixes})","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.560535Z","iopub.execute_input":"2024-04-14T15:15:54.561457Z","iopub.status.idle":"2024-04-14T15:15:54.578795Z","shell.execute_reply.started":"2024-04-14T15:15:54.561415Z","shell.execute_reply":"2024-04-14T15:15:54.577603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_context(reviews,keyword):\n    pattern = re.compile(f\"\\w+\\s{keyword}\\s\\w+\")\n    prefixes_suffixes = re.findall(pattern,reviews)\n    prefixes = []\n    suffixes = []\n    for p in prefixes_suffixes:\n        l = p.split(\" \")\n        prefixes.append(l[0].lower())\n        suffixes.append(l[-1].lower())\n    prefixes = [p for p in prefixes if p not in stop_words]\n    suffixes = [s for s in suffixes if s not in stop_words]\n    prefixes=pd.Series(prefixes).value_counts().head(5).index\n    suffixes=pd.Series(suffixes).value_counts().head(5).index\n    return pd.DataFrame({'prefixes':prefixes,'keyword':[f'{keyword}']*len(prefixes),'suffixes':suffixes})","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.580229Z","iopub.execute_input":"2024-04-14T15:15:54.581246Z","iopub.status.idle":"2024-04-14T15:15:54.59144Z","shell.execute_reply.started":"2024-04-14T15:15:54.581213Z","shell.execute_reply":"2024-04-14T15:15:54.590193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get_context(data[\"question_text\"].iloc[:1,:1000].astype(str),\"india\")","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.595545Z","iopub.execute_input":"2024-04-14T15:15:54.595948Z","iopub.status.idle":"2024-04-14T15:15:54.603777Z","shell.execute_reply.started":"2024-04-14T15:15:54.595903Z","shell.execute_reply":"2024-04-14T15:15:54.602701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\n\n# Download the NLTK stop words dataset\nnltk.download('stopwords')\n\n# Get the set of English stop words\nstop_words = set(stopwords.words('english'))","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.605375Z","iopub.execute_input":"2024-04-14T15:15:54.606232Z","iopub.status.idle":"2024-04-14T15:15:54.688375Z","shell.execute_reply.started":"2024-04-14T15:15:54.60619Z","shell.execute_reply":"2024-04-14T15:15:54.687163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#How Dependency Parsing Works?","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.690026Z","iopub.execute_input":"2024-04-14T15:15:54.690503Z","iopub.status.idle":"2024-04-14T15:15:54.696595Z","shell.execute_reply.started":"2024-04-14T15:15:54.690461Z","shell.execute_reply":"2024-04-14T15:15:54.695161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nactive = ['Hens lay eggs.',\n         'Birds build nests.',\n         'The batter hit the ball.',\n         'The computer transmitted a copy of the manual']\npassive = ['Eggs are laid by hens',\n           'Nests are built by birds',\n           'The ball was hit by the batter',\n           'A copy of the manual was transmitted by the computer.']","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.699057Z","iopub.execute_input":"2024-04-14T15:15:54.6996Z","iopub.status.idle":"2024-04-14T15:15:54.709401Z","shell.execute_reply.started":"2024-04-14T15:15:54.699482Z","shell.execute_reply":"2024-04-14T15:15:54.708018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for sent in active:\n    doc = nlp(sent)\n    displacy.render(doc, style = 'dep')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.711531Z","iopub.execute_input":"2024-04-14T15:15:54.712038Z","iopub.status.idle":"2024-04-14T15:15:54.798196Z","shell.execute_reply.started":"2024-04-14T15:15:54.711996Z","shell.execute_reply":"2024-04-14T15:15:54.797278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for sent in passive:\n    doc = nlp(sent)\n    displacy.render(doc, style = \"dep\")","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.799164Z","iopub.execute_input":"2024-04-14T15:15:54.799503Z","iopub.status.idle":"2024-04-14T15:15:54.88463Z","shell.execute_reply.started":"2024-04-14T15:15:54.799475Z","shell.execute_reply":"2024-04-14T15:15:54.883631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Summary:\n\n#     Spacy's dependency parser let's us visualise the relationships\n#     When a sentence is in passive voice there is always a presence if nsubjpass dependency relation\n\n# #","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.886152Z","iopub.execute_input":"2024-04-14T15:15:54.887136Z","iopub.status.idle":"2024-04-14T15:15:54.892546Z","shell.execute_reply.started":"2024-04-14T15:15:54.8871Z","shell.execute_reply":"2024-04-14T15:15:54.891119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NER (Name- Entity- Recognisation)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.894192Z","iopub.execute_input":"2024-04-14T15:15:54.894541Z","iopub.status.idle":"2024-04-14T15:15:54.903333Z","shell.execute_reply.started":"2024-04-14T15:15:54.894514Z","shell.execute_reply":"2024-04-14T15:15:54.902155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.904712Z","iopub.execute_input":"2024-04-14T15:15:54.905989Z","iopub.status.idle":"2024-04-14T15:15:54.914797Z","shell.execute_reply.started":"2024-04-14T15:15:54.905953Z","shell.execute_reply":"2024-04-14T15:15:54.913453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmodel = spacy.load(\"en_core_web_sm\") #load pre-trained model\n\n\n\n# processed_doc = model(data); #process input and perform NLP tasks\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:54.916355Z","iopub.execute_input":"2024-04-14T15:15:54.916749Z","iopub.status.idle":"2024-04-14T15:15:56.188043Z","shell.execute_reply.started":"2024-04-14T15:15:54.916688Z","shell.execute_reply":"2024-04-14T15:15:56.18696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"doc = data[\"question_text\"].iloc[:1000]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:56.189473Z","iopub.execute_input":"2024-04-14T15:15:56.18979Z","iopub.status.idle":"2024-04-14T15:15:56.196481Z","shell.execute_reply.started":"2024-04-14T15:15:56.189764Z","shell.execute_reply":"2024-04-14T15:15:56.193431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_docs = []\n\n# Iterate through each text in the data and process it using the model\nfor text in doc:\n    doc = model(text)\n    processed_docs.append(doc)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:15:56.198084Z","iopub.execute_input":"2024-04-14T15:15:56.19856Z","iopub.status.idle":"2024-04-14T15:16:07.139539Z","shell.execute_reply.started":"2024-04-14T15:15:56.198528Z","shell.execute_reply":"2024-04-14T15:16:07.138436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"doc.ents","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:16:07.140977Z","iopub.execute_input":"2024-04-14T15:16:07.141856Z","iopub.status.idle":"2024-04-14T15:16:07.151945Z","shell.execute_reply.started":"2024-04-14T15:16:07.141812Z","shell.execute_reply":"2024-04-14T15:16:07.149413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for token in doc:\n    \n    print(token.text, '--', token.pos_)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:16:07.154074Z","iopub.execute_input":"2024-04-14T15:16:07.155067Z","iopub.status.idle":"2024-04-14T15:16:07.168014Z","shell.execute_reply.started":"2024-04-14T15:16:07.15502Z","shell.execute_reply":"2024-04-14T15:16:07.166619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ngrammar = {\n    'S': [['S', '+', 'E'], ['E']],\n    'E': [['num'], ['(', 'S', ')']]\n}\n\n\ndef remove_left_recursion(grammar):\n    non_terminals = list(grammar.keys())\n    new_grammar = {}\n    \n    for A in non_terminals:\n        productions = grammar[A]\n        left_recursive = []\n        right_recursive = []\n        \n        for production in productions:\n            if production[0] == A:\n                left_recursive.append(production[1:])\n            else:\n                right_recursive.append(production)\n        \n        if left_recursive:\n            A_prime = A + \"'\"\n            new_grammar[A] = []\n            new_grammar[A_prime] = []\n            \n            for production in right_recursive:\n                new_grammar[A].append(production + [A_prime])\n                \n            new_grammar[A_prime].append(['ε'])\n            for production in left_recursive:\n                new_grammar[A_prime].append(production + [A_prime])\n        else:\n            new_grammar[A] = productions\n    \n    return new_grammar\n\nnew_grammar = remove_left_recursion(grammar)\nprint(\"Grammar before left recursion removal:\")\nfor key, value in grammar.items():\n    print(f\"{key} -> {' | '.join([' '.join(prod) for prod in value])}\")\nprint(\"Grammar after left recursion removal:\")\nfor key, value in new_grammar.items():\n    print(f\"{key} -> {' | '.join([' '.join(prod) for prod in value])}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:16:07.170243Z","iopub.execute_input":"2024-04-14T15:16:07.17062Z","iopub.status.idle":"2024-04-14T15:16:07.187176Z","shell.execute_reply.started":"2024-04-14T15:16:07.170591Z","shell.execute_reply":"2024-04-14T15:16:07.185689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom nltk import PCFG\nfrom nltk.parse import pchart\nfrom nltk.tokenize import word_tokenize\n\npcfg_grammar = PCFG.fromstring(\"\"\"\n    S -> NP VP [1.0]\n    NP -> Det N [0.7] | NP PP [0.3]\n    PP -> P NP [1.0]\n    VP -> V NP [0.6] | VP PP [0.4]\n    Det -> 'the' [0.8] | 'a' [0.2]\n    N -> 'man' [0.5] | 'telescope' [0.5]\n    V -> 'saw' [1.0]\n    P -> 'with' [1.0]\n\"\"\")\n\nparser = pchart.InsideChartParser(pcfg_grammar)\n\nsentence = \"the man saw a telescope with the telescope\"\n\ntokens = word_tokenize(sentence)\n\nfor i, tree in enumerate(parser.parse(tokens)):\n    print(tree)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:16:07.189023Z","iopub.execute_input":"2024-04-14T15:16:07.189391Z","iopub.status.idle":"2024-04-14T15:16:07.21084Z","shell.execute_reply.started":"2024-04-14T15:16:07.189362Z","shell.execute_reply":"2024-04-14T15:16:07.209204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnltk.download()\nfrom nltk.corpus import wordnet","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:16:07.21571Z","iopub.execute_input":"2024-04-14T15:16:07.21621Z","iopub.status.idle":"2024-04-14T15:22:10.263775Z","shell.execute_reply.started":"2024-04-14T15:16:07.21617Z","shell.execute_reply":"2024-04-14T15:22:10.261362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip /usr/share/nltk_data/corpora/wordnet.zip -d /usr/share/nltk_data/corpora/","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:22:10.265176Z","iopub.status.idle":"2024-04-14T15:22:10.266231Z","shell.execute_reply.started":"2024-04-14T15:22:10.265948Z","shell.execute_reply":"2024-04-14T15:22:10.26597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom nltk.corpus import wordnet\n\n# Function to identify ambiguous words in a sentence\ndef identify_ambiguous_words(sentence):\n    tokens = nltk.word_tokenize(sentence)\n    ambiguous_words = []\n    for token in tokens:\n        synsets = wordnet.synsets(token)\n        if len(synsets) > 1:  # If the word has more than one synset, it's ambiguous\n            ambiguous_words.append(token)\n    return ambiguous_words\n\n# Function to disambiguate words based on context\ndef disambiguate_word(word):\n    synsets = wordnet.synsets(word)\n    # For simplicity, let's assume we select the first synset as the most common one\n    if synsets:\n        return synsets[0].definition()\n    else:\n        return None  # Return None if no synsets found\n\n# Example sentence\nsentence = \"I saw a bat flying in the sky.\"\n\n# Identify ambiguous words\nambiguous_words = identify_ambiguous_words(sentence)\nprint(\"Ambiguous words:\")\nfor word in ambiguous_words:\n    print(word)\n\n# Disambiguate each ambiguous word and replace it in the sentence\ndisambiguated_sentence = sentence\nfor word in ambiguous_words:\n    disambiguated_word = disambiguate_word(word)\n    if disambiguated_word:\n        disambiguated_sentence = disambiguated_sentence.replace(word, disambiguated_word)\n\nprint(\"\\nDisambiguated sentence:\")\nprint(disambiguated_sentence)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:22:10.268144Z","iopub.status.idle":"2024-04-14T15:22:10.269026Z","shell.execute_reply.started":"2024-04-14T15:22:10.268716Z","shell.execute_reply":"2024-04-14T15:22:10.268741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\n\n# Example sentence\nsentence = \"The quick brown fox jumps over the lazy dog.\"\n\n# Tokenize the sentence\ntokens = nltk.word_tokenize(sentence)\n\n# Perform part-of-speech tagging\ntagged_tokens = nltk.pos_tag(tokens)\n\n# Define a chunk grammar to extract noun phrases (NP)\nchunk_grammar = r\"\"\"\n    NP: {<DT|JJ|NN.*>+}          # Chunk sequences of determiner, adjective, noun\n\"\"\"\n\n# Create a chunk parser with the defined grammar\nchunk_parser = nltk.RegexpParser(chunk_grammar)\n\n# Apply chunking to the tagged tokens\nchunks = chunk_parser.parse(tagged_tokens)\n\n# Print the chunks\nfor chunk in chunks:\n    if isinstance(chunk, nltk.tree.Tree) and chunk.label() == 'NP':\n        print(\" \".join([token[0] for token in chunk.leaves()]))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:22:10.270598Z","iopub.status.idle":"2024-04-14T15:22:10.271189Z","shell.execute_reply.started":"2024-04-14T15:22:10.270887Z","shell.execute_reply":"2024-04-14T15:22:10.270931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\n\ndef shallow_parse(sentence):\n    tokens = nltk.word_tokenize(sentence)\n    tagged_tokens = nltk.pos_tag(tokens)\n    chunk_grammar = r\"\"\"\n        NP: {<DT|JJ|NN.*>+}         \n    \"\"\"\n    chunk_parser = nltk.RegexpParser(chunk_grammar)\n    chunks = chunk_parser.parse(tagged_tokens)\n    noun_phrases = []\n    \n    for chunk in chunks:\n        if isinstance(chunk, nltk.tree.Tree) and chunk.label() == 'NP':\n            noun_phrases.append(\" \".join([token[0] for token in chunk.leaves()]))\n    \n    return noun_phrases\n\nquestions = [\n    \"How did Quebec nationalists see their province as a nation in 1960s?\",\n    \"What province adopted the National Flag of Canada in 1965?\",\n    \"Which province had a referendum to separate from Canada in 1980?\",\n    \"What was the first province to join Confederation?\",\n    \"Which province is the most populous?\",\n    \"What is Canada's second most populous province?\",\n    \"Which province is considered the cultural heart of English Canada?\",\n    \"What province is known for its beautiful coastlines?\"\n]\n\nfor i, question in enumerate(questions, 1):\n    print(f\"Question {i}: {question}\")\n    noun_phrases = shallow_parse(question)\n    print(\"Noun Phrases:\")\n    for np in noun_phrases:\n        print(\"-\", np)\n    print()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T15:22:10.273018Z","iopub.status.idle":"2024-04-14T15:22:10.27358Z","shell.execute_reply.started":"2024-04-14T15:22:10.273304Z","shell.execute_reply":"2024-04-14T15:22:10.273327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}