{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#First version to have successful rand_forest is V42\n#v64 first to group out rand forest based on pronoun type\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport csv\nimport json\nimport string\nimport keras\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\nfrom math import floor\nimport spacy\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.ensemble import RandomForestClassifier\nimport lightgbm as lgb\n\nimport time\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nimport regex as re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f1e93c7d85e122ea0b00418d2336e48b88b8bbe"},"cell_type":"code","source":"import nltk \nfrom nltk.corpus import stopwords, wordnet\nfrom nltk.tokenize import word_tokenize, sent_tokenize ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90848dfb31f4c60c412d1590307a272fa88ae426"},"cell_type":"code","source":"nlp = spacy.load('en_core_web_sm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8b8f6193e3fd14c6af7d5276dd4473bc41ec54e"},"cell_type":"code","source":"def word_locate(sentence, location): \n    count_words = 0\n    count_chars = 2 #2 is to count for the two spaces in the beginning\n    for word in sentence.split():\n        count_words += 1\n        if location == count_chars:\n            return word, count_words\n        count_chars += len(word)\n        count_chars += 1 #for space","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a99ac743d1d9a5e8e2fc853580180a1578c135a5"},"cell_type":"code","source":"def name_btwn_paran(sentence):\n    capture = \"\"\n    trigger_on = 0\n    for char in sentence:\n        if char == \")\":\n            trigger_on = 0\n        if trigger_on == 1:\n            capture += char\n        if char == \"(\":\n            trigger_on = 1\n    return capture","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba29b95b7b0eedc9121f3fb50d2ad91b767bf4be"},"cell_type":"code","source":"def which_name_first(sentence, name1, name2): #If name1 is first, return True\n    name1_check = 0\n    for word_punct in sentence.split():\n        for word_comma in word_punct.split(\";\"):\n            for word in word_comma.split(\",\"):\n                if word == name2 and name1_check == 0:\n                    return False\n                if word == name1:\n                    name1_check = 1\n    return True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea76e9ad2a9c0721a83d9861f0d0e596a0d0ff0b"},"cell_type":"code","source":"def curr_prev_sentence(sentence, loc):\n    current_sentence = \"\"\n    prev_sentence = \"\"\n    trunc_curr_sentence = \"\"\n    remainder_curr = \"\"\n    detect = 0\n    count = 0\n    for char in sentence:\n        count += 1\n        current_sentence += char\n        remainder_curr += char\n        if ((char == \".\" or char == \";\") and detect == 0 and sentence[count] != \",\"): #the last arguement to prevent ., as in sent #4\n            prev_sentence = current_sentence \n            current_sentence = \"\"\n        if char == \".\" and detect == 1:\n            return current_sentence, prev_sentence, trunc_curr_sentence, remainder_curr\n        if count == loc:\n            detect = 1\n            trunc_curr_sentence = current_sentence\n            remainder_curr = \"\"\n    return current_sentence, prev_sentence, trunc_curr_sentence, remainder_curr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54ece1edcc8031b0210f6fa8b7485d9d4a7f879a"},"cell_type":"code","source":"def remove_last_word(sentence):\n    new_sent = sentence.split()\n    new_sent = new_sent[:-1]\n    return \" \".join(new_sent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76f093c66ed25a9dd49b2c3a965c3b1c72a9897d"},"cell_type":"code","source":"def check_if_capital(word):\n    if word[0] in [\"A\",\"B\",\"C\",\"D\",\"E\",\"F\",\"G\",\"H\",\"I\",\"J\",\"K\",\"L\",\"M\",\"O\",\"P\",\"Q\",\"R\",\"S\",\"T\",\"U\",\"V\",\"W\",\"X\",\"Y\",\"Z\"]:\n        return True\n    else:\n        return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ea8283ddf446769245efef36dbe76ba153ae68b"},"cell_type":"code","source":"def list_of_name_words(tokenized):\n    names_list = []\n    for word_tuple in nltk.pos_tag(tokenized):\n        if word_tuple[1] == \"NNP\":\n            names_list.append(word_tuple[0])\n    return names_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ee705fd419885c98fc2a0f97f5954fd7307d36a"},"cell_type":"code","source":"def check_if_name(tokenized,word):\n    text = tokenized\n    for word_tuple in nltk.pos_tag(text):\n        if word_tuple[0] == word:\n            if word_tuple[1] == \"NNP\":\n                return True\n            else:\n                return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be028e13e5f3664b5455d275584f9cd8e8cbcb7e"},"cell_type":"code","source":"def find_name_words(sentence):\n    name = \"none\"\n    for word in sentence.split():\n        if check_if_capital(word):\n            return word\n    return name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdd4e46fa62b9b423178bc90fcbd7ac007865352"},"cell_type":"code","source":"def remove_first_word(sentence):\n    new_sent = sentence.split()\n    new_sent = new_sent[1:]\n    return \" \".join(new_sent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b612fad4260b0213b0ee30d0c65b4eb41acd912"},"cell_type":"code","source":"def find_nth_subj(doc, n): #finds subject number n\n    subject = \"none\"\n    count = 0\n    for token in doc:\n        if (token.dep_ == \"nsubj\" or token.dep_ == \"nsubjpass\"):\n            count += 1\n            if count == n:\n                subject = token.text\n    return subject","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"208f1f321017e0caba7a2ca2f78d430c7f4faba1"},"cell_type":"code","source":"def find_nth_dobj(doc, n): #finds direct object number n\n    dobj = \"none\"\n    count = 0\n    for token in doc:\n        if (token.dep_ == \"dobj\"):\n            count += 1\n            if count == n:\n                dobj = token.text\n    return dobj","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3e64ec88351d714da9df40135f936c9f4d1a247"},"cell_type":"code","source":"def find_nth_poss(doc, n): #finds possessing noun number n\n    poss = \"none\"\n    count = 0\n    for token in doc:\n        if (token.dep_ == \"poss\"):\n            count += 1\n            if count == n:\n                poss = token.text\n    return poss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0023cb6a761557173a1b0e7240cf98801a0d90e6"},"cell_type":"code","source":"def find_nth_appos(doc, n): #finds appos number n; sometimes Spacy mislabels nsubj as appos\n    appos = \"none\"\n    count = 0\n    for token in doc:\n        if (token.dep_ == \"appos\"):\n            count += 1\n            if count == n:\n                appos = token.text\n    return appos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b5fa3f00ad50948f3b699df5c189eba45b21669"},"cell_type":"code","source":"def check_if_poss_her(doc, pronoun): #tells whether it is her as in his or her as in him\n    #assumes only one her in the whole sentence (inaccurate?)\n    for token in doc:\n        if token.text == pronoun:\n            if token.dep_ == \"poss\":\n                return True\n            else:\n                return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e708ac21a1503112b9c7cad39951bfbda933d09"},"cell_type":"code","source":"with open('../input/gap-other-training-data/gap-training.tsv') as tsvfile:\n#with open('../input/gap-training-data/gap-development.tsv') as tsvfile:\n    \n    reader = csv.DictReader(tsvfile, dialect='excel-tab')\n    \n    train_ids= []\n    text_list = []\n    pronoun_list = []\n    pronoun_offset_list = []\n    correct_name_list = []\n    dict_of_all_list = []\n    \n    sent_num = 0\n    \n    p_f_s = [] #prev first subject\n    p_l_s = [] #prev last subject\n    p_f_o = [] #prev first object\n    p_l_o = [] #prev last object\n    tc_f_s = [] #trunc curr first subject\n    tc_f_o = [] #trunc curr first obj\n    tc_f_a = [] #trunc curr first aposs\n    tc_l_s = [] #trunc curr last subject\n    tc_l_o = [] #trunc curr last obj\n    tc_l_p = [] #trunc curr last poss\n    p_f_wp = [] #prev first word between paranthesis\n    tc_l_wp = [] #curr word between paranthesis\n    tc_l_nw = [] #last name word other than a subj in trunc curr\n    r_f_s = [] #remainder first subj\n    r_f_o = [] #remainder first object\n    r_f_a = [] #remainder first appos\n    p_f_a = [] #prev first appos\n    c_f_a = [] #curr first appos\n    poss_her = [] #possessive her true or false\n    pro_typ = [] #pronoun type\n    \n    p_f_s_clf = [] #prev first subject Random Forest labels\n    p_l_s_clf = [] #prev last subject Random Forest labels\n    p_f_o_clf = [] #prev first object Random Forest labels\n    p_l_o_clf = [] #prev last object Random Forest labels\n    tc_f_s_clf = [] #trunc curr first subject Random Forest labels\n    tc_f_o_clf = [] #trunc curr first obj Random Forest labels\n    tc_f_a_clf = [] #trunc curr first aposs Random Forest labels\n    tc_l_s_clf = [] #trunc curr last subject Random Forest labels\n    tc_l_o_clf = [] #trunc curr last obj Random Forest labels\n    tc_l_p_clf = [] #trunc curr last poss Random Forest labels\n    p_f_wp_clf = [] #prev first wb para Random Forest labels\n    tc_l_wp_clf = [] #curr wb para Random Forest labels\n    tc_l_nw_clf = [] #last name word in trunc curr Random Forest labels\n    r_f_s_clf = [] #remainder first subj Random Forest labels\n    r_f_o_clf = [] #remainder first object Random Forest labels\n    r_f_a_clf = [] #remainder first appos Random Forest labels\n    p_f_a_clf = [] #prev first appos Random Forest labels\n    c_f_a_clf = [] #curr first appos Random Forest labels\n    pro_typ_clf = [] #pronoun type for Rand Forest\n    train_idx = [] #training data indices. We do not want those with \"neither\" \n    \n    for row in reader:\n        \n        train_ids.append(row['ID'])\n        text = row['Text']\n        text_list.append(text)\n        dict_of_all = {}\n        \n        proffset = int(row['Pronoun-offset']) \n        pronoun_offset_list.append(proffset)\n        \n        pronoun = row['Pronoun']\n        pronoun_list.append(pronoun)\n        \n        if row['A-coref'] == 'TRUE':\n            correct_name_list.append(row['A'])\n            train_idx.append(sent_num)\n        elif row['B-coref'] == 'TRUE':\n            correct_name_list.append(row['B'])\n            train_idx.append(sent_num)\n        else:\n            correct_name_list.append('Neither')\n        \n        sent_num += 1\n        \n        curr, prev, trunc_curr, remainder = curr_prev_sentence(text, proffset)\n        curr_doc = nlp(curr)\n        prev_doc = nlp(prev) \n        curr_tok = word_tokenize(curr)\n        prev_tok = word_tokenize(prev)\n        trunc_curr_tok = word_tokenize(trunc_curr)\n        \n        #get first subj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_s = \"none\"\n        for n in [1,2,3,4,5]: #number of n is from common sense\n            dummy_p_f_s = find_nth_subj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_f_s) and get_p_f_s == \"none\":\n                get_p_f_s = dummy_p_f_s\n        \n        ####For sentence no. 5, spacy and nltk both failed to identify Collins as a propn.\n        ### therefore, we will add a new line here making sure we have a name.\n        \n        if get_p_f_s == \"none\":\n            if check_if_capital(find_nth_subj(prev_doc,1)):\n                get_p_f_s = find_nth_subj(prev_doc,1)\n\n        p_f_s.append(get_p_f_s)\n        \n        ### pfs Random forest classifier label special line:\n        if get_p_f_s in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_p_f_s: #last input of correct name\n            p_f_s_clf.append(1)\n        else:\n            p_f_s_clf.append(0)\n        \n        #get last  subj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_l_s = \"none\"\n        for n in [1,2,3,4,5]:\n            dummy_p_l_s = find_nth_subj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_l_s):\n                get_p_l_s = dummy_p_l_s\n        \n        p_l_s.append(get_p_l_s)\n        \n        ### pls Random forest classifier label special line:\n        if get_p_l_s in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_p_l_s: #last input of correct name\n            p_l_s_clf.append(1)\n        else:\n            p_l_s_clf.append(0)\n        \n        #get first  obj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_o = \"none\"\n        for n in [1,2,3,4,5]: \n            dummy_p_f_o = find_nth_dobj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_f_o) and get_p_f_o == \"none\":\n                get_p_f_o = dummy_p_f_o\n        \n        p_f_o.append(get_p_f_o)\n        \n        ### pfo Random forest classifier label special line:\n        if get_p_f_o in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_p_f_o: #last input of correct name\n            p_f_o_clf.append(1)\n        else:\n            p_f_o_clf.append(0)\n            \n        #get last  dobj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_l_o = \"none\"\n        for n in [1,2,3,4,5]: \n            dummy_p_l_o = find_nth_dobj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_l_o):\n                get_p_l_o = dummy_p_l_o\n        \n        p_l_o.append(get_p_l_o)\n        \n        ### plo Random forest classifier label special line:\n        if get_p_l_o in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_p_l_o: #last input of correct name\n            p_l_o_clf.append(1)\n        else:\n            p_l_o_clf.append(0)\n        \n        #get last  subj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_s = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_l_s = find_nth_subj(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_l_s)\\\n                    and (dummy_tc_l_s in trunc_curr): #this is slightly inaccurate but oh well\n                get_tc_l_s = dummy_tc_l_s \n            \n        tc_l_s.append(get_tc_l_s)\n        \n        ### tcls Random forest classifier label special line:\n        if get_tc_l_s in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_l_s: #last input of correct name\n            tc_l_s_clf.append(1)\n        else:\n            tc_l_s_clf.append(0)\n            \n        #get last  dobj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_o = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_l_o = find_nth_dobj(curr_doc,n)\n            if (dummy_tc_l_o in trunc_curr)\\\n                                        and check_if_name(curr_tok,dummy_tc_l_o): \n                get_tc_l_o = dummy_tc_l_o \n            \n        tc_l_o.append(get_tc_l_o)\n        \n        ### tclo Random forest classifier label special line:\n        if get_tc_l_o in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_l_o: #last input of correct name\n            tc_l_o_clf.append(1)\n        else:\n            tc_l_o_clf.append(0)\n        \n        #get last  poss in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_p = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_l_p = find_nth_poss(curr_doc,n)\n            if (dummy_tc_l_p in trunc_curr)\\\n                                        and check_if_name(curr_tok,dummy_tc_l_p): \n                get_tc_l_p = dummy_tc_l_p \n            \n        tc_l_p.append(get_tc_l_p)\n        \n        ### tclp Random forest classifier label special line:\n        if get_tc_l_p in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_l_p: #last input of correct name\n            tc_l_p_clf.append(1)\n        else:\n            tc_l_p_clf.append(0)\n        \n        #get first subj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_f_s = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_f_s = find_nth_subj(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_f_s) and get_tc_f_s == \"none\":\n                get_tc_f_s = dummy_tc_f_s \n            \n        tc_f_s.append(get_tc_f_s)\n        \n        ### tcfs Random forest classifier label special line:\n        if get_tc_f_s in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_f_s: #last input of correct name\n            tc_f_s_clf.append(1)\n        else:\n            tc_f_s_clf.append(0)\n            \n        #get first dobj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_f_o = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_f_o = find_nth_dobj(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_f_o) and get_tc_f_o == \"none\": \n                get_tc_f_o = dummy_tc_f_o \n            \n        tc_f_o.append(get_tc_f_o)\n        \n        ### tcfo Random forest classifier label special line:\n        if get_tc_f_o in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_f_o: #last input of correct name\n            tc_f_o_clf.append(1)\n        else:\n            tc_f_o_clf.append(0)\n    \n        #get last  non-subj name word  in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_nw = \"none\"\n        candidate = \"none\"\n        tc_name_words = list_of_name_words(trunc_curr_tok) \n        if len(tc_name_words) > 0:\n            candidate = tc_name_words[-1]\n        if candidate in get_tc_f_s or candidate in get_tc_l_s:\n            if len(tc_name_words) > 1:\n                candidate = tc_name_words[-1]\n        if check_if_name(curr_tok,candidate):\n            get_tc_l_nw = candidate\n        \n        tc_l_nw.append(get_tc_l_nw)\n        \n        ### tclnw Random forest classifier label special line:\n        if get_tc_l_nw in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_l_nw: #last input of correct name\n            tc_l_nw_clf.append(1)\n        else:\n            tc_l_nw_clf.append(0)\n        \n        #get first aposs in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_f_a = find_nth_appos(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_f_a) and get_tc_f_a == \"none\": \n                get_tc_f_a = dummy_tc_f_a \n            \n        tc_f_a.append(get_tc_f_a)\n        \n        ### tcfa Random forest classifier label special line:\n        if get_tc_f_a in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_f_a: #last input of correct name\n            tc_f_a_clf.append(1)\n        else:\n            tc_f_a_clf.append(0)\n    \n        #get word btwn paranthesis in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_wp = find_name_words(name_btwn_paran(prev))\n        \n        if check_if_name(prev_tok,get_p_f_wp): #Add only proper nouns into list\n            p_f_wp.append(get_p_f_wp)\n        else:\n            p_f_wp.append(\"none\")\n        \n        ### pfwp Random forest classifier label special line:\n        if get_p_f_wp in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_p_f_wp: #last input of correct name\n            p_f_wp_clf.append(1)\n        else:\n            p_f_wp_clf.append(0)\n            \n        #get word btwn paranthesis in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_wp = find_name_words(name_btwn_paran(curr))  \n        \n        if check_if_name(curr_tok,get_tc_l_wp): #Add only proper nouns into list\n            tc_l_wp.append(get_tc_l_wp)\n        else:\n            tc_l_wp.append(\"none\")\n            \n        ### tclwp Random forest classifier label special line:\n        if get_tc_l_wp in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_tc_l_wp: #last input of correct name\n            tc_l_wp_clf.append(1)\n        else:\n            tc_l_wp_clf.append(0)\n            \n        #get last subj in remainder @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_r_f_s = \"none\"\n        for n in [1,2,3,4,5,6,7,8]: #in the final version, each of the name subjects will be accunted for\n            dummy_r_f_s = find_nth_subj(curr_doc,n)\n            if dummy_r_f_s in remainder and check_if_name(curr_tok,dummy_r_f_s):\n                get_r_f_s = dummy_r_f_s \n            \n        r_f_s.append(get_r_f_s)\n        \n        ### rfs Random forest classifier label special line:\n        if get_r_f_s in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_r_f_s: #last input of correct name\n            r_f_s_clf.append(1)\n        else:\n            r_f_s_clf.append(0)\n            \n        #get last dobj in remainder @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_r_f_o = \"none\"\n        for n in [1,2,3,4,5,6,7,8]: #in the final version, each of the name objects will be accunted for\n            dummy_r_f_o = find_nth_dobj(curr_doc,n)\n            if dummy_r_f_o in remainder and check_if_name(curr_tok,dummy_r_f_o):\n                get_r_f_o = dummy_r_f_o \n            \n        r_f_o.append(get_r_f_o)\n        \n        ### rfo Random forest classifier label special line:\n        if get_r_f_o in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_r_f_o: #last input of correct name\n            r_f_o_clf.append(1)\n        else:\n            r_f_o_clf.append(0)\n            \n        #get last appos in remainder @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_r_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_r_f_a = find_nth_appos(curr_doc,n)\n            if dummy_r_f_a in remainder and check_if_name(curr_tok,dummy_r_f_a): \n                get_r_f_a = dummy_r_f_a \n            \n        r_f_a.append(get_r_f_a)\n        \n        ### rfa Random forest classifier label special line:\n        if get_r_f_a in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_r_f_a: #last input of correct name\n            r_f_a_clf.append(1)\n        else:\n            r_f_a_clf.append(0)\n        \n        #get first appos in current @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_c_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_c_f_a = find_nth_appos(curr_doc,n)\n            if check_if_name(curr_tok,dummy_c_f_a) and get_c_f_a == \"none\": \n                get_c_f_a = dummy_c_f_a \n            \n        c_f_a.append(get_c_f_a)\n        \n        ### cfa Random forest classifier label special line:\n        if get_c_f_a in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_c_f_a: #last input of correct name\n            c_f_a_clf.append(1)\n        else:\n            c_f_a_clf.append(0)\n        \n        #get first appos in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_p_f_a = find_nth_appos(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_f_a) and get_p_f_a == \"none\": \n                get_p_f_a = dummy_p_f_a \n            \n        p_f_a.append(get_p_f_a)\n        \n        ### pfa Random forest classifier label special line:\n        if get_p_f_a in correct_name_list[-1]\\\n                or correct_name_list[-1] in get_p_f_a: #last input of correct name\n            p_f_a_clf.append(1)\n        else:\n            p_f_a_clf.append(0)\n    \n        #check_if_poss_her\n        get_poss_her = check_if_poss_her(curr_doc, pronoun)\n        poss_her.append(get_poss_her)\n    \n        #rand_forest classifier for pronoun type:\n        if pronoun == \"he\" or pronoun == \"she\": \n            pro_typ.append(1)\n        elif pronoun == \"He\" or pronoun == \"She\": \n            pro_typ.append(2)\n        elif pronoun == \"his\" or (pronoun == \"her\" and get_poss_her): \n            pro_typ.append(3)\n        elif pronoun == \"him\" or (pronoun == \"her\" and not get_poss_her): \n            pro_typ.append(4)\n        elif pronoun == \"His\" or (pronoun == \"Her\" and get_poss_her): \n            pro_typ.append(5)\n        else:\n            pro_typ.append(6)\n        \n        dict_of_all[\"p_f_s\"] = get_p_f_s\n        dict_of_all[\"p_l_s\"] = get_p_l_s\n        dict_of_all[\"p_f_o\"] = get_p_f_o\n        dict_of_all[\"p_l_o\"] = get_p_l_o\n        dict_of_all[\"tc_f_s\"] = get_tc_f_s\n        dict_of_all[\"tc_f_o\"] = get_tc_f_o\n        dict_of_all[\"tc_f_a\"] = get_tc_f_a\n        dict_of_all[\"tc_l_s\"] = get_tc_l_s\n        dict_of_all[\"tc_l_o\"] = get_tc_l_o\n        dict_of_all[\"tc_l_p\"] = get_tc_l_p\n        dict_of_all[\"p_f_wp\"] = get_p_f_wp\n        dict_of_all[\"tc_l_wp\"] = get_tc_l_wp\n        dict_of_all[\"tc_l_nw\"] = get_tc_l_nw\n        dict_of_all[\"r_f_s\"] = get_r_f_s\n        dict_of_all[\"r_f_o\"] = get_r_f_o\n        dict_of_all[\"r_f_a\"] = get_r_f_a\n        dict_of_all[\"p_f_a\"] = get_p_f_a\n        dict_of_all[\"c_f_a\"] = get_c_f_a\n        dict_of_all[\"poss_her\"] = poss_her\n        \n        dict_of_all_list.append(dict_of_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d103ea90e98af22a01891b62b8770a225cac2a2c"},"cell_type":"code","source":"results_df = pd.DataFrame({\"correct\":correct_name_list})\n\nresults_df['pr_fsub'] = p_f_s\nresults_df['pr_lsub'] = p_l_s\nresults_df['pr_fobj'] = p_f_o\nresults_df['pr_lobj'] = p_l_o\nresults_df['tc_lsub'] = tc_l_s\nresults_df['tc_lobj'] = tc_l_o\nresults_df['tc_fsub'] = tc_f_s\nresults_df['tc_fobj'] = tc_f_o\nresults_df['tc_fapo'] = tc_f_a\nresults_df['tc_lnw'] = tc_l_nw\nresults_df['tc_lp'] = tc_l_p\nresults_df['re_sub'] = r_f_s\nresults_df['re_obj'] = r_f_o\nresults_df['re_app'] = r_f_a\nresults_df['pr_para'] = p_f_wp\nresults_df['tc_para'] = tc_l_wp\nresults_df['pr_app'] = p_f_a\nresults_df['cr_app'] = c_f_a\nresults_df['pronoun'] = pronoun_list\nresults_df['offset'] = pronoun_offset_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecf7d700932cf28fa9797d4fc60b3d116c4fd97a"},"cell_type":"code","source":"count = 0\nfor idx in range(20):\n    print(text_list[idx])\n    print(\"*********************\")\n    print(count)\n    print(\"@@@\")\n    count += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c803d3acaeb321810b43b222e44881d40227745"},"cell_type":"code","source":"#THE NEXT FEW CELLS ARE DEDICATED TO A PLAYGROUND FOR MANUAL LOGIC. RANDFOREST IS AFTER THAT. ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d1b0aa589fbf61b60d7e144df823a4820d3f2c5"},"cell_type":"code","source":"#### THIS PART IS INDEPENDENT FROM THE RANDOM FOREST SECTION.\ndef coref_logic(dict_of_all, pronoun):\n    guess = \"none\"\n    case_group = \"none\"\n    \n    #CASE-A: Pronoun he or she or He or She\n    if pronoun == \"he\" or pronoun == \"she\" or pronoun == \"He\" or pronoun == \"She\": \n        \n        #CASE-1\n        if dict_of_all[\"tc_f_s\"] == \"none\" and dict_of_all[\"p_f_s\"] != \"none\":\n            guess = dict_of_all[\"p_f_s\"] #pls will be = pfs if no other subj\n            case_group = \"A1\"\n        #CASE-2\n        elif dict_of_all[\"tc_f_s\"] != \"none\":\n            guess = dict_of_all[\"tc_f_s\"] #tcls will be = tcfs if no other subj\n            case_group = \"A2\"\n        #CASE-3\n        elif dict_of_all[\"tc_f_s\"] == \"none\" and dict_of_all[\"p_f_s\"] == \"none\":\n            if dict_of_all[\"tc_f_o\"] == \"none\" and dict_of_all[\"p_f_o\"] != \"none\":\n                guess = dict_of_all[\"p_f_o\"]\n                case_group = \"A3\"\n        #CASE-4\n        elif dict_of_all[\"tc_f_s\"] == \"none\" and dict_of_all[\"p_f_s\"] == \"none\":\n            if dict_of_all[\"tc_f_o\"] != \"none\" and dict_of_all[\"p_f_o\"] == \"none\":\n                guess = dict_of_all[\"tc_f_o\"]\n                case_group = \"A4\"\n    #CASE B: Pronoun his or her (possessive her):\n    if pronoun == \"his\" or (pronoun == \"her\" and dict_of_all[\"poss_her\"]): \n        \n        #CASE-1 #assuming the pronoun is also in the paranthesis\n        if dict_of_all[\"tc_l_p\"] != \"none\": \n            guess = dict_of_all[\"tc_l_p\"]\n            case_group = \"B1\"\n        #CASE-2 #assuming the pronoun is also in the paranthesis\n        elif dict_of_all[\"tc_l_wp\"] != \"none\": \n            guess = dict_of_all[\"tc_l_wp\"]\n            case_group = \"B2\"\n        #CASE-3\n        elif dict_of_all[\"r_f_a\"] != \"none\": \n            guess = dict_of_all[\"r_f_a\"]\n            case_group = \"B3\"\n        #CASE-4\n        elif dict_of_all[\"r_f_s\"] != \"none\": \n            guess = dict_of_all[\"r_f_s\"]\n            case_group = \"B4\"\n        #CASE-5\n        elif dict_of_all[\"tc_f_s\"] == \"none\" and dict_of_all[\"p_f_s\"] == \"none\"\\\n                    and dict_of_all[\"r_f_s\"] == \"none\" and dict_of_all[\"tc_f_o\"] != \"none\": \n            guess = dict_of_all[\"tc_f_o\"]\n            case_group = \"B5\"\n        #CASE-6\n        elif dict_of_all[\"tc_f_s\"] != \"none\" and dict_of_all[\"p_f_s\"] != \"none\"\\\n                    and dict_of_all[\"r_f_s\"] == \"none\" and dict_of_all[\"tc_f_o\"] == \"none\": \n            guess = dict_of_all[\"tc_f_s\"]\n            case_group = \"B6\"    \n        #CASE-7\n        elif dict_of_all[\"tc_f_s\"] == \"none\" and dict_of_all[\"p_f_s\"] != \"none\"\\\n                    and dict_of_all[\"r_f_s\"] == \"none\" and dict_of_all[\"tc_f_o\"] == \"none\": \n            guess = dict_of_all[\"p_f_s\"]\n            case_group = \"B7\"\n            \n    #CASE C: Pronoun his or her (object her):\n    if pronoun == \"him\" or (pronoun == \"her\" and not dict_of_all[\"poss_her\"]): \n        \n        #CASE-1\n        if dict_of_all[\"p_f_s\"] == \"none\" and dict_of_all[\"tc_f_o\"] == \"none\"\\\n                        and dict_of_all[\"tc_f_s\"] != \"none\": \n            guess = dict_of_all[\"tc_l_nw\"]\n            case_group = \"C1\"\n        \n        # to be continued..    \n        \n    return guess, case_group","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06692ab3b955a6c10b81c207e8dd62cc33085738"},"cell_type":"code","source":"guesses = []\ncase_groups = []\nfor dict_of_all, pronoun in zip(dict_of_all_list, pronoun_list):\n    guesses.append(coref_logic(dict_of_all, pronoun)[0])\n    case_groups.append(coref_logic(dict_of_all, pronoun)[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f393e9b5b84ca46413c396ad4b5b949ad944ce9"},"cell_type":"code","source":"guesses_df = pd.DataFrame({\"correct\":correct_name_list})\nguesses_df['guesses'] = guesses\nguesses_df['case_group'] = case_groups","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"984f0d6d6a8b96841e2121b0913d7bb19e5afb6b"},"cell_type":"code","source":"guesses_df.head(50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#RAND FOREST TRAINING DATA\ntr_p_f_s = [p_f_s_clf[idx] for idx in train_idx] \ntr_p_l_s = [p_l_s_clf[idx] for idx in train_idx]\ntr_p_f_o = [p_f_o_clf[idx] for idx in train_idx]\ntr_p_l_o = [p_l_o_clf[idx] for idx in train_idx]\ntr_tc_f_s = [tc_f_s_clf[idx] for idx in train_idx]\ntr_tc_f_o = [tc_f_o_clf[idx] for idx in train_idx]\ntr_tc_f_a = [tc_f_a_clf[idx] for idx in train_idx]\ntr_tc_l_s = [tc_l_s_clf[idx] for idx in train_idx]\ntr_tc_l_o = [tc_l_o_clf[idx] for idx in train_idx]\ntr_tc_l_p = [tc_l_p_clf[idx] for idx in train_idx]\ntr_p_f_wp = [p_f_wp_clf[idx] for idx in train_idx]\ntr_tc_l_wp = [tc_l_wp_clf[idx] for idx in train_idx]\ntr_tc_l_nw = [tc_l_nw_clf[idx] for idx in train_idx]\ntr_r_f_s = [r_f_s_clf[idx] for idx in train_idx]\ntr_r_f_o = [r_f_o_clf[idx] for idx in train_idx]\ntr_r_f_a = [r_f_a_clf[idx] for idx in train_idx]\ntr_p_f_a = [p_f_a_clf[idx] for idx in train_idx]\ntr_c_f_a = [c_f_a_clf[idx] for idx in train_idx]"},{"metadata":{"trusted":true,"_uuid":"5decb523ae584297ea9fe0601f7753c756c9cf79"},"cell_type":"code","source":"data_matrix = []\ndata_matrix1 = []\ndata_matrix2 = []\ndata_matrix3 = []\ndata_matrix4 = []\ndata_matrix5 = []\ndata_matrix6 = []\n\ntr_pro_t1_idxs = []\ntr_pro_t2_idxs = []\ntr_pro_t3_idxs = []\ntr_pro_t4_idxs = []\ntr_pro_t5_idxs = []\ntr_pro_t6_idxs = []\n\nfor idx in range(len(p_f_s)):\n    \n    data_vector = []\n    \n    if p_f_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)    \n    if p_l_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if p_f_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if p_l_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if tc_f_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if tc_f_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_p[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if p_f_wp[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_wp[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_nw[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if r_f_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if r_f_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if r_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if p_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if c_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if poss_her[idx] == True:\n        data_vector.append(1)\n    else:\n        data_vector.append(0)\n    #pronoun type, already numerical\n    if pro_typ[idx] == 1:\n        tr_pro_t1_idxs.append(idx)\n        data_matrix1.append(data_vector)\n    if pro_typ[idx] == 2:\n        tr_pro_t2_idxs.append(idx)\n        data_matrix2.append(data_vector)\n    if pro_typ[idx] == 3:\n        tr_pro_t3_idxs.append(idx)\n        data_matrix3.append(data_vector)\n    if pro_typ[idx] == 4:\n        tr_pro_t4_idxs.append(idx)\n        data_matrix4.append(data_vector)\n    if pro_typ[idx] == 5:\n        tr_pro_t5_idxs.append(idx)\n        data_matrix5.append(data_vector)\n    if pro_typ[idx] == 6:\n        tr_pro_t6_idxs.append(idx)\n        data_matrix6.append(data_vector)\n        \n    data_matrix.append(data_vector)\n    \nclf1_df = pd.DataFrame(data_matrix1, columns = dict_of_all_list[0].keys())\nclf2_df = pd.DataFrame(data_matrix2, columns = dict_of_all_list[0].keys())\nclf3_df = pd.DataFrame(data_matrix3, columns = dict_of_all_list[0].keys())\nclf4_df = pd.DataFrame(data_matrix4, columns = dict_of_all_list[0].keys())\nclf5_df = pd.DataFrame(data_matrix5, columns = dict_of_all_list[0].keys())\nclf6_df = pd.DataFrame(data_matrix6, columns = dict_of_all_list[0].keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f30a7bc70d07a05a4b6a35cc02d99c41d97686a8"},"cell_type":"code","source":"# Create a random forest Classifier. By convention, clf means 'Classifier'\nclf = RandomForestClassifier(n_estimators=150, n_jobs=2, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7062e845a5b9204674952d574ba80edd80b04b5c"},"cell_type":"code","source":"#Now make features for the test dataset. This requires doing alllllll those all over again.\n\n# IT IS JUST COPY-PASTE FROM THE TRAINING PROCEDURE (EXCEPT THE LABELS, WHICH WE DON'T NEED.)\n\nwith open('../input/gendered-pronoun-resolution/test_stage_1.tsv') as tsvfile:\n    \n    reader = csv.DictReader(tsvfile, dialect='excel-tab')\n    \n    test_ids= []\n    text_list = []\n    pronoun_list = []\n    pronoun_offset_list = []\n    dict_of_all_list = []\n    \n    sent_num = 1\n    \n    p_f_s = [] #prev first subject\n    p_l_s = [] #prev last subject\n    p_f_o = [] #prev first object\n    p_l_o = [] #prev last object\n    tc_f_s = [] #trunc curr first subject\n    tc_f_o = [] #trunc curr first obj\n    tc_f_a = [] #trunc curr first aposs\n    tc_l_s = [] #trunc curr last subject\n    tc_l_o = [] #trunc curr last obj\n    tc_l_p = [] #trunc curr last poss\n    p_f_wp = [] #prev first word between paranthesis\n    tc_l_wp = [] #curr word between paranthesis\n    tc_l_nw = [] #last name word other than a subj in trunc curr\n    r_f_s = [] #remainder first subj\n    r_f_o = [] #remainder first object\n    r_f_a = [] #remainder first appos\n    p_f_a = [] #prev first appos\n    c_f_a = [] #curr first appos\n    poss_her = [] #possessive her true or false\n    pro_typ = [] #pronoun type\n      \n    for row in reader:\n        \n        train_ids.append(row['ID'])\n        text = row['Text']\n        sent_num += 1\n        text_list.append(text)\n        dict_of_all = {}\n        \n        proffset = int(row['Pronoun-offset']) \n        pronoun_offset_list.append(proffset)\n        \n        pronoun = row['Pronoun']\n        pronoun_list.append(pronoun)\n              \n        curr, prev, trunc_curr, remainder = curr_prev_sentence(text, proffset)\n        curr_doc = nlp(curr)\n        prev_doc = nlp(prev) \n        curr_tok = word_tokenize(curr)\n        prev_tok = word_tokenize(prev)\n        trunc_curr_tok = word_tokenize(trunc_curr)\n        \n        #get first subj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_s = \"none\"\n        for n in [1,2,3,4,5]: #number of n is from common sense\n            dummy_p_f_s = find_nth_subj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_f_s) and get_p_f_s == \"none\":\n                get_p_f_s = dummy_p_f_s\n        \n        ####For sentence no. 5, spacy and nltk both failed to identify Collins as a propn.\n        ### therefore, we will add a new line here making sure we have a name.\n        \n        if get_p_f_s == \"none\":\n            if check_if_capital(find_nth_subj(prev_doc,1)):\n                get_p_f_s = find_nth_subj(prev_doc,1)\n\n        p_f_s.append(get_p_f_s)\n        \n        #get last  subj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_l_s = \"none\"\n        for n in [1,2,3,4,5]:\n            dummy_p_l_s = find_nth_subj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_l_s):\n                get_p_l_s = dummy_p_l_s\n        \n        p_l_s.append(get_p_l_s)\n                \n        #get first  obj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_o = \"none\"\n        for n in [1,2,3,4,5]: \n            dummy_p_f_o = find_nth_dobj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_f_o) and get_p_f_o == \"none\":\n                get_p_f_o = dummy_p_f_o\n        \n        p_f_o.append(get_p_f_o)\n                    \n        #get last  dobj in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_l_o = \"none\"\n        for n in [1,2,3,4,5]: \n            dummy_p_l_o = find_nth_dobj(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_l_o):\n                get_p_l_o = dummy_p_l_o\n        \n        p_l_o.append(get_p_l_o)\n                \n        #get last  subj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_s = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_l_s = find_nth_subj(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_l_s)\\\n                    and (dummy_tc_l_s in trunc_curr): #this is slightly inaccurate but oh well\n                get_tc_l_s = dummy_tc_l_s \n            \n        tc_l_s.append(get_tc_l_s)\n                    \n        #get last  dobj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_o = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_l_o = find_nth_dobj(curr_doc,n)\n            if (dummy_tc_l_o in trunc_curr)\\\n                                        and check_if_name(curr_tok,dummy_tc_l_o): \n                get_tc_l_o = dummy_tc_l_o \n            \n        tc_l_o.append(get_tc_l_o)\n                \n        #get last  poss in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_p = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_l_p = find_nth_poss(curr_doc,n)\n            if (dummy_tc_l_p in trunc_curr)\\\n                                        and check_if_name(curr_tok,dummy_tc_l_p): \n                get_tc_l_p = dummy_tc_l_p \n            \n        tc_l_p.append(get_tc_l_p)\n                \n        #get first subj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_f_s = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_f_s = find_nth_subj(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_f_s) and get_tc_f_s == \"none\":\n                get_tc_f_s = dummy_tc_f_s \n            \n        tc_f_s.append(get_tc_f_s)\n                    \n        #get first dobj in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_f_o = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_f_o = find_nth_dobj(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_f_o) and get_tc_f_o == \"none\": \n                get_tc_f_o = dummy_tc_f_o \n            \n        tc_f_o.append(get_tc_f_o)\n            \n        #get last  non-subj name word  in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_nw = \"none\"\n        candidate = \"none\"\n        tc_name_words = list_of_name_words(trunc_curr_tok) \n        if len(tc_name_words) > 0:\n            candidate = tc_name_words[-1]\n        if candidate in get_tc_f_s or candidate in get_tc_l_s:\n            if len(tc_name_words) > 1:\n                candidate = tc_name_words[-1]\n        if check_if_name(curr_tok,candidate):\n            get_tc_l_nw = candidate\n        \n        tc_l_nw.append(get_tc_l_nw)\n                \n        #get first aposs in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_tc_f_a = find_nth_appos(curr_doc,n)\n            if check_if_name(curr_tok,dummy_tc_f_a) and get_tc_f_a == \"none\": \n                get_tc_f_a = dummy_tc_f_a \n            \n        tc_f_a.append(get_tc_f_a)\n            \n        #get word btwn paranthesis in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_wp = find_name_words(name_btwn_paran(prev))\n        \n        if check_if_name(prev_tok,get_p_f_wp): #Add only proper nouns into list\n            p_f_wp.append(get_p_f_wp)\n        else:\n            p_f_wp.append(\"none\")\n                    \n        #get word btwn paranthesis in trunc curr @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_tc_l_wp = find_name_words(name_btwn_paran(curr))  \n        \n        if check_if_name(curr_tok,get_tc_l_wp): #Add only proper nouns into list\n            tc_l_wp.append(get_tc_l_wp)\n        else:\n            tc_l_wp.append(\"none\")\n                        \n        #get last subj in remainder @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_r_f_s = \"none\"\n        for n in [1,2,3,4,5,6,7,8]: #in the final version, each of the name subjects will be accunted for\n            dummy_r_f_s = find_nth_subj(curr_doc,n)\n            if dummy_r_f_s in remainder and check_if_name(curr_tok,dummy_r_f_s):\n                get_r_f_s = dummy_r_f_s \n            \n        r_f_s.append(get_r_f_s)\n                    \n        #get last dobj in remainder @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_r_f_o = \"none\"\n        for n in [1,2,3,4,5,6,7,8]: #in the final version, each of the name objects will be accunted for\n            dummy_r_f_o = find_nth_dobj(curr_doc,n)\n            if dummy_r_f_o in remainder and check_if_name(curr_tok,dummy_r_f_o):\n                get_r_f_o = dummy_r_f_o \n            \n        r_f_o.append(get_r_f_o)\n                    \n        #get last appos in remainder @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_r_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_r_f_a = find_nth_appos(curr_doc,n)\n            if dummy_r_f_a in remainder and check_if_name(curr_tok,dummy_r_f_a): \n                get_r_f_a = dummy_r_f_a \n            \n        r_f_a.append(get_r_f_a)\n               \n        #get first appos in current @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_c_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_c_f_a = find_nth_appos(curr_doc,n)\n            if check_if_name(curr_tok,dummy_c_f_a) and get_c_f_a == \"none\": \n                get_c_f_a = dummy_c_f_a \n            \n        c_f_a.append(get_c_f_a)\n                \n        #get first appos in prev @@@@@@@@@@@@@@@@@@@@@@@@@@@\n        get_p_f_a = \"none\"\n        for n in [1,2,3,4]:\n            dummy_p_f_a = find_nth_appos(prev_doc,n)\n            if check_if_name(prev_tok,dummy_p_f_a) and get_p_f_a == \"none\": \n                get_p_f_a = dummy_p_f_a \n            \n        p_f_a.append(get_p_f_a)\n            \n        #check_if_poss_her\n        poss_her.append(check_if_poss_her(curr_doc, pronoun))\n    \n        #rand_forest classifier for pronoun type:\n        if pronoun == \"he\" or pronoun == \"she\": \n            pro_typ.append(1)\n        elif pronoun == \"He\" or pronoun == \"She\": \n            pro_typ.append(2)\n        elif pronoun == \"his\" or (pronoun == \"her\" and get_poss_her): \n            pro_typ.append(3)\n        elif pronoun == \"him\" or (pronoun == \"her\" and not get_poss_her): \n            pro_typ.append(4)\n        elif pronoun == \"His\" or (pronoun == \"Her\" and get_poss_her): \n            pro_typ.append(5)\n        else:\n            pro_typ.append(6)\n    \n        dict_of_all[\"p_f_s\"] = get_p_f_s\n        dict_of_all[\"p_l_s\"] = get_p_l_s\n        dict_of_all[\"p_f_o\"] = get_p_f_o\n        dict_of_all[\"p_l_o\"] = get_p_l_o\n        dict_of_all[\"tc_f_s\"] = get_tc_f_s\n        dict_of_all[\"tc_f_o\"] = get_tc_f_o\n        dict_of_all[\"tc_f_a\"] = get_tc_f_a\n        dict_of_all[\"tc_l_s\"] = get_tc_l_s\n        dict_of_all[\"tc_l_o\"] = get_tc_l_o\n        dict_of_all[\"tc_l_p\"] = get_tc_l_p\n        dict_of_all[\"p_f_wp\"] = get_p_f_wp\n        dict_of_all[\"tc_l_wp\"] = get_tc_l_wp\n        dict_of_all[\"tc_l_nw\"] = get_tc_l_nw\n        dict_of_all[\"r_f_s\"] = get_r_f_s\n        dict_of_all[\"r_f_o\"] = get_r_f_o\n        dict_of_all[\"r_f_a\"] = get_r_f_a\n        dict_of_all[\"p_f_a\"] = get_p_f_a\n        dict_of_all[\"c_f_a\"] = get_c_f_a\n        dict_of_all[\"poss_her\"] = poss_her\n        \n        dict_of_all_list.append(dict_of_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e8231100586dc9d516f21fe7db28b11bfbb0bb6"},"cell_type":"code","source":"#AND NOW COPY-PASTE ALL THAT FEATURE EXTRACTION PROCEDURE FROM THE TRAINING CELLS:\n\ndata_matrix = []\ndata_matrix1 = []\ndata_matrix2 = []\ndata_matrix3 = []\ndata_matrix4 = []\ndata_matrix5 = []\ndata_matrix6 = []\n\nte_pro_t1_idxs = []\nte_pro_t2_idxs = []\nte_pro_t3_idxs = []\nte_pro_t4_idxs = []\nte_pro_t5_idxs = []\nte_pro_t6_idxs = []\n\nfor idx in range(len(p_f_s)):\n    \n    data_vector = []\n    \n    if p_f_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)    \n    if p_l_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if p_f_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if p_l_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if tc_f_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)\n    if tc_f_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_p[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if p_f_wp[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_wp[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if tc_l_nw[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if r_f_s[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if r_f_o[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if r_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if p_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if c_f_a[idx] == \"none\":\n        data_vector.append(0)\n    else:\n        data_vector.append(1)   \n    if poss_her[idx] == True:\n        data_vector.append(1)\n    else:\n        data_vector.append(0)\n    #pronoun type, already numerical\n    if pro_typ[idx] == 1:\n        te_pro_t1_idxs.append(idx)\n        data_matrix1.append(data_vector)\n    if pro_typ[idx] == 2:\n        te_pro_t2_idxs.append(idx)\n        data_matrix2.append(data_vector)\n    if pro_typ[idx] == 3:\n        te_pro_t3_idxs.append(idx)\n        data_matrix3.append(data_vector)\n    if pro_typ[idx] == 4:\n        te_pro_t4_idxs.append(idx)\n        data_matrix4.append(data_vector)\n    if pro_typ[idx] == 5:\n        te_pro_t5_idxs.append(idx)\n        data_matrix5.append(data_vector)\n    if pro_typ[idx] == 6:\n        te_pro_t6_idxs.append(idx)\n        data_matrix6.append(data_vector)\n    \n    data_matrix.append(data_vector)\n    \ntest1_df = pd.DataFrame(data_matrix1, columns = dict_of_all_list[0].keys())\ntest2_df = pd.DataFrame(data_matrix2, columns = dict_of_all_list[0].keys())\ntest3_df = pd.DataFrame(data_matrix3, columns = dict_of_all_list[0].keys())\ntest4_df = pd.DataFrame(data_matrix4, columns = dict_of_all_list[0].keys())\ntest5_df = pd.DataFrame(data_matrix5, columns = dict_of_all_list[0].keys())\ntest6_df = pd.DataFrame(data_matrix6, columns = dict_of_all_list[0].keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_n_pred(train_df, test_df, clf, train_y_list, train_y_idx):\n    features = train_df.columns[:-1]\n    train_y = [train_y_list[i] for i in train_y_idx]\n    train_y = np.asarray(train_y)\n    clf.fit(train_df[features], train_y)\n    return clf.predict(test_df[features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6e3b6cbabde5136893a95e530961ab629990c8b"},"cell_type":"code","source":"# CASE I : PRONOUN TYPE 1:\npred_C1_p_f_s = train_n_pred(clf1_df, test1_df, clf, p_f_s_clf, tr_pro_t1_idxs)\npred_C1_p_l_s = train_n_pred(clf1_df, test1_df, clf, p_l_s_clf, tr_pro_t1_idxs)\npred_C1_p_f_o = train_n_pred(clf1_df, test1_df, clf, p_f_o_clf, tr_pro_t1_idxs)\npred_C1_p_l_o = train_n_pred(clf1_df, test1_df, clf, p_l_o_clf, tr_pro_t1_idxs)\npred_C1_tc_f_s = train_n_pred(clf1_df, test1_df, clf, tc_f_s_clf, tr_pro_t1_idxs)\npred_C1_tc_f_o = train_n_pred(clf1_df, test1_df, clf, tc_f_o_clf, tr_pro_t1_idxs)\npred_C1_tc_f_a = train_n_pred(clf1_df, test1_df, clf, tc_f_a_clf, tr_pro_t1_idxs)\npred_C1_tc_l_s = train_n_pred(clf1_df, test1_df, clf, tc_l_s_clf, tr_pro_t1_idxs)\npred_C1_tc_l_o = train_n_pred(clf1_df, test1_df, clf, tc_l_o_clf, tr_pro_t1_idxs)\npred_C1_tc_l_p = train_n_pred(clf1_df, test1_df, clf, tc_l_p_clf, tr_pro_t1_idxs)\npred_C1_p_f_wp = train_n_pred(clf1_df, test1_df, clf, p_f_wp_clf, tr_pro_t1_idxs)\npred_C1_tc_l_wp = train_n_pred(clf1_df, test1_df, clf, tc_l_wp_clf, tr_pro_t1_idxs)\npred_C1_tc_l_nw = train_n_pred(clf1_df, test1_df, clf, tc_l_nw_clf, tr_pro_t1_idxs)\npred_C1_r_f_s = train_n_pred(clf1_df, test1_df, clf, r_f_s_clf, tr_pro_t1_idxs)\npred_C1_r_f_o = train_n_pred(clf1_df, test1_df, clf, r_f_o_clf, tr_pro_t1_idxs)\npred_C1_r_f_a = train_n_pred(clf1_df, test1_df, clf, r_f_a_clf, tr_pro_t1_idxs)\npred_C1_p_f_a = train_n_pred(clf1_df, test1_df, clf, p_f_a_clf, tr_pro_t1_idxs)\npred_C1_c_f_a = train_n_pred(clf1_df, test1_df, clf, c_f_a_clf, tr_pro_t1_idxs)\n\n# CASE II : PRONOUN TYPE 2:\npred_C2_p_f_s = train_n_pred(clf2_df, test2_df, clf, p_f_s_clf, tr_pro_t2_idxs)\npred_C2_p_l_s = train_n_pred(clf2_df, test2_df, clf, p_l_s_clf, tr_pro_t2_idxs)\npred_C2_p_f_o = train_n_pred(clf2_df, test2_df, clf, p_f_o_clf, tr_pro_t2_idxs)\npred_C2_p_l_o = train_n_pred(clf2_df, test2_df, clf, p_l_o_clf, tr_pro_t2_idxs)\npred_C2_tc_f_s = train_n_pred(clf2_df, test2_df, clf, tc_f_s_clf, tr_pro_t2_idxs)\npred_C2_tc_f_o = train_n_pred(clf2_df, test2_df, clf, tc_f_o_clf, tr_pro_t2_idxs)\npred_C2_tc_f_a = train_n_pred(clf2_df, test2_df, clf, tc_f_a_clf, tr_pro_t2_idxs)\npred_C2_tc_l_s = train_n_pred(clf2_df, test2_df, clf, tc_l_s_clf, tr_pro_t2_idxs)\npred_C2_tc_l_o = train_n_pred(clf2_df, test2_df, clf, tc_l_o_clf, tr_pro_t2_idxs)\npred_C2_tc_l_p = train_n_pred(clf2_df, test2_df, clf, tc_l_p_clf, tr_pro_t2_idxs)\npred_C2_p_f_wp = train_n_pred(clf2_df, test2_df, clf, p_f_wp_clf, tr_pro_t2_idxs)\npred_C2_tc_l_wp = train_n_pred(clf2_df, test2_df, clf, tc_l_wp_clf, tr_pro_t2_idxs)\npred_C2_tc_l_nw = train_n_pred(clf2_df, test2_df, clf, tc_l_nw_clf, tr_pro_t2_idxs)\npred_C2_r_f_s = train_n_pred(clf2_df, test2_df, clf, r_f_s_clf, tr_pro_t2_idxs)\npred_C2_r_f_o = train_n_pred(clf2_df, test2_df, clf, r_f_o_clf, tr_pro_t2_idxs)\npred_C2_r_f_a = train_n_pred(clf2_df, test2_df, clf, r_f_a_clf, tr_pro_t2_idxs)\npred_C2_p_f_a = train_n_pred(clf2_df, test2_df, clf, p_f_a_clf, tr_pro_t2_idxs)\npred_C2_c_f_a = train_n_pred(clf2_df, test2_df, clf, c_f_a_clf, tr_pro_t2_idxs)\n\n# CASE III : PRONOUN TYPE 3:\npred_C3_p_f_s = train_n_pred(clf3_df, test3_df, clf, p_f_s_clf, tr_pro_t3_idxs)\npred_C3_p_l_s = train_n_pred(clf3_df, test3_df, clf, p_l_s_clf, tr_pro_t3_idxs)\npred_C3_p_f_o = train_n_pred(clf3_df, test3_df, clf, p_f_o_clf, tr_pro_t3_idxs)\npred_C3_p_l_o = train_n_pred(clf3_df, test3_df, clf, p_l_o_clf, tr_pro_t3_idxs)\npred_C3_tc_f_s = train_n_pred(clf3_df, test3_df, clf, tc_f_s_clf, tr_pro_t3_idxs)\npred_C3_tc_f_o = train_n_pred(clf3_df, test3_df, clf, tc_f_o_clf, tr_pro_t3_idxs)\npred_C3_tc_f_a = train_n_pred(clf3_df, test3_df, clf, tc_f_a_clf, tr_pro_t3_idxs)\npred_C3_tc_l_s = train_n_pred(clf3_df, test3_df, clf, tc_l_s_clf, tr_pro_t3_idxs)\npred_C3_tc_l_o = train_n_pred(clf3_df, test3_df, clf, tc_l_o_clf, tr_pro_t3_idxs)\npred_C3_tc_l_p = train_n_pred(clf3_df, test3_df, clf, tc_l_p_clf, tr_pro_t3_idxs)\npred_C3_p_f_wp = train_n_pred(clf3_df, test3_df, clf, p_f_wp_clf, tr_pro_t3_idxs)\npred_C3_tc_l_wp = train_n_pred(clf3_df, test3_df, clf, tc_l_wp_clf, tr_pro_t3_idxs)\npred_C3_tc_l_nw = train_n_pred(clf3_df, test3_df, clf, tc_l_nw_clf, tr_pro_t3_idxs)\npred_C3_r_f_s = train_n_pred(clf3_df, test3_df, clf, r_f_s_clf, tr_pro_t3_idxs)\npred_C3_r_f_o = train_n_pred(clf3_df, test3_df, clf, r_f_o_clf, tr_pro_t3_idxs)\npred_C3_r_f_a = train_n_pred(clf3_df, test3_df, clf, r_f_a_clf, tr_pro_t3_idxs)\npred_C3_p_f_a = train_n_pred(clf3_df, test3_df, clf, p_f_a_clf, tr_pro_t3_idxs)\npred_C3_c_f_a = train_n_pred(clf3_df, test3_df, clf, c_f_a_clf, tr_pro_t3_idxs)\n\n# CASE IV : PRONOUN TYPE 4:\npred_C4_p_f_s = train_n_pred(clf4_df, test4_df, clf, p_f_s_clf, tr_pro_t4_idxs)\npred_C4_p_l_s = train_n_pred(clf4_df, test4_df, clf, p_l_s_clf, tr_pro_t4_idxs)\npred_C4_p_f_o = train_n_pred(clf4_df, test4_df, clf, p_f_o_clf, tr_pro_t4_idxs)\npred_C4_p_l_o = train_n_pred(clf4_df, test4_df, clf, p_l_o_clf, tr_pro_t4_idxs)\npred_C4_tc_f_s = train_n_pred(clf4_df, test4_df, clf, tc_f_s_clf, tr_pro_t4_idxs)\npred_C4_tc_f_o = train_n_pred(clf4_df, test4_df, clf, tc_f_o_clf, tr_pro_t4_idxs)\npred_C4_tc_f_a = train_n_pred(clf4_df, test4_df, clf, tc_f_a_clf, tr_pro_t4_idxs)\npred_C4_tc_l_s = train_n_pred(clf4_df, test4_df, clf, tc_l_s_clf, tr_pro_t4_idxs)\npred_C4_tc_l_o = train_n_pred(clf4_df, test4_df, clf, tc_l_o_clf, tr_pro_t4_idxs)\npred_C4_tc_l_p = train_n_pred(clf4_df, test4_df, clf, tc_l_p_clf, tr_pro_t4_idxs)\npred_C4_p_f_wp = train_n_pred(clf4_df, test4_df, clf, p_f_wp_clf, tr_pro_t4_idxs)\npred_C4_tc_l_wp = train_n_pred(clf4_df, test4_df, clf, tc_l_wp_clf, tr_pro_t4_idxs)\npred_C4_tc_l_nw = train_n_pred(clf4_df, test4_df, clf, tc_l_nw_clf, tr_pro_t4_idxs)\npred_C4_r_f_s = train_n_pred(clf4_df, test4_df, clf, r_f_s_clf, tr_pro_t4_idxs)\npred_C4_r_f_o = train_n_pred(clf4_df, test4_df, clf, r_f_o_clf, tr_pro_t4_idxs)\npred_C4_r_f_a = train_n_pred(clf4_df, test4_df, clf, r_f_a_clf, tr_pro_t4_idxs)\npred_C4_p_f_a = train_n_pred(clf4_df, test4_df, clf, p_f_a_clf, tr_pro_t4_idxs)\npred_C4_c_f_a = train_n_pred(clf4_df, test4_df, clf, c_f_a_clf, tr_pro_t4_idxs)\n\n# CASE V : PRONOUN TYPE V:\npred_C5_p_f_s = train_n_pred(clf5_df, test5_df, clf, p_f_s_clf, tr_pro_t5_idxs)\npred_C5_p_l_s = train_n_pred(clf5_df, test5_df, clf, p_l_s_clf, tr_pro_t5_idxs)\npred_C5_p_f_o = train_n_pred(clf5_df, test5_df, clf, p_f_o_clf, tr_pro_t5_idxs)\npred_C5_p_l_o = train_n_pred(clf5_df, test5_df, clf, p_l_o_clf, tr_pro_t5_idxs)\npred_C5_tc_f_s = train_n_pred(clf5_df, test5_df, clf, tc_f_s_clf, tr_pro_t5_idxs)\npred_C5_tc_f_o = train_n_pred(clf5_df, test5_df, clf, tc_f_o_clf, tr_pro_t5_idxs)\npred_C5_tc_f_a = train_n_pred(clf5_df, test5_df, clf, tc_f_a_clf, tr_pro_t5_idxs)\npred_C5_tc_l_s = train_n_pred(clf5_df, test5_df, clf, tc_l_s_clf, tr_pro_t5_idxs)\npred_C5_tc_l_o = train_n_pred(clf5_df, test5_df, clf, tc_l_o_clf, tr_pro_t5_idxs)\npred_C5_tc_l_p = train_n_pred(clf5_df, test5_df, clf, tc_l_p_clf, tr_pro_t5_idxs)\npred_C5_p_f_wp = train_n_pred(clf5_df, test5_df, clf, p_f_wp_clf, tr_pro_t5_idxs)\npred_C5_tc_l_wp = train_n_pred(clf5_df, test5_df, clf, tc_l_wp_clf, tr_pro_t5_idxs)\npred_C5_tc_l_nw = train_n_pred(clf5_df, test5_df, clf, tc_l_nw_clf, tr_pro_t5_idxs)\npred_C5_r_f_s = train_n_pred(clf5_df, test5_df, clf, r_f_s_clf, tr_pro_t5_idxs)\npred_C5_r_f_o = train_n_pred(clf5_df, test5_df, clf, r_f_o_clf, tr_pro_t5_idxs)\npred_C5_r_f_a = train_n_pred(clf5_df, test5_df, clf, r_f_a_clf, tr_pro_t5_idxs)\npred_C5_p_f_a = train_n_pred(clf5_df, test5_df, clf, p_f_a_clf, tr_pro_t5_idxs)\npred_C5_c_f_a = train_n_pred(clf5_df, test5_df, clf, c_f_a_clf, tr_pro_t5_idxs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0745f106aea89c1b41d75cc04b1087f54f298af"},"cell_type":"code","source":"#Now convert predictions to list of names: (The lists were over-written with test data.)\nlist_of_pred_names = []\nfor idx in range(len(p_f_s)):\n    current_predictions = []\n    \n    if idx in te_pro_t1_idxs:\n        pred_p_f_s = pred_C1_p_f_s[te_pro_t1_idxs.index(idx)]  \n        pred_p_l_s = pred_C1_p_l_s[te_pro_t1_idxs.index(idx)]\n        pred_p_f_o = pred_C1_p_f_o[te_pro_t1_idxs.index(idx)]\n        pred_p_l_o = pred_C1_p_l_o[te_pro_t1_idxs.index(idx)]\n        pred_tc_f_s = pred_C1_tc_f_s[te_pro_t1_idxs.index(idx)]\n        pred_tc_f_o = pred_C1_tc_f_o[te_pro_t1_idxs.index(idx)]\n        pred_tc_f_a = pred_C1_tc_f_a[te_pro_t1_idxs.index(idx)]\n        pred_tc_l_s = pred_C1_tc_l_s[te_pro_t1_idxs.index(idx)]\n        pred_tc_l_o = pred_C1_tc_l_o[te_pro_t1_idxs.index(idx)]\n        pred_tc_l_p = pred_C1_tc_l_p[te_pro_t1_idxs.index(idx)]\n        pred_p_f_wp = pred_C1_p_f_wp[te_pro_t1_idxs.index(idx)]\n        pred_tc_l_wp = pred_C1_tc_l_wp[te_pro_t1_idxs.index(idx)]\n        pred_tc_l_nw = pred_C1_tc_l_nw[te_pro_t1_idxs.index(idx)]\n        pred_r_f_s = pred_C1_r_f_s[te_pro_t1_idxs.index(idx)]\n        pred_r_f_o = pred_C1_r_f_o[te_pro_t1_idxs.index(idx)]\n        pred_r_f_a = pred_C1_r_f_a[te_pro_t1_idxs.index(idx)]\n        pred_p_f_a = pred_C1_p_f_a[te_pro_t1_idxs.index(idx)]\n        pred_c_f_a = pred_C1_c_f_a[te_pro_t1_idxs.index(idx)]\n    if idx in te_pro_t2_idxs:\n        pred_p_f_s = pred_C2_p_f_s[te_pro_t2_idxs.index(idx)]\n        pred_p_l_s = pred_C2_p_l_s[te_pro_t2_idxs.index(idx)]\n        pred_p_f_o = pred_C2_p_f_o[te_pro_t2_idxs.index(idx)]\n        pred_p_l_o = pred_C2_p_l_o[te_pro_t2_idxs.index(idx)]\n        pred_tc_f_s = pred_C2_tc_f_s[te_pro_t2_idxs.index(idx)]\n        pred_tc_f_o = pred_C2_tc_f_o[te_pro_t2_idxs.index(idx)]\n        pred_tc_f_a = pred_C2_tc_f_a[te_pro_t2_idxs.index(idx)]\n        pred_tc_l_s = pred_C2_tc_l_s[te_pro_t2_idxs.index(idx)]\n        pred_tc_l_o = pred_C2_tc_l_o[te_pro_t2_idxs.index(idx)]\n        pred_tc_l_p = pred_C2_tc_l_p[te_pro_t2_idxs.index(idx)]\n        pred_p_f_wp = pred_C2_p_f_wp[te_pro_t2_idxs.index(idx)]\n        pred_tc_l_wp = pred_C2_tc_l_wp[te_pro_t2_idxs.index(idx)]\n        pred_tc_l_nw = pred_C2_tc_l_nw[te_pro_t2_idxs.index(idx)]\n        pred_r_f_s = pred_C2_r_f_s[te_pro_t2_idxs.index(idx)]\n        pred_r_f_o = pred_C2_r_f_o[te_pro_t2_idxs.index(idx)]\n        pred_r_f_a = pred_C2_r_f_a[te_pro_t2_idxs.index(idx)]\n        pred_p_f_a = pred_C2_p_f_a[te_pro_t2_idxs.index(idx)]\n        pred_c_f_a = pred_C2_c_f_a[te_pro_t2_idxs.index(idx)]\n    if idx in te_pro_t3_idxs:\n        pred_p_f_s = pred_C3_p_f_s[te_pro_t3_idxs.index(idx)]\n        pred_p_l_s = pred_C3_p_l_s[te_pro_t3_idxs.index(idx)]\n        pred_p_f_o = pred_C3_p_f_o[te_pro_t3_idxs.index(idx)]\n        pred_p_l_o = pred_C3_p_l_o[te_pro_t3_idxs.index(idx)]\n        pred_tc_f_s = pred_C3_tc_f_s[te_pro_t3_idxs.index(idx)]\n        pred_tc_f_o = pred_C3_tc_f_o[te_pro_t3_idxs.index(idx)]\n        pred_tc_f_a = pred_C3_tc_f_a[te_pro_t3_idxs.index(idx)]\n        pred_tc_l_s = pred_C3_tc_l_s[te_pro_t3_idxs.index(idx)]\n        pred_tc_l_o = pred_C3_tc_l_o[te_pro_t3_idxs.index(idx)]\n        pred_tc_l_p = pred_C3_tc_l_p[te_pro_t3_idxs.index(idx)]\n        pred_p_f_wp = pred_C3_p_f_wp[te_pro_t3_idxs.index(idx)]\n        pred_tc_l_wp = pred_C3_tc_l_wp[te_pro_t3_idxs.index(idx)]\n        pred_tc_l_nw = pred_C3_tc_l_nw[te_pro_t3_idxs.index(idx)]\n        pred_r_f_s = pred_C3_r_f_s[te_pro_t3_idxs.index(idx)]\n        pred_r_f_o = pred_C3_r_f_o[te_pro_t3_idxs.index(idx)]\n        pred_r_f_a = pred_C3_r_f_a[te_pro_t3_idxs.index(idx)]\n        pred_p_f_a = pred_C3_p_f_a[te_pro_t3_idxs.index(idx)]\n        pred_c_f_a = pred_C3_c_f_a[te_pro_t3_idxs.index(idx)]\n    if idx in te_pro_t4_idxs:\n        pred_p_f_s = pred_C4_p_f_s[te_pro_t4_idxs.index(idx)]\n        pred_p_l_s = pred_C4_p_l_s[te_pro_t4_idxs.index(idx)]\n        pred_p_f_o = pred_C4_p_f_o[te_pro_t4_idxs.index(idx)]\n        pred_p_l_o = pred_C4_p_l_o[te_pro_t4_idxs.index(idx)]\n        pred_tc_f_s = pred_C4_tc_f_s[te_pro_t4_idxs.index(idx)]\n        pred_tc_f_o = pred_C4_tc_f_o[te_pro_t4_idxs.index(idx)]\n        pred_tc_f_a = pred_C4_tc_f_a[te_pro_t4_idxs.index(idx)]\n        pred_tc_l_s = pred_C4_tc_l_s[te_pro_t4_idxs.index(idx)]\n        pred_tc_l_o = pred_C4_tc_l_o[te_pro_t4_idxs.index(idx)]\n        pred_tc_l_p = pred_C4_tc_l_p[te_pro_t4_idxs.index(idx)]\n        pred_p_f_wp = pred_C4_p_f_wp[te_pro_t4_idxs.index(idx)]\n        pred_tc_l_wp = pred_C4_tc_l_wp[te_pro_t4_idxs.index(idx)]\n        pred_tc_l_nw = pred_C4_tc_l_nw[te_pro_t4_idxs.index(idx)]\n        pred_r_f_s = pred_C4_r_f_s[te_pro_t4_idxs.index(idx)]\n        pred_r_f_o = pred_C4_r_f_o[te_pro_t4_idxs.index(idx)]\n        pred_r_f_a = pred_C4_r_f_a[te_pro_t4_idxs.index(idx)]\n        pred_p_f_a = pred_C4_p_f_a[te_pro_t4_idxs.index(idx)]\n        pred_c_f_a = pred_C4_c_f_a[te_pro_t4_idxs.index(idx)]\n    if idx in te_pro_t5_idxs:\n        pred_p_f_s = pred_C5_p_f_s[te_pro_t5_idxs.index(idx)]\n        pred_p_l_s = pred_C5_p_l_s[te_pro_t5_idxs.index(idx)]\n        pred_p_f_o = pred_C5_p_f_o[te_pro_t5_idxs.index(idx)]\n        pred_p_l_o = pred_C5_p_l_o[te_pro_t5_idxs.index(idx)]\n        pred_tc_f_s = pred_C5_tc_f_s[te_pro_t5_idxs.index(idx)]\n        pred_tc_f_o = pred_C5_tc_f_o[te_pro_t5_idxs.index(idx)]\n        pred_tc_f_a = pred_C5_tc_f_a[te_pro_t5_idxs.index(idx)]\n        pred_tc_l_s = pred_C5_tc_l_s[te_pro_t5_idxs.index(idx)]\n        pred_tc_l_o = pred_C5_tc_l_o[te_pro_t5_idxs.index(idx)]\n        pred_tc_l_p = pred_C5_tc_l_p[te_pro_t5_idxs.index(idx)]\n        pred_p_f_wp = pred_C5_p_f_wp[te_pro_t5_idxs.index(idx)]\n        pred_tc_l_wp = pred_C5_tc_l_wp[te_pro_t5_idxs.index(idx)]\n        pred_tc_l_nw = pred_C5_tc_l_nw[te_pro_t5_idxs.index(idx)]\n        pred_r_f_s = pred_C5_r_f_s[te_pro_t5_idxs.index(idx)]\n        pred_r_f_o = pred_C5_r_f_o[te_pro_t5_idxs.index(idx)]\n        pred_r_f_a = pred_C5_r_f_a[te_pro_t5_idxs.index(idx)]\n        pred_p_f_a = pred_C5_p_f_a[te_pro_t5_idxs.index(idx)]\n        pred_c_f_a = pred_C5_c_f_a[te_pro_t5_idxs.index(idx)]\n    if idx in te_pro_t6_idxs:\n        pred_p_f_s = pred_C6_p_f_s[te_pro_t6_idxs.index(idx)]\n        pred_p_l_s = pred_C6_p_l_s[te_pro_t6_idxs.index(idx)]\n        pred_p_f_o = pred_C6_p_f_o[te_pro_t6_idxs.index(idx)]\n        pred_p_l_o = pred_C6_p_l_o[te_pro_t6_idxs.index(idx)]\n        pred_tc_f_s = pred_C6_tc_f_s[te_pro_t6_idxs.index(idx)]\n        pred_tc_f_o = pred_C6_tc_f_o[te_pro_t6_idxs.index(idx)]\n        pred_tc_f_a = pred_C6_tc_f_a[te_pro_t6_idxs.index(idx)]\n        pred_tc_l_s = pred_C6_tc_l_s[te_pro_t6_idxs.index(idx)]\n        pred_tc_l_o = pred_C6_tc_l_o[te_pro_t6_idxs.index(idx)]\n        pred_tc_l_p = pred_C6_tc_l_p[te_pro_t6_idxs.index(idx)]\n        pred_p_f_wp = pred_C6_p_f_wp[te_pro_t6_idxs.index(idx)]\n        pred_tc_l_wp = pred_C6_tc_l_wp[te_pro_t6_idxs.index(idx)]\n        pred_tc_l_nw = pred_C6_tc_l_nw[te_pro_t6_idxs.index(idx)]\n        pred_r_f_s = pred_C6_r_f_s[te_pro_t6_idxs.index(idx)]\n        pred_r_f_o = pred_C6_r_f_o[te_pro_t6_idxs.index(idx)]\n        pred_r_f_a = pred_C6_r_f_a[te_pro_t6_idxs.index(idx)]\n        pred_p_f_a = pred_C6_p_f_a[te_pro_t6_idxs.index(idx)]\n        pred_c_f_a = pred_C6_c_f_a[te_pro_t6_idxs.index(idx)]\n    \n    if pred_p_f_s == 1:\n        current_predictions.append(p_f_s[idx])\n    if pred_p_l_s == 1:\n        current_predictions.append(p_l_s[idx])\n    if pred_p_f_o == 1:\n        current_predictions.append(p_f_o[idx])\n    if pred_p_l_o == 1:\n        current_predictions.append(p_l_o[idx])\n    if pred_tc_f_s == 1:\n        current_predictions.append(tc_f_s[idx])\n    if pred_tc_f_o == 1:\n        current_predictions.append(tc_f_o[idx])\n    if pred_tc_f_a == 1:\n        current_predictions.append(tc_f_a[idx])\n    if pred_tc_l_s == 1:\n        current_predictions.append(tc_l_s[idx])\n    if pred_tc_l_o == 1:\n        current_predictions.append(tc_l_o[idx])\n    if pred_tc_l_p == 1:\n        current_predictions.append(tc_l_p[idx])\n    if pred_p_f_wp == 1:\n        current_predictions.append(p_f_wp[idx])\n    if pred_tc_l_wp == 1:\n        current_predictions.append(tc_l_wp[idx])\n    if pred_tc_l_nw == 1:\n        current_predictions.append(tc_l_nw[idx])\n    if pred_r_f_s == 1:\n        current_predictions.append(r_f_s[idx])\n    if pred_r_f_o == 1:\n        current_predictions.append(r_f_o[idx])\n    if pred_r_f_a == 1:\n        current_predictions.append(r_f_a[idx])\n    if pred_p_f_a == 1:\n        current_predictions.append(p_f_a[idx])\n    if pred_c_f_a == 1:\n        current_predictions.append(c_f_a[idx])\n        \n    list_of_pred_names.append(current_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cdc5c6757d20ac8c1b197b3d48ddb021c0691df"},"cell_type":"code","source":"#Compare Random Forest preds with A and B: \nwith open('../input/gendered-pronoun-resolution/test_stage_1.tsv') as tsvfile:\n    \n    reader = csv.DictReader(tsvfile, dialect='excel-tab')\n    count_idx = 0\n    test_ids = []\n    results_A = []\n    results_B = []\n    results_N = []\n    \n    for row in reader:\n        \n        result_A = 0.33\n        result_B = 0.33\n        result_N = 0.33\n        \n        test_ids.append(row['ID'])\n        \n        num_A = 0\n        num_B = 0\n            \n        for name in list_of_pred_names[count_idx]:    \n            if name in row['A']:\n                num_A += 1 \n            elif name in row['B']:\n                num_B += 1\n                \n        if num_A >= 1:\n            result_A = 0.56 + num_A*0.2 #numbers are arbitrary\n        if num_B >= 1:\n            result_B = 0.56 + num_B*0.2 #numbers are arbitrary\n        if num_A > num_B:\n            result_A += 0.15 \n            result_B -= 0.15\n        if num_A == 0 and num_B == 0:\n            result_N = 0.86\n            result_A = 0.1\n            result_B = 0.1\n        \n        results_A.append(result_A)\n        results_B.append(result_B)\n        results_N.append(result_N)\n        \n        count_idx += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c677b852f6e2bf17dfadb89178e97dd8c4ef16aa"},"cell_type":"code","source":"from sklearn import metrics\n# Model Accuracy, how often is the classifier correct?\n#print(\"Accuracy:\",metrics.accuracy_score(y, pred_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36778274cd6744ecf152641bebb414b7f008a7d6"},"cell_type":"code","source":"his_her_list = []\ncount = 0\nfor pronoun in pronoun_list:\n    if pronoun == \"his\" or pronoun == \"her\":\n        his_her_list.append(count)\n    count += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb7f44799fcbfd093616915757be4b6f47f29218"},"cell_type":"code","source":"### rules for \"her\" or \"his\": # represents sent num (each sent diff rule):\n#1 if no subj or obj in trunc curr, then not the prev subj but the prev obj\n#2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ff217e17f6e582eca4384ec2618ec9d2acfcd38"},"cell_type":"code","source":"analyze = \"Slant Magazine's Sal Cinquemani viewed the album as formulaic and ``competently, often frustratingly more of the same from an artist who still seems capable of much more.'' Greg Kot of the Chicago Tribune perceived ``formula production and hack songwriting'', but complimented Pink's personality and its ``handful'' of worthy tracks. In his list for The Barnes & Noble Review, Robert Christgau named The Truth About Love the fourth best album of 2012.\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb0f5ecd7545524776056c3dd570e01b41cef09e"},"cell_type":"code","source":"curr, prev, trunc_curr, remainder = curr_prev_sentence(analyze, pronoun_offset_list[5])\nanalyze_para_lst = find_name_words(name_btwn_paran(trunc_curr))\ntok = word_tokenize(curr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a9c150689d699d58b262dfeb045ea5f942749d9"},"cell_type":"code","source":"doc = nlp(curr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0732869d19183783cb8db558ca681340b1591985"},"cell_type":"code","source":"list_of_name_words(tok)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05272328ca33d68406230153ea5a0564135895fd"},"cell_type":"markdown","source":"print(text_list[0])\nprint(\"0\")\nprint(text_list[1])\nprint(\"1\")\nprint(text_list[2])\nprint(\"2\")\nprint(text_list[3])\nprint(\"3\")\nprint(text_list[5])\nprint(\"5\")\nprint(text_list[7])\nprint(\"7\")\nprint(text_list[14])\nprint(\"14\")\nprint(text_list[16])\nprint(\"16\")\nprint(text_list[18])\nprint(\"18\")\nprint(text_list[19])\nprint(\"19\")\nprint(text_list[23])\nprint(\"23\")\nprint(text_list[24])\nprint(\"24\")\nprint(text_list[25])\nprint(\"25\")\nprint(text_list[26])\nprint(\"26\")\nprint(text_list[27])\nprint(\"27\")\nprint(text_list[28])\nprint(\"28\")\nprint(text_list[29])\nprint(\"29\")\nprint(text_list[220])\nprint(\"220\")\nprint(text_list[221])\nprint(\"221\")\nprint(text_list[224])\nprint(\"224\")\nprint(text_list[226])\nprint(\"226\")\nprint(text_list[228])\nprint(\"228\")\nprint(text_list[229])\nprint(\"229\")\nprint(text_list[230])\nprint(\"230\")\nprint(text_list[231])\nprint(\"231\")\nprint(text_list[232])\nprint(\"232\")\nprint(text_list[233])\nprint(\"233\")\nprint(text_list[234])\nprint(\"234\")\nprint(text_list[237])\nprint(\"237\")\nprint(text_list[240])\nprint(\"240\")\nprint(text_list[245])\nprint(\"245\")\nprint(text_list[246])\nprint(\"246\")\nprint(text_list[247])\nprint(\"247\")\nprint(text_list[249])\nprint(\"249\")"},{"metadata":{"trusted":true,"_uuid":"e0297401b85a16b1e83a68455ad771fc0c95b081"},"cell_type":"code","source":"curr_tok = word_tokenize(curr)\nfor n in [1,2,3,4]:\n    dummy_r_f_a = find_nth_appos(doc,n)\n    #if dummy_r_f_a in remainder and check_if_name(curr_tok,dummy_r_f_a): \n    print(dummy_r_f_a) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b2e538c633a98cc7d0f64b7b05983febf706dd5"},"cell_type":"code","source":"out_df = pd.DataFrame({\"ID\":test_ids})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2e0a79063a21df55ce65f61c8f6a6fbda78383c"},"cell_type":"code","source":"out_df['A'] = results_A\nout_df['B'] = results_B\nout_df['NEITHER'] = results_N","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68c5ad418ef20acac49415ff61ae8ed6b0c6e474"},"cell_type":"code","source":"out_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}