{"cells":[{"metadata":{"_uuid":"43e5b67b871476c80ee94d6bbae58a17f13bd595"},"cell_type":"markdown","source":"# changing\n\nhttps://www.kaggle.com/cristianossd/tf-idf-approach-on-insincere-questions"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\nfrom nltk import word_tokenize\nfrom scipy.sparse import coo_matrix\nfrom nltk.corpus import stopwords\nstopWords = set(stopwords.words('english'))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')#[:50000]\ntest=pd.read_csv('../input/test.csv')#[:10000]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"323be5bfdf6a73920f2dddbefc6e35957da33bc7"},"cell_type":"code","source":"df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b067c94a1a89074f908ebd860eda5fc050dc4faa"},"cell_type":"markdown","source":"### don't  Resample\n\nTrying undersampling strategy:"},{"metadata":{"_uuid":"00c3a7b82b1e53845e2a656dec1d833fe70770ec"},"cell_type":"markdown","source":"### change tfidf to countvectorizer binary and use 90k features\nuse all words test + train\nusually one uses all train words and omits all new test words"},{"metadata":{"trusted":true,"_uuid":"5d934e74507db59219320522f61ecefd4108d1e7"},"cell_type":"code","source":"\n\n\ntf_vectorizer =CountVectorizer(binary=True,strip_accents='unicode',max_features=90000).fit(df['question_text'].append(test['question_text']))\nlistOfWords = tf_vectorizer.get_feature_names()\ndictOfWords = { listOfWords[i]:i for i in range(0, len(listOfWords) ) }\ntf_vectorizer.transform(df['question_text'])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ef7b1bfe3f8872cfdfd5556996165ca967d1780"},"cell_type":"markdown","source":"**co-occurrance snippet**"},{"metadata":{"trusted":true,"_uuid":"84ec098651898609da054b41017b61f77d06fdd2"},"cell_type":"code","source":"from nltk import word_tokenize\nfrom scipy.sparse import coo_matrix\n\ndef create_cooccurrence_matrix(filename,tokenizer,window_size,vocabulary):\n    #vocabulary={}\n    data=[]\n    row=[]\n    col=[]\n    for sentence in filename:\n        sentence=sentence.strip()\n        #print(sentence)\n        tokens=[token for token in tokenizer(sentence) if token!=u\"\"]\n        for pos,token in enumerate(tokens):\n            i=vocabulary.setdefault(token,len(vocabulary))\n            start=max(0,pos-window_size)\n            end=min(len(tokens),pos+window_size+1)\n            for pos2 in range(start,end):\n                if pos2==pos: \n                    continue\n                j=vocabulary.setdefault(tokens[pos2],len(vocabulary))\n                data.append(1.); row.append(i); col.append(j);\n    \n    cooccurrence_matrix=coo_matrix((data,(row,col)))\n    return vocabulary,cooccurrence_matrix\n#voca,coo=create_cooccurrence_matrix(df.question_text.str.lower().values,word_tokenize,100,dictOfWords)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5aa77e23c9d494903577260cc289c943ec0f1a74"},"cell_type":"markdown","source":"**splitting and vectorizing**"},{"metadata":{"trusted":true,"_uuid":"1c5216d7038b6a6fc7cdd7d0194cf4cf8f5e949b"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\n\n\nX_train, X_test, y_train, y_test = train_test_split(df['question_text'],\n                                                    df['target'],\n                                                    test_size=0.2)\n#tf_vectorizer = TfidfVectorizer().fit(df_under['question_text'])\n#tf_vectorizer = CountVectorizer(tokenizer=word_tokenize,stop_words).fit(df['question_text'])\nX_train = tf_vectorizer.transform(X_train)\nX_test = tf_vectorizer.transform(X_test)\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ebd777f9dc8db3568f0639de61f83c7d96726a3a"},"cell_type":"markdown","source":"**logistic training**"},{"metadata":{"trusted":true,"_uuid":"04a680bab8f3a8fe2ad8802b1d2761bf03987be9"},"cell_type":"code","source":"#from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.linear_model import LogisticRegression\nclf=LogisticRegression(C=1.0,multi_class='multinomial',penalty='l2', solver='saga',n_jobs=-1)\nclf.fit(X_train, y_train)\n#clf = MultinomialNB().fit(X_train, y_train)\npredicted = clf.predict(X_test)\nnp.mean(predicted == y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.datasets import make_moons, make_circles, make_classification\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.gaussian_process import GaussianProcessClassifier\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\nfrom sklearn.svm import LinearSVC\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import Perceptron\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.neighbors import NearestCentroid\nfrom sklearn.linear_model import RidgeClassifier\nfrom sklearn.naive_bayes import BernoulliNB, MultinomialNB\nfrom sklearn.metrics import f1_score,confusion_matrix\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nfrom sklearn.neural_network import MLPClassifier\n\nnames = [\"LR\",#\"MLP\",\n        #\"SVC\",        #\"SVC3\",\n        \"XGB\",\n         \"Passive-Aggressive\",    \n        \"linearSVC\",\"NearestCentroid\",\n        \"multNB\",\"bernouilliNB\",\"Ridge Classifier\",\n         \"Perceptron\",#\"kNN\",\n\n         \"SGD modeL2\",\"SGD elast\",\n         #\"Nearest Neighbors\",# \"Linear SVM\", \n         #\"RBF SVM\", #\"Gaussian Process\",\n         \"Decision Tree\", #\"Random Forest\", #\"Neural Net\",\n        \"AdaBoost\",\n         #\"Naive Bayes\" #, \"QDA\"\n        ]\n\nclassifiers = [\n    LogisticRegression(),\n    #MLPClassifier(),\n    #SVC(kernel='linear'),\n    #SVC(kernel='sigmoid'),\n    XGBClassifier(learning_rate=0.1,n_estimators=100),\n    PassiveAggressiveClassifier(max_iter=50, tol=1e-3),    \n    LinearSVC(penalty=\"l2\", dual=False,tol=1e-3),\n    NearestCentroid(),\n    MultinomialNB(alpha=.01),\n    BernoulliNB(alpha=.01),\n    RidgeClassifier(tol=1e-2, solver=\"sag\"),\n    Perceptron(max_iter=50, tol=1e-3),\n    #KNeighborsClassifier(n_neighbors=10),\n\n    SGDClassifier(alpha=.0001, max_iter=50,penalty=\"l2\"),\n    SGDClassifier(alpha=.0001, max_iter=50,penalty=\"elasticnet\"),\n    #KNeighborsClassifier(5),\n    \n    #SVC(kernel=\"linear\", C=0.025),\n    #SVC(gamma=2, C=1),\n    #GaussianProcessClassifier(1.0 * RBF(1.0)),\n    DecisionTreeClassifier(max_depth=5),\n    #RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),\n    #MLPClassifier(alpha=1, max_iter=1000),\n    AdaBoostClassifier(),\n    #GaussianNB(),\n    #QuadraticDiscriminantAnalysis()\n    ]\n\n#or countmatrix or tfidfmatrix\n#X_train, X_test, y_train, y_test = train_test_split(countmatrix, y, test_size=0.2, random_state=42)\n    # iterate over classifiers\nfor name, clf in zip(names, classifiers):\n    clf.fit(X_train, y_train)\n    score = clf.score(X_test, y_test)\n    y_pred=clf.predict(X_test)\n    print(name,score,f1_score(y_test,y_pred))\n    print('Confusion matrix:', confusion_matrix(y_pred, y_test)  )  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd65d4ac8c6a4a35c95cd78687345601af4107a6"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\n\nf1_score(y_test, predicted,average=None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80790e9798092407b93d62547d568ca34d1e2e36"},"cell_type":"markdown","source":"### Submission dataset"},{"metadata":{"trusted":true,"_uuid":"6d2a83008b4a507d9c606a5e0f120b00c0d2caf4"},"cell_type":"code","source":"df_test = pd.read_csv('../input/test.csv')\nX_submission = tf_vectorizer.transform(df_test['question_text'])\npredicted_test = clf.predict(X_submission)\n\ndf_test['prediction'] = predicted_test\nsubmission = df_test.drop(columns=['question_text'])\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50fba64d48f018a5db8f80f1e0657aca58848005"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}