{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n# print(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn import model_selection\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\n# import lightgbm as lgb\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f40da9ab348d34e6b4b3863c44c919193cda8b83","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntrain_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7eff3bb329b873fd09c1bed020a1ae0a45fd5f6","trusted":true},"cell_type":"code","source":"from sklearn.pipeline import make_pipeline\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nimport nltk\nnltk.download('punkt')\nnltk.download('stopwords')\nnltk.download('wordnet')\n\nvect = TfidfVectorizer()\nsklearn_tokenizer = vect.build_tokenizer()\n\nvect = TfidfVectorizer(tokenizer= sklearn_tokenizer,stop_words='english')\nclf = SGDClassifier(loss = 'log',class_weight = 'balanced')\nmodel = Pipeline([('vect', vect), ('clf', clf)])\npipe = Pipeline([('vect', vect), ('clf', clf)])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_text = train_df['question_text']\ntest_text = test_df['question_text']\ny_train=train_df['target']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pipe.fit(train_text,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import cross_val_predict, cross_val_score, \\\n    GridSearchCV, RandomizedSearchCV, train_test_split, KFold\nfrom sklearn.linear_model import SGDRegressor\n\n# Creating X, y Variables\n\n\nall_text = train_text.append(test_text)\n#tfidf_vectorizer = TfidfVectorizer()\n#tfidf_vectorizer.fit(all_text)\n\ngrid = {\n       # 'vect__max_features':[100, 2000],\n       # 'vect__ngram_range': [(1, 1), (1, 2), (2, 2)],\n        'vect__min_df' : [0.015, 0.035],# отбрасывает все слова, которые имеют повторяемость ниже чем x\n        \n      # 'vect_lowercase': [True, False],# переводит все заглавные буквы из слов из Заглавных букв в прописные\n       'vect__stop_words':[{'english'},None],# определение словаря стоп-слов, если None, то словарь определяется в процессе обучения\n        'clf__loss' : ['hinge','squared_hinge','perceptron'], # функция потерь, так как используем метрику roc_auc то нужна функция, которая выдаёт вероятностные знач принадлежности к классу\n        'clf__alpha' : [0.005,0.0005], #параметр регуляризации (1/lambda)=c чем меньше тем сильнее регуляризация\n       \n        }\n\nsearch = RandomizedSearchCV(pipe, param_distributions=grid, cv = 5,scoring='f1', verbose =1, n_jobs = -1)\nsearch.fit(train_text,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\npipe_2=search.estimator\npipe_2.fit(train_text,y_train)\ndef print_report(pipe):\n    \n    y_pred = pipe.predict(train_df.question_text.values)\n    report = metrics.classification_report(y_train, y_pred\n       )\n    print(report)\n    print(\"accuracy: {:0.3f}\".format(metrics.accuracy_score(y_train, y_pred)))\n    print(\"rog_auc: {:0.3f}\".format(metrics.roc_auc_score(y_train,y_pred)))\n    print(\"f1: {:0.3f}\".format(metrics.f1_score(y_train,y_pred)))\nprint_report(pipe_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = pipe_2.predict(test_df.question_text.values);\ny_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_text_features_cv = tfidf_vectorizer.transform(train_text)\ntest_text_features_cv = tfidf_vectorizer.transform(test_text)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57412eb0502d577266e0348ec051567b2258cc1c","trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"92bb17af86b299abad59336231564a93ca0396ba","trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a067fef47c398a598bc75f95e444121aaa15c240","trusted":true},"cell_type":"code","source":"train_text = train_df['question_text']\ntest_text = test_df['question_text']\ntrain_target = train_df['target']\nall_text = train_text.append(test_text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58a71472c9bc073f8c9fa28baa4c8f84a95faf6c","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fac0fbe600a3a260a0b32553e21273d1e833a10","trusted":true},"cell_type":"code","source":"submission1 = pd.DataFrame.from_dict({'qid': test_df['qid']})\nsubmission1['prediction'] = y_pred\nsubmission1.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aca5a6db0f61f935ccfd3e326cd489ced3dd582c","trusted":true},"cell_type":"code","source":"submission1['prediction']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1 = pd.DataFrame.from_dict({'qid': test_df['qid']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.size(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":1}