{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n%pylab inline\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"quora_train = pd.read_csv(\"../input/train.csv\")\nquora_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55366858ee32033dc8c11cffc7b28d959a0a9bf3"},"cell_type":"code","source":"quora_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87c8302f505a259670214b562ea56128f37ac06d"},"cell_type":"code","source":"print(f\"Number of neitral texts: {len(quora_train[quora_train['target']==0])}\")\nprint(f\"Number of neitral texts: {len(quora_train[quora_train['target']==1])}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f5f2f47d1927b786656c7c63c4ee0bfe632818d"},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"trusted":true,"_uuid":"536844dcb7bdfa677695625d918c401a64e800a4"},"cell_type":"code","source":"import nltk\nnltk.download('punkt')\nnltk.download('stopwords')\nnltk.download('wordnet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc824fcc4b6d500c9c02162dd98b62b83c2f27f4"},"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nvect = TfidfVectorizer()\nsklearn_tokenizer = vect.build_tokenizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61b7426045fd2b0c323a051c783760898600d680"},"cell_type":"code","source":"sklearn_tokenizer(quora_train.loc[2][\"question_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac0c33ff718d9f76130a68bbfefc58c34f52709e"},"cell_type":"code","source":"word_tokenize(quora_train.loc[2][\"question_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71a8b8212fdf9fa55529031119f8447d9726d1b3"},"cell_type":"code","source":"from nltk.corpus import stopwords\n\nstopwords.words(\"english\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ecb72fd21668102048fc8ac25db530f9ce55273a"},"cell_type":"markdown","source":"## Learning"},{"metadata":{"trusted":true,"_uuid":"f2a878925768904baf494053060a6701443b46ee"},"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.model_selection import StratifiedKFold\n\nvect = TfidfVectorizer(stop_words=stopwords.words(\"english\"))\nclf = SGDClassifier(loss=\"modified_huber\", max_iter=6, class_weight = \"balanced\")\nmodel = Pipeline([('vect', vect), ('clf', clf)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41f96d7c154f281b219bd999c33773cbab6e225a"},"cell_type":"code","source":"%%time\npreds = cross_val_predict(model, quora_train.question_text.values, quora_train.target.values,\n                          cv=StratifiedKFold(4), n_jobs=-1,\n                          method='predict_proba')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"776d3e2594e326e531cd57c4eed5e4e4ce325f96"},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, classification_report, f1_score\n\nroc_auc_score(quora_train.target.values, preds[:,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f370338b6e8c81a5008e4e40e5dfbc3e7dc3c3a"},"cell_type":"code","source":"thresholds = np.arange(0.05, 0.95, 0.05)\npred_arr = []\nfor threshold in thresholds:\n    pred_arr.append(list(map(lambda x: 1 if x[1]>threshold else 0, preds)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebeb6fd184ff18813a3c5d39d517887f967e6559"},"cell_type":"code","source":"i = 0\nfor pred in pred_arr:\n    print(f\"F1-score = {f1_score(quora_train.target.values, np.array(pred))} with threshold = {thresholds[i]}\")\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b780fa34860b8ac3541940b68909301d9417d61b"},"cell_type":"code","source":"best_predictions = list(map(lambda x: 1 if x[1]>0.7 else 0, preds))\nprint(classification_report(quora_train.target.values, best_predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8396e44956db6e90d6a318b23f4413f67448255"},"cell_type":"markdown","source":"## Объясним с помощью eli5"},{"metadata":{"trusted":true,"_uuid":"e0814613ffc91c2ec9eb0965639d49d668e281bc"},"cell_type":"code","source":"import eli5\nmodel.fit(quora_train.question_text.values, quora_train.target.values)\neli5.show_weights(model, top=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"910d03ebfb24fd14da8798733d6fc0c995924603"},"cell_type":"markdown","source":"## Стемминг, Лематизация"},{"metadata":{"trusted":true,"_uuid":"d98d9391f35febf96bba6c4330fe07cb7d9aac8d"},"cell_type":"code","source":"from nltk.stem import SnowballStemmer, WordNetLemmatizer, LancasterStemmer\nfrom functools import lru_cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83a10122814eac3e93ff92bfbe35414be9702733"},"cell_type":"code","source":"@lru_cache(maxsize=2048)\ndef lemmatize_word(word):\n    parts = ['a','v','n','r']\n    lemmatizer = WordNetLemmatizer()\n    for part in parts:\n        temp = lemmatizer.lemmatize(word, part)\n        if temp != word:\n            return temp\n    return word    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f04d1c10247d5a43b4b17c2fe8d0658952447dba"},"cell_type":"code","source":"stemmer = SnowballStemmer('english')\nprint(lemmatize_word('evening'))\nprint(stemmer.stem('evening'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7715bab1de20072f34f787540dbc5362945949c"},"cell_type":"code","source":"def lemmatize_sentence(sentence, tokinizer):\n    return list(map(lemmatize_word, tokinizer(sentence)))\n\ndef stemmatize_sentence(sentence, tokinizer):\n    stemmer = SnowballStemmer('english')\n    return list(map(stemmer.stem, tokinizer(sentence)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e94b4000482818cfee92235de5513e210bb00c94"},"cell_type":"code","source":"%%time\n\nlemmatized_data = list(map(lambda t: lemmatize_sentence(t, sklearn_tokenizer), quora_train.question_text.values))\nstemmatized_data = list(map(lambda t: stemmatize_sentence(t, sklearn_tokenizer), quora_train.question_text.values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54253bc146ebf4c871ec9875599c9f71507270b0"},"cell_type":"code","source":"%%time\n\ninv_lemmatized_data = list(map(lambda t: \" \".join(t), lemmatized_data))\ninv_stemmatized_data = list(map(lambda t: \" \".join(t), stemmatized_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"151376795c6611c3b4938882466e5526be82c1ab"},"cell_type":"code","source":"stemming_preds = cross_val_predict(model, inv_stemmatized_data, quora_train.target.values,\n                          cv=StratifiedKFold(4), n_jobs=4, verbose=1,\n                          method='predict_proba')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"656bd90d65b14459bb89c9f8b964ed3d92451a15"},"cell_type":"code","source":"lemming_preds = cross_val_predict(model, inv_lemmatized_data, quora_train.target.values,\n                          cv=StratifiedKFold(4), n_jobs=4, verbose=1,\n                          method='predict_proba')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47fc4c2cc9088e96bf0810f23bec7a47adb8b7bf"},"cell_type":"code","source":"lemming_pred_arr = []\nstemming_pred_arr = []\nfor threshold in thresholds:\n    lemming_pred_arr.append(list(map(lambda x: 1 if x[1]>threshold else 0, lemming_preds)))\n    stemming_pred_arr.append(list(map(lambda x: 1 if x[1]>threshold else 0, stemming_preds)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83729da75407a17837c94f6e247a6dc9bcf0378b"},"cell_type":"code","source":"i = 0\nprint(\"Lemmatization\")\nfor pred in lemming_pred_arr:\n    print(f\"F1-score = {f1_score(quora_train.target.values, pred)} with threshold = {thresholds[i]}\")\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea5d3595eddd5dd0490b3e016f084483e2e29916"},"cell_type":"code","source":"i = 0\nprint(\"Stemming\")\nfor pred in stemming_pred_arr:\n    print(f\"F1-score = {f1_score(quora_train.target.values, pred)} with threshold = {thresholds[i]}\")\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26ee2b198ead76d69ec083cff91ce070722494b1"},"cell_type":"markdown","source":"## Вроде как стемминг дал лучший результат, чем изначальные данные или лемминизация. Так что будем дальше учиться на них "},{"metadata":{"_uuid":"b662230a4665f73a124761ba0bd2d0aadedc6959"},"cell_type":"markdown","source":"## Творчество гыгыгы"},{"metadata":{"trusted":true,"_uuid":"837606cbdc97149b14e6c300ae221c4d13e83a0c"},"cell_type":"code","source":"model.get_params().keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37f945ca4f29403d42a66708a5956401701cfaa2"},"cell_type":"code","source":"# зададим сетку параметров для перебора модельки\n\"\"\"Я уже перебрал(207 минут!!!), получил такие лучшие параметры, \n{'clf__l1_ratio': 0.15,\n 'clf__loss': 'modified_huber',\n 'clf__max_iter': 3,\n 'vect__max_df': 0.8,\n 'vect__min_df': 0,\n 'vect__stop_words': None}\"\"\"\n\"\"\"\nВот старая сетка параметров. Теперь переберу быстрее, чтобы дебильный kernel смог закоммитить\nparam_grid = {\n    'clf__l1_ratio': [0.15, 0.35],\n    'clf__loss': ['log', 'modified_huber'],\n    'clf__max_iter': [3, 5, 7],\n    'vect__stop_words': [None, stopwords.words(\"english\")],\n    'vect__max_df': [0.8, 1.0],\n    'vect__min_df': [0, 100]\n}\"\"\"\n\nparam_grid = {\n    'clf__max_iter': [2, 3],\n    'vect__max_df': [0.7, 0.8]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aeee092de4b7a13fb508808adfe69475de177754"},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nnew_vect = TfidfVectorizer(lowercase=True)\nnew_clf = SGDClassifier(class_weight = \"balanced\", loss='modified_huber')\nnew_model = Pipeline([('vect', new_vect), ('clf', new_clf)])\nrealy_long_search = GridSearchCV(new_model, param_grid, cv = StratifiedKFold(4), verbose=1, n_jobs=4, scoring='f1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06abd663ecb1fb75b98a2a1388d18588a832ab72"},"cell_type":"code","source":"realy_long_search.fit(inv_stemmatized_data, quora_train.target.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b178daa124e40e074785fe68a6e555a0e4c3bdd5"},"cell_type":"code","source":"realy_long_search.best_params_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bcd4c459d33e1c333afe8f9e2339420166503a6"},"cell_type":"code","source":"best_sgd = realy_long_search.best_estimator_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"feff93d4ed37bcf8c6a15dacc8650db73e944c91"},"cell_type":"code","source":"best_predictions = cross_val_predict(best_sgd, inv_stemmatized_data, quora_train.target.values,\n                          cv=StratifiedKFold(4), n_jobs=4, verbose=1,\n                          method='predict_proba')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9035d3d2d1988c9150a1f7849ad0486bc20bf8d"},"cell_type":"code","source":"best_pred_arr = []\nfor threshold in thresholds:\n    best_pred_arr.append(list(map(lambda x: 1 if x[1]>threshold else 0, best_predictions)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1601aaf52809d71394ce03c32120da1b6df54e87"},"cell_type":"code","source":"i = 0\nprint(\"Best model scores\")\nfor pred in best_pred_arr:\n    print(f\"F1-score = {f1_score(quora_train.target.values, pred)} with threshold = {thresholds[i]:.2f}\")\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3c429a3ef3b7cc591aab567dc46f77efcb810d7"},"cell_type":"code","source":"best_threshold = 0.75","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b46db6e66bdbcb19af907607f543d71c1a3f4ec1"},"cell_type":"markdown","source":"## Я пока сохраню результаты, ибо усталь, позже продолжу"},{"metadata":{"trusted":true,"_uuid":"da2eed68f55da31e39244197292f7ccdd56619d5"},"cell_type":"code","source":"submission_example = pd.read_csv(\"../input/sample_submission.csv\")\nsubmission_example.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b950998fe2d7b1a83de996be4b198a739939f65f"},"cell_type":"code","source":"test = pd.read_csv(\"../input/test.csv\")\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed6ee28f6932fd714a2eb4814a96807a3a1d6198"},"cell_type":"code","source":"%%time\n\ntest_stemmatized_data = list(map(lambda t: stemmatize_sentence(t, sklearn_tokenizer), test.question_text.values))\ntest_inv_stemmatized_data = list(map(lambda t: \" \".join(t), test_stemmatized_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b1b31216173c33f018112054f673a7986e30ca5"},"cell_type":"code","source":"test_predictions = best_sgd.predict_proba(test_inv_stemmatized_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b378e3428b08cbfee2745f589a0695a7890dc7a"},"cell_type":"code","source":"out_predictions = list(map(lambda x: 1 if x[1]>best_threshold else 0, test_predictions))\nout_predictions[200:220]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b585850a4c7605c3608277c89f78f5883aa55e91"},"cell_type":"code","source":"answers = pd.DataFrame(np.transpose([test[\"qid\"].values, out_predictions]), columns=[\"qid\", \"prediction\"])\nanswers.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"e587585048e726e68f32641d658ef325ecefe5c1"},"cell_type":"code","source":"answers.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ea1b6b6465cb9bd92636f4e2464c30344be1199"},"cell_type":"code","source":"len(answers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cf875856d95ee6ff8638ab7babb4b1b869350ec"},"cell_type":"code","source":"\"\"\"from IPython.display import HTML\nimport base64\n\n# function that takes in a dataframe and creates a text link to  \n# download it (will only work for files < 2MB or so)\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv()\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n# create a random sample dataframe\n\n# create a link to download the dataframe\ncreate_download_link(answers.iloc[30000:])\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f010cfc6f07965df3220024e85f6c1ba5b09200c"},"cell_type":"markdown","source":"## Штуки с векторами. Будут потом"},{"metadata":{"trusted":true,"_uuid":"25e3b813ada94d42bcf6392f74106aa66698b62b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5a954f0edb95d43139bb1e2cd8971ccd957604d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}