{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import classification_report\n\nfrom scipy.sparse import hstack","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfa72073519c60dc240157bf87aa62d61b16aa43"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv').fillna(' ')\ntest = pd.read_csv('../input/test.csv').fillna(' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ac5241ba2e8cb6044f09421fb1599d0a7de37d4"},"cell_type":"code","source":"def prepear_data(train, test):\n    \n#     yes_df = train[train['target'] == 1]\n#     no_df = train[train['target'] == 0]\n#     train_df = yes_df.append(yes_df).append(no_df.loc[:yes_df.shape[0]])\n    train_df = train\n    \n#     print('target: positive: {} negative: {}'.format(yes_df.shape[0], no_df.shape[0]))\n    \n    \n    train_texts = train_df['question_text']\n    test_texts = test['question_text']\n    all_texts = pd.concat([train_texts, test_texts])\n\n    word_vectorizer = TfidfVectorizer(\n        sublinear_tf=True,\n        strip_accents='unicode',\n        analyzer='word',\n        token_pattern=r'\\w{1,}',\n        stop_words='english',\n        ngram_range=(1, 2),\n        max_features=10000)\n\n    word_vectorizer.fit(all_texts)\n\n    char_vectorizer = TfidfVectorizer(\n        sublinear_tf=True,\n        strip_accents='unicode',\n        analyzer='char',\n        stop_words='english',\n        ngram_range=(2, 6),\n        max_features=10000)\n\n    char_vectorizer.fit(all_texts)\n\n    \n    train_word_features = word_vectorizer.transform(train_texts)\n    test_word_features = word_vectorizer.transform(test_texts)\n    \n    train_char_features = char_vectorizer.transform(train_texts)\n    test_char_features = char_vectorizer.transform(test_texts)\n\n    train_features = hstack([train_char_features, train_word_features])\n    test_features = hstack([test_char_features, test_word_features])\n    \n    train_target = train_df['target']\n    \n    return train_features, train_target, test_features\n#     return train_word_features, train_target, test_word_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63c3dd48cbbddbed2ee9d237becc36b24d875eeb"},"cell_type":"code","source":"train_features, train_target, test_features = prepear_data(train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f9cef0894e51f21468a554de232a30215454779"},"cell_type":"code","source":"classifier = LogisticRegression(C=0.1, solver='sag')\nchosen_scoring = 'recall' # 'roc_auc', 'accuracy', 'recall'\ncv_score = np.mean(cross_val_score(classifier, train_features, train_target, cv=8, scoring=chosen_scoring))\nprint('CV score ({}) for target 1 is {}'.format(chosen_scoring, cv_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"299e6b37348280ea6f7883268ae8ca7c2756154f"},"cell_type":"code","source":"classifier.fit(train_features, train_target)\npredicted_targets = classifier.predict(train_features)\nprint(classification_report(train_target, predicted_targets))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18051dc59c56303102588ff0a8bd87ec563ea453"},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission['prediction'] = classifier.predict(test_features)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb943d2eafd391f550c38e22fb26ef5cf3f5c0c3"},"cell_type":"code","source":"submission.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fe167a853a667e2eedff7f9cb3aba9c038f6f1e"},"cell_type":"code","source":"print(submission[submission['prediction']>0.5].shape, submission[submission['prediction'] < 0.5].shape)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}