{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Reading train and test data\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fee0e5a3b943a54c19275f26ab0ee5f0c174c32d"},"cell_type":"code","source":"print(\"Train data shape: \",train.shape)\nprint(\"Test data shape: \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3632617b142f12945c218a1654bc7034bccbfbf"},"cell_type":"code","source":"#Class count\ncount_class_0, count_class_1 = train.target.value_counts()\nprint(\"Count of class 0\", count_class_0)\nprint(\"Count of class 1\", count_class_1)\n\nclass_0 = train[train['target'] == 0]\nclass_1 = train[train['target'] == 1]\nprint(class_0.shape)\nprint(class_1.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f84ba651dc5016e0aab4a4d21125524a6db07bc"},"cell_type":"markdown","source":"Data is highly biased as approx ~6% values belongs to class 1(Insincere) only, rest belongs to class 0(Sincere)."},{"metadata":{"trusted":true,"_uuid":"fea5fbc086abcc3bf5d7f2ff93120422e29a461a"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79099aa603d04ea6423c3b7091ca5073e8cc84ac"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e72471a0bedb4bf4ee9673803035f8674fffa497"},"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\nimport re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cc551ecd1d857339db21eb05c0c5bfb3f07a69b"},"cell_type":"code","source":"#Remove bad symbols and stopwords from test and train data\nREPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\nBAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')\nSTOPWORDS = set(stopwords.words('english'))\n\ndef text_prepare(text):\n    \"\"\"\n        text: a string\n        \n        return: modified initial string\n    \"\"\"\n    text = text.lower()   # lowercase text\n    text = REPLACE_BY_SPACE_RE.sub(\" \", text)     # replace REPLACE_BY_SPACE_RE symbols by space in text\n    text = BAD_SYMBOLS_RE.sub(\"\", text)     # delete symbols which are in BAD_SYMBOLS_RE from text\n    \n    \n    resultwords = [word for word in text.split() if word not in STOPWORDS]  # delete stopwords from text\n    text = ' '.join(resultwords)\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21200d0aed8f24392c2a7cba93ad01d667858dbf"},"cell_type":"code","source":"\nX_train_class_0 = class_0['question_text'].values\nX_train_class_1 = class_1['question_text'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24563e3d159ebc7698ce858314da9df98006690f"},"cell_type":"code","source":"X_train_class_0 = [text_prepare(x) for x in X_train_class_0]\nX_train_class_1 = [text_prepare(x) for x in X_train_class_1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94476219a865ad351320921c346cf25f7c5d3e66"},"cell_type":"code","source":"#Word counts of n-grams of both the classes\nfrom nltk.util import ngrams\nfrom collections import Counter\nwords_counts_class_0_unigram = Counter()\nwords_counts_class_1_unigram = Counter()\nwords_counts_class_0_bigram = Counter()\nwords_counts_class_1_bigram = Counter()\nwords_counts_class_0_trigram = Counter()\nwords_counts_class_1_trigram = Counter()\n\nfor sentence in X_train_class_0:\n  token = [word for word in sentence.split()]\n  words_counts_class_0_unigram.update(x for x in ngrams(token, 1))\n  words_counts_class_0_bigram.update(x for x in ngrams(token, 2))\n  words_counts_class_0_trigram.update(x for x in ngrams(token, 3))\n\nfor sentence in X_train_class_1:\n  token = [word for word in sentence.split()]\n  words_counts_class_1_unigram.update(x for x in ngrams(token, 1))\n  words_counts_class_1_bigram.update(x for x in ngrams(token, 2))\n  words_counts_class_1_trigram.update(x for x in ngrams(token, 3))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fadd4f7adfb0c561db559390e2deb06f655d154"},"cell_type":"code","source":"words_counts_class_0_unigram.most_common(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc456f3498336a0390376afdd287a05b9a20714d"},"cell_type":"code","source":"words_counts_class_1_unigram.most_common(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0323d9ad7f267ab6b68e4cc63d95b9244c71248"},"cell_type":"code","source":"words_counts_class_0_bigram.most_common(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d148b68a86d366a906827ece46c76ac880b3421"},"cell_type":"code","source":"words_counts_class_1_bigram.most_common(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f0337e38e35bc4518c67d33c45a2cfff6f12661"},"cell_type":"code","source":"words_counts_class_0_trigram.most_common(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a9dfe30ebc1471a2739bff690644435f28bb88"},"cell_type":"code","source":"words_counts_class_1_trigram.most_common(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"550591cf9b31116fc11e2ff0c0ec8a6d5142830b"},"cell_type":"code","source":"print('Average character length of Sincere questions in train is {0:.0f}.'.format(np.mean(class_0['question_text'].apply(lambda x: len(x)))))\nprint('Average character length of Insincere questions in train is {0:.0f}.'.format(np.mean(class_1['question_text'].apply(lambda x: len(x)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc6a8c82f4beff1ef63e1ba399a65933b4f58f40"},"cell_type":"code","source":"print('Average word length of Sincere questions in train is {0:.0f}.'.format(np.mean(class_0['question_text'].apply(lambda x: len(x.split())))))\nprint('Average word length of Insincere questions in train is {0:.0f}.'.format(np.mean(class_1['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91e5e7ec3381c4cc801eccf5320511dd12a05e17"},"cell_type":"code","source":"print('Max word length of Sincere questions in train is {0:.0f}.'.format(np.max(class_0['question_text'].apply(lambda x: len(x.split())))))\nprint('Max word length of Insincere questions in train is {0:.0f}.'.format(np.max(class_1['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30253de7fbe0e93f667000b1749aca38ad6986a9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}