{"cells":[{"metadata":{"_uuid":"3c6bb4c0d3d16017e21508f838f72d0fffec4b7b"},"cell_type":"markdown","source":"# Qura EDA\n对数据的细节进行分析:文本的长度分布,文本中unique单词的数目,文本中非英文单词数据的展示."},{"metadata":{"trusted":true,"_uuid":"9841beb9b2ee033786c835fda344dc95f7741c7e"},"cell_type":"code","source":"# 加载软件包\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"514a086b8564b35f1f4f2496a2530887a68a98cb"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a57c1f11555f94e5be7d7496964ccb90dd959a5"},"cell_type":"code","source":"#列名\nprint(\"train:\",train_df.columns.values,\"test:\",test_df.columns.values)\ntrain_sentences = train_df[\"question_text\"].apply(lambda x: x.split()).values\ntest_sentences = test_df[\"question_text\"].apply(lambda x: x.split()).values\nprint(train_sentences.shape,test_sentences.shape)\n\ntrain_length = [len(sentence) for sentence in train_sentences]\ntest_length = [len(sentence) for sentence in test_sentences]\ntrain_length_unique = [len(set(sentence)) for sentence in train_sentences]\ntest_length_unique = [len(set(sentence)) for sentence in test_sentences]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a90705c154f745477b0457efc7af9562884be41d"},"cell_type":"code","source":"train_length_df = pd.Series(train_length)\ntest_length_df = pd.Series(test_length)\nlength1 = train_length_df.value_counts(sort=False)\nlength2 = test_length_df.value_counts(sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"738a48461b64b496f001b7ffe7a185fe93fd2192"},"cell_type":"code","source":"# 没有经过处理的train句子的长度\nplt.figure(figsize=(12, 8))\nlength1.plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08e4cdaefae5dc35f46db0dc38b3ae62750a82d7"},"cell_type":"code","source":"# 没有经过处理的test句子的长度\nplt.figure(figsize=(12, 8))\nlength2.plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc573e6f759ef101f15703a3ce5afc943842697e"},"cell_type":"code","source":"length1_unique = pd.Series(train_length_unique).value_counts(sort=False)\nlength2_unique = pd.Series(test_length_unique).value_counts(sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d50aa89d4ce05208320015f283696aa4bd3fe0b"},"cell_type":"code","source":"# 没有经过处理的train句子的长度\nplt.figure(figsize=(12, 8))\nlength1_unique.plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83d9fdc58fa15da8c57f9a86ec2946b92dcd921c"},"cell_type":"code","source":"# 没有经过处理的test句子的长度\nplt.figure(figsize=(12, 8))\nlength2_unique.plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"028eaae571874383144cc45ece7ff9ccd8c2b2e4"},"cell_type":"markdown","source":"以上的结果表明,在长度上,句子中重复的单词重复对于句子长度的分布并不是很明显!"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}