{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom pandas.core.frame import DataFrame\nfrom gensim.models.word2vec import Word2Vec\nimport nltk.data\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nimport datetime\n\nfrom xgboost.sklearn import XGBClassifier\nimport gc\n\nstopw=[]\nfor w in stopwords.words('english'):\n    stopw.append(w)\ndef process(address,scri): #数据处理\n    pf=[]\n    tf = pd.read_csv(address)\n    count=0\n    for line in tf[scri]:\n        line1= re.sub(r'[^a-zA-Z]',' ' ,line)  #去标点符号\n        line2 = line1.lower().split()           #小写化，分割单词\n        line3= [w for w in line2 if w not in stopw]\n        pf.append(line3)\n        count+=1\n        if count%10000==0: print(\"+\",end ='')\n    return pf\n\nprint(\"处理训练数据......\")\npf=process(\"../input/train.csv\",\"question_text\")\nprint(\"处理测试数据......\")\ntf=process(\"../input/test.csv\",\"question_text\")\n\ndel stopw\ngc.collect()\nprint(\"读取标签......\")\ntrain_lable = pd.read_csv(\"../input/train.csv\")[\"target\"].head(300000)\ntest_qid = pd.read_csv(\"../input/test.csv\")[\"qid\"]\ndf=DataFrame({\"question_test\":pf})\nall_df=DataFrame({\"question_test\":pf+tf})\ntesdata=DataFrame({\"question_test\":tf})\ndel pf \ngc.collect()\ndel tf\ngc.collect()\nprint(\"训练稠密矩阵.....\")\nnum_features = 100    # Word vector dimensionality\nmin_word_count = 10   # Minimum word count\nnum_workers = 4       # Number of threads to run in parallel\ncontext = 10          # Context window size\ndownsampling = 1e-3   # Downsample setting for frequent words\nVec = Word2Vec(all_df[\"question_test\"],workers=num_workers, \\\n            size=num_features, min_count = min_word_count, \\\n            window = context, sample = downsampling)\ndef to_review_vector(review):\n    array = np.array([Vec[w] for w in review if w in Vec])\n    return pd.Series(array.mean(axis=0))\nprint(\"生成训练集X......\")\ntrain_data_feature=pd.DataFrame()\nfor i in range(4):\n    train_data_feature1=df.question_test[300000*i:(i+1)*300000].apply(to_review_vector)\n    print(\"finish{}\",i)\n    train_data_feature=pd.concat([train_data_feature,train_data_feature1])\ntrain_data_feature1=df.question_test[1200000:].apply(to_review_vector)\ntrain_data_feature=pd.concat([train_data_feature,train_data_feature1])\nprint(\"finish X......\")\n\nprint(\"生成测试集x......\")\ntest_data_feature=tesdata.question_test.apply(to_review_vector)\nprint(\"finish x......\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c9f4e06a7aa0b360b5867bdcf8c4c4771dde97d"},"cell_type":"code","source":"del df\ngc.collect()\ndel tesdata\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"849881baa97d32e12c74c4788f5d99771f6506e0"},"cell_type":"code","source":"model_xgb = XGBClassifier()\nprint(\"分类模型训练中......\")\nfor i in range(4):\n    model_xgb.fit(train_data_feature[300000*i:300000*(i+1)],train_lable[300000*i:300000*(i+1)])\nmodel_xgb.fit(train_data_feature[1200000:],train_lable[1200000:])\nprint(\"分类模型训练完毕！\")\n\n\nresult=model_xgb.predict(test_data_feature)\n\nsubmit_df = pd.DataFrame({\"qid\": test_qid, \"prediction\": result})\nsubmit_df.to_csv(\"submission.csv\", index=False)\n\n\nprint(\"结束\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e34701409406d878b70a27db36611ce883607ca5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f803aed0f618a1e60dc1b258679fc0196b1fa891"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"186abeb9dfaa97e820711f34f79e06037aee2d8b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92fe94b2bdca7b0cefe77f926a32211aab7acbf5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4253ca6dce29536d3e1aa6c9cbbd27541036cb6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3ba997585cb3253732f8c699c61d55791543e50"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"198956af420bb69b130be51783d6569dca465eb6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}