{"cells":[{"metadata":{"_uuid":"b378e1a8ecf10ee93d4e363f4eb0e78fd2db246b","trusted":false},"cell_type":"code","source":"%%time\nfrom scipy.sparse import coo_matrix, hstack\nimport numpy as np\nimport pandas as pd\nfrom sklearn import *\nimport lightgbm as lgb\nimport string\n\ntrain = pd.read_csv('../input/train.csv', encoding='latin-1').fillna('') \ntest = pd.read_csv('../input/test.csv', encoding='latin-1').fillna('')\nprint(train.shape, test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef7fea85d3e7959745c27c89f5c8268cf442ccee","trusted":false},"cell_type":"code","source":"%%time\ntrainn = train[train['target']==1]\ntrainp = train[train['target']==0].tail(trainn.shape[0] * 3)\nprint(trainn.shape, trainp.shape)\ntrain = pd.concat((trainn, trainp), axis=0).reset_index(drop=True)\ntrainn = []; trainp = [];\nprint(train.shape, test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9939ace532034d438a0f92ff4363b7025e611d76"},"cell_type":"code","source":"#https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\npuncts = list(set([',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ] + list(string.punctuation)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fa50605c7ed9a8a96efa8e0c95dd54a14a76ecaa"},"cell_type":"code","source":"#https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cb2ec853f228d5997389a75589f1b924b92a9ef2"},"cell_type":"code","source":"def clean_str(df):\n    df['question_text'] = df['question_text'].str.lower()\n    df['question_text'] = df['question_text'].map(lambda x: (' ').join([mispell_dict[w] if w in mispell_dict else w for w in x.split(\" \")]))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"628559ce9581d26f4bb3df9182473bc847ec1f02","trusted":false},"cell_type":"code","source":"%%time\ntrain = clean_str(train)\ntest = clean_str(test)\n\ndef starndard_text_features(df):\n    col = [c for c in df.columns]\n    df['len'] = df['question_text'].map(lambda x: len(str(x)))\n    df['wc'] = df['question_text'].map(lambda x: len(str(x).split(' ')))\n    df['wcu'] = df['question_text'].map(lambda x: len(set(str(x).split(' '))))\n    df[\"pc\"] = df['question_text'].map(lambda x: len([c for c in str(x) if c in puncts]))\n    df[\"mwl\"] = df['question_text'].map(lambda x: np.mean([len(w) for w in str(x).split()]))\n    df['wcu%'] = df['wcu'] / df['wc']\n    df['pc%'] = df['pc'] / df['len']\n    col = [c for c in df.columns if c not in col]\n    return df[col]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3dce740f726e2e2423233d42e68ffb020733b7d9","trusted":false},"cell_type":"code","source":"%%time\ntfidf = feature_extraction.text.TfidfVectorizer(stop_words='english', ngram_range=(2, 5), max_features=10000)\ncvect = feature_extraction.text.CountVectorizer(stop_words='english', ngram_range=(2, 5), max_features=10000)\ntfidf_char = feature_extraction.text.TfidfVectorizer(sublinear_tf=True, strip_accents='unicode', analyzer='char_wb', token_pattern=r'\\w{1,}', stop_words='english', ngram_range=(2, 5), max_features=10000)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43ac47546422a567c3affbf38b5598e89c84bebd","trusted":false},"cell_type":"code","source":"%%time\ntfidf.fit(pd.concat((train['question_text'], test['question_text'])).values.astype(str))\ntrainf = hstack([starndard_text_features(train), tfidf.transform(train['question_text'])]); print(trainf.shape)\ntestf = hstack([starndard_text_features(test), tfidf.transform(test['question_text'])]); print(testf.shape)\n\ncvect.fit(pd.concat((train['question_text'], test['question_text'])).values.astype(str))\ntrainf = hstack([trainf, cvect.transform(train['question_text'])]); print(trainf.shape)\ntestf = hstack([testf, cvect.transform(test['question_text'])]); print(testf.shape)\n\ntfidf_char.fit(pd.concat((train['question_text'], test['question_text'])).values.astype(str))\ntrainf = hstack([trainf, tfidf_char.transform(train['question_text'])]); print(trainf.shape)\ntestf = hstack([testf, tfidf_char.transform(test['question_text'])]); print(testf.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"%%time\nx1, x2, y1, y2 = model_selection.train_test_split(trainf, train['target'], test_size=0.2, random_state=3)\n\ndef lgb_f1(preds, y):\n    y = y.get_label()\n    score = metrics.f1_score(y, (preds>0.5).astype(int))\n    return 'f1', score, True\n\nparams = {'learning_rate': 0.2, 'max_depth': 8, 'objective': 'binary', 'boosting_type': 'gbdt', 'metric': 'binary_error', 'num_leaves': 32, 'feature_fraction': 0.9,'bagging_fraction': 0.8, 'bagging_freq': 5}\nmodel = lgb.train(params, lgb.Dataset(x1, label=y1), 1000, lgb.Dataset(x2, label=y2), early_stopping_rounds=50,  feval=lgb_f1,  verbose_eval=20)\ntest['prediction'] = (model.predict(testf, num_iteration=model.best_iteration) > 0.5).astype(int)\ntest[['qid', 'prediction']].to_csv('submission.csv', index=False)\nprint(test.prediction.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"07444866658bbc8f3de68f6ca064afb767af53c0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"13473320d6e57353d9a41228a21e88ae425a79c5"},"cell_type":"code","source":"#Early stopping, best iteration is:\n#[517]\tvalid_0's binary_error: 0.098116\tvalid_0's f1: 0.795103","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}