{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport time\nfrom contextlib import contextmanager\nfrom functools import lru_cache\nos.environ['OMP_NUM_THREADS'] = '4'\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.1f} s')\n    gc.collect();\n    \ndef reduce_memory(df):\n    for c in df.columns:\n        if df[c].dtype=='int':\n            if df[c].min()<0:\n                if df[c].abs().max()<2**7:\n                    df[c] = df[c].astype('int8')\n                elif df[c].abs().max()<2**15:\n                    df[c] = df[c].astype('int16')\n                elif df[c].abs().max()<2**31:\n                    df[c] = df[c].astype('int32')\n                else:\n                    continue\n            else:\n                if df[c].max()<2**8:\n                    df[c] = df[c].astype('uint8')\n                elif df[c].max()<2**16:\n                    df[c] = df[c].astype('uint16')\n                elif df[c].max()<2**32:\n                    df[c] = df[c].astype('uint32')\n                else:\n                    continue\n        if df[c].dtype=='float64':\n            df[c] = df[c].astype('float32')\n    return df\nDATA_DIR = '../input/avito-demand-prediction/'\ntarget_col = 'deal_probability'\nos.listdir(DATA_DIR)","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"usecols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', \n           'title', 'description']\ntrain = pd.read_csv(DATA_DIR+'train.csv', usecols=usecols+[target_col])\ntest = pd.read_csv(DATA_DIR+'test.csv', usecols=usecols)","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58bec0fdfd29c2fd263b6d5ee51237f1b4017b91","collapsed":true},"cell_type":"code","source":"y = train[target_col].values\ndel train[target_col]; gc.collect()\ntrain_num = len(train)\ndf = pd.concat([train, test], ignore_index=True)\ndel train, test; gc.collect()","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0207f7cb003a7d41ce6113604709abb545e15a1a","collapsed":true},"cell_type":"code","source":"df['title'].isnull().sum(), df['description'].isnull().sum()","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"50089345baee301ce3dd3a49ded8f3b76aee828d"},"cell_type":"code","source":"df['description'].fillna('unknown', inplace=True)","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d9e4a2c821a881a0d33d3506b6f26ee64025fc52"},"cell_type":"code","source":"# df = df.head(1000) #for debug","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc493989556d2b7c12084e25dcb1fa1054f691db","collapsed":true},"cell_type":"code","source":"context_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3']\ndf['context'] = ''\nfor i, c in tqdm(enumerate(context_cols), total=len(context_cols)):\n    if i>0:\n        df['context'] += ' '\n    df['context'] = df['context'] + df[c].fillna('').str.lower()","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18b3f500eadfa6e827a7ee47db55b9c93122ca9a","collapsed":true},"cell_type":"code","source":"%%time\nimport re\nimport string\nfrom nltk.corpus import stopwords\nfrom nltk import word_tokenize\nSTOPWORDS = stopwords.words('russian')\npuncs = str(string.punctuation).replace('-', '')\ntranslator = str.maketrans(puncs, len(puncs)*' ')\ndef proc_text(s):\n    s = s.translate(translator)\n    s = s.strip(' ').lower().split(' ')\n    s = [w for w in s if w!='' and w not in STOPWORDS]\n    s = ' '.join(s)\n    return s\ndf['context'] = df['context'].apply(lambda s: proc_text(s))\ndf['desc'] = df['description'].copy()","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3835771d9d4afafb495b43a043cce2e49c89d26e","collapsed":true},"cell_type":"code","source":"df = df.assign(**{'text': df['title'] + ' ' + df['desc']})\ndf = df[['context', 'text', 'title', 'desc']]\ngc.collect();\ndf.head()","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"268c8ae11e33da93ec928a09c6910d949f539027"},"cell_type":"markdown","source":"## Reference\n- https://www.analyticsvidhya.com/blog/2018/02/the-different-methods-deal-text-data-predictive-python/"},{"metadata":{"_uuid":"45260185750db2d253f601de88c96bd95a17d287"},"cell_type":"markdown","source":"## 1. Basic Feature Extraction\n### 1.1 Number of Words"},{"metadata":{"trusted":true,"_uuid":"c86e1c2db7528e0ad0835d38623c6d84576c31ad","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nWord_'\nfunc = lambda x: len(x.split(' '))\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].apply(func)\n    })\nnew_col = df[fprefix+'title']+df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46e7225091686e45aad969142b71cda486c35fa3","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"80640e0667adaea8fc7728a07afb96a6344832a1"},"cell_type":"markdown","source":"### 1.2 Number of characters"},{"metadata":{"trusted":true,"_uuid":"03faf7f20f00a62f2fd87d3b5a5eb5ef684685fe","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nChar_'\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].str.len()\n    })\nnew_col = df[fprefix+'title']+df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58d6cad48e6b10c88688ed9c3b4f7de9cee29ab1","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":13,"outputs":[]},{"metadata":{"_uuid":"cfdb70734b6c6b69ae73bea570f96ebe7d69b9ef"},"cell_type":"markdown","source":"### 1.3 Average Word Length"},{"metadata":{"trusted":true,"_uuid":"fd63a615ffa7f207846848df0f3a60d3487faf9e","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'AvgWordLen_'\n\ndef avg_word_len(sentence):\n    words = sentence.split()\n    return (sum(len(word) for word in words)/len(words))\nfunc = lambda x: avg_word_len(x)\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].apply(func)\n    })\nnew_col = df[fprefix+'title']*df['nWord_'+'title']+\\\n            df[fprefix+'desc']*df['nWord_'+'desc']\nnew_col = new_col / df['nWord_'+'text']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e1b1092b5516ad542e0b442d2a3f2863874cc7d","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":15,"outputs":[]},{"metadata":{"_uuid":"6dc06879defe290b6b5dbc09c23a5d9ba9e533a0"},"cell_type":"markdown","source":"### 1.4 Number of stopwords"},{"metadata":{"trusted":true,"_uuid":"f82b27a2cece7ff3991e00943c7c1f590e09a580","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nStop_'\nfunc = lambda x: len([x for x in x.split() if x in set(STOPWORDS)])\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].apply(func)\n    })\nnew_col = df[fprefix+'title'] + df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"786d3d8d2d6a0c7363310b82e51642f590ff7c8b","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":17,"outputs":[]},{"metadata":{"_uuid":"91606058c2a13f088df45e2a4c18bcd63f678bb3"},"cell_type":"markdown","source":"### 1.5 Number of special characters"},{"metadata":{"trusted":true,"_uuid":"4abd9966eafaee1a942b80b928eb1d3a1914fc76","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nSpec_'\nfunc = lambda s: len([c for c in str(s) if c in string.punctuation])\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].apply(func)\n    })\nnew_col = df[fprefix+'title'] + df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":18,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4122363001ab2131231419ff70b86cf46092dfad","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"c80ef655758e33d4769470bca1e9151ace7a622a"},"cell_type":"markdown","source":"### 1.6 Number of numerics"},{"metadata":{"trusted":true,"_uuid":"f53cc1f49477f7dab8238a3a953ade64d8ff60ba","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nNum_'\nfunc = lambda s: len(re.findall(r'[0-9]', s))\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].apply(func)\n    })\nnew_col = df[fprefix+'title'] + df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":20,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2b7d5ddcf2bc813e77a1c4b95cee48c24b67a1d","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":21,"outputs":[]},{"metadata":{"_uuid":"a6734c24ace7d409fd1e9b6f698d6f99d43d6778"},"cell_type":"markdown","source":"### 1.7 Number of Uppercase words"},{"metadata":{"trusted":true,"_uuid":"5e64d1f5ac952f81e5b960d1f94b131263b0e6a1","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nUpper_'\nfunc = lambda s: len(re.findall(r'[A-Я]', s))\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df[col].apply(func)\n    })\nnew_col = df[fprefix+'title'] + df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":22,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc1bacb9c0d9f849b9df231097fcfc1bde7c8608","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":23,"outputs":[]},{"metadata":{"_uuid":"2d6ff52bc54621c1428f8bb2645ebb13b142757c"},"cell_type":"markdown","source":"### 1.8 Number of words in Context (Keywords)"},{"metadata":{"trusted":true,"_uuid":"37e1784ca7344b38de3eb653abe4f2107de91d9d","collapsed":true},"cell_type":"code","source":"%%time\nfprefix = 'nKey_'\ndef num_keywords(x, col, candidate_col='context'):\n    words = [w for w in proc_text(x[col]).split() \\\n          if w in set(x[candidate_col].split())]\n    return len(words)\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        fprefix+col: df.apply(lambda x: num_keywords(x, col), axis=1)\n    })\nnew_col = df[fprefix+'title'] + df[fprefix+'desc']\ndf = df.assign(**{\n    fprefix+'text': new_col\n})","execution_count":24,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1dd7a780ccff233ebc30b4362a701f21cccaca4","collapsed":true},"cell_type":"code","source":"plt.figure(figsize=[12, 4])\nfor i, col in enumerate(['text', 'title', 'desc']):\n    plt.subplot(1, 3, i+1)\n    sns.distplot(df[fprefix+col])\n    plt.legend([fprefix+col])\n    plt.grid()","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"adddcbfb42acf4503279984b1982ddb2ed2aa36b"},"cell_type":"markdown","source":"## 2. Basic Pre-processing\n### 2.1 Lower & Removing Punctuation, Stop Words"},{"metadata":{"trusted":true,"_uuid":"d619bfe6dce2761ff949d4bdc74d31f4fb5df525","collapsed":true},"cell_type":"code","source":"%%time\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        col: df[col].apply(lambda x: proc_text(x))\n    })\nnew_col = df['title'] + ' ' + df['desc']\ndf = df.assign(**{\n    'text': new_col\n})","execution_count":26,"outputs":[]},{"metadata":{"_uuid":"c01a6556c067bbb2d7aa334407b197cb0f681a34"},"cell_type":"markdown","source":"### 2.2 Common word removal"},{"metadata":{"trusted":true,"_uuid":"8d9a5874ffd0bc38a6f6b8c2fa942119547abf2b","collapsed":true},"cell_type":"code","source":"top_k = 10\nfreq = pd.Series(' '.join(df['text']).split()).value_counts()[:top_k]\nfreq","execution_count":27,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9801a9ff3f53fa93f7e272c91b43500403909342","collapsed":true},"cell_type":"code","source":"# %%time\n# freq = set(list(freq.index))\n# func = lambda x: \" \".join(x for x in x.split() if x not in freq)\n# for col in ['title', 'desc']:\n#     df = df.assign(**{\n#         col: df[col].apply(func)\n#     })\n# new_col = df['title'] + ' ' + df['desc']\n# df = df.assign(**{\n#     'text': new_col\n# })","execution_count":28,"outputs":[]},{"metadata":{"_uuid":"f1259c22cf16e5819644cd0a3747f87cabceba3d"},"cell_type":"markdown","source":"### 2.3 Rare words removal"},{"metadata":{"trusted":true,"_uuid":"b1b171ac6a086989051f9efa022b8f86a356e955","collapsed":true},"cell_type":"code","source":"# freq = pd.Series(' '.join(df['text']).split()).value_counts()[-top_k:]\n# freq","execution_count":29,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbdf177c78d3df772154ecc7835eb08bd015ba8c","collapsed":true},"cell_type":"code","source":"# %%time\n# freq = set(list(freq.index))\n# func = lambda x: \" \".join(x for x in x.split() if x not in freq)\n# for col in ['title', 'desc']:\n#     df = df.assign(**{\n#         col: df[col].apply(func)\n#     })\n# new_col = df['title'] + ' ' + df['desc']\n# df = df.assign(**{\n#     'text': new_col\n# })","execution_count":30,"outputs":[]},{"metadata":{"_uuid":"9a2de790bee0fe8077211d8703c3e45f23c12640"},"cell_type":"markdown","source":"### 2.4 Spelling correction (TIME CONSUMING!)"},{"metadata":{"trusted":true,"_uuid":"2896d12bd38a63e5418bde2336f25f3058a44a6a","collapsed":true},"cell_type":"code","source":"# %%time\n# from textblob import TextBlob\n# func = lambda x: str(TextBlob(x).correct())\n# for col in ['title', 'desc']:\n#     df = df.assign(**{\n#         col: df[col].apply(func)\n#     })\n# new_col = df['title'] + ' ' + df['desc']\n# df = df.assign(**{\n#     'text': new_col\n# })","execution_count":31,"outputs":[]},{"metadata":{"_uuid":"c0ce73ae9be27c19b7f270d40f93b0dd069d4558"},"cell_type":"markdown","source":"### 2.5 Lemmatization"},{"metadata":{"trusted":true,"_uuid":"0a1d2e6818c9aa5d643433dd3d8d039a55860cfa","collapsed":true},"cell_type":"code","source":"%%time\n# from textblob import Word\n# func = lambda x: \" \".join([Word(word).lemmatize() for word in x.split()]) #slow\nfrom nltk.stem import WordNetLemmatizer\nwnl = WordNetLemmatizer()\nfunc = lambda x: \" \".join([wnl.lemmatize(word) for word in x.split()])\nfor col in ['title', 'desc']:\n    df = df.assign(**{\n        col: df[col].apply(func)\n    })\nnew_col = df['title'] + ' ' + df['desc']\ndf = df.assign(**{\n    'text': new_col\n})","execution_count":32,"outputs":[]},{"metadata":{"_uuid":"e01e203c7f131da530cf5d16d90f285ffa7a56ac"},"cell_type":"markdown","source":"## 3. Advance Text Processing\n### 3.1 N-grams"},{"metadata":{"trusted":true,"_uuid":"18d059d87146216a787ecc7194f0a50db316989b","collapsed":true},"cell_type":"code","source":"%%time\nfrom sklearn.pipeline import FeatureUnion\nfrom sklearn.feature_extraction.text \\\n    import CountVectorizer, TfidfVectorizer\ntfidf_params = {\n    'stop_words': STOPWORDS,\n    'analyzer': 'word',\n    'token_pattern': r'\\w{1,}',\n    'sublinear_tf': True,\n    'dtype': np.float32,\n    'norm': 'l2',\n    'smooth_idf':False\n}\nvectorizer = FeatureUnion([\n        ('title+desc', TfidfVectorizer(\n            ngram_range=(1, 2),\n            max_features=15000,\n            **tfidf_params,\n            preprocessor=lambda x: x['text'])), #desc text\n        ('title', CountVectorizer(\n            ngram_range=(1, 2),\n            stop_words = STOPWORDS,\n            preprocessor=lambda x: x['title']))\n])\nwith timer('vectorizer fitting'):\n    vectorizer.fit(df.to_dict('records'))\nwith timer('vectorizer transforming'):\n    text_vector = vectorizer.transform(df.to_dict('records'))","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e4b61746eb3c83e59b35ae864d3594663fc12993"},"cell_type":"code","source":"from scipy import sparse\ndf['vecSum'] = np.array(text_vector.sum(1))\ndf['vecNPosEle'] = (text_vector>0).sum(1)","execution_count":34,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"792de6e55f0776c006b1b4abfb60f2dbd164d704","collapsed":true},"cell_type":"code","source":"from sklearn.linear_model import Ridge\nfrom sklearn.metrics import mean_squared_error\ndl_feat_idx = pd.read_csv(\n    '../input/adp-prepare-kfold-text/textdata.csv', usecols=['eval_set', 'label'])\ntrain_num = (dl_feat_idx['eval_set']!=10).sum()\neval_sets = dl_feat_idx['eval_set'][:train_num].values\ny = dl_feat_idx['label'][:train_num].values\ndel dl_feat_idx; gc.collect()","execution_count":35,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e4d98d88f3b9e7c23c461e70ef2ed17686a0def","collapsed":true},"cell_type":"code","source":"def get_ridge_pred(use_all_feats=False):\n    ridge = Ridge(**{'alpha':30.0, 'fit_intercept':True, 'normalize':use_all_feats, 'copy_X':True,\n                    'max_iter':None, 'tol':0.0001, 'solver':'auto', 'random_state':2018})\n    pred_train = np.zeros(train_num,)\n    pred_test = np.zeros(len(df)-train_num,)\n    if use_all_feats:\n        X = sparse.hstack([sparse.csr_matrix(df.values[:, 4:].astype('float32')),\n                           text_vector])\n        X = sparse.csr_matrix(X)\n    else:\n        X = text_vector\n    for valid_fold in range(10):\n        print('processing fold %d...'%valid_fold)\n        mask_val = eval_sets==valid_fold\n        mask_tr = ~mask_val\n        ridge.fit(X[:train_num][mask_tr], y[mask_tr])\n        pred_train[mask_val] = ridge.predict(X[:train_num][mask_val])\n        pred_test += ridge.predict(X[train_num:])\n    pred_test = pred_test / 10\n    print('Ridge RMSE:', np.sqrt(mean_squared_error(y, pred_train)))\n    return pred_train, pred_test","execution_count":51,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b39977ef849ad51faf20bfd4c668688e77e06fee","collapsed":true},"cell_type":"code","source":"%%time\npred_train, pred_test = get_ridge_pred()","execution_count":37,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a059490d998d2c4ec6b6fcc15c00336e855ffbf","collapsed":true},"cell_type":"code","source":"df['ridge_textvec'] = np.hstack([pred_train, pred_test])","execution_count":38,"outputs":[]},{"metadata":{"_uuid":"97b608e4909ef53aee2ab04c0b866b2561cd0a80"},"cell_type":"markdown","source":"### 3.2 Sentiment Analysis (SLOW)"},{"metadata":{"trusted":true,"_uuid":"04a1a0f8f2288b67a33e76791e66f9418b8c6429","collapsed":true},"cell_type":"code","source":"# %%time\n# from textblob import TextBlob\n# res = df['text'].apply(lambda x: TextBlob(x).sentiment)\n# res = np.array(np.array(res).tolist())\n# df['sentPop'] = res[:, 0]\n# df['sentSub'] = res[:, 1]\n# del res; gc.collect();","execution_count":39,"outputs":[]},{"metadata":{"_uuid":"3e66dbef757a78591a4760bff8b7f98e905b181a"},"cell_type":"markdown","source":"# Combination & Interaction\n## (Comment them to save space)"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8bcdcf8332536bbf13f6bea71508aa07f9e29f0a"},"cell_type":"code","source":"df['rW_title_desc'] = df['nWord_title'] / df['nWord_desc']\ndf['rW_title_text'] = df['nWord_title'] / df['nWord_text']\ndf['rC_title_desc'] = df['nChar_title'] / df['nChar_desc']\ndf['rC_title_text'] = df['nChar_title'] / df['nChar_text']","execution_count":40,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a74c12dd03646216fb41b8610146315652756384"},"cell_type":"code","source":"df['rUp_title'] = df['nUpper_title'] / df['nChar_title']\ndf['rUp_desc'] =  df['nUpper_desc'] / df['nChar_desc']\ndf['rUp_text'] =  df['nUpper_text'] / df['nChar_text']\ndf['rNum_title'] = df['nNum_title'] / df['nChar_title']\ndf['rNum_desc'] =  df['nNum_desc'] / df['nChar_desc']\ndf['rNum_text'] =  df['nNum_text'] / df['nChar_text']\ndf['rSpec_title'] = df['nSpec_title'] / df['nChar_title']\ndf['rSpec_desc'] =  df['nSpec_desc'] / df['nChar_desc']\ndf['rSpec_text'] =  df['nSpec_text'] / df['nChar_text']","execution_count":41,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6fab9b54f985383c490870654a5b8c121f430121"},"cell_type":"code","source":"df['rStop_title'] = df['nStop_title'] / df['nWord_title']\ndf['rStop_desc'] =  df['nStop_desc'] / df['nWord_desc']\ndf['rStop_text'] =  df['nStop_text'] / df['nWord_text']\ndf['rKey_title'] = df['nKey_title'] / df['nWord_title']\ndf['rKey_desc'] =  df['nKey_desc'] / df['nWord_desc']\ndf['rKey_text'] =  df['nKey_text'] / df['nWord_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95dd5f4c7acc5a13332b672ed648e92b357ca539","collapsed":true},"cell_type":"code","source":"df.head(2).T","execution_count":42,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f71498fc6c5f54ee6c4b2b0b53db38c7f93cb832","collapsed":true},"cell_type":"code","source":"for c in ['context', 'text', 'title', 'desc']:\n    if c in df.columns:\n        del df[c]\ngc.collect();","execution_count":43,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7fe3e8c18ef8a8433d476c9f12ab708ef97f9a3","collapsed":true},"cell_type":"code","source":"df.columns.tolist()","execution_count":44,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c112a2e6821843d051b59e25be750580894e7c7d","collapsed":true},"cell_type":"code","source":"textmeta_cols = df.columns.tolist()","execution_count":45,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7a8716b1c775792236c8db96fd6657f06da8307","collapsed":true},"cell_type":"code","source":"df.info(verbose=False)","execution_count":46,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e882d605e56dc0bc99709346b0e71d74e1979ee2","collapsed":true},"cell_type":"code","source":"df.describe().T","execution_count":47,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5712ab1ed9e2b95e8430f03616cd4730af6587b5","collapsed":true},"cell_type":"code","source":"df.to_csv('textmeta.csv', index=False)","execution_count":48,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}