{"cells":[{"metadata":{"_uuid":"26fcc5aba0351480376a4d82cfac0bd5724db2c5"},"cell_type":"markdown","source":"## waht this notebook do:\n* to predict train or test\n* extract fft feats\n\n## what this notebook do not:\n* to predict Power Line Fault\n\n## findings\n* high auc ~ 0.99\n* so train and test has different distribution?"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nimport pyarrow.parquet as pq\nimport gc\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6af0eb586664eb2083047d87b8711c49504dfad9"},"cell_type":"code","source":"smooth = 1000\nkaggle_notebook = True","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"if kaggle_notebook:\n    read_line = 500\n    test_start = 8712\n    train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(read_line)]).to_pandas().values.T\n    test = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(test_start,test_start+read_line)]).to_pandas().values.T\n    meta_tr = pd.read_csv('../input/metadata_train.csv', nrows=read_line)\n    meta_te = pd.read_csv('../input/metadata_test.csv', nrows=read_line)\nelse:\n    train = pq.read_pandas('../input/train.parquet').to_pandas().values.T\n    test = pq.read_pandas('../input/test.parquet').to_pandas().values.T\n    meta_tr = pd.read_csv('../input/metadata_train.csv')\n    meta_te = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb49d6dcb9422c84d67ac8b5b9e67fd51a8913c9"},"cell_type":"markdown","source":"### down sampling"},{"metadata":{"trusted":true,"_uuid":"b9cac68afea1bd5a30022b1ba82f917806e6cdf2"},"cell_type":"code","source":"def smooth_data(arr):\n    avarr = np.zeros((arr.shape[0],int(arr.shape[1]/smooth)))\n    for i in range(arr.shape[0]):\n        for j in range(int(arr.shape[1]/smooth)):\n            avarr[i,j] = np.mean(arr[i,smooth*j:smooth*j+smooth-1])\n    return avarr\ntrn = smooth_data(train)\ntst = smooth_data(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"534acd09a5d635a700ac8d92666c348f685da8b3"},"cell_type":"markdown","source":"### convert by fft"},{"metadata":{"trusted":true,"_uuid":"2f3de46139371681bc9b177c3c085453613d9bf1"},"cell_type":"code","source":"def conv_fft(arr):\n    farr = np.zeros((arr.shape[0],int(arr.shape[1])))\n    for i in range(arr.shape[0]):\n        farr[i,:] = np.log(1+np.abs(np.fft.fft(arr[i])))\n    return farr\nftrn = conv_fft(trn)\nftst = conv_fft(tst)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbf1a911f3ea0e3f7fe6b8058fe8cc693179a27b"},"cell_type":"markdown","source":"### add same feats"},{"metadata":{"trusted":true,"_uuid":"84045c3f6157759c0e9073f4d2e92b475273f9ae"},"cell_type":"code","source":"trn_df = pd.DataFrame(ftrn)\ntst_df = pd.DataFrame(ftst)\n\n#target\ntrn_df['is_test'] = 0\ntst_df['is_test'] = 1\n\n# additional feats\ntrn_df['mean'] = np.mean(train,axis=1)\ntrn_df['max'] = np.max(train,axis=1)\ntrn_df['min'] = np.min(train,axis=1)\ntrn_df['phase'] = meta_tr.phase\ntst_df['mean'] = np.mean(test,axis=1)\ntst_df['max'] = np.max(test,axis=1)\ntst_df['min'] = np.min(test,axis=1)\ntst_df['phase'] = meta_te.phase\n\ndf = pd.concat([trn_df,tst_df])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"691e92e21b198169c93a05296ff823e2a905d54b"},"cell_type":"markdown","source":"### training"},{"metadata":{"trusted":true,"_uuid":"64636e3d8405c652da75488a38db98d21983c8a6"},"cell_type":"code","source":"params = {'num_leaves': 80,\n         'objective':'binary',\n         'learning_rate': 0.01,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.8,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.8 ,\n         \"metric\": 'auc'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e845befa0f1640d7415e96110f937b06ea3e06f","scrolled":true},"cell_type":"code","source":"n_folds = 10\nkfolds = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=110)\nvalid_preds = np.zeros(df.shape[0])\nfor n, (trn_idx, val_idx) in enumerate(kfolds.split(df, df['is_test'])):\n    tr = df.iloc[trn_idx].copy()\n    va = df.iloc[val_idx].copy()\n    tr_y = tr['is_test']\n    va_y = va['is_test']\n    del tr['is_test'], va['is_test']\n    lgtrain = lgb.Dataset(tr, label=tr_y)\n    lgvalid = lgb.Dataset(va, label=va_y)\n    valid_names=['train','valid']\n    valid_sets=[lgtrain, lgvalid]\n    bst = lgb.train(params,\n                     lgtrain,\n                     valid_sets=valid_sets,\n                     num_boost_round=100000,\n                     early_stopping_rounds=100,\n                     verbose_eval=100,\n                     )\n    valid_preds[val_idx] = bst.predict(va)\nscore = roc_auc_score(df['is_test'], valid_preds)\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"98ce53e73af7b16ed07f2c477df6fabb9914dbf0"},"cell_type":"code","source":"_ =lgb.plot_importance(bst, max_num_features=100, importance_type ='gain', figsize=(15,50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfd675503f19d068c95b57f802876e8322d09046"},"cell_type":"code","source":"meta_tr['pred'] = valid_preds[:len(trn_df)]\nmeta_tr.to_csv('metadata_train_for_AdversarialValidation.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8497e70c178ec61734b71b8a1770f27a938d96ee"},"cell_type":"markdown","source":"### Let's check whether we get the same result, if we split test to test1 and test2 and try to predict if it is test1 or test2?"},{"metadata":{"trusted":true,"_uuid":"b496ee1ce777a35c403c174547113a5fb909d0ce"},"cell_type":"code","source":"del train, test, trn, tst, ftrn, ftst\ngc.collect()\nread_line = 1000\ntest_start = 8712\ntest = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(test_start,test_start+read_line)]).to_pandas().values.T\nmeta_te = pd.read_csv('../input/metadata_test.csv', nrows=read_line)\ntrain, test, meta_tr, meta_te = train_test_split(test, meta_te, test_size=0.5, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bd73d7bbf089b8fc6a244235d3eb9046afd8f28"},"cell_type":"code","source":"trn = smooth_data(train)\ntst = smooth_data(test)\nftrn = conv_fft(trn)\nftst = conv_fft(tst)\ntrn_df = pd.DataFrame(ftrn)\ntst_df = pd.DataFrame(ftst)\ntrn_df['is_test'] = 0\ntst_df['is_test'] = 1\ntrn_df['mean'] = np.mean(train,axis=1)\ntrn_df['max'] = np.max(train,axis=1)\ntrn_df['min'] = np.min(train,axis=1)\ntrn_df['phase'] = meta_tr.phase.values\ntst_df['mean'] = np.mean(test,axis=1)\ntst_df['max'] = np.max(test,axis=1)\ntst_df['min'] = np.min(test,axis=1)\ntst_df['phase'] = meta_te.phase.values\ndf = pd.concat([trn_df,tst_df])\n\nparams = {'num_leaves': 80,\n         'objective':'binary',\n         'learning_rate': 0.01,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.8,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.8 ,\n         \"metric\": 'auc'}\n\nn_folds = 10\nkfolds = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=110)\nvalid_preds = np.zeros(df.shape[0])\nfor n, (trn_idx, val_idx) in enumerate(kfolds.split(df, df['is_test'])):\n    tr = df.iloc[trn_idx].copy()\n    va = df.iloc[val_idx].copy()\n    tr_y = tr['is_test']\n    va_y = va['is_test']\n    del tr['is_test'], va['is_test']\n    lgtrain = lgb.Dataset(tr, label=tr_y)\n    lgvalid = lgb.Dataset(va, label=va_y)\n    valid_names=['train','valid']\n    valid_sets=[lgtrain, lgvalid]\n    bst = lgb.train(params,\n                     lgtrain,\n                     valid_sets=valid_sets,\n                     num_boost_round=100000,\n                     early_stopping_rounds=100,\n                     verbose_eval=100,\n                     )\n    valid_preds[val_idx] = bst.predict(va)\nscore = roc_auc_score(df['is_test'], valid_preds)\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e44ca74d8c33bc5636c921f04670080f4ec36d0"},"cell_type":"code","source":"_ =lgb.plot_importance(bst, max_num_features=100, importance_type ='gain', figsize=(15,50))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}