{"cells":[{"metadata":{"_uuid":"c89b92b58338cb48c21fdb5e8135e92511d25182"},"cell_type":"markdown","source":"## Adversarial validation\n\nBecause of the obvious problems with the validation in this competition, I was thinking about preparing [adversarial validation](http://fastml.com/adversarial-validation-part-one/)\n\n*Work in progress!*"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from tqdm import tqdm\nimport time\n\nimport numpy as np\nimport pandas as pd\n\nimport lightgbm as lgb\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR\nfrom sklearn.metrics import f1_score, accuracy_score, classification_report, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nimport tsfresh.feature_extraction.feature_calculators as tff\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7971b536a764172714581aee01df21a836f8af7b"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40182917e6b7ad6b8bd90bcbcc61d7dc222cad94"},"cell_type":"markdown","source":"I'm going to use some popuular features, together with some different properties of the signal."},{"metadata":{"trusted":true,"_uuid":"76dbfa4f2dd1097b9e56d15d4bcb7e0a14bb9a6f"},"cell_type":"code","source":"def extract_features(X_all, x, seg_id):\n    \n    X_all.loc[seg_id, 'ave'] = x.mean()\n    X_all.loc[seg_id, 'std'] = x.std()\n    X_all.loc[seg_id, 'max'] = x.max()\n    X_all.loc[seg_id, 'min'] = x.min()\n    X_all.loc[seg_id, 'abs_energy'] = np.dot(x, x)\n    X_all.loc[seg_id, 'sum_of_reoccurring_data_points'] = tff.sum_of_reoccurring_data_points(x)\n    X_all.loc[seg_id, 'sum_of_reoccurring_values'] = tff.sum_of_reoccurring_values(x)\n    X_all.loc[seg_id, 'count_above_mean'] = tff.count_above_mean(x)\n\n    return X_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"090553faffc125ca8b3af41b9aa6603471a67235"},"cell_type":"code","source":"rows = 150_000\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_train_data = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min', 'abs_energy'])\nttf_train = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])\n\nfor seg_id in tqdm(range(segments)):\n    seg = train.iloc[seg_id*rows:seg_id*rows+rows]\n    x = seg['acoustic_data'].values.astype(float)\n    ttf_train.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]\n\n    X_train_data = extract_features(X_train_data, x, seg_id)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e377788a0614b9c16ca490461d8a5526224dff88"},"cell_type":"markdown","source":"The same with the test set"},{"metadata":{"trusted":true,"_uuid":"62c1d7bceda93ec5d412b1a18f5b452539c13ae1"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f1016914b7f0eab35834d4b1985578696a1bcda"},"cell_type":"code","source":"X_test_data = pd.DataFrame(columns=X_train_data.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ce3d4df9c609f6c73862bc9f5ae285c5a9b125f"},"cell_type":"code","source":"for seg_id in tqdm(X_test_data.index):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv', dtype={'acoustic_data': np.int16})\n    \n    x = seg['acoustic_data'].values.astype(float)\n    X_test_data = extract_features(X_test_data, x, seg_id)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5de8665674c440bf65e8a1701b48982222095b07"},"cell_type":"markdown","source":"Now we can prepare the cross-validation, where the training examples have label 0, and test examples have label 1."},{"metadata":{"trusted":true,"_uuid":"fb9adf839df659ed5d1a6075da57ed29aa41320b"},"cell_type":"code","source":"X = pd.concat([X_train_data, X_test_data])\ny = np.append(np.zeros((X_train_data.shape[0], )), np.ones(X_test_data.shape[0], ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1070a6be8dafac481dc5338418610ac2105e8e7a"},"cell_type":"code","source":"n_fold = 5\nkf = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=11)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03f88a22f347c86ac9f300603b498269d9f7eb57"},"cell_type":"code","source":"lgb_classifier_params = {'num_leaves': 100,\n                         'min_data_in_leaf': 120,\n                         'objective': 'binary',\n                         'max_depth': -1,\n                         'learning_rate': 0.1,\n                         \"boosting\": \"gbdt\",\n                         \"metric\": 'auc',\n                         \"verbosity\": -1,\n                         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"658c9041b04159e7c5fd4c09f7e28882c3bc274f"},"cell_type":"code","source":"f1s_valid = []\nf1s_train = []\nall_correctly_recognized = []\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(X, y)):\n    print('\\nFold', fold_n, 'started at', time.ctime())\n    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n    y_train, y_valid = y[train_index], y[valid_index]\n    model = lgb.LGBMClassifier(**lgb_classifier_params, n_estimators=10000, n_jobs=-1)\n    model.fit(X_train, y_train, \n              eval_set=[(X_train, y_train), (X_valid, y_valid)],\n              verbose=100, early_stopping_rounds=200)\n\n    y_pred_valid = np.where(model.predict(X_valid, num_iteration=model.best_iteration_) > 0.5, 1, 0)\n    y_pred_train = np.where(model.predict(X_train, num_iteration=model.best_iteration_) > 0.5, 1, 0)\n\n    f1s_valid.append(f1_score(y_valid, y_pred_valid))\n    f1s_train.append(f1_score(y_train, y_pred_train))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce5b595908e7705f7f246dd95270f3100a3d755d"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"b09a4ddbbc675fc4f21b8de86aab50963e8d5de9"},"cell_type":"code","source":"print('CV mean train score: {0:.4f}, std: {1:.4f}.'.format(np.mean(f1s_train), np.std(f1s_train)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f63bc7383372bbf32492b164385911a73ceec1a"},"cell_type":"code","source":"print('CV mean valid score: {0:.4f}, std: {1:.4f}.'.format(np.mean(f1s_valid), np.std(f1s_valid)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d2dee3d88b724035628e22fcaef8f4e67cf5d9d"},"cell_type":"markdown","source":"Classification report for the last fold"},{"metadata":{"trusted":true,"_uuid":"22f56c723f6ecef00506c421cd0c900e98520b9e"},"cell_type":"code","source":"print(classification_report(y_valid, y_pred_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c0577062d7584c0f82d5dd8fa9c1f6add5182bd"},"cell_type":"code","source":"print(classification_report(y_valid, y_pred_valid))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29276fa4f237d822bd090d264b8e51bd820ead3b"},"cell_type":"markdown","source":"The datasets can be distinguished quite good.\nLet's check which features are most important"},{"metadata":{"trusted":true,"_uuid":"381f3ed0e6f3eee4e75c89f4a985af9b869a51e6"},"cell_type":"code","source":"features_importance = pd.DataFrame(sorted(zip(model.feature_importances_, X_train.columns.values)),  columns=['Value', 'Feature'])\n\nplt.figure(figsize=(20, 10))\nsns.barplot(x=\"Value\", y=\"Feature\", data=features_importance.sort_values(by=\"Value\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0958d56f993e53cb7d5e677b4a69b0cd600fe46"},"cell_type":"markdown","source":"We can visualize the distribution of the most interesting features"},{"metadata":{"trusted":true,"_uuid":"0a4d5c6578fc119812e02b6631f04c5ad8105586"},"cell_type":"code","source":"def compare_splits_feature(column, quantile=1.0):\n    \n    q = X[column].quantile(quantile)\n    plt.figure(figsize=(13, 7))\n    plt.title(column)\n    plt.ylabel('Value')\n    plt.xlabel('Split')\n    sns.violinplot(x='split', y=column, data=X[X[column] < q])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf2104d436141a55720778939662bf4574ea024f"},"cell_type":"code","source":"X['split'] = y\ncompare_splits_feature('sum_of_reoccurring_data_points', 0.9)\ncompare_splits_feature('ave')\ncompare_splits_feature('count_above_mean')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b44551254fe190f35368f2d1354394607876d66"},"cell_type":"markdown","source":"There are some different properties in the test and training data. I'd like to understand them better! The next step is to analyze what training samples are similar to the test samples, and from which part of the earthquake they come.\n\nWe saved correctly recognized indices in correctly_recognized list. Let's plot what part of the signal was found correct.\n\nI base the analysis on the last validation split."},{"metadata":{"trusted":true,"_uuid":"364eea1439666b808e8d4d47bc9273152fcaabd3"},"cell_type":"code","source":"pointer = np.where(valid_index < len(X_train_data))[0]\nvalid_ids_from_train_data = valid_index[pointer]\nX_valid_from_train_data = X_train_data.iloc[valid_ids_from_train_data, :]\ncorrect = y_valid[pointer] == y_pred_valid[pointer]\ncorrectly_recognized = np.where(correct == True)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d56963f8bc681ba704fe70305006c340d210ea22"},"cell_type":"code","source":"recognized = np.zeros((len(X_valid_from_train_data), ))\nrecognized[correctly_recognized] = 1\nX_valid_from_train_data['correctly_recognized'] = recognized\nX_valid_from_train_data['time_to_failure'] = ttf_train.iloc[valid_ids_from_train_data, :].values\nplt.figure(figsize=(13, 7))\nplt.title('Correctly recognized training examples vs time_to_failure')\nplt.ylabel('Value')\nplt.xlabel('Split')\nsns.violinplot(x='correctly_recognized', y='time_to_failure',\n               data=X_valid_from_train_data, ) #scale='count'\nplt.xticks([0, 1], ['False', 'True'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"744d4c63a7b15ac13a9e61187d4698c599d02be4"},"cell_type":"markdown","source":"Most of the samples from the training data are recognized correctly. I'm disappointed because:\n- Plots show there are long samples in incorrectly recognized signals\n- The distribution of mistakes time to failure is wide. The peak doesn't tell me anything\n\nWork in progress"},{"metadata":{"_uuid":"67415dbc01a3616bd336f2432967e7ec3227e4fa"},"cell_type":"markdown","source":"### Dear Kagglers\n\nIf you like my kernel maybe you'll like my other work for this competition:\nhttps://www.kaggle.com/davids1992/watching-frequencies-in-the-data\n\nUpvotes are my motivation :)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}