{"cells":[{"metadata":{"_uuid":"db42d866cc0dbee0a9244682ba497a83ec7adb6f"},"cell_type":"markdown","source":"This kernel shows dependency between each feature extracted by [Feature Engineering Kernel](https://www.kaggle.com/elvenmonk/lanl-feature-engineering) and TTF value [being predicted](https://www.kaggle.com/elvenmonk/lanl-ttf-error) for training data of [LANL Earthquake Prediction](https://www.kaggle.com/c/LANL-Earthquake-Prediction) competition.\nUse it if you want to review features before running [regression](https://www.kaggle.com/elvenmonk/lanl-regression) or [GPLearn regression](https://www.kaggle.com/elvenmonk/lanl-gp-regression)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy import stats\nfrom sklearn.preprocessing import StandardScaler\n\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nplt.rcParams['figure.figsize'] = [25, 625]\nimport json\n\nX_test = pd.read_csv('../input/lanl-feature-transform/test_features.csv', index_col=[0])\nX_tr = pd.read_csv('../input/lanl-feature-transform/train_features.csv', dtype=np.float64)\nY_tr = pd.read_csv('../input/lanl-ttf-error/time_prediction.csv', dtype=np.float64)\nprint(X_tr.shape)\nprint(np.nonzero(X_tr.values == -np.inf))\n\nX_tr = X_tr\nY_tr = Y_tr\nprint(X_tr.shape)\nprint(Y_tr.shape)\n\nscaler = StandardScaler()\nscaler.fit(X_tr)\nX_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns)\nX_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)\n\nY_scaler = StandardScaler()\nY_scaler.fit(Y_tr)\nY_train_scaled = pd.DataFrame(Y_scaler.transform(Y_tr), columns=Y_tr.columns)\nN = Y_tr.shape[0]\nT = np.arange(N)\ngood_columns = {column: stats.pearsonr(X_tr[column], (Y_tr.values[:,0]/Y_tr.values[:,1]).ravel())[0] for column in X_tr.columns if abs(stats.pearsonr(X_tr[column], (Y_tr.values[:,0]/Y_tr.values[:,1]).ravel())[0]) > 0.8}\nprint(json.dumps(good_columns, indent='\\t', separators=(',', ':\\t')))\ngood_columns = {column: stats.pearsonr(X_tr[column], Y_tr.values[:,1].ravel())[0] for column in X_tr.columns if abs(stats.pearsonr(X_tr[column], Y_tr.values[:,1].ravel())[0]) > 0.17}\nprint(json.dumps(good_columns, indent='\\t', separators=(',', ':\\t')))\nM = len(X_tr.columns)\nR2 = (M+1)//2\nR3 = (M+2)//3\nR5 = (M+4)//5\nprint(M,R2,R3,R5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9952a272ce65b28c3c100c7602673b885292008"},"cell_type":"markdown","source":"First lets see how features depend on TTF directly"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"2bb240940f0095809ae91d3d9fc8ad1a651ede0f"},"cell_type":"code","source":"plt.figure(figsize=(25, 3*R5))\nfor i, column in tqdm(enumerate(X_tr.columns)):\n    plt.subplot(R5, 5, i + 1)\n    plt.title(column)\n    plt.axis([0,1,-3,3])\n    plt.scatter(Y_tr.values[37:,0]/Y_tr.values[37:,1], X_train_scaled[column][37:], s=1, c=Y_tr.values[37:,1], cmap='gist_rainbow')\n    plt.scatter(np.full_like(X_test_scaled[column], 15), X_test_scaled[column], s=1, c='b')\nplt.show()\n#plt.savefig('features.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, lets see correlation between ttf and best correlated collumn for each individual earthquake."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(25, 3*R2))\nbounds = [0, 37, 333, 697, 925, 1250, 1457, 1638, 2052, 2255, 2502, 2795, 3078, 3305, 3525, 3903, 4146, 4193]\nn = len(bounds) - 1\ngood_columns2 = X_tr.columns[[103,238]]\nprint(good_columns2)\n#m = len(good_columns)\nfor j, column in tqdm(enumerate(X_tr.columns)):\n    plt.subplot(R2, 2, j + 1)\n    plt.title('{0}. {1}'.format(j, column))\n    plt.axis([0,1,-3,3])\n    for i in range(n):\n        c = (Y_tr.values[bounds[i],1]-7)/10\n        plt.plot(Y_tr.values[bounds[i]:bounds[i+1],0]/Y_tr.values[bounds[i]:bounds[i+1],1], X_train_scaled[column][bounds[i]:bounds[i+1]], c=plt.cm.gist_rainbow(c))\nplt.show()\n#plt.savefig('features.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec306af91bbdab0e0664e5a87069d58b48b308bd"},"cell_type":"markdown","source":"Next series of plots represents correlation between features and TTF values (scaled) throughout experiment."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"60f3438bb4bac4a22e9fe056ada86cab4865e4ba"},"cell_type":"code","source":"plt.figure(figsize=(30, 3*R2))\nfor i, column in tqdm(enumerate(X_tr.columns)):\n    plt.subplot(R2, 2, i + 1)\n    plt.title(column)\n    plt.axis([0,4200,-2,2])\n    plt.plot(T, X_train_scaled[column], T, Y_train_scaled-1)\nplt.show()\n#plt.savefig('features.png')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}