{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport time\nimport datetime\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom scipy.signal import convolve\nfrom scipy import stats\n\nimport shap","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split train data into Dataframe containing rows of 150000 data points.\n\nrows = 150000\nsegments = int(np.floor(train.shape[0] / rows))\nX_train = pd.DataFrame(index=range(4194), dtype=np.float64, columns = range(rows))\ny_tr = pd.DataFrame(index=range(4194), dtype=np.float64, columns=range(1))\n\nfor segment in tqdm_notebook(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    X_train.iloc[segment] = seg['acoustic_data'].values\n    y_tr.iloc[segment] = seg['time_to_failure'].values[-1]\n    \nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_tr.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataframe to create features from test data\n\nX_tr = pd.DataFrame(index=range(4194), dtype=np.float64)\n\nfor segment in tqdm_notebook(X_train.index):\n    X_tr.loc[segment, 'mean'] = X_train.loc[segment].mean()\n    X_tr.loc[segment,'max'] = X_train.loc[segment].max()\n    X_tr.loc[segment,'min'] = X_train.loc[segment].min()\n    X_tr.loc[segment, 'std'] = X_train.loc[segment].std()\n    X_tr.loc[segment, 'var'] = X_train.loc[segment].var()\n    X_tr.loc[segment, 'kurt'] = X_train.loc[segment].kurtosis()\n    X_tr.loc[segment, 'skew'] = X_train.loc[segment].skew()\n    X_tr.loc[segment, 'med'] = X_train.loc[segment].median()\n    X_tr.loc[segment, 'sum'] = X_train.loc[segment].sum()\n    X_tr.loc[segment, 'q91'] = np.quantile(X_train.loc[segment], 0.91)\n    X_tr.loc[segment, 'q92'] = np.quantile(X_train.loc[segment], 0.92)\n    X_tr.loc[segment, 'q93'] = np.quantile(X_train.loc[segment], 0.93)\n    X_tr.loc[segment, 'q94'] = np.quantile(X_train.loc[segment], 0.94)\n    X_tr.loc[segment, 'q95'] = np.quantile(X_train.loc[segment], 0.95)\n    X_tr.loc[segment, 'q96'] = np.quantile(X_train.loc[segment], 0.96)\n    X_tr.loc[segment, 'q97'] = np.quantile(X_train.loc[segment], 0.97)\n    X_tr.loc[segment, 'q98'] = np.quantile(X_train.loc[segment], 0.98)\n    X_tr.loc[segment, 'q99'] = np.quantile(X_train.loc[segment], 0.99)\n    X_tr.loc[segment, 'q01'] = np.quantile(X_train.loc[segment], 0.01)\n    X_tr.loc[segment, 'q02'] = np.quantile(X_train.loc[segment], 0.02)\n    X_tr.loc[segment, 'q03'] = np.quantile(X_train.loc[segment], 0.03)\n    X_tr.loc[segment, 'q04'] = np.quantile(X_train.loc[segment], 0.04)\n    X_tr.loc[segment, 'q05'] = np.quantile(X_train.loc[segment], 0.05)\n    X_tr.loc[segment, 'q06'] = np.quantile(X_train.loc[segment], 0.06)\n    X_tr.loc[segment, 'q07'] = np.quantile(X_train.loc[segment], 0.07)\n    X_tr.loc[segment, 'q08'] = np.quantile(X_train.loc[segment], 0.08)\n    X_tr.loc[segment, 'q09'] = np.quantile(X_train.loc[segment], 0.09)\n    X_tr.loc[segment, 'q999'] = np.quantile(X_train.loc[segment],0.999)\n    X_tr.loc[segment, 'q001'] = np.quantile(X_train.loc[segment],0.001)\n\n    \nX_tr.head(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_tr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_tr)\nX_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading test files into a dataframe,\n# with rows as file names and columns as the acoustic signal data. \n\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nX = pd.DataFrame(columns=range(0,150000), dtype=np.float64, index=submission.index)\n\nfor seg_id in tqdm_notebook(X.index):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    X.loc[seg_id,] = seg['acoustic_data'].values\n\nX.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove index name seg_id\ndel X.index.name\nX.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = pd.DataFrame(index=X.index, dtype=np.float64)\n\nfor row in tqdm_notebook(X.index):\n    X_test.loc[row, 'mean'] = X.loc[row].mean()\n    X_test.loc[row,'max'] = X.loc[row].max()\n    X_test.loc[row,'min'] = X.loc[row].min()\n    X_test.loc[row, 'std'] = X.loc[row].std()\n    X_test.loc[row, 'var'] = X.loc[row].var()\n    X_test.loc[row, 'kurt'] = X.loc[row].kurtosis()\n    X_test.loc[row, 'skew'] = X.loc[row].skew()\n    X_test.loc[row, 'med'] = X.loc[row].median()\n    X_test.loc[row, 'sum'] = X.loc[row].sum()\n    X_test.loc[row, 'q91'] = np.quantile(X.loc[row], 0.91)\n    X_test.loc[row, 'q92'] = np.quantile(X.loc[row], 0.92)\n    X_test.loc[row, 'q93'] = np.quantile(X.loc[row], 0.93)\n    X_test.loc[row, 'q94'] = np.quantile(X.loc[row], 0.94)\n    X_test.loc[row, 'q95'] = np.quantile(X.loc[row], 0.95)\n    X_test.loc[row, 'q96'] = np.quantile(X.loc[row], 0.96)\n    X_test.loc[row, 'q97'] = np.quantile(X.loc[row], 0.97)\n    X_test.loc[row, 'q98'] = np.quantile(X.loc[row], 0.98)\n    X_test.loc[row, 'q99'] = np.quantile(X.loc[row], 0.99)\n    X_test.loc[row, 'q01'] = np.quantile(X.loc[row], 0.01)\n    X_test.loc[row, 'q02'] = np.quantile(X.loc[row], 0.02)\n    X_test.loc[row, 'q03'] = np.quantile(X.loc[row], 0.03)\n    X_test.loc[row, 'q04'] = np.quantile(X.loc[row], 0.04)\n    X_test.loc[row, 'q05'] = np.quantile(X.loc[row], 0.05)\n    X_test.loc[row, 'q06'] = np.quantile(X.loc[row], 0.06)\n    X_test.loc[row, 'q07'] = np.quantile(X.loc[row], 0.07)\n    X_test.loc[row, 'q08'] = np.quantile(X.loc[row], 0.08)\n    X_test.loc[row, 'q09'] = np.quantile(X.loc[row], 0.09)\n    X_test.loc[row, 'q999'] = np.quantile(X.loc[row],0.999)\n    X_test.loc[row, 'q001'] = np.quantile(X.loc[row],0.001)\n\nX_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_test)\nX_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# RF regressor model\n\nfrom sklearn.ensemble import RandomForestRegressor\n\nforest = RandomForestRegressor(n_estimators=1000, max_depth = 100, min_samples_leaf = 30, min_samples_split = 30)\nforest.fit(X_train_scaled.values, y_tr.values)\n\n#print(regr_lanl.feature_importances_)\n\nprediction_rf = forest.predict(X_test_scaled.values)\n\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nsubmission['time_to_failure'] = prediction_rf\nprint(submission.head())\nsubmission.to_csv('submission_rf.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print the JS visualization code to the notebook\nshap.initjs()\n\n#X1,y = shap.datasets.adult()\n#X_display,y_display = shap.datasets.adult(display=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"explainer = shap.TreeExplainer(forest)\nshap_values = explainer.shap_values(X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.force_plot(explainer.expected_value, shap_values[0,:], X_train_scaled.iloc[0,:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.force_plot(explainer.expected_value, shap_values[4,:], X_train_scaled.iloc[4,:])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.summary_plot(shap_values,X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.summary_plot(shap_values, X_train_scaled, plot_type=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name in X_tr.columns:\n    shap.dependence_plot(name, shap_values, X_train_scaled)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}