{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"## JerusML meetup #12: Hands on Kaggle for everyone ##\n## Credit for 'inversion' who's Kernel was used as a skeleton ##\n\nimport numpy as np # Python package that effiecently performs computations on n-dimentional data types.\nimport pandas as pd # Python package useful to hold and persent data. \n\nimport matplotlib.pyplot as plt # Used to plot graphs of the data.\nfrom tqdm import tqdm # Show progess bar for example it can show the progess of a loop.\n\n#Sklearn is a pyhton packages that provides tools for data mining and analysis.\nfrom sklearn.preprocessing import StandardScaler \nfrom sklearn.svm import NuSVR\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Read the provides sesmic data \ntrain = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64}) #train is of type DataFrame.\n# **Short explantion about imports**","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now we can perform some simple operations to view the data.\n#Train holds the data as a DataFrame variable which is in the form of a table.  \ntrain.head() #Shows the first 5 entries of the table. \n#train.head(10) #Shows the first 10 entries\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pandas doesn't show us all the decimals\npd.options.display.precision = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# much better! (Note another way to show the first 10 entries).\ntrain[0:10]\n#train[0:20:2] #Or the first 10 even entries.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[0:10]['acoustic_data']\n#train[0:10]['time_to_failure']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now, lets try to understand the data. \n#First lets see what are the dimentions of our train data.\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#function for plotting based on both features\ndef plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title=\"Acoustic data and time to failure\"):\n    fig, ax1 = plt.subplots(figsize=(12, 8))\n    plt.title(title)\n    plt.plot(train_ad_sample_df, color='r')\n    ax1.set_ylabel('acoustic data', color='r')\n    plt.legend(['acoustic data'], loc=(0.01, 0.95))\n    ax2 = ax1.twinx()\n    plt.plot(train_ttf_sample_df, color='b')\n    ax2.set_ylabel('time to failure', color='b')\n    plt.legend(['time to failure'], loc=(0.01, 0.9))\n    plt.grid(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We can sample the data set to be able to plot it and maybe see patterns.\ntrain_ad_sample_df = train['acoustic_data'].values[::100]\ntrain_ttf_sample_df = train['time_to_failure'].values[::100]\n\nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)\ndel train_ad_sample_df\ndel train_ttf_sample_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lets zoom into one 'quake'\ntest = train[0:6_000_000]\ntrain_ad_sample_df = test['acoustic_data'].values[::100]\ntrain_ttf_sample_df = test['time_to_failure'].values[::100]\n\nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)\ndel train_ad_sample_df\ndel train_ttf_sample_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We choose to divide the data into segments of 150,000 since the test data is provided in segements of this size.\nrows = 150_000\nsegments = int(np.floor(train.shape[0] / rows))\n\nprint(segments) #We get 4194 segments.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now we would like to extract useful insights on each segments - those insights will be called features.\n\n#We now defined two new data types, each holds the extracted features of each segments.\n\n# Will holds information about the acoustic data of each seagment.\nX_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min']) \ny_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['time_to_failure'])\n\n#For each consequent 150,000 rows, extract features and save them a new row in X_train and y_train.\nfor segment in tqdm(range(segments)):\n    #Extract consquent rows. \n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = seg['acoustic_data'].values\n    y = seg['time_to_failure'].values\n    \n    #Extract simple statistical features on the data. \n    #Could we extract other interesting features? Would it help?\n    X_train.loc[segment, 'ave'] = x.mean()\n    X_train.loc[segment, 'std'] = x.std()\n    X_train.loc[segment, 'max'] = x.max()\n    X_train.loc[segment, 'min'] = x.min()\n\n    #For each segment we choose to take the last time to failure as the segment's time to failure as specified.\n    y_train.loc[segment, 'time_to_failure'] = y[-1]\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# StandardScaler is used to normalize/standardize (mean = 0 and standard deviation = 1) your features\n# before applying the machine learning techniques\nscaler = StandardScaler()\nscaler.fit(X_train)\nX_train_scaled = scaler.transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learn the data.\n# We are interested in predicting a value from a segment of samples.\n# Therefore we are looking for a regression model, which can use the extracted features from the segment and make a prediction.\n# We will apply Support Vector Regression in order to make predictions.\nsvm = NuSVR()\n#Learns the data.\nsvm.fit(X_train_scaled, y_train.values.flatten())\n# Generate the predictions.\ny_pred = svm.predict(X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plot the average features of data points vs. \nplt.plot(X_train_scaled[:,[0]], y_train.values.flatten(), 'ro')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show the correlation between the predicted and the actual time to failure.\nplt.figure(figsize=(6, 6))\nplt.scatter(y_train.values.flatten(), y_pred)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = mean_absolute_error(y_train.values.flatten(), y_pred)\nprint(f'Score: {score:0.3f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read the output file for submission. \nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a data frame to hold the test results.\nX_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read each test segment and extract features from each.\nfor seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    \n    X_test.loc[seg_id, 'ave'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict the result and save to submission file.\nX_test_scaled = scaler.transform(X_test)\nsubmission['time_to_failure'] = svm.predict(X_test_scaled)\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}