{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv',nrows = 10**5, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\n                    \n                    #nrows=6000000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pandas doesn't show us all the decimals\npd.options.display.precision = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.rename({\"acoustic_data\": \"signal\", \"time_to_failure\": \"time\"}, axis=\"columns\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.fillna(train.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Create a training file with simple derived features\n\nrows = 150_000\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min','kurt','skew','median','var','sum','q1','q19','iqr','diff'\\\n                                'q2','Q1','q3','q4','Q2','q6','q7','Q3','q8','q81','q82','q9','q10','p1',\\\n                                'hmean','gmean','time_to_failure'])\ny_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['time_to_failure'])\n\nfor segment in tqdm(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = seg['signal'].values\n    y = seg['time'].values[-1]\n    \n    y_train.loc[segment, 'time_to_failure'] = y\n    \n    X_train.loc[segment, 'ave'] = np.mean(x)\n    X_train.loc[segment, 'std'] = np.std(x)\n    X_train.loc[segment, 'max'] = np.max(x)\n    X_train.loc[segment, 'min'] = np.min(x)\n    X_train.loc[segment, 'time_to_failure'] = y\n    X_train.loc[segment, 'skew'] = ((x-x.mean())/x.std() ** 3).mean()\n    X_train.loc[segment,'kurt'] =  ((x-x.mean())/x.std() ** 4).mean()\n    X_train.loc[segment, 'sum'] = np.sum(x)\n    X_train.loc[segment, 'median'] = np.median(x)\n    X_train.loc[segment, 'var'] = np.var(x)\n    #X_train.loc[segment, 'mode'] = x.mode\n    #X_train.loc[segment, 'log'] = ln(x)\n    X_train.loc[segment, 'q1'] = np.quantile(x,.1)\n    X_train.loc[segment, 'q19'] = np.quantile(x,.19)\n    X_train.loc[segment, 'q2'] = np.quantile(x,.2)\n    X_train.loc[segment, 'Q1'] = np.quantile(x,.25)\n    X_train.loc[segment, 'q3'] = np.quantile(x,.3)\n    X_train.loc[segment, 'q4'] = np.quantile(x,.4)\n    X_train.loc[segment, 'Q2'] = np.quantile(x,.5)\n    X_train.loc[segment, 'q6'] = np.quantile(x,.6)\n    X_train.loc[segment, 'q7'] = np.quantile(x,.7)\n    X_train.loc[segment, 'Q3'] = np.quantile(x,.75)\n    X_train.loc[segment, 'q8'] = np.quantile(x,.8)\n    X_train.loc[segment, 'q81'] = np.quantile(x,.81)\n    X_train.loc[segment, 'q82'] = np.quantile(x,.82)\n    X_train.loc[segment, 'q9'] = np.quantile(x,.9)\n    X_train.loc[segment, 'q10'] = np.quantile(x,1) #81,82\n    X_train.loc[segment, 'iqr'] = np.quantile(x,.75) - np.quantile (x,.25)\n    X_train.loc[segment, 'p1'] = np.percentile(x,1)\n    X_train.loc[segment, 'p2'] = np.percentile(x,2)\n    X_train.loc[segment, 'p3'] = np.percentile(x,3)\n    X_train.loc[segment, 'p4'] = np.percentile(x,4)\n    X_train.loc[segment, 'p5'] = np.percentile(x,5)\n    X_train.loc[segment, 'p6'] = np.percentile(x,6)\n    X_train.loc[segment, 'p7'] = np.percentile(x,7)\n    X_train.loc[segment, 'p8'] = np.percentile(x,8)\n    X_train.loc[segment, 'p9'] = np.percentile(x,9)\n    X_train.loc[segment, 'p10'] = np.percentile(x,10)\n    X_train.loc[segment, 'p11'] = np.percentile(x,11)\n    X_train.loc[segment, 'p12'] = np.percentile(x,12)\n    X_train.loc[segment, 'p13'] = np.percentile(x,13)\n    X_train.loc[segment, 'p14'] = np.percentile(x,14)\n    X_train.loc[segment, 'p15'] = np.percentile(x,15)\n    X_train.loc[segment, 'p16'] = np.percentile(x,16)\n    X_train.loc[segment, 'p17'] = np.percentile(x,17)\n    X_train.loc[segment, 'p18'] = np.percentile(x,18)\n    X_train.loc[segment, 'p19'] = np.percentile(x,19)\n    X_train.loc[segment, 'p20'] = np.percentile(x,20)\n    X_train.loc[segment, 'p21'] = np.percentile(x,21)\n    X_train.loc[segment, 'p22'] = np.percentile(x,22)\n    X_train.loc[segment, 'p23'] = np.percentile(x,23)\n    X_train.loc[segment, 'p24'] = np.percentile(x,24)\n    X_train.loc[segment, 'p25'] = np.percentile(x,25)\n    X_train.loc[segment, 'p26'] = np.percentile(x,26)\n    X_train.loc[segment, 'p27'] = np.percentile(x,27)\n    X_train.loc[segment, 'p28'] = np.percentile(x,28)\n    X_train.loc[segment, 'p29'] = np.percentile(x,29)\n    X_train.loc[segment, 'p30'] = np.percentile(x,30)\n    X_train.loc[segment, 'p31'] = np.percentile(x,31)\n    X_train.loc[segment, 'p32'] = np.percentile(x,32)\n    X_train.loc[segment, 'p33'] = np.percentile(x,33)\n    X_train.loc[segment, 'p34'] = np.percentile(x,34)\n    X_train.loc[segment, 'p35'] = np.percentile(x,35)\n    X_train.loc[segment, 'p36'] = np.percentile(x,36)\n    X_train.loc[segment, 'p37'] = np.percentile(x,37)\n    X_train.loc[segment, 'p38'] = np.percentile(x,38)\n    X_train.loc[segment, 'p39'] = np.percentile(x,39)\n    X_train.loc[segment, 'p40'] = np.percentile(x,40)\n    X_train.loc[segment, 'p41'] = np.percentile(x,41)\n    X_train.loc[segment, 'p42'] = np.percentile(x,42)\n    X_train.loc[segment, 'p43'] = np.percentile(x,43)\n    X_train.loc[segment, 'p44'] = np.percentile(x,44)\n    X_train.loc[segment, 'p45'] = np.percentile(x,45)\n    X_train.loc[segment, 'p46'] = np.percentile(x,46)\n    X_train.loc[segment, 'p47'] = np.percentile(x,47)\n    X_train.loc[segment, 'p48'] = np.percentile(x,48)\n    X_train.loc[segment, 'p49'] = np.percentile(x,49)\n    X_train.loc[segment, 'p50'] = np.percentile(x,50)\n    X_train.loc[segment, 'p51'] = np.percentile(x,51)\n    X_train.loc[segment, 'p52'] = np.percentile(x,52)\n    X_train.loc[segment, 'p53'] = np.percentile(x,53)\n    X_train.loc[segment, 'p54'] = np.percentile(x,54)\n    X_train.loc[segment, 'p55'] = np.percentile(x,55)\n    X_train.loc[segment, 'p56'] = np.percentile(x,56)\n    X_train.loc[segment, 'p57'] = np.percentile(x,57)\n    X_train.loc[segment, 'p58'] = np.percentile(x,58)\n    X_train.loc[segment, 'p59'] = np.percentile(x,59)\n    X_train.loc[segment, 'p60'] = np.percentile(x,60)\n    X_train.loc[segment, 'p61'] = np.percentile(x,61)\n    X_train.loc[segment, 'p62'] = np.percentile(x,62)\n    X_train.loc[segment, 'p63'] = np.percentile(x,63)\n    X_train.loc[segment, 'p64'] = np.percentile(x,64)\n    X_train.loc[segment, 'p65'] = np.percentile(x,65)\n    X_train.loc[segment, 'p66'] = np.percentile(x,66)\n    X_train.loc[segment, 'p67'] = np.percentile(x,67)\n    X_train.loc[segment, 'p68'] = np.percentile(x,68)\n    X_train.loc[segment, 'p69'] = np.percentile(x,69)\n    X_train.loc[segment, 'p70'] = np.percentile(x,70)\n    X_train.loc[segment, 'p71'] = np.percentile(x,71)\n    X_train.loc[segment, 'p72'] = np.percentile(x,72)\n    X_train.loc[segment, 'p73'] = np.percentile(x,73)\n    X_train.loc[segment, 'p74'] = np.percentile(x,74)\n    X_train.loc[segment, 'p75'] = np.percentile(x,75)\n    X_train.loc[segment, 'p76'] = np.percentile(x,76)\n    X_train.loc[segment, 'p77'] = np.percentile(x,77)\n    X_train.loc[segment, 'p78'] = np.percentile(x,78)\n    X_train.loc[segment, 'p79'] = np.percentile(x,79)\n    X_train.loc[segment, 'p80'] = np.percentile(x,80)\n    X_train.loc[segment, 'p81'] = np.percentile(x,81)\n    X_train.loc[segment, 'p82'] = np.percentile(x,82)\n    X_train.loc[segment, 'p83'] = np.percentile(x,83)\n    X_train.loc[segment, 'p84'] = np.percentile(x,84)\n    X_train.loc[segment, 'p85'] = np.percentile(x,85)\n    X_train.loc[segment, 'p86'] = np.percentile(x,86)\n    X_train.loc[segment, 'p87'] = np.percentile(x,87)\n    X_train.loc[segment, 'p88'] = np.percentile(x,88)\n    X_train.loc[segment, 'p89'] = np.percentile(x,89)\n    X_train.loc[segment, 'p90'] = np.percentile(x,90)\n    X_train.loc[segment, 'p91'] = np.percentile(x,91)\n    X_train.loc[segment, 'p92'] = np.percentile(x,92)\n    X_train.loc[segment, 'p93'] = np.percentile(x,93)\n    X_train.loc[segment, 'p94'] = np.percentile(x,94)\n    X_train.loc[segment, 'p95'] = np.percentile(x,95)\n    X_train.loc[segment, 'p96'] = np.percentile(x,96)\n    X_train.loc[segment, 'p97'] = np.percentile(x,97)\n    X_train.loc[segment, 'p98'] = np.percentile(x,98)\n    X_train.loc[segment, 'p99'] = np.percentile(x,99)\n    X_train.loc[segment, 'p100'] = np.percentile(x,100)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from scipy import stats\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p1'], X_train['time_to_failure'])\nprint(\"p1: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p2'], X_train['time_to_failure'])\nprint(\"p2: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p3'], X_train['time_to_failure'])\nprint(\"p3: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p4'], X_train['time_to_failure'])\nprint(\"p4: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p5'], X_train['time_to_failure'])\nprint(\"p5: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p6'], X_train['time_to_failure'])\nprint(\"p6: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p7'], X_train['time_to_failure'])\nprint(\"p7: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p8'], X_train['time_to_failure'])\nprint(\"p8: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p9'], X_train['time_to_failure'])\nprint(\"p9: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p10'], X_train['time_to_failure'])\nprint(\"p10: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p11'], X_train['time_to_failure'])\nprint(\"p11: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p12'], X_train['time_to_failure'])\nprint(\"p12: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p13'], X_train['time_to_failure'])\nprint(\"p13: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p14'], X_train['time_to_failure'])\nprint(\"p14: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p15'], X_train['time_to_failure'])\nprint(\"p15: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p16'], X_train['time_to_failure'])\nprint(\"p16: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p17'], X_train['time_to_failure'])\nprint(\"p17: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p18'], X_train['time_to_failure'])\nprint(\"p18: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p19'], X_train['time_to_failure'])\nprint(\"p19: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p20'], X_train['time_to_failure'])\nprint(\"p20: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p21'], X_train['time_to_failure'])\nprint(\"p21: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p22'], X_train['time_to_failure'])\nprint(\"p22: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p23'], X_train['time_to_failure'])\nprint(\"p23: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p24'], X_train['time_to_failure'])\nprint(\"p24: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p25'], X_train['time_to_failure'])\nprint(\"p25: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p26'], X_train['time_to_failure'])\nprint(\"p26: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p27'], X_train['time_to_failure'])\nprint(\"p27: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p28'], X_train['time_to_failure'])\nprint(\"p28: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p29'], X_train['time_to_failure'])\nprint(\"p29: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p30'], X_train['time_to_failure'])\nprint(\"p30: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p31'], X_train['time_to_failure'])\nprint(\"p31: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p32'], X_train['time_to_failure'])\nprint(\"p32: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p33'], X_train['time_to_failure'])\nprint(\"p33: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p34'], X_train['time_to_failure'])\nprint(\"p34: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p35'], X_train['time_to_failure'])\nprint(\"p35: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p36'], X_train['time_to_failure'])\nprint(\"p36: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p37'], X_train['time_to_failure'])\nprint(\"p37: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p38'], X_train['time_to_failure'])\nprint(\"p38: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p39'], X_train['time_to_failure'])\nprint(\"p39: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p40'], X_train['time_to_failure'])\nprint(\"p40: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p41'], X_train['time_to_failure'])\nprint(\"p41: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p42'], X_train['time_to_failure'])\nprint(\"p42: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p43'], X_train['time_to_failure'])\nprint(\"p43: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p44'], X_train['time_to_failure'])\nprint(\"p44: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p45'], X_train['time_to_failure'])\nprint(\"p45: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p46'], X_train['time_to_failure'])\nprint(\"p46: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p47'], X_train['time_to_failure'])\nprint(\"p47: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p48'], X_train['time_to_failure'])\nprint(\"p48: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p49'], X_train['time_to_failure'])\nprint(\"p49: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p50'], X_train['time_to_failure'])\nprint(\"p50: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p51'], X_train['time_to_failure'])\nprint(\"p51: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p52'], X_train['time_to_failure'])\nprint(\"p52: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p53'], X_train['time_to_failure'])\nprint(\"p53: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p54'], X_train['time_to_failure'])\nprint(\"p54: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p55'], X_train['time_to_failure'])\nprint(\"p55: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p56'], X_train['time_to_failure'])\nprint(\"p56: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p57'], X_train['time_to_failure'])\nprint(\"p57: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p58'], X_train['time_to_failure'])\nprint(\"p58: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p59'], X_train['time_to_failure'])\nprint(\"p59: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p60'], X_train['time_to_failure'])\nprint(\"p60: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p61'], X_train['time_to_failure'])\nprint(\"p61: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p62'], X_train['time_to_failure'])\nprint(\"p62: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p63'], X_train['time_to_failure'])\nprint(\"p63: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p64'], X_train['time_to_failure'])\nprint(\"p64: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p65'], X_train['time_to_failure'])\nprint(\"p65: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p66'], X_train['time_to_failure'])\nprint(\"p66: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p67'], X_train['time_to_failure'])\nprint(\"p67: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p68'], X_train['time_to_failure'])\nprint(\"p68: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p69'], X_train['time_to_failure'])\nprint(\"p69: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p70'], X_train['time_to_failure'])\nprint(\"p70: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p71'], X_train['time_to_failure'])\nprint(\"p71: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p72'], X_train['time_to_failure'])\nprint(\"p72: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p73'], X_train['time_to_failure'])\nprint(\"p73: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p74'], X_train['time_to_failure'])\nprint(\"p74: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p75'], X_train['time_to_failure'])\nprint(\"p75: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p76'], X_train['time_to_failure'])\nprint(\"p76: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p77'], X_train['time_to_failure'])\nprint(\"p77: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p78'], X_train['time_to_failure'])\nprint(\"p78: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p79'], X_train['time_to_failure'])\nprint(\"p79: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p80'], X_train['time_to_failure'])\nprint(\"p80: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p81'], X_train['time_to_failure'])\nprint(\"p81: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p82'], X_train['time_to_failure'])\nprint(\"p82: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p83'], X_train['time_to_failure'])\nprint(\"p83: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p84'], X_train['time_to_failure'])\nprint(\"p84: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p85'], X_train['time_to_failure'])\nprint(\"p85: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p86'], X_train['time_to_failure'])\nprint(\"p86: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p87'], X_train['time_to_failure'])\nprint(\"p87: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p88'], X_train['time_to_failure'])\nprint(\"p88: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p89'], X_train['time_to_failure'])\nprint(\"p89: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p90'], X_train['time_to_failure'])\nprint(\"p90: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#1\npearson_coef, p_value = stats.pearsonr(X_train['p91'], X_train['time_to_failure'])\nprint(\"p91: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p92'], X_train['time_to_failure'])\nprint(\"p92: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p93'], X_train['time_to_failure'])\nprint(\"p93: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p94'], X_train['time_to_failure'])\nprint(\"p94: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p95'], X_train['time_to_failure'])\nprint(\"p95: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p96'], X_train['time_to_failure'])\nprint(\"p96: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p97'], X_train['time_to_failure'])\nprint(\"p97: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p98'], X_train['time_to_failure'])\nprint(\"p98: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p99'], X_train['time_to_failure'])\nprint(\"p99: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n#1\npearson_coef, p_value = stats.pearsonr(X_train['p100'], X_train['time_to_failure'])\nprint(\"p100: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from scipy import stats\n\n#1\npearson_coef, p_value = stats.pearsonr(X_train['ave'], X_train['time_to_failure'])\nprint(\"ave: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#2\npearson_coef, p_value = stats.pearsonr(X_train['std'], X_train['time_to_failure'])\nprint(\"std: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#3\n\npearson_coef, p_value = stats.pearsonr(X_train['kurt'], X_train['time_to_failure'])\nprint(\"kurt: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#4\npearson_coef, p_value = stats.pearsonr(X_train['max'], X_train['time_to_failure'])\nprint(\"max: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#5\npearson_coef, p_value = stats.pearsonr(X_train['min'], X_train['time_to_failure'])\nprint(\"min: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#6\npearson_coef, p_value = stats.pearsonr(X_train['skew'], X_train['time_to_failure'])\nprint(\"skew: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#7\npearson_coef, p_value = stats.pearsonr(X_train['sum'], X_train['time_to_failure'])\nprint(\"sum: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#8\npearson_coef, p_value = stats.pearsonr(X_train['median'], X_train['time_to_failure'])\nprint(\"median: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#9\npearson_coef, p_value = stats.pearsonr(X_train['var'], X_train['time_to_failure'])\nprint(\"var: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#10\npearson_coef, p_value = stats.pearsonr(X_train['q1'], X_train['time_to_failure'])\nprint(\"q1: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#11\npearson_coef, p_value = stats.pearsonr(X_train['q19'], X_train['time_to_failure'])\nprint(\"q19: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#12\npearson_coef, p_value = stats.pearsonr(X_train['q2'], X_train['time_to_failure'])\nprint(\"q2: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#13\npearson_coef, p_value = stats.pearsonr(X_train['Q1'], X_train['time_to_failure'])\nprint(\"Q1: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#14\npearson_coef, p_value = stats.pearsonr(X_train['q3'], X_train['time_to_failure'])\nprint(\"q3: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#15\npearson_coef, p_value = stats.pearsonr(X_train['q4'], X_train['time_to_failure'])\nprint(\"q4: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#16\npearson_coef, p_value = stats.pearsonr(X_train['Q2'], X_train['time_to_failure'])\nprint(\"Q2: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#17\npearson_coef, p_value = stats.pearsonr(X_train['q6'], X_train['time_to_failure'])\nprint(\"q6: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#18\npearson_coef, p_value = stats.pearsonr(X_train['q7'], X_train['time_to_failure'])\nprint(\"q7: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#19\npearson_coef, p_value = stats.pearsonr(X_train['Q3'], X_train['time_to_failure'])\nprint(\"Q3: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#20\npearson_coef, p_value = stats.pearsonr(X_train['q8'], X_train['time_to_failure'])\nprint(\"q8: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#21\npearson_coef, p_value = stats.pearsonr(X_train['q81'], X_train['time_to_failure'])\nprint(\"q81: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#22\npearson_coef, p_value = stats.pearsonr(X_train['q82'], X_train['time_to_failure'])\nprint(\"q82: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#23\npearson_coef, p_value = stats.pearsonr(X_train['q9'], X_train['time_to_failure'])\nprint(\"q9: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#24\npearson_coef, p_value = stats.pearsonr(X_train['q10'], X_train['time_to_failure'])\nprint(\"q10: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)\n\n#25\npearson_coef, p_value = stats.pearsonr(X_train['iqr'], X_train['time_to_failure'])\nprint(\"iqr: The Pearson Correlation Coefficient is\", pearson_coef, \" with a P-value of P =\", p_value)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train['time_to_failure'].unique\n#X_train['signal'].describe()\n#train['signal'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train=X_train[X_train['ave']<=5.20555]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a cloumn featuring binary values on the basis of accident risks\n#X_train['outcome'] =[1 if 4.519464272770625 else 0 for x in X_train['std']]\n#train['outcome'] =[1 if 4.884113333333334 else 0 for x in train['signal']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"x_train=X_train [['ave', 'std', 'max', 'min','kurt','skew','median','var','sum','q1','q19',\\\n                                'q2','Q1','q3','q4','Q2','q6','q7','Q3','q8','q81','q82','q9','q10','iqr',\\\n                  'p1','p2','p3','p4','p5','p6','p7','p8','p9','p10','p11','p12','p13','p14','p15','p16',\\\n                 'p17','p18','p19','p20','p21','p22','p23','p24','p25','p26','p27','p28','p29','p30',\\\n                  'p31','p32','p33','p34','p35','p36','p37','p38','p39','p40','p41','p42','p43','p44',\\\n                  'p45','p46','p47','p48','p49','p50','p51','p52','p53','p54','p55','p56','p57','p58',\\\n                  'p59','p60','p61','p62','p63','p64','p65','p66','p67','p68',\\\n                  'p69','p70','p71','p72','p73','p74','p75','p76','p77','p78','p79','p80','p81',\\\n                'p82','p83','p84','p85','p86','p87','p88','p89','p90','p91','p92','p93','p94','p95',\\\n                'p96','p97','p98','p99','p100']] =2.011"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#x_train=X_train [[ 'std', 'max', 'min','q1','q19',\\\n #                               'q2','Q1','q3','q4','Q2','q6','q7','Q3','q8','q81','q82','q9','q10','iqr',\\\n  #                'p1','p2','p3','p4','p5','p6','p7','p8','p9','p10','p11','p12','p13','p14','p15','p16',\\\n   #              'p17','p18','p19','p20','p21','p22','p23','p24','p25','p26','p27','p28','p29','p30',\\\n    #            'p69','p70','p71','p72','p73','p74','p75','p76','p77','p78','p79','p80','p81',\\\n     #           'p82','p83','p84','p85','p86','p87','p88','p89','p90','p91','p92','p93','p94','p95',\\\n      #          'p96','p97','p98','p99']] =2.020\n   \n#x_train=X_train [['q1','q19','q2','Q1','q3','q4','Q2','q6','q7','Q3','q8','q81','q82','q9','q10','iqr']]#=2.062\n#x_train=X_train [['q1','q19','q2','q8','q81','q82','q9','min','max','std','iqr']]#=2.075\n#x_train=X_train [['q1','q19','q2','q8','q81','q82','q9','iqr']] # = 2.075\nx_train=X_train[['p1','p2','p3','p4','p5','p6','p7','p8','p9','p10','p11','p12','p13','p14','p15','p16',\\\n                 'p17','p18','p19','p20','p21','p22','p23','p24','p25','p26','p27','p28','p29','p30',\\\n                'p69','p70','p71','p72','p73','p74','p75','p76','p77','p78','p79','p80','p81',\\\n                'p82','p83','p84','p85','p86','p87','p88','p89','p90','p91','p92','p93','p94','p95',\\\n                'p96','p97','p98','p99',]] #=2.019\n\nx_train.head()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"x_train=X_train[['p1','p2','p3','p4','p5','p6','p7','p8','p9','p10','p11','p12','p13','p14','p15','p16',\\\n                 'p17','p18','p19','p20','p21','p22','p23','p24','p25','p26','p27','p28','p29','p30',\\\n                'p69','p70','p71','p72','p73','p74','p75','p76','p77','p78','p79','p80','p81',\\\n                'p82','p83','p84','p85','p86','p87','p88','p89','p90','p91','p92','p93','p94','p95',\\\n                'p96','p97','p98','p99',]] #=2.019"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"x_train.head()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"x_train=x_train.shift(periods=6)#, fill_value=0)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train=X_train [[ 'p13','p19','p80','p81','p82','p86','p87','p88', 'p89', 'p90']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def df_derived_by_shift(df,lag=0,NON_DER=[]):\n    df = df.copy()\n    if not lag:\n        return df\n    cols ={}\n    for i in range(1,lag+1):\n        for x in list(df.columns):\n            if x not in NON_DER:\n                if not x in cols:\n                    cols[x] = ['{}_{}'.format(x, i)]\n                else:\n                    cols[x].append('{}_{}'.format(x, i))\n    for k,v in cols.items():\n        columns = v\n        dfn = pd.DataFrame(data=None, columns=columns, index=df.index)    \n        i = 1\n        for c in columns:\n            dfn[c] = df[k].shift(periods=i)\n            i+=1\n        df = pd.concat([df, dfn], axis=1, join_axes=[df.index])\n    return df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"NON_DER = ['',]\ndf_new = df_derived_by_shift(x_train, 12, NON_DER)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"x_train.head()"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train=x_train.fillna(x_train.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#numpy.corrcoef(df['C'][1:-1], df['C'][2:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#def df_autocorr(df, lag=1, axis=0):\n #   \"\"\"Compute full-sample column-wise autocorrelation for a DataFrame.\"\"\"\n  #  return df.apply(lambda col: col.autocorr(lag), axis=axis)\n#d1 = DataFrame(np.random.randn(100, 6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train['min'].autocorr(lag=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y_train.head()\n#y_train=X_train[['time_to_failure']]\n#y_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = train[['signal']]\ny_train = train[['time']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(x_train)\nX_train_scaled = scaler.transform(x_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"svm = NuSVR()\nsvm.fit(X_train_scaled, y_train.values.flatten())\ny_pred = svm.predict(X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nplt.scatter(y_train.values.flatten(), y_pred)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = mean_absolute_error(y_train.values.flatten(), y_pred)\nprint(f'Score: {score:0.3f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = pd.DataFrame(columns=x_train.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"for seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    #X_test.loc[sed_id] = x\n    X_test.loc[seg_id, 'p1'] = np.percentile(x,1)\n    X_test.loc[seg_id, 'p2'] = np.percentile(x,2)\n    X_test.loc[seg_id, 'p3'] = np.percentile(x,3)\n    X_test.loc[seg_id, 'p4'] = np.percentile(x,4)\n    X_test.loc[seg_id, 'p5'] = np.percentile(x,5)\n    X_test.loc[seg_id, 'p6'] = np.percentile(x,6)\n    X_test.loc[seg_id, 'p7'] = np.percentile(x,7)\n    X_test.loc[seg_id, 'p8'] = np.percentile(x,8)\n    X_test.loc[seg_id, 'p9'] = np.percentile(x,9)\n    X_test.loc[seg_id, 'p10'] = np.percentile(x,10)\n    X_test.loc[seg_id, 'p11'] = np.percentile(x,11)\n    X_test.loc[seg_id, 'p12'] = np.percentile(x,12)\n    X_test.loc[seg_id, 'p13'] = np.percentile(x,13)\n    X_test.loc[seg_id, 'p14'] = np.percentile(x,14)\n    X_test.loc[seg_id, 'p15'] = np.percentile(x,15)\n    X_test.loc[seg_id, 'p16'] = np.percentile(x,16)\n    X_test.loc[seg_id, 'p17'] = np.percentile(x,17)\n    X_test.loc[seg_id, 'p18'] = np.percentile(x,18)\n    X_test.loc[seg_id, 'p19'] = np.percentile(x,19)\n    X_test.loc[seg_id, 'p20'] = np.percentile(x,20)\n    X_test.loc[seg_id, 'p21'] = np.percentile(x,21)\n    X_test.loc[seg_id, 'p22'] = np.percentile(x,22)\n    X_test.loc[seg_id, 'p23'] = np.percentile(x,23)\n    X_test.loc[seg_id, 'p24'] = np.percentile(x,24)\n    X_test.loc[seg_id, 'p25'] = np.percentile(x,25)\n    X_test.loc[seg_id, 'p26'] = np.percentile(x,26)\n    X_test.loc[seg_id, 'p27'] = np.percentile(x,27)\n    X_test.loc[seg_id, 'p28'] = np.percentile(x,28)\n    X_test.loc[seg_id, 'p29'] = np.percentile(x,29)\n    X_test.loc[seg_id, 'p30'] = np.percentile(x,30)\n    \n    X_test.loc[seg_id, 'p69'] = np.percentile(x,69)\n    X_test.loc[seg_id, 'p70'] = np.percentile(x,70)\n    X_test.loc[seg_id, 'p71'] = np.percentile(x,71)\n    X_test.loc[seg_id, 'p72'] = np.percentile(x,72)\n    X_test.loc[seg_id, 'p73'] = np.percentile(x,73)\n    X_test.loc[seg_id, 'p74'] = np.percentile(x,74)\n    X_test.loc[seg_id, 'p75'] = np.percentile(x,75)\n    X_test.loc[seg_id, 'p76'] = np.percentile(x,76)\n    X_test.loc[seg_id, 'p77'] = np.percentile(x,77)\n    X_test.loc[seg_id, 'p78'] = np.percentile(x,78)\n    X_test.loc[seg_id, 'p79'] = np.percentile(x,79)\n    X_test.loc[seg_id, 'p80'] = np.percentile(x,80)\n    X_test.loc[seg_id, 'p81'] = np.percentile(x,81)\n    X_test.loc[seg_id, 'p82'] = np.percentile(x,82)\n    X_test.loc[seg_id, 'p83'] = np.percentile(x,83)\n    X_test.loc[seg_id, 'p84'] = np.percentile(x,84)\n    X_test.loc[seg_id, 'p85'] = np.percentile(x,85)\n    X_test.loc[seg_id, 'p86'] = np.percentile(x,86)\n    X_test.loc[seg_id, 'p87'] = np.percentile(x,87)\n    X_test.loc[seg_id, 'p88'] = np.percentile(x,88)\n    X_test.loc[seg_id, 'p89'] = np.percentile(x,89)\n    X_test.loc[seg_id, 'p90'] = np.percentile(x,90)\n    X_test.loc[seg_id, 'p91'] = np.percentile(x,91)\n    X_test.loc[seg_id, 'p92'] = np.percentile(x,92)\n    X_test.loc[seg_id, 'p93'] = np.percentile(x,93)\n    X_test.loc[seg_id, 'p94'] = np.percentile(x,94)\n    X_test.loc[seg_id, 'p95'] = np.percentile(x,95)\n    X_test.loc[seg_id, 'p96'] = np.percentile(x,96)\n    X_test.loc[seg_id, 'p97'] = np.percentile(x,97)\n    X_test.loc[seg_id, 'p98'] = np.percentile(x,98)\n    X_test.loc[seg_id, 'p99'] = np.percentile(x,99)\n    "},{"metadata":{"trusted":true},"cell_type":"markdown","source":"for seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    \n    #X_test.loc[seg_id, 'ave'] = x.mean()\n    #X_test.loc[seg_id, 'std'] = np.std(x)\n    #X_test.loc[seg_id, 'max'] = np.max(x)\n    #X_test.loc[seg_id, 'min'] = np.min(x)\n    #X_test.loc[seg_id, 'median'] = np.median(x)\n    #X_test.loc[seg_id, 'var'] = np.var(x)\n    #X_test.loc[seg_id, 'kurt'] = ((x-x.mean())/x.std() ** 4).mean()\n    #X_test.loc[seg_id, 'skew'] = ((x-x.mean())/x.std() ** 3).mean()\n    #X_test.loc[seg_id, 'sum'] = x.sum()\n    #X_test.loc[seg_id, 'q1'] = np.quantile(x,.1)\n    #X_test.loc[seg_id, 'q19'] = np.quantile(x,.19)\n    #X_test.loc[seg_id, 'q2'] = np.quantile(x,.2)\n    #X_test.loc[seg_id, 'Q1'] = np.quantile(x,.25)\n    #X_test.loc[seg_id, 'q3'] = np.quantile(x,.3)\n    #X_test.loc[seg_id, 'q4'] = np.quantile(x,.4)\n    #X_test.loc[seg_id, 'Q2'] = np.quantile(x,.5)\n    #X_test.loc[seg_id, 'q6'] = np.quantile(x,.6)\n    #X_test.loc[seg_id, 'q7'] = np.quantile(x,.7)\n    #X_test.loc[seg_id, 'Q3'] = np.quantile(x,.75)\n    #X_test.loc[seg_id, 'q8'] = np.quantile(x,.8)\n    #X_test.loc[seg_id, 'q81'] = np.quantile(x,.81)\n    #X_test.loc[seg_id, 'q82'] = np.quantile(x,.82)\n    #X_test.loc[seg_id, 'q9'] = np.quantile(x,.9)\n    #X_test.loc[seg_id, 'q10'] = np.quantile(x,1)\n    #X_test.loc[seg_id, 'iqr'] = np.quantile(x,.75) - np.quantile (x,.25)\n    \n    X_test.loc[seg_id, 'p1'] = np.percentile(x,1)\n    X_test.loc[seg_id, 'p2'] = np.percentile(x,2)\n    X_test.loc[seg_id, 'p3'] = np.percentile(x,3)\n    X_test.loc[seg_id, 'p4'] = np.percentile(x,4)\n    X_test.loc[seg_id, 'p5'] = np.percentile(x,5)\n    X_test.loc[seg_id, 'p6'] = np.percentile(x,6)\n    X_test.loc[seg_id, 'p7'] = np.percentile(x,7)\n    X_test.loc[seg_id, 'p8'] = np.percentile(x,8)\n    X_test.loc[seg_id, 'p9'] = np.percentile(x,9)\n    X_test.loc[seg_id, 'p10'] = np.percentile(x,10)\n    X_test.loc[seg_id, 'p11'] = np.percentile(x,11)\n    X_test.loc[seg_id, 'p12'] = np.percentile(x,12)\n    X_test.loc[seg_id, 'p13'] = np.percentile(x,13)\n    X_test.loc[seg_id, 'p14'] = np.percentile(x,14)\n    X_test.loc[seg_id, 'p15'] = np.percentile(x,15)\n    X_test.loc[seg_id, 'p16'] = np.percentile(x,16)\n    X_test.loc[seg_id, 'p17'] = np.percentile(x,17)\n    X_test.loc[seg_id, 'p18'] = np.percentile(x,18)\n    X_test.loc[seg_id, 'p19'] = np.percentile(x,19)\n    X_test.loc[seg_id, 'p20'] = np.percentile(x,20)\n    X_test.loc[seg_id, 'p21'] = np.percentile(x,21)\n    X_test.loc[seg_id, 'p22'] = np.percentile(x,22)\n    X_test.loc[seg_id, 'p23'] = np.percentile(x,23)\n    X_test.loc[seg_id, 'p24'] = np.percentile(x,24)\n    X_test.loc[seg_id, 'p25'] = np.percentile(x,25)\n    X_test.loc[seg_id, 'p26'] = np.percentile(x,26)\n    X_test.loc[seg_id, 'p27'] = np.percentile(x,27)\n    X_test.loc[seg_id, 'p28'] = np.percentile(x,28)\n    X_test.loc[seg_id, 'p29'] = np.percentile(x,29)\n    X_test.loc[seg_id, 'p30'] = np.percentile(x,30)\n    \n    #X_test.loc[seg_id, 'p31'] = np.percentile(x,31)\n    #X_test.loc[seg_id, 'p32'] = np.percentile(x,32)\n    #X_test.loc[seg_id, 'p33'] = np.percentile(x,33)\n    #X_test.loc[seg_id, 'p34'] = np.percentile(x,34)\n    #X_test.loc[seg_id, 'p35'] = np.percentile(x,35)\n    #X_test.loc[seg_id, 'p36'] = np.percentile(x,36)\n    #X_test.loc[seg_id, 'p37'] = np.percentile(x,37)\n    #X_test.loc[seg_id, 'p38'] = np.percentile(x,38)\n    #X_test.loc[seg_id, 'p39'] = np.percentile(x,39)\n    #X_test.loc[seg_id, 'p40'] = np.percentile(x,40)\n    X_test.loc[seg_id, 'p41'] = np.percentile(x,41)\n    X_test.loc[seg_id, 'p42'] = np.percentile(x,42)\n    X_test.loc[seg_id, 'p43'] = np.percentile(x,43)\n    X_test.loc[seg_id, 'p44'] = np.percentile(x,44)\n    X_test.loc[seg_id, 'p45'] = np.percentile(x,45)\n    X_test.loc[seg_id, 'p46'] = np.percentile(x,46)\n    X_test.loc[seg_id, 'p47'] = np.percentile(x,47)\n    X_test.loc[seg_id, 'p48'] = np.percentile(x,48)\n    X_test.loc[seg_id, 'p49'] = np.percentile(x,49)\n    X_test.loc[seg_id, 'p50'] = np.percentile(x,50)\n    \n    X_test.loc[seg_id, 'p51'] = np.percentile(x,51)\n    X_test.loc[seg_id, 'p52'] = np.percentile(x,52)\n    X_test.loc[seg_id, 'p53'] = np.percentile(x,53)\n    X_test.loc[seg_id, 'p54'] = np.percentile(x,54)\n    X_test.loc[seg_id, 'p55'] = np.percentile(x,55)\n    X_test.loc[seg_id, 'p56'] = np.percentile(x,56)\n    X_test.loc[seg_id, 'p57'] = np.percentile(x,57)\n    X_test.loc[seg_id, 'p58'] = np.percentile(x,58)\n    X_test.loc[seg_id, 'p59'] = np.percentile(x,59)\n    X_test.loc[seg_id, 'p60'] = np.percentile(x,60)\n     \n    X_test.loc[seg_id, 'p61'] = np.percentile(x,61)\n    X_test.loc[seg_id, 'p62'] = np.percentile(x,62)\n    X_test.loc[seg_id, 'p63'] = np.percentile(x,63)\n    X_test.loc[seg_id, 'p64'] = np.percentile(x,64)\n    X_test.loc[seg_id, 'p65'] = np.percentile(x,65)\n    X_test.loc[seg_id, 'p66'] = np.percentile(x,66)\n    X_test.loc[seg_id, 'p67'] = np.percentile(x,67)\n    X_test.loc[seg_id, 'p68'] = np.percentile(x,68)\n   \n    \n    X_test.loc[seg_id, 'p69'] = np.percentile(x,69)\n    X_test.loc[seg_id, 'p70'] = np.percentile(x,70)\n    X_test.loc[seg_id, 'p71'] = np.percentile(x,71)\n    X_test.loc[seg_id, 'p72'] = np.percentile(x,72)\n    X_test.loc[seg_id, 'p73'] = np.percentile(x,73)\n    X_test.loc[seg_id, 'p74'] = np.percentile(x,74)\n    X_test.loc[seg_id, 'p75'] = np.percentile(x,75)\n    X_test.loc[seg_id, 'p76'] = np.percentile(x,76)\n    X_test.loc[seg_id, 'p77'] = np.percentile(x,77)\n    X_test.loc[seg_id, 'p78'] = np.percentile(x,78)\n    X_test.loc[seg_id, 'p79'] = np.percentile(x,79)\n    X_test.loc[seg_id, 'p80'] = np.percentile(x,80)\n    X_test.loc[seg_id, 'p81'] = np.percentile(x,81)\n    X_test.loc[seg_id, 'p82'] = np.percentile(x,82)\n    X_test.loc[seg_id, 'p83'] = np.percentile(x,83)\n    X_test.loc[seg_id, 'p84'] = np.percentile(x,84)\n    X_test.loc[seg_id, 'p85'] = np.percentile(x,85)\n    X_test.loc[seg_id, 'p86'] = np.percentile(x,86)\n    X_test.loc[seg_id, 'p87'] = np.percentile(x,87)\n    X_test.loc[seg_id, 'p88'] = np.percentile(x,88)\n    X_test.loc[seg_id, 'p89'] = np.percentile(x,89)\n    X_test.loc[seg_id, 'p90'] = np.percentile(x,90)\n    X_test.loc[seg_id, 'p91'] = np.percentile(x,91)\n    X_test.loc[seg_id, 'p92'] = np.percentile(x,92)\n    X_test.loc[seg_id, 'p93'] = np.percentile(x,93)\n    X_test.loc[seg_id, 'p94'] = np.percentile(x,94)\n    X_test.loc[seg_id, 'p95'] = np.percentile(x,95)\n    X_test.loc[seg_id, 'p96'] = np.percentile(x,96)\n    X_test.loc[seg_id, 'p97'] = np.percentile(x,97)\n    X_test.loc[seg_id, 'p98'] = np.percentile(x,98)\n    X_test.loc[seg_id, 'p99'] = np.percentile(x,99)\n    X_test.loc[seg_id, 'p100'] = np.percentile(x,100)"},{"metadata":{"trusted":true},"cell_type":"code","source":"for seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    print (x)\n    X_test.loc[seg_id,'signal'] = np.median(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_scaled = scaler.transform(X_test)\nsubmission['time_to_failure'] = svm.predict(X_test_scaled)\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}