{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_dt = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c6b772e505c8972d1b8bcdc6ad5db7afe59eb64"},"cell_type":"code","source":"print(train_dt.head())\nprint(train_dt.tail())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fe4c872f1b80218f803d0356d5236f421c4a624"},"cell_type":"code","source":"train_acoustic_small = train_dt['acoustic_data'].values[::50]\ntrain_ttf_small = train_dt['time_to_failure'].values[::50]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"697d346786cf9aee881cda16cf7305a3d6a632a0"},"cell_type":"code","source":"train_acoustic_small.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5c6edcd2aecf1b6d4c598c38773f37eef89c7a6"},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title('Acoustic_data and time_to_failure (sampled)')\nplt.plot(train_acoustic_small, color='b')\nax1.set_ylabel('acoustic_data', color='b')\nplt.legend(['acoustic_data'])\nax2 = ax1.twinx()\nplt.plot(train_ttf_small, color='r')\nax2.set_ylabel('time_to_failure', color='r')\nplt.legend(['time_to_failure'], loc=(0.875, 0.9))\nplt.grid(False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc170da1b56ff261539ddb9379bb060bd5fe7748"},"cell_type":"code","source":"del train_ttf_small\ndel train_acoustic_small","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e8de361d7b996cd72b8b4ead20cc8eafedf44f9"},"cell_type":"code","source":"train_ttf_epoch = train_dt['time_to_failure'].values[:100000]\ntrain_acoustic_epoch = train_dt['acoustic_data'].values[:100000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4061ec951dad0c03cefc652d9f67676c01cebeb1"},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title('Acoustic_data and time_to_failure(s) (sampled)')\nplt.plot(train_acoustic_epoch, color='b')\nax1.set_ylabel('acoustic_data', color='b')\nplt.legend(['acoustic_data'])\nax2 = ax1.twinx()\nplt.plot(train_ttf_epoch, color='r')\nax2.set_ylabel('time_to_failure', color='r')\nplt.legend(['time_to_failure'], loc=(0.875, 0.9))\nplt.grid(False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0efe008fe38dca74f098a47908a58cb68068b252"},"cell_type":"code","source":"#train_ttf_epoch[:100]\n# occurance of ttf == \n# 1.4690996: 108,\n# 1.4690998: 109,\n# 1.4690999: 108,\n# 1.4691: 41\nunique, counts = np.unique(train_ttf_epoch[:1000], return_counts=True)\ndict(zip(unique, counts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#delete train_ttf/acoustic_epoch\ndel train_ttf_epoch\ndel train_acoustic_epoch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e0ff0759bfe8366a771e35b97953232a1057cda"},"cell_type":"code","source":"# sample first 1% of the data\nonepercent = int(len(train_dt)*0.01)\ntrain_ttf_one = train_dt['time_to_failure'].values[:onepercent]\ntrain_acoustic_one = train_dt['acoustic_data'].values[:onepercent]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c2dcda8dc85febde9e473afaa1a87edcfa57d50"},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title('Acoustic_data and time_to_failure(s) (sampled) one percent')\nplt.plot(train_acoustic_one, color='b')\nax1.set_ylabel('acoustic_data', color='b')\nplt.legend(['acoustic_data'])\nax2 = ax1.twinx()\nplt.plot(train_ttf_one, color='r')\nax2.set_ylabel('time_to_failure', color='r')\nplt.legend(['time_to_failure'], loc=(0.875, 0.9))\nplt.grid(False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"984dcdf754e12d430af26ec1872baac7aa808fcb"},"cell_type":"code","source":"#8th percent\ntrain_ttf_one = train_dt['time_to_failure'].values[onepercent*7:onepercent*8]\ntrain_acoustic_one = train_dt['acoustic_data'].values[onepercent*7:onepercent*8]\ntrain_acoustic_one.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21b502116ead1852194065b8b15f5ba4da5e6172"},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title('Acoustic_data and time_to_failure(s) (sampled) one percent')\nplt.plot(train_acoustic_one, color='b')\nax1.set_ylabel('acoustic_data', color='b')\nplt.legend(['acoustic_data'])\nax2 = ax1.twinx()\nplt.plot(train_ttf_one, color='r')\nax2.set_ylabel('time_to_failure', color='r')\nplt.legend(['time_to_failure'], loc=(0.875, 0.9))\nplt.grid(False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c44f3be2132309468be7357a75b0d40a8a18d9c"},"cell_type":"markdown","source":"# seems like there is a certain time gap between the ttf reaches 0 and acoustic data has high value\n"},{"metadata":{"trusted":true,"_uuid":"3dbec5351c4b881d6635b4f203866f8092a19c1a"},"cell_type":"code","source":"# returns time difference between ttf is minmum and when acoustic data is unusually high\ndef timegap(acoustic,ttf):\n    ttf_spike = np.argmax(np.abs(acoustic))\n    ttf_min = np.argmin(ttf)\n    return {\"min_ttf\":ttf[ttf_min],\n           \"ttf_spike\":ttf[ttf_spike],\n           \"difference\":np.abs(ttf[ttf_min]-ttf[ttf_spike])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7745677cb6723cd840e975178f462f188121ecf9"},"cell_type":"code","source":"timegaps = []\ntimegaps.append(timegap(train_acoustic_one,train_ttf_one))\nprint(timegaps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b646c67afb7c132ae0688a0a06a149aa7f85043"},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title('Acoustic_data and time_to_failure (sampled)')\nplt.plot(train_acoustic_small, color='b')\nax1.set_ylabel('acoustic_data', color='b')\nplt.legend(['acoustic_data'])\nax2 = ax1.twinx()\nplt.plot(train_ttf_small, color='r')\nax2.set_ylabel('time_to_failure', color='r')\nplt.legend(['time_to_failure'], loc=(0.875, 0.9))\nplt.grid(False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f5836d2a390336f46bfecd80c12f30bfcb95a15"},"cell_type":"code","source":"# going to filter out low amplitude by changing to 0\n# takes in acoustic data and an int amplitude\n# will change acoustic data to 0 if less than amplitude value\ndef low_amp_filter(acou,amp):\n    toReturn = []\n    i=0\n    while i < len(acou):\n        if np.abs(acou[i])<amp:\n            toReturn.append(0)\n        else: # when value is higher than amp\n            #print(acou[i])\n            toReturn.append(acou[i])\n        i+=1\n    return toReturn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d895801fd69c3ccdb59271cd71a4a4ce410c344"},"cell_type":"code","source":"acou_filtered = low_amp_filter(train_acoustic_small,800)\nacou_filtered_np = np.array(acou_filtered)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb30cfd26ebd3cff979fe0ce543ee1861523d138"},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title('Acoustic_data and time_to_failure (sampled & filtered)')\nplt.plot(acou_filtered, color='b')\nax1.set_ylabel('acoustic_data', color='b')\nplt.legend(['acoustic_data'])\nax2 = ax1.twinx()\nplt.plot(train_ttf_small, color='r')\nax2.set_ylabel('time_to_failure', color='r')\nplt.legend(['time_to_failure'], loc=(0.875, 0.9))\nplt.grid(False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# loop backward and find ttf values for when it is lowest and collects the time when acoustic is not 0 (After filtered)"},{"metadata":{"trusted":true,"_uuid":"7ed17f5e1bcb4d901ca6d40585ba756e847b4531"},"cell_type":"code","source":"# get indices of ttf when it reaches the minimum\n# when ttf value increases \ndef ttf_low(ttf):\n    toreturn = []\n    i = 0\n    while i < len(ttf)-2:\n        diff = ttf[i+1]-ttf[i]\n        #print(diff)\n        if diff > 1.0:\n            #store the index and the ttf value\n            tmp = np.array([i,ttf[i]])\n            toreturn.append(tmp)\n        i+=1\n    return toreturn\n\n\nttf_low_idx = ttf_low(train_ttf_small)\nttf_low_idx = np.array(ttf_low_idx)\nttf_low_idx.shape\n\n#indices of ttf when it is right before reaching 0\nttf_low_idx[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bf8cbee275f9217dfe35675c003896144cd5639"},"cell_type":"code","source":"# counts the number of high acoustic values before ttf reaches 0\n\"\"\"\nacou: filtered acoustic data\nttf_idx: array of indices of ttf right before it resets\n\"\"\"\ndef acou_counter(acou,ttf_idx,ttf):\n    i=0\n    toreturn = []\n    for indices in ttf_idx:\n        #indices = index of ttf right before it resets\n        tmp = []\n        ttf_ind = int(indices)\n        while i < ttf_ind:\n            #loop thru right before i == indices -1\n            #get the index of acou where not 0\n            if acou[i] != 0:\n                acou_dict = {\n                    \"index\": i,\n                    \"acoustic_value\": acou[i],\n                    \"acou_ttf\": ttf[i],\n                    \"this.ttf\": ttf[ttf_ind],\n                    \"this.ttf_ind\": ttf_ind\n                }\n                tmp.append(acou_dict)\n            i+=1\n            \n        toreturn.append(tmp)\n    return toreturn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f62cc1aa056b411713668d5ba9eb52017c4ac35a"},"cell_type":"code","source":"acou_idx = acou_counter(np.array(acou_filtered),ttf_low_idx[:,0],train_ttf_small)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acou_idx[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acou_idx[0][0]\nprint(type(acou_idx),type(acou_idx[0]),type(acou_idx[0][0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73369a2f1e8de34d5c46118fdf98bb00cef62933"},"cell_type":"code","source":"for i in range(len(acou_idx)):\n    acou0 = acou_idx[i]\n    print(str(i+1)+\"th\")\n    for d in range(len(acou0)):\n        print(acou0[d][\"acou_ttf\"] - acou0[d][\"this.ttf\"])\ndel acou0    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<p>we can see there are some time differences between the spikes(wether it is a spike right before or not)</p>\n<p>therefore we need to get rid when acoustic is not right before ttf is lowest</p>"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#acou_diction = list(list(dict))\ndef timegap_avg(acou_diction):\n    toreturn = []\n    for sublist in acou_diction:\n        dif_list = []\n        for diction in sublist:\n            dif_list.append(diction[\"acou_ttf\"] - diction[\"this.ttf\"])\n        toreturn.append(np.mean(dif_list))\n    return toreturn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"timeavg = timegap_avg(acou_idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"timeavg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acou_idx[0][0].keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}