{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport gc\nimport seaborn as sns\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c46bd9c62b5fffd7131228d44f836653788effc"},"cell_type":"markdown","source":"Let's look at train data's statistical properties"},{"metadata":{"trusted":true,"_uuid":"15326d3179cc6174c48584b0acd000ee9398ad62"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"075def976cf4af01a5539a807ac584976d80c3a9","scrolled":true},"cell_type":"code","source":"train.rename({\"acoustic_data\": \"signal\", \"time_to_failure\": \"time\"}, axis=\"columns\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aaeef02f49666a74eb9c8e8335c7a6c8d9565f48"},"cell_type":"markdown","source":"Since the data is huge, we sample %1 of it and plot. We can see that signal peaks usually happen right before earthquakes but there are no signal peaks caused by earthquake, that data is not included."},{"metadata":{"trusted":true,"_uuid":"268c50edb05c8ebb4d6f2f816e218d56913c3240"},"cell_type":"code","source":"train_sample_signal = train['signal'].values[::100]\ntrain_sample_time = train['time'].values[::100]\n\nfig, ax1 = plt.subplots(figsize=(20,8))\n\nplt.title(\"Signal and time to failure with %1 of data\")\nplt.plot(train_sample_signal, color = 'burlywood')\nax2 = ax1.twinx()\nplt.plot(train_sample_time, color = 'g')\n\ndel train_sample_signal\ndel train_sample_time\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"acbc6d5fc639a964a21c77718b45392427519fd5"},"cell_type":"markdown","source":"We can see signal data distribution below. The outliers are caused by earthquakes."},{"metadata":{"trusted":true,"_uuid":"5c928648cdf3dc91b97964c3d442c13dbfa09cb4"},"cell_type":"code","source":"train_sample = train.sample(frac=0.01)\n\nplt.figure(figsize=(12,6))\nplt.title(\"Signal data histogram\")\nax = sns.distplot(train_sample['signal'], label='Signal')\n\ndel train_sample\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ab210149febb58a1dda5660ab4f16b8cc25fba1"},"cell_type":"markdown","source":"Let's see signal distribution without outliers."},{"metadata":{"trusted":true,"_uuid":"2bae2b75ba407588d16e70cef0185326654b85f6"},"cell_type":"code","source":"train_sample = train.sample(frac=0.01)\nplt.figure(figsize=(10,5))\nplt.title(\"Signal distribution without outliers\")\ntmp = train_sample.signal[train_sample.signal.between(-25, 25)]\nax = sns.distplot(tmp, label='Signal', kde=False)\n\ndel train_sample\ndel tmp\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1b0e87e82bc35cc3c078211250a1590077f8235"},"cell_type":"markdown","source":"Let's examine signal and time to failure data more closely."},{"metadata":{"trusted":true,"_uuid":"b46e1a02eb70f0b0e51634f2e8bfa397837cdb3a"},"cell_type":"code","source":"train_signal_million = train['signal'].values[:1000000]\ntrain_time_million = train['time'].values[:1000000]\n\nfig, ax1 = plt.subplots(figsize=(20, 8))\nplt.title(\"first 1 million rows\")\nplt.plot(train_signal_million, color = 'burlywood')\nax2 = plt.twinx()\nplt.plot(train_time_million, color = 'g')\n\ndel train_signal_million\ndel train_time_million\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f1716173ac039132739ffbd9df8420685620d0c"},"cell_type":"code","source":"train_signal_thousand = train['signal'].values[:100000]\ntrain_time_thousand = train['time'].values[:100000]\n\nfig, ax1 = plt.subplots(figsize=(20, 8))\nplt.title(\"first 100 thousand rows\")\nplt.plot(train_signal_thousand, color = 'burlywood')\nax2 = plt.twinx()\nplt.plot(train_time_thousand, color = 'g')\n\ndel train_signal_thousand\ndel train_time_thousand\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"692eae00c2b1b56799af539f5bf3d26a64aab4ad"},"cell_type":"code","source":"train_signal_one = train['signal'].values[:10000]\ntrain_time_one = train['time'].values[:10000]\n\nfig, ax1 = plt.subplots(figsize=(20, 8))\nplt.title(\"first ten thousand rows\")\nplt.plot(train_signal_one, color = 'burlywood')\nax2 = plt.twinx()\nplt.plot(train_time_one, color = 'g')\n\ndel train_signal_one\ndel train_time_one\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b39cbf38da3c38ccd74b9e91b93c883380a9ec71"},"cell_type":"markdown","source":"Let's check test files as well."},{"metadata":{"trusted":true,"_uuid":"c5ae0b69efc3dc2d2eb419c194a9f8bd46c4995b"},"cell_type":"code","source":"test_files = os.listdir(\"../input/test\")\nlen(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bbb9916b5982818c09ebec3b11c0bee4dbcc296"},"cell_type":"code","source":"seg = pd.read_csv(\"../input/test/seg_004cd2.csv\")\n\nfig, ax1 = plt.subplots(figsize=(20, 8))\nplt.title(\"example test data\")\nplt.plot(seg, color = 'g')\n\nseg2 = pd.read_csv(\"../input/test/seg_00c35b.csv\")\n\nfig2, ax1 = plt.subplots(figsize=(20, 8))\nplt.title(\"example test data\")\nplt.plot(seg2, color = 'g')\n\nseg3 = pd.read_csv(\"../input/test/seg_00cc91.csv\")\n\nfig3, ax1 = plt.subplots(figsize=(20, 8))\nplt.title(\"example test data\")\nplt.plot(seg3, color = 'g')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7059b46d5f8f757b8d89d43c72d747a4254cfcdf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}