{"cells":[{"metadata":{"scrolled":true,"trusted":true,"_uuid":"e8ce37d6ac9553f3160f5804f167a683b3bba5a1"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom plotly.offline import init_notebook_mode\ninit_notebook_mode()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4663afc4705258127e361e2cce07290371ded132"},"cell_type":"markdown","source":"## Train metadata"},{"metadata":{"trusted":true,"_uuid":"23ec1df5c51e35682f126122a485cb48b8a83ca5"},"cell_type":"code","source":"train = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb98b5012d78d3d8801545d0e9b4cfb2134d9084"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99f3b6452673555db8f9579e8f34e3956eba2732"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab03a8584d43f2b53b953ffd4671b613fce8f853"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"894c30de03ee9f12c5916e0705afbf0988ed1a22"},"cell_type":"code","source":"print(train.signal_id.dtype)\nprint(train.id_measurement.dtype)\nprint(train.phase.dtype)\nprint(train.target.dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd9892e8caaeadb89c7a6811db19605dc1ddc949"},"cell_type":"code","source":"train.phase.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"450919665285cd8ea24117b8f96bf0cfaf5cc9ce"},"cell_type":"markdown","source":"Check for missing values"},{"metadata":{"trusted":true,"_uuid":"fa08aba750151b0568c73abdef48886357662582"},"cell_type":"code","source":"train.isna().any()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a869eb2ffc428338e1f77b00463e3d6429ae1447"},"cell_type":"markdown","source":"Check distribution of target"},{"metadata":{"trusted":true,"_uuid":"c7f4c17ef3139467075f3128cf3f31e207c836ea"},"cell_type":"code","source":"train.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95dcf3d7139040a6a00ee4df22ebf308e40aa79d"},"cell_type":"code","source":"train['target'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72ab57ca60117a400c4d71661f20be0047577074"},"cell_type":"markdown","source":"Check how many ids have at least 1 fault"},{"metadata":{"trusted":true,"_uuid":"91d30fbfd009649bcb2e1c67e52569762d7a0e31"},"cell_type":"code","source":"train.groupby([\"id_measurement\"]).sum().query(\"target > 0\").shape[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f59a6264e645cbb539114d5e98e567178aa9895"},"cell_type":"markdown","source":"How many ids in total?"},{"metadata":{"trusted":true,"_uuid":"916621f16944f0aece1aa7958f591b2d2cdf38ba"},"cell_type":"code","source":"train['id_measurement'].unique().shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1003f57490155af921c59ea338d2d39e48d49e7"},"cell_type":"code","source":"print('{} out of {} ids contain a fault in at least one of three phases. This is {:.0f}%'.format(\n      train.groupby([\"id_measurement\"]).sum().query(\"target > 0\").shape[0],\n      train['id_measurement'].unique().shape[0],\n      (train.groupby([\"id_measurement\"]).sum().query(\"target > 0\").shape[0]*100)/train['id_measurement'].unique().shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fa4c00f6758cf353b0884bac3691e1fd311685d"},"cell_type":"code","source":"train.groupby([\"id_measurement\"]).sum()['target'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"99f5cd57e66018aa553496edcb29b2c5ffb57bec"},"cell_type":"markdown","source":"Looks like if a fault exists, all three lines usually have a fault simultaneously"},{"metadata":{"trusted":true,"_uuid":"72558354c38654cfbe821782d731bbaeb0ab2848"},"cell_type":"code","source":"train.groupby([\"id_measurement\"]).sum()['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34ade223bb7828757c13cfdfc02b33e3d9d65f02"},"cell_type":"markdown","source":"## Test metadata"},{"metadata":{"trusted":true,"_uuid":"d64ac5d421d15978dd60190ff37749ffc84e5d61"},"cell_type":"code","source":"test = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"447848a4bba70461953d5621190b58f00a4cf777"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4e7b64fa0a42e1e556c01c681fc1081eca802b3"},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9584fff9bbc09d0da9dbf9a65f1b165c7b0da6c4"},"cell_type":"code","source":"test.phase.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c68ff067d2ee9b7b07b7a591ac727b9784c306e2"},"cell_type":"markdown","source":"# Train signal data"},{"metadata":{"trusted":true,"_uuid":"20eb818e69b6932a05b1a72bf2fd74bd35149092"},"cell_type":"code","source":"train_sig = pd.read_parquet('../input/train.parquet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55eb61b4e24ad28fcb49f688cf0893dc74fd25e6"},"cell_type":"code","source":"train_sig.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea5be3263d79aefab331e42540ba5905b3d6e06b"},"cell_type":"code","source":"train_sig.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1b5fd9d22a054fac231848b0575531b03a38f55"},"cell_type":"markdown","source":"Signals with no fault"},{"metadata":{"trusted":true,"_uuid":"44237d515dd1d84766619a349fd66299b4bd4700"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,10))\nfor i in range(3):\n    sns.lineplot(train_sig.index, train_sig[str(i)])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af8e87758f2cd58bc1dcac1fccab93005e496e57"},"cell_type":"markdown","source":"Signals with faults"},{"metadata":{"trusted":true,"_uuid":"2b7910e39dced3df50fb26962f73581b126aa1ff"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,10))\nfor i in range(3,6):\n    sns.lineplot(train_sig.index, train_sig[str(i)])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15f24a677de4a75160073a2d96dcb2c6de4fe976"},"cell_type":"markdown","source":"Presumably the fault lies at the beginning of the signals in the second plot but there is not a huge amount of difference between the two graphs. Looks like a significant amount of interference in the power lines in the first plot have led to noisy signals, much like:\n\n![samples](https://storage.googleapis.com/kaggle-forum-message-attachments/445388/10942/samples.png)"},{"metadata":{"_uuid":"b4691755c349a425d4800bbfedfe44b23120b4ba"},"cell_type":"markdown","source":"From [this](https://www.kaggle.com/c/vsb-power-line-fault-detection/discussion/75771) discussion and other [EDA kernels posted](https://www.kaggle.com/go1dfish/basic-eda), it looks like this isn't an isolated case. It would be worth looking at these signals again after wavelet transformations and denoising as per Tomas' problem description thread."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}