{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Overview\n#### electric transmission lines --> fault --> Partial discharge\n\n#### each signal --> 800,000 measurements of a power line's voltage, taken over 20 milliseconds\n\n#### underlying electric grid operates at 50Hz\n\n#### 3-phase power scheme. / 3-phases are measured simultaneously\n\n\n\n# data description in homepage\n## Metadata_[train/test].csv\n#### id_measurement: ID code for a trio of signals recorded at the same time\n#### signal_id: foreign key for the signal data. (unique across both train and test)\n#### phase: the phase ID code within the signal trio. *** The phases may or may not all be impacted by a fault on the line.\n#### target: 0 (undamaged)/ 1 (fault)\n\n## parquet\n#### each column contains one signal: 800,000 int8 measurments.\n#### (Note that this is different than our usual data orientation of one row per observation)"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm_notebook as tqdm\nfrom numba import jit, int32\nplt.style.use(\"fast\")\n\nimport warnings\nwarnings.filterwarnings(\"ignore\") \n\nplt.style.use('seaborn')\nsns.set(font_scale=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8651efabbea48c200375725477bccf8439f2098"},"cell_type":"markdown","source":"## read parquet file as df_train"},{"metadata":{"trusted":true,"_uuid":"d2f36aa094ac1f983c26cd1b42b89277c3c183cc"},"cell_type":"code","source":"df_train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(3000)]).to_pandas()\n# df_train = pq.read_pandas('../input/train.parquet').to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4de36066bf2482597106b3bf5675a4a45f8702b"},"cell_type":"markdown","source":"## Read test data (test.parquet)\n#### We cannot read test parquet in the same way with train parquet.\n#### The first ID in train is '0' but the first ID in test is '8712'.\n\n#### cannot use below\n###### df_test = pq.read_pandas(os.path.join(cwd, 'test.parquet'), columns=[str(i) for i in range(1000)]).to_pandas()\n###### df_test = pq.read_pandas(os.path.join(cwd, 'test.parquet')).to_pandas()\n\n#### We should read test.parquet like below line\n###### df_test = pq.read_pandas('../input/train.parquet', columns=[str(i+8712) for i in range(1000)]).to_pandas()"},{"metadata":{"_uuid":"a8bfaf5effc41475050afbd560fb96bffcb8d06d"},"cell_type":"markdown","source":"#### train parquet data check"},{"metadata":{"trusted":true,"_uuid":"f6fbacd9b3dd4a19caa9d22568df676293bd04e7"},"cell_type":"code","source":"print(df_train.info())\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c8fcd1b29e21e2d174937f6d90db828e2d9e827"},"cell_type":"markdown","source":"### null check"},{"metadata":{"trusted":true,"_uuid":"e79aebfe07cba0a41ac47fa3246cc91b9f6474d0"},"cell_type":"code","source":"df_train.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65e66bed21c4ff583dacea84b890907d1628afd0"},"cell_type":"markdown","source":"## read Metadata_[train/test].csv as meta_train & meta_test"},{"metadata":{"trusted":true,"_uuid":"756efa652722612b50149987d05076d66ab13f66"},"cell_type":"code","source":"meta_train = pd.read_csv('../input/metadata_train.csv')\nmeta_test = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0068c757f767030f8ad7d5f391e0dacb7cf383dd"},"cell_type":"markdown","source":"#### Metadata_train"},{"metadata":{"trusted":true,"_uuid":"c6e63956fb6223aae0a3158e745ff0d8ec32af63"},"cell_type":"code","source":"meta_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b5c1f6457e9f1e111edc46c12ab855cc5dc8999"},"cell_type":"code","source":"meta_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0581caeec538ee79955000bbef27676f557637ea"},"cell_type":"code","source":"meta_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c8a36942d77a24233c953aca371f341d201f6f3"},"cell_type":"markdown","source":"#### Metadata_test"},{"metadata":{"trusted":true,"_uuid":"2e4a19cd1dcb9c80f52d386f655988318d5dadec"},"cell_type":"code","source":"meta_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19c16360fe184632dbfea14edd209028afcc1a06"},"cell_type":"code","source":"meta_test.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46a5c31ff8aeb8f39ec7045a791732e2093aa4f2"},"cell_type":"code","source":"meta_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5229956f2cd62665aa37e711a2d3f009725c816"},"cell_type":"code","source":"meta_train['target'].unique()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"060bf32cc1286fd1a6212735104707175350aa26"},"cell_type":"markdown","source":"### null check"},{"metadata":{"trusted":true,"_uuid":"33ad20c6f9e9c0ec8f02e78fff9e824c5fb5f1c4"},"cell_type":"code","source":"print(meta_train.isnull().sum())\nprint(meta_test.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d3381e2fbbf398305972d543efd170ecf0e1897"},"cell_type":"markdown","source":"## Number & ratio of normal and fault samples"},{"metadata":{"trusted":true,"_uuid":"85a1133b51b38040d669a343caabf9f972361931"},"cell_type":"code","source":"meta_train['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7153c0e568eef89074c2096480f57892e7df4e30"},"cell_type":"code","source":"print('Normal sample number: {:d}'.format(meta_train['target'].value_counts()[0]))\nprint('Fault sample number: {:d}'.format(meta_train['target'].value_counts()[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e87d5a0573ac4da047b7d50efe02f9dab73fa5be"},"cell_type":"code","source":"print('Normal sample ratio: {:.3f} %'.format(meta_train['target'].value_counts()[0]/len(meta_train)*100))\nprint('Fault sample ratio: {:.3f} %'.format(meta_train['target'].value_counts()[1]/len(meta_train)*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7646c8e8082baaac6edd8eec9e7568d7cc89e8eb"},"cell_type":"code","source":"f, ax = plt.subplots(1, 3, figsize=(24, 8))\n\nax[0].pie(meta_train['target'].value_counts(), explode=[0, 0.1], autopct='%1.3f%%', shadow=True)\nax[0].set_title('Pie plot - Fault')\nax[0].set_ylabel('')\n\nsns.countplot('target', data=meta_train, ax=ax[1])\nax[1].set_title('Count plot - Fault')\n\nsns.countplot('target', data=meta_train, hue ='phase', ax=ax[2])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5612e7a6394b7e9633d2cb9e04e97bc423d9b0fa"},"cell_type":"markdown","source":"## --> very small number of fault data: imbalanced dataset"},{"metadata":{"trusted":true,"_uuid":"c4e90af6db1d5758d2ac1a20521ff1c14c8a0b13"},"cell_type":"code","source":"meta_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8a08b3af59463f1e03b573ed9b7dfe536c8ecf3"},"cell_type":"code","source":"diff = meta_train.groupby(['id_measurement']).sum().query(\"target != 3 & target != 0\")\nprint('not all fault or normal data number: {:d}'.format(diff.shape[0]))\ndiff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5500f80e52ef4b2ff871a07ea40fd4f5394c5f82"},"cell_type":"code","source":"pd.crosstab(meta_train['phase'], meta_train['target'], margins=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed622c486b6acf8c37b1778d83ff3be7e1bb9933"},"cell_type":"code","source":"meta_train[['phase', 'target']].groupby(['phase'], as_index=True).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7dbdf004ef4a1748b74af91f0bf327fd91289f6"},"cell_type":"code","source":"meta_train[['phase', 'target']].groupby(['phase'], as_index=True).mean().sort_values(by='target', ascending=False).plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77fcf1ea66c6752228eb18d10bffd1d6bdeb1e88"},"cell_type":"code","source":"targets = meta_train.groupby('id_measurement', as_index=True)[['target','id_measurement']].mean()\ntargets.iloc[67,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84fa3d36fe388c8ae11017fe1f5d27f0d6e8d28a"},"cell_type":"code","source":"targets.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7970a60298e33f264f274dc738800c943af63100"},"cell_type":"code","source":"sns.countplot(x='target',data=round(targets, 2))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4cbc84910b687b56a34999fa56b97212e42a4c04"},"cell_type":"markdown","source":"### There is little difference in the number of faults (or fault rate) with respect to the phases\n### Data with the same id is not always all fault or normal.\n\n### as data description, \"phase: the phase ID code within the signal trio. The phases may or may not all be impacted by a fault on the line.\""},{"metadata":{"trusted":true,"_uuid":"88d32d6a85414a5946abc17d9a5169c123b2ea24"},"cell_type":"code","source":"meta_train['phase'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bd54f4810f8782af1c0ca8ff7d1f91c6842d5d0"},"cell_type":"code","source":"meta_train['phase'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"487113824afd4c9c541d34b6bd7796949de52401"},"cell_type":"code","source":"meta_train.groupby('phase').count().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cae57a7b01ef4b22774630a19903b8cc269ae80"},"cell_type":"code","source":"meta_train.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"231cb273412609b92de25057ebc54475d88d6ead"},"cell_type":"code","source":"plt.figure(figsize = (10,10))\n\ncolormap = plt.cm.summer_r\nsns.heatmap(meta_train.corr(), linewidths=0.1, vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True, annot_kws={\"size\": 16})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f31851ddd2f2101950eba7a228296206c80956c"},"cell_type":"markdown","source":"# Normal and Fault data visualization"},{"metadata":{"trusted":true,"_uuid":"e7500cc28c0b1ff8028d9bffac3a7e11683a2139"},"cell_type":"code","source":"#np.unique(meta_train.loc[meta_train['target']==1, 'id_measurement'].values)\n# Fault measurement ID\nfault_mid = meta_train.loc[meta_train['target']==1, 'id_measurement'].unique()\n# Normal measurement ID\nnormal_mid = meta_train.loc[meta_train['target']==0, 'id_measurement'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e705688b682f361ba29cfdb82fd46e59d19d9e78"},"cell_type":"code","source":"print(fault_mid.shape)\nprint(normal_mid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"185c22c51cb25975b87590b6466f9960c71caf1d"},"cell_type":"code","source":"print('8712//3={:d}'.format(8712//3))\nprint('2748+194-38={:d} (38 overlapped samples (some phases are normal, the others are fault))'.format(2748+194-38))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0610cc28cb7c3941844a04bbf5591c5ce614660b"},"cell_type":"markdown","source":"## There are 38 overlapping data. That is, for the 38 measure IDs, the normal phase and the fault phase exist simultaneously."},{"metadata":{"trusted":true,"_uuid":"dccffef30188daa644fd304e6dce6ac76b44335f"},"cell_type":"code","source":"print(meta_train.signal_id.unique())\nprint(meta_train.id_measurement.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ad61930c6a4fdb08e3218195e54dc98e43c9c36"},"cell_type":"code","source":"meta_train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f532ccf44a0ce590ddacd2f913a49055aae76f91"},"cell_type":"code","source":"fault_mid[1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a64d093f4b94db5603d0859524ccbea58ca84a3"},"cell_type":"markdown","source":"## N/F signal ID_sid"},{"metadata":{"trusted":true,"_uuid":"0fd7f2c25a30370e0ec31b9c83819957ee00b0d9"},"cell_type":"code","source":"fault_sid = meta_train.loc[meta_train.id_measurement == fault_mid[0], 'signal_id']\nnormal_sid = meta_train.loc[meta_train.id_measurement == normal_mid[0], 'signal_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d73c29fe7bce214eda07f49822e9881326f6d72"},"cell_type":"code","source":"print(fault_sid)\nprint(normal_sid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3d4b8f0a2a3ecb60d83c505c3d01e427885bb10"},"cell_type":"code","source":"fault_sample = df_train.iloc[:, fault_sid]\nnormal_sample = df_train.iloc[:, normal_sid]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67ed4995e6f2c480624f7e24b90fd2a047d969b6"},"cell_type":"markdown","source":"# Plotting the normal and fault signals"},{"metadata":{"_uuid":"2d896b69e70d636fee5d81fc0de78be7f8bce9c6"},"cell_type":"markdown","source":"## Normal sample"},{"metadata":{"trusted":true,"_uuid":"6c07fc6c32b777bab55b2eddfa1e1fc8453580ae"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(normal_sample, alpha=0.7);\nplt.ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e1e93fdd06f3f29e9115851b368167d06f559ca"},"cell_type":"markdown","source":"## Fault sample"},{"metadata":{"trusted":true,"_uuid":"386ccd72c2b909aef23304684214e926acb725ec"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(fault_sample, alpha=0.8);\nplt.ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eda1fc49ddc1641b10a9c30758efa441f72eb9b9"},"cell_type":"markdown","source":"# Faltiron\n#### reference: https://www.kaggle.com/miklgr500/flatiron"},{"metadata":{"trusted":true,"_uuid":"32f0840917c610f9f733c7c6803636477a1fe64c"},"cell_type":"code","source":"@jit('float32(float32[:,:], int32, int32)')\ndef flatiron(x, alpha=100., beta=1):\n    new_x = np.zeros_like(x)\n    zero = x[0]\n    for i in range(1, len(x)):\n        zero = zero*(alpha-beta)/alpha + beta*x[i]/alpha\n        new_x[i] =  x[i] - zero\n    return new_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e6926a9e1ab5ff454bf60216ae30500b07f6d6b"},"cell_type":"code","source":"fault_sample_filt = flatiron(fault_sample.values)\nnormal_sample_filt = flatiron(normal_sample.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fe4eb7fb3ba16ad2b0977f7cac90675a7852b10"},"cell_type":"code","source":"f, ax = plt.subplots(2, 2, figsize=(24, 16))\n\nax[0, 0].plot(fault_sample, alpha=0.8)\nax[0, 0].set_title('fault signal')\nax[0, 0].set_ylim([-100, 100])\n\nax[0, 1].plot(fault_sample_filt, alpha=0.5)\nax[0, 1].set_title('filtered fault signal')\nax[0, 1].set_ylim([-100, 100])\n\nax[1, 0].plot(normal_sample, alpha=0.7)\nax[1, 0].set_title('normal signal')\nax[1, 0].set_ylim([-100, 100])\n\nax[1, 1].plot(normal_sample_filt, alpha=0.5)\nax[1, 1].set_title('filtered normal signal')\nax[1, 1].set_ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df576072bbc301f16d862bfe7fbd985353148153"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}