{"cells":[{"metadata":{"_uuid":"32cae686122612c1984395b99f29975076b0329b"},"cell_type":"markdown","source":"Title:  Power line fault detection EDA and pre processing  \nData: https://www.kaggle.com/c/vsb-power-line-fault-detection  \nAuthor: [Virksaab](https://www.kaggle.com/virksaab)  \nDate:   28 December, 2018"},{"metadata":{"trusted":true,"_uuid":"fbdd535184b4f36a39d58c2266a9707f10b84d4e"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f5ef49d973cb2f6aac81cb6cb233d43893c953d"},"cell_type":"markdown","source":"### Paths to data and metadata"},{"metadata":{"trusted":true,"_uuid":"10cd97477b7310415af6aa0dae20f42048a3aa48"},"cell_type":"code","source":"PARENT_DATA_DIR_PATH = '../input'\nMETADATA_TRAIN_FILE_PATH = os.path.join(PARENT_DATA_DIR_PATH, \"metadata_train.csv\")\nTRAIN_DATA_FILE_PATH = os.path.join(PARENT_DATA_DIR_PATH, \"train.parquet\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8eaf247ffce42f0c0940e019308f5a628c3747e"},"cell_type":"markdown","source":"### Train metadata"},{"metadata":{"_uuid":"016e3624a7bfddf7ef099f55af3af1538cf4abad"},"cell_type":"markdown","source":"    Target:\n        0 : undamaged\n        1 : fault"},{"metadata":{"trusted":true,"_uuid":"563fe3e835c906c0dd9c6ce88ac868a017a0e696"},"cell_type":"code","source":"metadata_train = pd.read_csv(METADATA_TRAIN_FILE_PATH)\nprint(\"#samples:\", len(metadata_train))\nmetadata_train.head(15)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b5431c376c9b0fb1044a56c1e5601064f9ddd24"},"cell_type":"markdown","source":"Above table shows that 3 phases have same target of each signal.\nThat means if one phase is damaged, others follow.\nSo, each class's total signal_ids should be the multiple of 3 as shown below."},{"metadata":{"trusted":true,"_uuid":"3e15e23cdc86595fb216cc1ada9ca6f9f51e2dd7"},"cell_type":"code","source":"metadata_train.target.value_counts()/3","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3663f34c1210d521a9b77351415ed25975aca5bf"},"cell_type":"markdown","source":"### Load train data"},{"metadata":{"_uuid":"18dc6cd0c58777e9f1aea92cf447eebc374e4161"},"cell_type":"markdown","source":"#### BIG DATA FILE!! RAM USAGE: 7.9GB on ubuntu\n(uncomment below cell to load full data)"},{"metadata":{"trusted":true,"_uuid":"cdf28e92eee7ecd553ac466fc6388c7e04e537d9"},"cell_type":"code","source":"# traindataDF = pq.read_pandas(TRAIN_DATA_FILE_PATH).to_pandas()\n# traindataDF.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8aa563d451c35323b9e17f024b973f4db5a0603"},"cell_type":"markdown","source":"#### Below cell loads sample data"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"5b9f4cb0000848ede749b3cdcdb11552c7087f7a"},"cell_type":"code","source":"traindataDFsample = pq.read_pandas(TRAIN_DATA_FILE_PATH, columns=[str(i) for i in range(15)]).to_pandas()          \ntraindataDFsample.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b254223a40c4530ca295994a59793903e701ab44"},"cell_type":"code","source":"traindataDFsample.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60d7715a935c1d8257dc534bab25a0a8cba4c7b8"},"cell_type":"code","source":"traindataDFsample.tail(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65550d5b9b19b11922453b6e1438729063935cfd"},"cell_type":"markdown","source":"### Plot 3 phase signals with target"},{"metadata":{"trusted":true,"_uuid":"d53778be8352f1945c7a3292d542a1f6377950e5"},"cell_type":"code","source":"traindataDFsample.iloc[:,:3].plot(title=\"3 phase, Target 0\", figsize=(15,5));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a53c452c6f2758ea9be8f0c35f42c0cec7058ff8"},"cell_type":"code","source":"traindataDFsample.iloc[:,3:6].plot(title=\"3 phase, Target 1\", figsize=(15,5));","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6d8b9cfd663977946cab6836dedaf7049a2bb2b"},"cell_type":"markdown","source":"### Plotting 0 phase with target 0 and 1"},{"metadata":{"trusted":true,"_uuid":"e590adf6d8e34e4caf90d572a4f885ac23278d09"},"cell_type":"code","source":"traindataDFsample.iloc[:,0].plot(title=\"phase 0, Target 0\", figsize=(15,5));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b3a4b98ced995dd515e2ddedf95064862d45f32"},"cell_type":"code","source":"traindataDFsample.iloc[:,3].plot(title=\"phase 0, Target 1\", figsize=(15,5));","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e800e663fc5728a473b4e7f3e2c7a1ab5f80624d"},"cell_type":"markdown","source":"### Group signals metadata accroding to target"},{"metadata":{"trusted":true,"_uuid":"5206ba1fe19592fed455a6bae7056e14eb53adb5"},"cell_type":"code","source":"target0df = metadata_train[metadata_train['target'] == 0]\ntarget1df = metadata_train[metadata_train['target'] == 1]\nprint(\"target0data shape:\", target0df.shape)\nprint(\"target1data shape:\", target1df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fceee0433e8cacf8188f9b5f3b69d623fd6b8c25"},"cell_type":"markdown","source":"#### Load some target 0 (undamaged) signals and visualize"},{"metadata":{"trusted":true,"_uuid":"78681c21487b0d17cc1044ead6e9cb2efe02e52c"},"cell_type":"code","source":"nSamples = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92cc1516fce9c852bb2d249640644a7f2fbf8429"},"cell_type":"code","source":"target0samplecols = [str(i) for i in list(target0df.iloc[:nSamples].signal_id)]\ntarget0sampledata = pq.read_pandas(TRAIN_DATA_FILE_PATH, columns=target0samplecols).to_pandas()\ntarget0sampledata.plot(title=\"Target 0\", figsize=(15,10))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c7c539237ae6db1093105307e32061b893753f64"},"cell_type":"markdown","source":"#### Load some target 1 (fault) signals and visualize"},{"metadata":{"trusted":true,"_uuid":"02c0578cd6fa980d8897a898224750019fcdf3ef","scrolled":true},"cell_type":"code","source":"target1samplecols = [str(i) for i in list(target1df.iloc[:nSamples].signal_id)]\ntarget1sampledata = pq.read_pandas(TRAIN_DATA_FILE_PATH, columns=target1samplecols).to_pandas()\ntarget1sampledata.plot(title=\"Target 1\", figsize=(15,10))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0de95aa2274b2905a4e35a2914bb43f85466f471"},"cell_type":"markdown","source":"### Reduce values per colm by averaging over 8\nIt'll help reducing the computation without throwing any information"},{"metadata":{"trusted":true,"_uuid":"d6d9279593cd36bd5e293d3f8a922fb6f17be958"},"cell_type":"code","source":"sample = traindataDFsample.iloc[:,3]\nsample.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b853ae0aa8c7aba145fb7fd26e4b3f6e2f51e544"},"cell_type":"code","source":"def reduce_sample(_sample, avgOver=8):\n    preVal = 0\n    processed_sample_list = []\n    for index in range(avgOver, _sample.shape[0]+avgOver, avgOver):\n        tmpdf = _sample.iloc[preVal:index]\n        avgVal = tmpdf.sum()/avgOver\n        processed_sample_list.append(avgVal)\n        preVal = index\n    return pd.Series(processed_sample_list)\nprocessed_sample = reduce_sample(sample, 8)\nprocessed_sample.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45cc3b6b36c39cce9184888a43f0e9c9b2f44665"},"cell_type":"code","source":"processed_sample.plot(title=\"reduced sample\", figsize=(15,5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a03f848ea098aa94860bed3bacc22d1969784227"},"cell_type":"markdown","source":"### Apply reduction on some samples and visualize the results"},{"metadata":{"trusted":true,"_uuid":"14b2a529f71466f17b2f9f98181ea4c3cfd05690"},"cell_type":"code","source":"# TARGET 0 (UNDAMAGED) SAMPLES\nreducedtarget0sampleDF = pd.DataFrame()\nfor col in range(target0sampledata.shape[1]):\n    tmp_pdSeries = reduce_sample(target0sampledata.iloc[:,col])\n    reducedtarget0sampleDF[str(col)] = tmp_pdSeries\nreducedtarget0sampleDF.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a27ce2147d125cc8cfaa3c23027a2d322c950840"},"cell_type":"code","source":"# TARGET 1 (FAULT) SAMPLES\nreducedtarget1sampleDF = pd.DataFrame()\nfor col in range(target1sampledata.shape[1]):\n    tmp_pdSeries = reduce_sample(target1sampledata.iloc[:,col])\n    reducedtarget1sampleDF[str(col)] = tmp_pdSeries\nreducedtarget1sampleDF.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d09ee824850e44dd22a77350153af412532839c8"},"cell_type":"code","source":"reducedtarget0sampleDF.plot(title=\"Reduced target 0\", figsize=(15,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b8bec6772643b74ae5c10c138128bbd4279717e"},"cell_type":"code","source":"reducedtarget1sampleDF.plot(title=\"Reduced target 1\", figsize=(15,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e37559c35b1617ca1211d4bb49b56cc813e1e6f"},"cell_type":"code","source":"reducedtarget1sampleDF.iloc[:,0].plot(title=\"Reduced target 1 single signal\", figsize=(15,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59c27d9e71b2c9c134ce362c10c368553e66f7ef"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0d2257c043b1249b750fed2ef57dd3963ebb90e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad477b9d4deb526e081a7598e4f3f41d2101ca94"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eee43d6a54d5556abba5d02e94589a3948fb6eba"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}