{"cells":[{"metadata":{"_uuid":"951d773a559b21cc79f1477d5fee8bd5633d0fd0"},"cell_type":"markdown","source":"Title:  Power line fault detection pre processing  \nData source: https://www.kaggle.com/c/vsb-power-line-fault-detection  \nAuthor: [Virksaab](https://www.kaggle.com/virksaab)   \nDate:   30 December, 2018"},{"metadata":{"trusted":true,"_uuid":"16888ef27f06b94cd60c0cc515ee87dc0f0bcb77"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nfrom scipy import signal as sps\nfrom sklearn.model_selection import train_test_split\nfrom statistics import mode\n\nplt.style.use('ggplot')\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ccf54d641a5d0dc270ac7bde8ecaaaec908addb"},"cell_type":"code","source":"# PARAMETERS\nBATCH_SIZE = 8\nEPOCHS = 10","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1747f1182167c3303eb4d5245f3012cb47176e9b"},"cell_type":"markdown","source":"### Paths to data and metadata"},{"metadata":{"trusted":true,"_uuid":"ad2ed885ad607b0d4805ea704e654f1225f39d01"},"cell_type":"code","source":"PARENT_DATA_DIR_PATH = '../input'\nMETADATA_TRAIN_FILE_PATH = os.path.join(PARENT_DATA_DIR_PATH, \"metadata_train.csv\")\nTRAIN_DATA_FILE_PATH = os.path.join(PARENT_DATA_DIR_PATH, \"train.parquet\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"257946172c1a899e914838e1cfc93b5cbf907dc1"},"cell_type":"markdown","source":"### Train metadata"},{"metadata":{"_uuid":"aa3727ae85aa5f80985865748f06acf9c4d9f580"},"cell_type":"markdown","source":"    Target:\n        0 : undamaged\n        1 : fault"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"021b85287e9b5bf37754769493f14cbd3370f982"},"cell_type":"code","source":"metadata_train = pd.read_csv(METADATA_TRAIN_FILE_PATH)\nprint(\"#samples:\", len(metadata_train))\nmetadata_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da573190e3706c0aa626d89b7d09732dbefa082c"},"cell_type":"code","source":"# For equal number of samples for each class\ntarget0DF = metadata_train[metadata_train.target == 0]\ntarget1DF = metadata_train[metadata_train.target == 1]\nmetadata_train = pd.concat([target0DF.iloc[:len(target1DF), :], target1DF])\nmetadata_train.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8bc27782adb4180c967a7eee1ea382859e68a1d3"},"cell_type":"markdown","source":"### Combine 3 phases to one signal"},{"metadata":{"trusted":true,"_uuid":"ab216d71da7792ce3bbcda73c3476c6bbbb2add1"},"cell_type":"code","source":"gridDF = metadata_train.groupby('id_measurement')\nmetadataList = []\nfor name, group in gridDF:\n    if len(group.signal_id) == 3:\n    #     print(name, group)\n        ids = list(map(lambda x: str(x), group.signal_id))\n        target = mode(group.target)\n    #     print(\"IDs:\", ids)\n    #     print(\"Target:\", target)\n        metadataList.append({'IDs':ids, 'target': target})\nmetadataList[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80201647e7821b8034527063fd064e67e8f12f8c"},"cell_type":"markdown","source":"### Train-val split"},{"metadata":{"trusted":true,"_uuid":"0b848f8a6cddf5d259b276c604e447fd16a4062e"},"cell_type":"code","source":"trainMeta, valMeta = train_test_split(metadataList, test_size=.10)\nprint(\"trainMDDF shape:\", len(trainMeta))\nprint(\"valMDDF shapeL\", len(valMeta))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d822c94119285658ce645ae6afd541413b3b8541"},"cell_type":"code","source":"# VERIFY THAT TRAIN AND VALIDATION SET HAVE BOTH CLASSES DATA.\nt0 = 0\nt1 = 0\nfor pair in trainMeta:\n    if pair['target'] == 0:\n        t0 += 1\n    else:\n        t1 += 1\nprint(\"train set: #t0={}, #t1={}\".format(t0, t1))\nt0 = 0\nt1 = 0\nfor pair in valMeta:\n    if pair['target'] == 0:\n        t0 += 1\n    else:\n        t1 += 1\nprint(\"val set: #t0={}, #t1={}\".format(t0, t1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf91326b3081239fcda4fdfda60611a6c3bc198b"},"cell_type":"markdown","source":"### Load data"},{"metadata":{"trusted":true,"_uuid":"b9185c4a56cf29914d737e1715d5bc5d644010fe"},"cell_type":"code","source":"data = pq.read_pandas(TRAIN_DATA_FILE_PATH).to_pandas()\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28dd4a10f7ddf947ad7869401f590f155bbd05ad"},"cell_type":"markdown","source":"### Get data in batches"},{"metadata":{"trusted":true,"_uuid":"e8c1e5c0c258458c2c7e87422dc51c9e89e72170"},"cell_type":"code","source":"def get_data_batch(dataDF, metaList, batchsize=2, loop=False, denoise=True, combine_phases=True):\n    \"\"\"\n    Args:\n    dataDF (pandas.DataFrame): The train.parquet file dataframe\n    metaList(pandas.DataFrame): The training metadata file\n    batchsize(int): Number of samples in a batch\n    loop(boolean): True, For training with keras fit_generator; False, for validation.\n    denoise(boolean): True, reduce signal noise\n    combine_phase(boolean): True, return shape:(batchsize, 800000, 1); False, return shape:(batchsize, 800000, 3)\n    \"\"\"\n    # set filter\n    b, a = sps.butter(3, 0.5, btype='highpass', analog=False)\n    while True:\n        counter = 0\n        signals_list = []\n        target_list = []\n        try:\n            for index in range(len(metaList)):\n                counter += 1\n                sample_ids = metaList[index]['IDs']\n                sample_target = metaList[index]['target']\n                # OneHot encoding\n                sample_targetOH = np.zeros(2)\n                sample_targetOH[sample_target] = 1\n                sample_signal = dataDF[sample_ids]\n                if denoise:\n                    # DeNoise\n                    for colname in sample_signal:\n                        noisy_signal = sample_signal[colname]\n                        sample_signal[colname] = sps.filtfilt(b, a, noisy_signal)\n                if combine_phases:\n                    # combine phases to one signal\n                    sample_signal = sample_signal.mean(1)\n                    signals_list.append(np.expand_dims(sample_signal.values.reshape(-1, 1), 0))\n                    target_list.append(np.expand_dims(sample_targetOH, 0))\n                    if counter == batchsize:\n                        yield np.concatenate(signals_list), np.concatenate(target_list)\n                        counter = 0\n                        signals_list.clear()\n                        target_list.clear()\n                else:\n                    signals_list.append(np.expand_dims(sample_signal.values, 0))\n                    target_list.append(np.expand_dims(sample_targetOH, 0))\n                    if counter == batchsize:\n                        yield np.concatenate(signals_list), np.concatenate(target_list)\n                        counter = 0\n                        signals_list.clear()\n                        target_list.clear()\n        except:\n            pass\n        if not loop:\n            break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dca7c45bd7f79cae053fed0b2be88e69738873c1"},"cell_type":"markdown","source":"### Visualize"},{"metadata":{"trusted":true,"_uuid":"82d6549bf8f38be1f78d644ce8877fe87e92c61d","scrolled":false},"cell_type":"code","source":"sb, tb = next(get_data_batch(data, trainMeta, batchsize=BATCH_SIZE, loop=False))\nfor sample_signal, sample_target in zip(sb, tb):\n    plt.figure(figsize=(15,5))\n    plt.plot(sample_signal);\n    plt.title(\"Label: \"+str(np.argmax(sample_target)))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db48efe84021955dab9b0f933cb067a1eae11a9e"},"cell_type":"markdown","source":"### Sample data fetching from batch"},{"metadata":{"trusted":true,"_uuid":"c6c32c0e3cf541a16a9cc607ffdbd198d60a4512"},"cell_type":"code","source":"sb, tb = next(get_data_batch(data, trainMeta, batchsize=BATCH_SIZE, loop=False))\nprint(\"Sample signal batch:\", sb.shape)\nprint(\"Sample target batch:\", tb.shape) # Target are OneHot encodeed","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddd2167c89cf7ddf46f473c3318131a6fefa5ebb"},"cell_type":"markdown","source":"### Your algorithm...."},{"metadata":{"_uuid":"096b47b78fb4c85741887ba149a2daabcf6adb3c"},"cell_type":"markdown","source":"## Thanks for reading."},{"metadata":{"trusted":true,"_uuid":"aee06dd687686fbe748d6bdfd8e289188a374278"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}