{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport os\nfrom matplotlib import pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0693bc343168489f5b9729048722a31eb4c1d4bf"},"cell_type":"markdown","source":"Load the training parquet file, this file contains signal measurements, each column contains aa single 800,000 measurement signal"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pq.read_pandas('../input/train.parquet').to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"169313d1067a9fa4af4199078b2f3813e2563973"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e9c0a92b62ccee5a6f84ad87d7c599ac5e45fb2"},"cell_type":"markdown","source":"Plotting first few signals from the training set"},{"metadata":{"trusted":true,"_uuid":"58e99c168075cfd5ce6492579a943bd401185b81"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(train.iloc[:, :5]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e959b46363fa929aa89bdea4942da115bd42814a"},"cell_type":"markdown","source":"Loading the meta files"},{"metadata":{"trusted":true,"_uuid":"f83c643f674ac2820538fcfc43fb68572726b9b7"},"cell_type":"code","source":"meta = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e304f1d6ef78efca3bdf96fe55a6358192df8ec"},"cell_type":"code","source":"meta.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2498086a66c3bdb1072b49164ab45ad81119d145"},"cell_type":"code","source":"meta.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd8facf45fed7088a89c2da6981c1aca971977d3"},"cell_type":"code","source":"meta.corr()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be9d3b61dccef1dd251373c09488a5f7fcc1ea5c"},"cell_type":"markdown","source":"It can be seen that `phase` and `target` are independent of `signal_id` and `id_measurement` and are independent of each other"},{"metadata":{"trusted":true,"_uuid":"16e6f02af4ebb6184666f53bf7bddb5726dd2bd8"},"cell_type":"code","source":"meta.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24f5439e312851014ddc1c2be614599f03bf108a"},"cell_type":"markdown","source":"So, the fault meter detects faults on each phase (three: 0, 1, 2) and each detection has a unique id called `id_measurement`.\n"},{"metadata":{"_uuid":"cb201ad26bb61c1ff247fa9a06bf2c553b7890e2"},"cell_type":"markdown","source":"Let's plot some positive and negative samples"},{"metadata":{"trusted":true,"_uuid":"78f22ca6c1672fdaf2a3108ff24ba79501b9e3a4"},"cell_type":"code","source":"# get positive and negative `id_measurement`s\npositive_mid = np.unique(meta.loc[meta.target == 1, 'id_measurement'].values)\nnegative_mid = np.unique(meta.loc[meta.target == 0, 'id_measurement'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80e25fff7cde26b9029c68a40273377dc36eaad8"},"cell_type":"code","source":"# get one positive and one negative signal_id\npid = meta.loc[meta.id_measurement == positive_mid[0], 'signal_id']\nnid = meta.loc[meta.id_measurement == negative_mid[0], 'signal_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b268c6e7a66fe128ad9f9abb3f957166e12326a3"},"cell_type":"code","source":"positive_sample = train.iloc[:, pid]\nnegative_sample = train.iloc[:, nid]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7ee18c2afb35aa75a4cd9df5813862ac9d8c5e3"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(positive_sample);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4b6a1ccb49d4e9a99313706d52e5ac7c163738e"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(negative_sample);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d67aea6621a5e8cd5583402bf104e966c8f17bf6"},"cell_type":"markdown","source":"Analysing class imbalance"},{"metadata":{"trusted":true,"_uuid":"abcf954311f9ad998510e3eabab79b135772964d"},"cell_type":"code","source":"meta['target'].value_counts().plot(kind='bar');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1b1c11b0e9a215d9de8c20c02799085225d6120"},"cell_type":"code","source":"meta['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"325b5d94dfb842ff6066fd029bcacfa656234421"},"cell_type":"markdown","source":"We can see, there is a huge class imbalance in the data"},{"metadata":{"trusted":true,"_uuid":"ce46969dee29250b251f6f5cf6a01e016908309c"},"cell_type":"code","source":"meta['phase'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f846ef06d12335e66728d2c8c8aca2481e76d8c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aadda45d901c1e366ccf6cdada2c4e90c4bd7105"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}