{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel uses 1D convolutions on signals from power lines to identify partial faults","metadata":{"_uuid":"c97895bf-6e47-450a-9188-78df043954da","_cell_guid":"73e59b0b-adb1-48d9-9fcd-0801484288e8","trusted":true}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\n# import os\n\n# os.listdir('../input/vsb-power-line-fault-detection')","metadata":{"_uuid":"73c72c75-90ef-491d-92f0-b470ccf7f561","_cell_guid":"dc838021-97e8-448a-bc6a-abc8d898314d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:52:55.911899Z","iopub.execute_input":"2023-06-25T19:52:55.912200Z","iopub.status.idle":"2023-06-25T19:52:56.227252Z","shell.execute_reply.started":"2023-06-25T19:52:55.912134Z","shell.execute_reply":"2023-06-25T19:52:56.226435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read the parquet file. The full length of each signal is 800000. We will halve it to 400000 readings to create the pipeline.","metadata":{"_uuid":"17fb53b4-1e6a-4541-a2a5-86695382551c","_cell_guid":"2a55c497-31a4-44e6-8a1b-b56bbac9ac42","trusted":true}},{"cell_type":"code","source":"subset_train = pq.read_pandas('../input/vsb-power-line-fault-detection/train.parquet',columns=[str(i) for i in range(5000)]).to_pandas() #, columns=[str(i) for i in range(10)]).to_pandas()","metadata":{"_uuid":"0b2332d4-15eb-45cf-a1e4-ebc95847d18d","_cell_guid":"4de6b42f-7104-4174-9892-e6f2a5d71045","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:52:56.228661Z","iopub.execute_input":"2023-06-25T19:52:56.228953Z","iopub.status.idle":"2023-06-25T19:53:44.853074Z","shell.execute_reply.started":"2023-06-25T19:52:56.228902Z","shell.execute_reply":"2023-06-25T19:53:44.852179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_train = subset_train.iloc[200000:600000,:]\nsubset_train.info()","metadata":{"_uuid":"45063a7e-649a-4659-8171-aefbd5e028f5","_cell_guid":"6fc031ec-6577-4ca8-9fb4-72e1ab005b7e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:44.854343Z","iopub.execute_input":"2023-06-25T19:53:44.854662Z","iopub.status.idle":"2023-06-25T19:53:45.074256Z","shell.execute_reply.started":"2023-06-25T19:53:44.854609Z","shell.execute_reply":"2023-06-25T19:53:45.073222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now read the metadata file.","metadata":{"_uuid":"42d3475d-2a0b-44d1-9b90-31646c19c3d2","_cell_guid":"0599d4e2-1a18-4939-8c5b-b11e128f27a9","trusted":true}},{"cell_type":"code","source":"metadata_train = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')\nmetadata_train.info()","metadata":{"_uuid":"92e384da-9d2b-4512-b5a0-ad27c7af85e1","_cell_guid":"d30902ea-eb72-4756-bd1f-b336ffea86e8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:45.075872Z","iopub.execute_input":"2023-06-25T19:53:45.076443Z","iopub.status.idle":"2023-06-25T19:53:45.109880Z","shell.execute_reply.started":"2023-06-25T19:53:45.076117Z","shell.execute_reply":"2023-06-25T19:53:45.109123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import plotting libraries and create some basic plots.","metadata":{"_uuid":"3d8c6adc-f39c-4a84-8b15-16e8438c8546","_cell_guid":"9d7c5738-4b7e-44ce-97b9-8d60aa0f3246","trusted":true}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"_uuid":"9c0c0daa-7ee9-4b77-ba80-a0476f963737","_cell_guid":"4c18e10b-675b-4d27-8673-2d2a8028dfb0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:45.111288Z","iopub.execute_input":"2023-06-25T19:53:45.111751Z","iopub.status.idle":"2023-06-25T19:53:45.904309Z","shell.execute_reply.started":"2023-06-25T19:53:45.111555Z","shell.execute_reply":"2023-06-25T19:53:45.903268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a plot of the target values. As expected, a faulty power line is a kind of rare event. Let's visualize some negative and positive (faulty) signals.","metadata":{"_uuid":"c29b2b2c-f9b9-499a-b4f5-dd40eb7953e0","_cell_guid":"a6311caa-462b-4d73-a0f5-4e77c86611aa","trusted":true}},{"cell_type":"markdown","source":"At least from these couple of plots, we notice that the faulty signals (last 2) have relatively more outliers than the non-faulty ones. We will analyze this further with more data.\n\nLet's separate the positive and negative signals for further analysis. I'm going to reduce the sample sizes to make sure we don't run out of memory limits.","metadata":{"_uuid":"317061a7-2602-43c2-9632-bf8e5a04a176","_cell_guid":"1de385d3-aac2-4239-99b6-f9e7554c66f7","trusted":true}},{"cell_type":"code","source":"import numpy as np\n\n### Temporarily reduce data size to build the pipeline\nS_decimation = subset_train.iloc[0:25000:8,:]\nsmall_subset_train = S_decimation\nsmall_subset_train = small_subset_train.transpose()\nsmall_subset_train.index = small_subset_train.index.astype(np.int32)\ntrain_dataset = metadata_train.join(small_subset_train, how='right')\n\n### Uncomment the following to train on the full dataset\n# subset_train = subset_train.transpose()\n# subset_train.index = subset_train.index.astype(np.int32)\n# train_dataset = metadata_train.join(subset_train, how='right')","metadata":{"_uuid":"a8524e68-8eb1-415f-b454-bf9875277a27","_cell_guid":"a97a6770-88dc-46b2-a8be-32fcb63a944a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:45.906116Z","iopub.execute_input":"2023-06-25T19:53:45.906701Z","iopub.status.idle":"2023-06-25T19:53:46.190431Z","shell.execute_reply.started":"2023-06-25T19:53:45.906648Z","shell.execute_reply":"2023-06-25T19:53:46.189568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples = train_dataset[train_dataset['target']==1]\npositive_samples = positive_samples.iloc[:,3:]\npositive_samples.info()","metadata":{"_uuid":"567aaf5c-dd37-4c66-8823-8a6bc5fbc87f","_cell_guid":"6c1c79b8-420c-4706-953f-f1887373c78c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:46.191890Z","iopub.execute_input":"2023-06-25T19:53:46.192432Z","iopub.status.idle":"2023-06-25T19:53:46.405131Z","shell.execute_reply.started":"2023-06-25T19:53:46.192362Z","shell.execute_reply":"2023-06-25T19:53:46.404243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.shape","metadata":{"_uuid":"120b3cab-3b82-445b-b093-e10f49a4dc99","_cell_guid":"d45a329f-eaa7-4735-b9b0-d7f314309cb6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:46.406618Z","iopub.execute_input":"2023-06-25T19:53:46.407140Z","iopub.status.idle":"2023-06-25T19:53:46.416590Z","shell.execute_reply.started":"2023-06-25T19:53:46.407087Z","shell.execute_reply":"2023-06-25T19:53:46.415565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's visualize the positive (faulty) signals using a boxplot for several of them.","metadata":{"_uuid":"3ac34e0b-9a82-40a2-bff7-5048e7dc970e","_cell_guid":"c6a16bea-90ea-4f96-8397-7c496bc6f520","trusted":true}},{"cell_type":"markdown","source":"We see that the data values differ a lot. Let's normalize the data first, this will also be needed for training some type of models later.","metadata":{"_uuid":"e52e1e27-d0a7-4b3b-98a7-d9725e4b1388","_cell_guid":"295c2a2d-f9cd-4c96-b6eb-eccfb5f6bcdf","trusted":true}},{"cell_type":"code","source":"# Normalize the data set\nfrom sklearn.preprocessing import StandardScaler\ny_train_pos = positive_samples.iloc[:, 0]\nX_train_pos = positive_samples.iloc[:, 1:]\nscaler = StandardScaler()         \nscaler.fit(X_train_pos.T)           \nX_train_pos = scaler.transform(X_train_pos.T).T","metadata":{"_uuid":"4557243f-3f00-4f80-9e5f-a35d87b057a8","_cell_guid":"7c6de163-a828-4c0a-9330-833b579427f2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:46.418191Z","iopub.execute_input":"2023-06-25T19:53:46.418740Z","iopub.status.idle":"2023-06-25T19:53:46.808832Z","shell.execute_reply.started":"2023-06-25T19:53:46.418678Z","shell.execute_reply":"2023-06-25T19:53:46.807801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize the boxplots again using this normalized data.","metadata":{"_uuid":"61dd95f3-6ed2-4f67-8004-8070c9218f15","_cell_guid":"c52f6cbf-cb90-4593-8f12-c1f16539d90e","trusted":true}},{"cell_type":"markdown","source":"Again we notice that there are a lot of outliers in the positive (faulty) signals.\n\nNow let's extract the negative (non-faulty) samples and visualize the same boxplots, and see if we can notice any apparent difference.","metadata":{"_uuid":"4a063d40-ddf7-43f5-b19f-6a20568a500e","_cell_guid":"2349af2a-2948-42cd-817d-6b7eb2da32bf","trusted":true}},{"cell_type":"code","source":"negative_samples = train_dataset[train_dataset['target']==0]\nnegative_samples = negative_samples.iloc[:,3:]\n\ny_train_neg = negative_samples.iloc[:, 0]\nX_train_neg = negative_samples.iloc[:, 1:]\nscaler.fit(X_train_neg.T)\nX_train_neg = scaler.transform(X_train_neg.T).T","metadata":{"_uuid":"90f3aaf0-32d9-439f-9024-58dfc573d0e7","_cell_guid":"d069b77d-66f2-4850-aad8-2d2ddc8f97e2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:46.810270Z","iopub.execute_input":"2023-06-25T19:53:46.810909Z","iopub.status.idle":"2023-06-25T19:53:47.627946Z","shell.execute_reply.started":"2023-06-25T19:53:46.810542Z","shell.execute_reply":"2023-06-25T19:53:47.625920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_samples.shape","metadata":{"_uuid":"9b82830f-95e7-4f92-a5d1-b836127dc3da","_cell_guid":"429c2a00-8992-4996-88a3-e0ec6b2bc454","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:47.629373Z","iopub.execute_input":"2023-06-25T19:53:47.629667Z","iopub.status.idle":"2023-06-25T19:53:47.635679Z","shell.execute_reply.started":"2023-06-25T19:53:47.629617Z","shell.execute_reply":"2023-06-25T19:53:47.634691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The negative (non-faulty) signals have much fewer outliers, and their magnitudes also seem to be very low. Seems like the number of outliers could be a promising feature.\n\nNow let's create the test/train split for training a Conv1D model.","metadata":{"_uuid":"dc78b727-b5da-4c39-b82b-91852ef9ff1f","_cell_guid":"2ac46f05-e9e6-43b1-a7c7-bc15919dfd96","trusted":true}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train_pos, X_valid_pos, y_train_pos, y_valid_pos = train_test_split(X_train_pos, y_train_pos, \n                                                                    test_size=0.2,\n                                                                    random_state = 0,\n                                                                    shuffle=True)\n\nX_train_neg, X_valid_neg, y_train_neg, y_valid_neg = train_test_split(X_train_neg, y_train_neg, \n                                                                    test_size=0.2,\n                                                                    random_state = 0,\n                                                                    shuffle=True)","metadata":{"_uuid":"e594cc57-9698-46d5-834e-93f32326eb80","_cell_guid":"cad75a05-0e6d-4b94-aa96-b8d1ccfe3811","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:47.637114Z","iopub.execute_input":"2023-06-25T19:53:47.637690Z","iopub.status.idle":"2023-06-25T19:53:48.268800Z","shell.execute_reply.started":"2023-06-25T19:53:47.637639Z","shell.execute_reply":"2023-06-25T19:53:48.268009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_pos.shape, X_train_neg.shape","metadata":{"_uuid":"ad16894b-ccf4-47af-a3be-c765946fed75","_cell_guid":"4c49a594-931c-4433-a09e-ff9fbb871aac","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.270323Z","iopub.execute_input":"2023-06-25T19:53:48.270631Z","iopub.status.idle":"2023-06-25T19:53:48.276670Z","shell.execute_reply.started":"2023-06-25T19:53:48.270579Z","shell.execute_reply":"2023-06-25T19:53:48.275821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we know, the positive samples are fewer, so we will only select a subset of negative samples for training.","metadata":{"_uuid":"e04ae334-9c6d-424c-bb41-90d0cb0d3f59","_cell_guid":"573342c8-aad4-4265-a0b5-d492be05d661","trusted":true}},{"cell_type":"code","source":"# Combine positive and negative samples for training...\ndef combine_positive_and_negative_samples(pos_samples, neg_samples, y_pos, y_neg):\n    X_combined = np.concatenate((pos_samples, neg_samples)) \n                                                    # don't select all negative samples, to\n                                                    # keep the samples balanced\n    y_combined = np.concatenate((y_pos, y_neg))\n    #X_train_combined.shape, y_train_combined.shape\n    combined_samples = np.hstack((X_combined, y_combined.reshape(y_combined.shape[0],1)))\n    np.random.shuffle(combined_samples)\n    return combined_samples\n\n# Only use 500 negative samples, to create a balanced dataset with the positive samples...\ntrain_samples = combine_positive_and_negative_samples(X_train_pos, X_train_neg[:395, :], y_train_pos, y_train_neg[:395])        ###################\nX_train = train_samples[:,:-1]\ny_train = train_samples[:,-1]\nX_train.shape, y_train.shape","metadata":{"_uuid":"94be01e8-ee0b-4887-b668-2e6a49476167","_cell_guid":"f5f72a53-0961-48d3-900a-6084747c71c3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.278056Z","iopub.execute_input":"2023-06-25T19:53:48.278608Z","iopub.status.idle":"2023-06-25T19:53:48.309535Z","shell.execute_reply.started":"2023-06-25T19:53:48.278557Z","shell.execute_reply":"2023-06-25T19:53:48.308507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples.shape","metadata":{"_uuid":"6c4028a2-6142-4c6f-a7c8-b6ef1c3dbcab","_cell_guid":"f04a48d1-49e7-4fcf-a2ce-2b5a5937bfb2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.310792Z","iopub.execute_input":"2023-06-25T19:53:48.311245Z","iopub.status.idle":"2023-06-25T19:53:48.317216Z","shell.execute_reply.started":"2023-06-25T19:53:48.311034Z","shell.execute_reply":"2023-06-25T19:53:48.315902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid_pos.shape, X_valid_neg.shape","metadata":{"_uuid":"0636f73f-0715-4198-97c9-8db48cf4438b","_cell_guid":"61477f31-7115-40a7-a3fa-a47ada834819","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.319435Z","iopub.execute_input":"2023-06-25T19:53:48.320062Z","iopub.status.idle":"2023-06-25T19:53:48.330591Z","shell.execute_reply.started":"2023-06-25T19:53:48.319948Z","shell.execute_reply":"2023-06-25T19:53:48.329529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the validation set\n#X_valid_combined = np.concatenate((X_valid_pos, X_valid_neg[:500,:])) # don't select all negative samples, to\n                                                  # keep the samples balanced\n#y_valid_combined = np.concatenate((y_valid_pos, y_valid_neg[:500]))\n#X_valid_combined.shape, y_valid_combined.shape\n#validation_samples = np.hstack((X_valid_combined, y_valid_combined.reshape(y_valid_combined.shape[0],1)))\n#np.random.shuffle(validation_samples)\n\nvalidation_samples = combine_positive_and_negative_samples(X_valid_pos, X_valid_neg[:98,:], y_valid_pos, y_valid_neg[:98])    #################\nX_valid = validation_samples[:,:-1]\ny_valid = validation_samples[:,-1]\nX_valid.shape, y_valid.shape","metadata":{"_uuid":"4db4f9c7-0587-4490-8647-415479100306","_cell_guid":"e406288b-6106-4444-9eb1-a6e9e21a9559","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.332040Z","iopub.execute_input":"2023-06-25T19:53:48.332482Z","iopub.status.idle":"2023-06-25T19:53:48.346206Z","shell.execute_reply.started":"2023-06-25T19:53:48.332283Z","shell.execute_reply":"2023-06-25T19:53:48.345169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_valid.shape","metadata":{"_uuid":"88c727db-257a-43b5-92b7-c560a25bb6f0","_cell_guid":"1b3cbdd0-961d-4357-8970-15c2994341e6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.348060Z","iopub.execute_input":"2023-06-25T19:53:48.348418Z","iopub.status.idle":"2023-06-25T19:53:48.354786Z","shell.execute_reply.started":"2023-06-25T19:53:48.348329Z","shell.execute_reply":"2023-06-25T19:53:48.353981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_samples.shape","metadata":{"_uuid":"4e40c44c-84b8-4514-81a5-e1e861b18112","_cell_guid":"ae599fd5-d07c-4a72-a5f2-db12791d8a29","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.356165Z","iopub.execute_input":"2023-06-25T19:53:48.356747Z","iopub.status.idle":"2023-06-25T19:53:48.364490Z","shell.execute_reply.started":"2023-06-25T19:53:48.356687Z","shell.execute_reply":"2023-06-25T19:53:48.363627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A 1-D ConvNet would be an interesting model to try out on this signal. Earlier we saw that there are a lot of outliers in fauty signals. Since the actual signal value differs at different times, the outliers are relative to this mean signal value. A 1-D ConvNet can analyze the signal in various windows of increasing lengths and create high-level features out of that to classify on.","metadata":{"_uuid":"38e5fc40-dbea-4be9-80b5-8918cadefa52","_cell_guid":"351f50e4-d14f-480c-abe3-5b3a0240a63a","trusted":true}},{"cell_type":"code","source":"# Reshape training and validation data for keras input layer\nX_train = X_train.reshape(-1,1,3125, 1)\nX_valid = X_valid.reshape(-1,1,3125, 1)\n\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape\n#print(X_train)","metadata":{"_uuid":"f73d9303-163b-4715-a6c2-1ad60a283bbd","_cell_guid":"b5adc6c5-1bb1-492f-8cbd-b890953ba853","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.365972Z","iopub.execute_input":"2023-06-25T19:53:48.366426Z","iopub.status.idle":"2023-06-25T19:53:48.375616Z","shell.execute_reply.started":"2023-06-25T19:53:48.366354Z","shell.execute_reply":"2023-06-25T19:53:48.374675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid = X_valid.astype(np.float32)\ny_valid = y_valid.astype(np.float32)\nX_train = X_train.astype(np.float32)\ny_train = y_train.astype(np.float32)\nX_train.dtype","metadata":{"_uuid":"55a6addc-6b52-430c-8689-55c1a1aadde7","_cell_guid":"0964e829-e81a-4b59-a2ec-1b9d84984759","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.376961Z","iopub.execute_input":"2023-06-25T19:53:48.377456Z","iopub.status.idle":"2023-06-25T19:53:48.388797Z","shell.execute_reply.started":"2023-06-25T19:53:48.377389Z","shell.execute_reply":"2023-06-25T19:53:48.387818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('X_valid.npy',X_valid)\nnp.save('y_valid.npy',y_valid)\nnp.save('X_train.npy',X_train)\nnp.save('y_train.npy',y_train)","metadata":{"_uuid":"4d3dc7c1-38d9-4a47-8d55-1ea74afc60f8","_cell_guid":"baa7e6fd-cf94-44a9-ad3d-15dfe895ad7e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.390234Z","iopub.execute_input":"2023-06-25T19:53:48.390773Z","iopub.status.idle":"2023-06-25T19:53:48.407245Z","shell.execute_reply.started":"2023-06-25T19:53:48.390477Z","shell.execute_reply":"2023-06-25T19:53:48.406531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nX_valid = np.load('X_valid.npy')\ny_valid = np.load('y_valid.npy')\nX_train = np.load('X_train.npy')\ny_train = np.load('y_train.npy')","metadata":{"_uuid":"e55162bd-3ec6-4364-981e-e85cd76b83c9","_cell_guid":"d630a4d7-b6d9-4c5c-883c-2ce1d615115e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.409875Z","iopub.execute_input":"2023-06-25T19:53:48.410103Z","iopub.status.idle":"2023-06-25T19:53:48.424485Z","shell.execute_reply.started":"2023-06-25T19:53:48.410055Z","shell.execute_reply":"2023-06-25T19:53:48.423760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(X_train, axis = 0), np.max(X_train, axis = 0), np.min(X_train, axis = 0), np.std(X_train, axis = 0)","metadata":{"_uuid":"6f3ca6e7-9074-44fa-a762-9a58086d7713","_cell_guid":"1745f71b-bf50-4058-95e6-a1a900822535","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.426226Z","iopub.execute_input":"2023-06-25T19:53:48.426808Z","iopub.status.idle":"2023-06-25T19:53:48.445620Z","shell.execute_reply.started":"2023-06-25T19:53:48.426479Z","shell.execute_reply":"2023-06-25T19:53:48.444619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(y_valid), np.sum(y_train)","metadata":{"_uuid":"a387beca-5208-4546-b450-6404a8872bd9","_cell_guid":"af478e41-2151-4634-b098-c0963c4f1d8e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.447214Z","iopub.execute_input":"2023-06-25T19:53:48.447770Z","iopub.status.idle":"2023-06-25T19:53:48.453229Z","shell.execute_reply.started":"2023-06-25T19:53:48.447461Z","shell.execute_reply":"2023-06-25T19:53:48.452516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_normalize(data):\n    mu = np.mean(data,axis=0)\n    std = np.std(data,axis=0)\n    return (data - mu)/std","metadata":{"_uuid":"bce6703e-5a46-4cfe-a870-ffd854cc16c9","_cell_guid":"718ff6cc-f942-4f49-be27-d5ea2fc06f77","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.454500Z","iopub.execute_input":"2023-06-25T19:53:48.455189Z","iopub.status.idle":"2023-06-25T19:53:48.461590Z","shell.execute_reply.started":"2023-06-25T19:53:48.455130Z","shell.execute_reply":"2023-06-25T19:53:48.460816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"X_valid = feature_normalize(X_valid)\nX_train = feature_normalize(X_train)","metadata":{"_uuid":"b00f1011-ff0f-4e4c-a2f7-0573815d01cb","_cell_guid":"6017214b-6184-4da7-9a61-ef7d70079723","trusted":true}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import datasets, transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as Data\nprint(torch.__version__)","metadata":{"_uuid":"07d1d84f-dd88-4ce2-9fcd-5d51647e3752","_cell_guid":"d9f39fb8-e3b9-41f3-b006-470ffc817c0d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:48.462917Z","iopub.execute_input":"2023-06-25T19:53:48.463448Z","iopub.status.idle":"2023-06-25T19:53:49.341149Z","shell.execute_reply.started":"2023-06-25T19:53:48.463376Z","shell.execute_reply":"2023-06-25T19:53:49.340377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class torch_Dataset(Data.Dataset): \n    def __init__(self, x, y):\n        self.x = torch.from_numpy(x)\n        self.y = torch.from_numpy(y)\n    def __getitem__(self, index):\n        data = (self.x[index], self.y[index])\n        return data\n    def __len__(self):\n        return len(self.y)","metadata":{"_uuid":"d9f02708-3c02-4b82-bdfc-f0b125fc0d4c","_cell_guid":"ce0db3dc-0e33-42ca-8064-3140717a6863","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:49.343347Z","iopub.execute_input":"2023-06-25T19:53:49.343905Z","iopub.status.idle":"2023-06-25T19:53:49.350302Z","shell.execute_reply.started":"2023-06-25T19:53:49.343844Z","shell.execute_reply":"2023-06-25T19:53:49.349682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loader(train_data, batch_size, shuffle):\n    return torch.utils.data.DataLoader(train_data, batch_size, shuffle)","metadata":{"_uuid":"85f1605f-d4f6-4969-a89f-60b2e81b3211","_cell_guid":"04810d38-0379-4f11-810c-b023d3678d1f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:49.351691Z","iopub.execute_input":"2023-06-25T19:53:49.352334Z","iopub.status.idle":"2023-06-25T19:53:49.365393Z","shell.execute_reply.started":"2023-06-25T19:53:49.352182Z","shell.execute_reply":"2023-06-25T19:53:49.364496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_dataset = torch_Dataset(X_train, y_train)\ntest_dataset = torch_Dataset(X_valid, y_valid)\ntrain_loader = training_loader(Train_dataset, batch_size=1, shuffle=True)\ntest_loader = training_loader(test_dataset, batch_size=1, shuffle=True)","metadata":{"_uuid":"1405b274-270c-4a12-838e-8219429906c7","_cell_guid":"35e03b1f-9f2b-499a-b6da-5eee84bd4931","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:49.367035Z","iopub.execute_input":"2023-06-25T19:53:49.367593Z","iopub.status.idle":"2023-06-25T19:53:49.382586Z","shell.execute_reply.started":"2023-06-25T19:53:49.367325Z","shell.execute_reply":"2023-06-25T19:53:49.381805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class LSTM(nn.Module):\n#     def __init__(self):\n#         super(LSTM, self).__init__()\n#         self.lstm = nn.LSTM(1, 128, batch_first=True)\n#         self.fc1 = nn.Linear(128, 64)\n#         self.rl1 = nn.ReLU()\n#         self.do1 = nn.Dropout(0.2)\n#         self.fc2 = nn.Linear(64, 1)\n\n#     def forward(self, x):\n#         x = x.view(1, 3125, 1)\n#         _, (x, _) = self.lstm(x)\n#         x = x[-1, :, :]\n#         x = self.fc1(x)\n#         x = self.rl1(x)\n#         x = self.do1(x)\n#         x = self.fc2(x)\n#         x = torch.sigmoid(x)\n#         return x\n","metadata":{"_uuid":"6a243c52-e95c-4bf5-bd6c-6a235ab7e71c","_cell_guid":"59031d2d-7d85-4fb4-8c22-0899acc64a2a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:49.384330Z","iopub.execute_input":"2023-06-25T19:53:49.384895Z","iopub.status.idle":"2023-06-25T19:53:49.392981Z","shell.execute_reply.started":"2023-06-25T19:53:49.384838Z","shell.execute_reply":"2023-06-25T19:53:49.392096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LSTM(nn.Module):\n    def __init__(self):\n        super(LSTM,self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=(16,1), padding=0, stride=(2,1))\n        self.bn1 = nn.BatchNorm2d(32)\n        self.rl1 =  nn.ReLU()\n        self.pool1 = nn.MaxPool2d((2,1))\n        self.do1 =    nn.Dropout(0.2)\n        self.conv2 = nn.Conv2d(32, 128, kernel_size=(16,1), padding=0, stride=(2,1))\n        self.bn2 = nn.BatchNorm2d(128)\n        self.rl2 =  nn.ReLU()\n        self.pool2 = nn.MaxPool2d((2,1))\n        self.do2 =    nn.Dropout(0.2)\n        self.conv3 = nn.Conv2d(128, 256, kernel_size=(10,1), padding=0, stride=(2,1))\n        self.bn3 = nn.BatchNorm2d(256)\n        self.rl3 =  nn.ReLU()\n        self.pool3 = nn.MaxPool2d((2,1))\n        self.do3 =    nn.Dropout(0.2)\n        self.conv6 = nn.Conv2d(256, 64, kernel_size=(10,1), padding=0, stride=(2,1))\n        self.rl6 =  nn.ReLU()\n        \n        self.fc1   = nn.Linear(1152, 512)\n        self.rl7 =  nn.ReLU()\n        self.do7 =    nn.Dropout(0.2)\n        self.fc2   = nn.Linear(512, 128)\n        self.rl8 =  nn.ReLU()\n        self.do8 =    nn.Dropout(0.2)\n        self.fc3   = nn.Linear(128, 1)\n      \n            \n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.rl1(x)\n        x = self.pool1(x)\n        x = self.do1(x)\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.rl2(x)\n        x = self.pool2(x)\n        x = self.do2(x)\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.rl3(x)\n        x = self.pool3(x)\n        x = self.do3(x)\n        x = self.conv6(x)\n        x = self.rl6(x)\n        x = x.view(-1,1152)\n   \n        x = self.fc1(x)\n        x = self.rl7(x)\n        x = self.do7(x)\n        x = self.fc2(x)\n        x = self.rl8(x)\n        x = self.do8(x)\n        x = self.fc3(x)\n       # print(x.shape)\n        x = torch.sigmoid(x)\n        \n        return x","metadata":{"_uuid":"29a1d7a0-2e5b-4078-9cbb-c30bde976ce4","_cell_guid":"e85f1f69-f19a-497b-b256-9eaa047cda2e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:49.394446Z","iopub.execute_input":"2023-06-25T19:53:49.394978Z","iopub.status.idle":"2023-06-25T19:53:49.404496Z","shell.execute_reply.started":"2023-06-25T19:53:49.394705Z","shell.execute_reply":"2023-06-25T19:53:49.403433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class LSTM(nn.Module):\n#     def __init__(self):\n#         super(LSTM, self).__init__()\n        \n#         self.lstm = nn.LSTM(input_size=1, hidden_size=128, num_layers=2, batch_first=True, dropout=0.2)\n        \n#         self.fc1 = nn.Linear(128, 64)\n#         self.relu = nn.ReLU()\n#         self.fc2 = nn.Linear(64, 1)\n#         self.sigmoid = nn.Sigmoid()\n        \n#     def forward(self, x):\n#         x = x.view(1, 3125, 1)\n\n#         # Pass the input through the LSTM layer\n#         _, (h_n, _) = self.lstm(x)\n\n#         # Extract the last hidden state of the LSTM layer\n#         x = h_n[-1, :, :]\n\n#         # Apply fully connected layers\n#         x = self.fc1(x)\n#         x = self.relu(x)\n#         x = self.fc2(x)\n#         x = self.sigmoid(x)\n\n#         return x\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T19:53:49.405681Z","iopub.execute_input":"2023-06-25T19:53:49.405960Z","iopub.status.idle":"2023-06-25T19:53:49.417706Z","shell.execute_reply.started":"2023-06-25T19:53:49.405899Z","shell.execute_reply":"2023-06-25T19:53:49.416676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.nn as nn\n\n# class LSTM(nn.Module):\n#     def __init__(self):\n#         super(LSTM, self).__init__()\n        \n#         self.lstm1 = nn.LSTM(input_size=1, hidden_size=128, num_layers=2, batch_first=True, dropout=0.2)\n#         self.dropout1 = nn.Dropout(0.2)\n        \n#         self.lstm2 = nn.LSTM(input_size=128, hidden_size=256, num_layers=2, batch_first=True, dropout=0.2)\n#         self.dropout2 = nn.Dropout(0.2)\n        \n#         self.lstm3 = nn.LSTM(input_size=256, hidden_size=512, num_layers=2, batch_first=True, dropout=0.2)\n#         self.dropout3 = nn.Dropout(0.2)\n        \n#         self.lstm4 = nn.LSTM(input_size=512, hidden_size=256, num_layers=2, batch_first=True, dropout=0.2)\n#         self.dropout4 = nn.Dropout(0.2)\n        \n#         self.lstm5 = nn.LSTM(input_size=256, hidden_size=128, num_layers=2, batch_first=True, dropout=0.2)\n#         self.dropout5 = nn.Dropout(0.2)\n        \n#         self.lstm6 = nn.LSTM(input_size=128, hidden_size=64, num_layers=2, batch_first=True, dropout=0.2)\n        \n#         self.fc1 = nn.Linear(64, 32)\n#         self.relu = nn.ReLU()\n#         self.dropout6 = nn.Dropout(0.2)\n#         self.fc2 = nn.Linear(32, 1)\n#         self.sigmoid = nn.Sigmoid()\n        \n#     def forward(self, x):\n#         # Reshape the input tensor to (batch_size, input_size, sequence_length)\n#         x = x.view(-1, 16, 3125)\n        \n#         # Pass the input through the LSTM layers\n#         _, (h_n1, _) = self.lstm1(x)\n#         x = self.dropout1(h_n1[-1, :, :])\n        \n#         _, (h_n2, _) = self.lstm2(x)\n#         x = self.dropout2(h_n2[-1, :, :])\n        \n#         _, (h_n3, _) = self.lstm3(x)\n#         x = self.dropout3(h_n3[-1, :, :])\n        \n#         _, (h_n4, _) = self.lstm4(x)\n#         x = self.dropout4(h_n4[-1, :, :])\n        \n#         _, (h_n5, _) = self.lstm5(x)\n#         x = self.dropout5(h_n5[-1, :, :])\n        \n#         _, (h_n6, _) = self.lstm6(x)\n        \n#         # Extract the last hidden state of the final LSTM layer\n#         x = h_n6[-1, :, :]\n        \n#         # Apply fully connected layers\n#         x = self.fc1(x)\n#         x = self.relu(x)\n#         x = self.dropout6(x)\n#         x = self.fc2(x)\n#         x = self.sigmoid(x)\n        \n#         return x\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T19:53:49.419058Z","iopub.execute_input":"2023-06-25T19:53:49.419568Z","iopub.status.idle":"2023-06-25T19:53:49.431054Z","shell.execute_reply.started":"2023-06-25T19:53:49.419369Z","shell.execute_reply":"2023-06-25T19:53:49.430227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    if torch.cuda.is_available():\n        device = torch.device( \"cuda\")\n    else:\n        device = torch.device( \"cpu\")\n    print(f\"training on {device} device.\")\n    model = LSTM().to(device)\n    optimizer = optim.Adam(model.parameters(), lr=0.0001) #1e-2)\n    loss_fn = nn.BCELoss()\n\n    print(model)","metadata":{"_uuid":"ce78f947-167b-4d85-8b09-8f40b107e7b2","_cell_guid":"9de2f4d6-f632-4790-8740-33b80392fa51","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:49.434414Z","iopub.execute_input":"2023-06-25T19:53:49.434724Z","iopub.status.idle":"2023-06-25T19:53:53.845781Z","shell.execute_reply.started":"2023-06-25T19:53:49.434666Z","shell.execute_reply":"2023-06-25T19:53:53.844802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loop(n_epochs, optimizer, model, loss_fn, train_loader, dev):\n    for epoch in range(1, n_epochs + 1):\n        loss_train = 0.0\n        for imgs, labels in train_loader:\n            imgs = imgs.to(device=dev)\n            #imgs = imgs.float() \n            labels = labels.to(device=dev)\n            outputs = model(imgs)\n            #outputs = outputs.squeeze(-1)       \n            #print(\"train output size \",outputs.size())\n           # print(\"label output size \",labels.size())\n            #print(outputs)\n            optimizer.zero_grad()\n            loss = loss_fn(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            loss_train += loss.item()\n        if epoch == 1 or epoch % 2 == 0:\n            print('{0} Epoch {1:3d}, Training loss {2:.6f}'.format(\n                datetime.datetime.now(), epoch,\n                loss_train / len(train_loader)))\n    return","metadata":{"_uuid":"02e6ea5c-9617-45e5-8524-248ad1039c42","_cell_guid":"d5413f1d-8ec2-4627-aee5-1d1c1ec17c6b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:53.849925Z","iopub.execute_input":"2023-06-25T19:53:53.852088Z","iopub.status.idle":"2023-06-25T19:53:53.862077Z","shell.execute_reply.started":"2023-06-25T19:53:53.852028Z","shell.execute_reply":"2023-06-25T19:53:53.861040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    import warnings\n    import datetime\n    warnings.filterwarnings('ignore') \n    training_loop(\n        n_epochs = 10, #100,\n        optimizer = optimizer,\n        model = model,\n        loss_fn = loss_fn,\n        train_loader = train_loader,\n        dev = device\n    )","metadata":{"_uuid":"8d8eb197-7385-406d-b337-756f8b683ffc","_cell_guid":"87860c48-c764-4f1a-ac6c-07fe094db1f4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T19:53:53.864379Z","iopub.execute_input":"2023-06-25T19:53:53.864937Z","iopub.status.idle":"2023-06-25T20:02:18.053423Z","shell.execute_reply.started":"2023-06-25T19:53:53.864870Z","shell.execute_reply":"2023-06-25T20:02:18.052255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint('Trained model written to ', 'torch_chkp.pt')\ntorch.save(model.state_dict(), 'torch_chkp.pt')\nloaded_model = LSTM()  # <1>\nloaded_model.load_state_dict(torch.load('torch_chkp.pt'))","metadata":{"_uuid":"71f17cd6-648d-422d-a50a-d279594125d4","_cell_guid":"bec4e974-805c-49c3-9d96-5fed9ff7c564","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:02:18.054926Z","iopub.execute_input":"2023-06-25T20:02:18.055266Z","iopub.status.idle":"2023-06-25T20:02:18.071646Z","shell.execute_reply.started":"2023-06-25T20:02:18.055193Z","shell.execute_reply":"2023-06-25T20:02:18.070802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(model, train_loader, val_loader, dev):\n    accdict = {}\n    for name, loader in [(\"train dataset\", train_loader), (\"test dataset  \", val_loader)]:\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for imgs, labels in loader:\n                imgs = imgs.float() \n                imgs = imgs.to(device=dev)\n                labels = labels.to(device=dev)\n                outputs = model(imgs)\n                \n                predicted = torch.max(outputs) # <1>\n                #print(predicted)\n                if(predicted>0.5):\n                   #print(predicted)\n                    falt_detected =1\n                else:\n                    falt_detected =0\n                total += labels.shape[0]\n                correct += int((falt_detected == labels).sum())\n                #print(\"predict value:\", falt_detected, \"real value:\" , labels)\n                \n        print(\"Accuracy {0}: {1:.2f}(%)\".format(name , 100*(correct/total)))\n        accdict[name] = correct / total\n    return accdict","metadata":{"_uuid":"6bbbd502-79ae-465c-beed-1abfd154ffba","_cell_guid":"89ac94a2-9e5b-455c-aa89-95b5f7806edb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:02:18.074692Z","iopub.execute_input":"2023-06-25T20:02:18.075308Z","iopub.status.idle":"2023-06-25T20:02:18.084265Z","shell.execute_reply.started":"2023-06-25T20:02:18.075245Z","shell.execute_reply":"2023-06-25T20:02:18.083099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(model, train_loader, val_loader, dev):\n    accdict = {}\n    for name, loader in [(\"train dataset\", train_loader), (\"test dataset  \", val_loader)]:\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for imgs, labels in loader:\n                imgs = imgs.float() \n                imgs = imgs.to(device=dev)\n                labels = labels.to(device=dev)\n                outputs = model(imgs)\n                \n                predicted = torch.max(outputs) # <1>\n                #print(predicted)\n  \n    return accdict","metadata":{"_uuid":"8ccef51c-0518-447c-8c66-ede82167c771","_cell_guid":"2778d594-ca2a-4241-8e83-ff94d9de0f2c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:02:18.085807Z","iopub.execute_input":"2023-06-25T20:02:18.086487Z","iopub.status.idle":"2023-06-25T20:02:18.100245Z","shell.execute_reply.started":"2023-06-25T20:02:18.086154Z","shell.execute_reply":"2023-06-25T20:02:18.099238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate(model, train_loader, test_loader, device)","metadata":{"_uuid":"243cfc5f-ca63-42b8-8388-32eabbb61dd1","_cell_guid":"27bb0ed1-a203-413d-9424-275a152746b9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:02:18.101540Z","iopub.execute_input":"2023-06-25T20:02:18.101817Z","iopub.status.idle":"2023-06-25T20:02:44.573104Z","shell.execute_reply.started":"2023-06-25T20:02:18.101755Z","shell.execute_reply":"2023-06-25T20:02:44.572336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport collections\nloaded_model = LSTM()  # <1>\nloaded_model.load_state_dict(torch.load('torch_chkp.pt'))\nall_acc_dict = collections.OrderedDict()\ntime_start = time.time()\nall_acc_dict[\"baseline\"] = validate(model, train_loader, test_loader, device)\ntime_end = time.time()\ntimetotal = time_end - time_start\nrunTotal = len(train_loader)\nfps = float(runTotal / timetotal)\nprint(\"FPS=%.2f, total frames = %.0f , time=%.4f seconds\" %(fps,runTotal, timetotal))","metadata":{"_uuid":"41f86738-dd5f-4eed-9459-5a193266db07","_cell_guid":"b3576284-da65-408d-94cd-12210eafcc56","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:02:44.574519Z","iopub.execute_input":"2023-06-25T20:02:44.575048Z","iopub.status.idle":"2023-06-25T20:03:11.129000Z","shell.execute_reply.started":"2023-06-25T20:02:44.574992Z","shell.execute_reply":"2023-06-25T20:03:11.128210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\n\ndef plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n    \n    plt.imshow(cm, cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i in range(cm.shape[0]):\n        for j in range(cm.shape[1]):\n            plt.text(j, i, format(cm[i, j], fmt), \n                    horizontalalignment=\"center\",\n                    color=\"white\" if cm[i, j] > thresh else \"black\")\n    plt.tight_layout()\n    plt.show()\n\ndef get_true_labels_and_predicted_labels(model, test_loader):\n    true_labels = []\n    predicted_labels = []\n    with torch.no_grad():\n        for images, labels in test_loader:\n            images = images.to('cuda:0')\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            true_labels.append(labels.item())\n            predicted_labels.append(predicted.item())\n\n    return true_labels, predicted_labels\n","metadata":{"_uuid":"bca7b394-8d58-42a1-8aed-e22420ba9b51","_cell_guid":"23148cb7-cc39-4913-8e68-e4c8a296304c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:03:11.130286Z","iopub.execute_input":"2023-06-25T20:03:11.130765Z","iopub.status.idle":"2023-06-25T20:03:11.142838Z","shell.execute_reply.started":"2023-06-25T20:03:11.130710Z","shell.execute_reply":"2023-06-25T20:03:11.141936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true_labels, predicted_labels = get_true_labels_and_predicted_labels(model, test_loader)\n\ncm = confusion_matrix(true_labels, predicted_labels)\nplot_confusion_matrix(cm, classes=[\"Fault\", \"Normal\"])","metadata":{"_uuid":"bca7b394-8d58-42a1-8aed-e22420ba9b51","_cell_guid":"23148cb7-cc39-4913-8e68-e4c8a296304c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-25T20:03:11.144319Z","iopub.execute_input":"2023-06-25T20:03:11.144826Z","iopub.status.idle":"2023-06-25T20:03:16.756150Z","shell.execute_reply.started":"2023-06-25T20:03:11.144710Z","shell.execute_reply":"2023-06-25T20:03:16.755253Z"},"trusted":true},"execution_count":null,"outputs":[]}]}