{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":false,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport scipy as sp\nimport pandas as pd\nimport os\n\nimport random\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\npd.options.display.precision = 15\n\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom fastai.tabular import * \nfrom tqdm import tqdm_notebook\nfrom fastai.callbacks import *","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"bc314853a64c3ee3b7b14dd2ad99c06e77e16561","trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32},nrows=6e8)","execution_count":2,"outputs":[{"output_type":"stream","text":"CPU times: user 2min 1s, sys: 10.1 s, total: 2min 11s\nWall time: 2min 12s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"min = -100\nmax = 100\nspread = 110\ndef get_counts(sequence): \n counts = [0]*spread\n unique_count = np.unique(sequence, return_counts=True)\n for i in range(0,len(unique_count[0])):\n val = unique_count[0][i]\n count = unique_count[1][i]\n r = count*val\n if val <= min:\n counts[0] += r\n elif val >= max:\n counts[-1] += r\n else:\n counts[int(val/2)+int(spread/2)] += r\n\n return counts","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interval = 75000\ncounts = [get_counts(train.acoustic_data.values[i:i+150000]) for i in tqdm_notebook(range(0,len(train),interval))]\nttfs = [train.time_to_failure.values[i] for i in range(0,len(train),interval)]\ndel train\n\nlabels = [\"D\"+str(i) for i in range(0,len(counts[0]))]\n\ndf = pd.DataFrame(counts, columns=labels)\nttf_df = pd.DataFrame(ttfs, columns=[\"expected\"])\ndf = df.join(ttf_df)","execution_count":4,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=8000), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7dee40d8ce7e4c8c9c05120f7f4ac664"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(3)","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":" D0 D1 D2 D3 ... D107 D108 D109 expected\n0 0 0 0 0 ... 0 0 206 1.469099998474121\n1 -3169 0 0 0 ... 0 0 5113 1.449998617172241\n2 -3703 0 0 0 ... 0 0 5326 1.430797219276428\n\n[3 rows x 111 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
D0D1D2D3D4D5D6D7D8D9D10D11D12D13D14D15D16D17D18D19D20D21D22D23D24D25D26D27D28D29D30D31D32D33D34D35D36D37D38D39...D71D72D73D74D75D76D77D78D79D80D81D82D83D84D85D86D87D88D89D90D91D92D93D94D95D96D97D98D99D100D101D102D103D104D105D106D107D108D109expected
0000000-980-95-92-9100-85-82-161-7800-1460-138-67-194-63-60-233-170-271-527-100-242-373-400-213-443-271-511-553-618...1399134310591273688594102156043745552548939617642225265265137141218757615680828534489093009800002061.469099998474121
1-316900000-296-194-95-93-900-348-85-411-240-158-230-223-653-425-273-201-64-250-180-235-282-326-578-759-96-93-445-550-812-536-584-725-717...185516231604131011777236666976304074732196213523623753211332748424376722283144818308726554337137919398000051131.449998617172241
2-370300000-296-291-190-93-90-88-348-85-411-320-158-230-223-653-495-273-201-64-376-180-293-282-326-683-809-290-233-490-719-892-688-619-966-1303...24392101185917301665934128811161068356473438792587602375321200479913437672228548481838487354543371568386196000053261.430797219276428
\n
"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"ae3de367d7834583fa6c6957788dc86d7b77d4d2"},"cell_type":"code","source":"path =\"../tmp\"\ntry:\n os.makedirs(path)\nexcept:\n pass","execution_count":6,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"tpath = \"../input/test\"\nfiles = os.listdir(tpath)\ni = 0\ntest_id = []\ntest_df = pd.DataFrame(dtype=np.float64, columns=df.columns.values[:-1])\nfor f in tqdm_notebook(files):\n seg = pd.read_csv(f'{tpath}/{f}')\n converted = get_counts(seg.acoustic_data.values)\n test_df.loc[i] = converted\n test_id.append(f.replace(\".csv\", \"\"))\n i+=1","execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=2624), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"09597cf7c05a40fda24811d49bcac6f3"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"num = len(df)\ninterval = int(num/100)\nvalues = int(num/(5*100))\nvalid_idx = []\nfor i in range(0,len(df)-values,interval):\n for j in range(0,values-1):\n valid_idx.append(i+j)","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ttfs = np.array([df.iloc[i].expected for i in valid_idx])","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = TabularDataBunch.from_df(path, df, \"expected\", valid_idx=valid_idx, test_df=test_df, procs=[Normalize])\n# data = TabularDataBunch.from_df(path, df, \"expected\", valid_idx=valid_idx, procs=[Normalize])","execution_count":10,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* spread 200 - 2.02\n* spread 300 - "},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nbest_learn = None\nbest_mae = 9999\n\nfor i in range(0, 99):\n learn = tabular_learner(data=data, layers=[200,100], metrics=mae, ps=0.5, y_range=(-1,15))\n learn.callbacks = [SaveModelCallback(learn, every='improvement', mode='min', name='best')]\n learn.fit_one_cycle(20, 1e-2)\n gc.collect()\n\n preds = learn.get_preds(DatasetType.Valid)[0].numpy().flatten()\n new_mae = np.abs(valid_ttfs-preds).mean()\n if new_mae < best_mae or not best_learn:\n best_learn = learn\n best_mae = new_mae\n print(f'Run {i} - Best MAE: {best_mae}')","execution_count":11,"outputs":[{"output_type":"stream","text":"Run 19 - Best MAE: 2.0077382553915184\nCPU times: user 5min 6s, sys: 2min 33s, total: 7min 39s\nWall time: 8min 30s\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = best_learn.get_preds(DatasetType.Test)[0].numpy().flatten()","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tpath = \"../input/test\"\nfiles = os.listdir(tpath)\nfiles = [f.replace(\".csv\",\"\") for f in files]\nfiles[:3]","execution_count":13,"outputs":[{"output_type":"execute_result","execution_count":13,"data":{"text/plain":"['seg_0b082e', 'seg_9e7dff', 'seg_b6c10d']"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = pd.DataFrame({\"seg_id\":files, \"time_to_failure\":preds})\nresults.head()","execution_count":14,"outputs":[{"output_type":"execute_result","execution_count":14,"data":{"text/plain":" seg_id time_to_failure\n0 seg_0b082e 9.671259880065918\n1 seg_9e7dff 3.582855701446533\n2 seg_b6c10d 7.680974960327148\n3 seg_4435bd 4.013928413391113\n4 seg_c09a41 3.563020706176758","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
seg_idtime_to_failure
0seg_0b082e9.671259880065918
1seg_9e7dff3.582855701446533
2seg_b6c10d7.680974960327148
3seg_4435bd4.013928413391113
4seg_c09a413.563020706176758
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"results.to_csv('submission.csv',index=False)","execution_count":15,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}