{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## **We'll solve the task here**","metadata":{"_cell_guid":"7fd53b38-a1b5-469d-9099-698eceb83a2a","_uuid":"53f5f69d-5001-4803-b4cf-55b3a7004b1e","id":"tZ58ADSLXg7g"}},{"cell_type":"markdown","source":"### importing packages","metadata":{"_cell_guid":"5f432694-ded1-4bba-9990-b6c3252de365","_uuid":"93e62e61-813a-47f5-a932-65e9286d2531","id":"GFI2bJ0FXpGr"}},{"cell_type":"code","source":"# !pip install metrics # this package is not used\nimport pandas as pd\nimport numpy as np\nfrom sklearn import preprocessing\nimport logging\nimport sys\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.figure_factory as ff\nfrom plotly.offline import init_notebook_mode, iplot, plot\nimport plotly.graph_objs as go\n# This is needed so that the plotted figures appear embedded in this notebook:\n#%matplotlib inline    \n#import matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier, export_graphviz\nfrom sklearn.ensemble import BaggingRegressor, RandomForestRegressor\nlogging.basicConfig(stream=sys.stdout, level=logging.INFO)\n\nSEED = 42","metadata":{"_cell_guid":"77158992-e3dc-450c-ae25-97e1d4248d0a","_uuid":"95c84f60-48a9-45cc-9569-3931b3d83fe8","executionInfo":{"elapsed":392,"status":"ok","timestamp":1643663158888,"user":{"displayName":"Geva Bidner","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11625105523359315411"},"user_tz":-120},"id":"sXUmOU4EW519","jupyter":{"outputs_hidden":false},"scrolled":true,"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:41:36.565333Z","iopub.execute_input":"2022-08-07T17:41:36.565856Z","iopub.status.idle":"2022-08-07T17:41:36.574855Z","shell.execute_reply.started":"2022-08-07T17:41:36.565815Z","shell.execute_reply":"2022-08-07T17:41:36.573366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### import the file. Make sure to upload it to the files of the colab","metadata":{"_cell_guid":"6e34c4c0-e56d-44ab-b429-4ee40fc27ca5","_uuid":"2be48f4c-8e0b-4e1f-a340-b15b2c30d00d","id":"qRWVKizYYAOI"}},{"cell_type":"code","source":"df_holidays_events = pd.read_csv(\"../input/store-sales-time-series-forecasting/holidays_events.csv\",\n                                 parse_dates = ['date'])\n\ndf_oil = pd.read_csv(\"../input/store-sales-time-series-forecasting/oil.csv\",\n                     parse_dates = ['date'])\n\ndf_stores = pd.read_csv(\"../input/store-sales-time-series-forecasting/stores.csv\")\n\ndf_test = pd.read_csv(\"../input/store-sales-time-series-forecasting/test.csv\",\n                      parse_dates = ['date'])\n\ndf_train = pd.read_csv(\"../input/store-sales-time-series-forecasting/train.csv\",\n                      parse_dates = ['date'])\n\ndf_sample_submission = pd.read_csv(\"../input/store-sales-time-series-forecasting/sample_submission.csv\")\n\ndf_transactions = pd.read_csv(\"../input/store-sales-time-series-forecasting/transactions.csv\", parse_dates = ['date'])","metadata":{"_cell_guid":"15ec3f45-cc01-450e-a06a-04767dace5a9","_uuid":"6dbe8d9d-db09-42a9-873a-1005653c139e","executionInfo":{"elapsed":42,"status":"ok","timestamp":1643663159329,"user":{"displayName":"Geva Bidner","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11625105523359315411"},"user_tz":-120},"id":"Ln8aXYiFXeup","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:41:42.294680Z","iopub.execute_input":"2022-08-07T17:41:42.295173Z","iopub.status.idle":"2022-08-07T17:41:44.750939Z","shell.execute_reply.started":"2022-08-07T17:41:42.295137Z","shell.execute_reply":"2022-08-07T17:41:44.749707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### view basic info from the file","metadata":{"_cell_guid":"52fd897f-5a20-4be4-b3a2-ef213231f25f","_uuid":"7bd44e35-aa6a-4b52-8863-bd1d46ebd1b9","id":"9Bz8GVruX8WO"}},{"cell_type":"markdown","source":"##### detecting the problems in of different types in the Data Frame\nThis gives us a good idea on which columns have problems in them","metadata":{"_cell_guid":"973b5410-b382-4f01-ba60-bb62d1700b65","_uuid":"144135d0-ae41-4c4f-bd69-ef5a96152411","id":"KFLHCfCHYaPj"}},{"cell_type":"code","source":"dfs = {\"df_holidays_events\": df_holidays_events, \"df_oil\": df_oil, \"df_stores\": df_stores, \"df_transactions\": df_transactions, \"df_test\": df_test, \"df_train\": df_train, \"df_sample_submission\": df_sample_submission}","metadata":{"_cell_guid":"a3fae76e-f7e9-4954-b8e3-d714eb941ed9","_uuid":"3952f867-6b24-4e9b-90e2-7a09b4f3a291","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:41:59.538623Z","iopub.execute_input":"2022-08-07T17:41:59.539058Z","iopub.status.idle":"2022-08-07T17:41:59.547893Z","shell.execute_reply.started":"2022-08-07T17:41:59.539025Z","shell.execute_reply":"2022-08-07T17:41:59.546418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_view = [\"df_holidays_events\", \"df_oil\", \"df_stores\", \"df_transactions\"]","metadata":{"_cell_guid":"e8a00c49-7b44-4495-9f1d-1c4d622c1b14","_uuid":"9183ed3d-7b6a-4630-a25c-05ab3020219f","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:00.253802Z","iopub.execute_input":"2022-08-07T17:42:00.255197Z","iopub.status.idle":"2022-08-07T17:42:00.260089Z","shell.execute_reply.started":"2022-08-07T17:42:00.255139Z","shell.execute_reply":"2022-08-07T17:42:00.258904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for info_name in dfs:\n    if info_name in to_view:\n        print(\"\\n\" + info_name + \"\\n\")\n        dfs[info_name].info()","metadata":{"_cell_guid":"e04f8b75-06ff-4f52-a909-beac3211acbd","_uuid":"d9406dcc-82fc-46c4-8a3f-6091a1f2f138","executionInfo":{"elapsed":42,"status":"ok","timestamp":1643663159331,"user":{"displayName":"Geva Bidner","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11625105523359315411"},"user_tz":-120},"id":"xGp24COEX7pG","jupyter":{"outputs_hidden":false},"outputId":"8ba03f87-20da-4430-abc3-f6527f38a967","collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:01.136971Z","iopub.execute_input":"2022-08-07T17:42:01.137739Z","iopub.status.idle":"2022-08-07T17:42:01.181626Z","shell.execute_reply.started":"2022-08-07T17:42:01.137688Z","shell.execute_reply":"2022-08-07T17:42:01.180377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### By looking at the headlines and their types we can tell that there are some missing values on df_oil data frame. We also notice that the types of the objects are sometimes \"OBJECT\" and not string as expected. We can guess that some of the values in there are not defined currectly","metadata":{"_cell_guid":"0170194c-6f8a-4ec0-8afa-c9311b8c86bf","_uuid":"e9afbd06-28a0-49b1-963b-531c22b56aaa","id":"530E-1r8ZB9o"}},{"cell_type":"code","source":"print(info_name)\ndfs[info_name].describe()","metadata":{"_cell_guid":"48b80084-e406-4222-bee9-110882fa43a6","_uuid":"15fd6f61-e6a9-4198-9c66-e305e1631044","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:04.474400Z","iopub.execute_input":"2022-08-07T17:42:04.475876Z","iopub.status.idle":"2022-08-07T17:42:04.500685Z","shell.execute_reply.started":"2022-08-07T17:42:04.475832Z","shell.execute_reply":"2022-08-07T17:42:04.499374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for info_name in dfs:\n    if info_name in to_view:\n        print(\"\\n\" + info_name + \"\\n\")\n        des = dfs[info_name].describe()\n        print(des)","metadata":{"_cell_guid":"12244f8e-bd14-483b-a162-89dce0090df1","_uuid":"57c93dbe-6100-411b-8fcf-1a91e926a1b7","executionInfo":{"elapsed":38,"status":"ok","timestamp":1643663159332,"user":{"displayName":"Geva Bidner","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11625105523359315411"},"user_tz":-120},"id":"t0Kjpx3qZBbe","jupyter":{"outputs_hidden":false},"outputId":"15a80491-4557-42e3-a4ff-03daf671a906","collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:05.656365Z","iopub.execute_input":"2022-08-07T17:42:05.656834Z","iopub.status.idle":"2022-08-07T17:42:05.713105Z","shell.execute_reply.started":"2022-08-07T17:42:05.656798Z","shell.execute_reply":"2022-08-07T17:42:05.711697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for info_name in dfs:\n    if info_name in to_view:\n        print(\"\\n\" + info_name + \"\\n\")\n        print(dfs[info_name].head(10))","metadata":{"_cell_guid":"bbd57dd8-9cd9-49d3-b420-b061b9cc6d33","_uuid":"98be1e75-3e11-4861-a8e3-6f3278fa8706","executionInfo":{"elapsed":45,"status":"ok","timestamp":1643663159345,"user":{"displayName":"Geva Bidner","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11625105523359315411"},"user_tz":-120},"id":"482udZR5ZKLX","jupyter":{"outputs_hidden":false},"outputId":"270d21f3-01e9-4a7b-830a-c17fbe8aed51","collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:09.599896Z","iopub.execute_input":"2022-08-07T17:42:09.600382Z","iopub.status.idle":"2022-08-07T17:42:09.622400Z","shell.execute_reply.started":"2022-08-07T17:42:09.600344Z","shell.execute_reply":"2022-08-07T17:42:09.620918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As we expected - there're missing values in the oil dataframe.","metadata":{"_cell_guid":"1e7e1103-c20f-4e25-8c01-1ee4b804c11e","_uuid":"0f5c3811-ccce-4c01-8af3-974ac99aaffb"}},{"cell_type":"markdown","source":"### Handling With the Data\n##### creating a backup copy of data frames","metadata":{"_cell_guid":"4e4a0146-0bf7-41ce-92fd-f4b293f375af","_uuid":"28c06ef2-3521-402e-9724-72abba7b703a","id":"U8Oym2UXXVf6"}},{"cell_type":"code","source":"df_holidays_events_proccesed = df_holidays_events.copy()\ndf_oil_proccesed = df_oil.copy()\ndf_stores_proccesed = df_stores.copy()\ndf_transactions_proccesed = df_transactions.copy()","metadata":{"_cell_guid":"e1f91671-ab6b-481d-bf07-206b59ad22d8","_uuid":"d48925b8-2afb-44d6-95a1-7925e564514b","executionInfo":{"elapsed":40,"status":"ok","timestamp":1643663159346,"user":{"displayName":"Geva Bidner","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"11625105523359315411"},"user_tz":-120},"id":"kzfV5NtjZvnN","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:14.317169Z","iopub.execute_input":"2022-08-07T17:42:14.317807Z","iopub.status.idle":"2022-08-07T17:42:14.330721Z","shell.execute_reply.started":"2022-08-07T17:42:14.317756Z","shell.execute_reply":"2022-08-07T17:42:14.329564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's view the data","metadata":{"_cell_guid":"211e2bcb-e511-40de-9075-1628bc80e9dd","_uuid":"709b3ddd-5b34-49c6-b620-9c39a0415236","id":"ncNx_xNUQ2vj"}},{"cell_type":"markdown","source":"##### importing plotly lib for data viewing","metadata":{"_cell_guid":"1e9e1328-af86-4711-97a1-396cbc1a85fb","_uuid":"f28867e2-42cd-4979-aaa8-b76be1f81c6f"}},{"cell_type":"code","source":"","metadata":{"_cell_guid":"fdcbf51f-d0c3-4469-802b-fa50bf6a1a32","_uuid":"66b8fa5d-b718-4e9e-92f1-9e7949942b3b","jupyter":{"outputs_hidden":false},"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### let's make sure that we have all the dates","metadata":{"_cell_guid":"d175f119-b143-4472-a588-c0c08585f020","_uuid":"2b20c4ea-673d-429b-a3f2-188a1cefc26a"}},{"cell_type":"code","source":"# import numpy as np\nfrom datetime import datetime\nfrom datetime import datetime, timedelta\n\n\ndf_oil_fixed = df_oil.copy()\ndf_oil_fixed[\"date\"]= pd.to_datetime(df_oil_fixed[\"date\"])\n\ncount_add = 0\nfor date_idx, date_val in enumerate(df_oil_fixed['date']):\n    current_date = date_val\n    if date_idx == 0:\n        next_date = current_date\n    if date_idx != 0: \n        count_idx = date_idx + count_add\n        while(str(current_date).strip() != str(next_date).strip()):\n#             print(next_date)\n            # insert new line to df\n            new_row = [str(next_date).strip(),  np.nan]\n            df_oil_fixed = pd.DataFrame(np.insert(df_oil_fixed.values, count_idx, new_row, axis=0))\n            next_date = next_date + timedelta(days=1)\n            count_idx += 1\n            count_add +=1\n#             print(current_date)\n\n    next_date = next_date + timedelta(days=1)\n    \ndf_oil_fixed.columns = df_oil.columns\ndf_oil_fixed.head(10)","metadata":{"_cell_guid":"6b372608-b56e-4bed-aa93-ea1ca7305cd7","_uuid":"dd4129af-e97f-429d-9c7f-3bde1529635c","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:42:28.634806Z","iopub.execute_input":"2022-08-07T17:42:28.635201Z","iopub.status.idle":"2022-08-07T17:42:51.286810Z","shell.execute_reply.started":"2022-08-07T17:42:28.635169Z","shell.execute_reply":"2022-08-07T17:42:51.285340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We'll begin with the oil dataframe since we suspect it has missing values","metadata":{"_cell_guid":"58b4ea9e-bf25-4543-9dc9-6f6ee7cc0210","_uuid":"f1c03e7b-0d39-4fda-9bf0-2ac54e7edda9"}},{"cell_type":"code","source":"fig = px.line(df_oil_fixed, x='date', y=\"dcoilwtico\")\nfig.update_layout(title = \"Oil by Date\")\nfig.show()","metadata":{"_cell_guid":"9aec9387-ebc3-4a83-a09e-7f99b3966525","_uuid":"38ecfb8a-7b60-47f6-b3d2-3fcc18c5ab2a","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:00.827189Z","iopub.execute_input":"2022-08-07T17:43:00.827645Z","iopub.status.idle":"2022-08-07T17:43:00.937978Z","shell.execute_reply.started":"2022-08-07T17:43:00.827609Z","shell.execute_reply":"2022-08-07T17:43:00.936701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### This graph confirms our suspection - we have missing values! Let's fix those","metadata":{"_cell_guid":"0694bd88-2eef-4fd9-bdb2-749b2dd4662e","_uuid":"407c7b94-992b-4d83-b9e7-d90561bf2121"}},{"cell_type":"code","source":"df_oil_fixed = df_oil_fixed.set_index('date')\ndf_oil_fixed = df_oil_fixed['dcoilwtico'].resample('D').sum().reset_index()\ndf_oil_fixed = df_oil_fixed.replace({0:np.nan})\ndf_oil_fixed['dcoilwtico_interpolate'] = df_oil_fixed['dcoilwtico'].interpolate(limit_direction = 'both')","metadata":{"_cell_guid":"e069a158-f172-46a2-ba5c-9e58dccdec13","_uuid":"7f64a7e6-dc5c-4abf-9d13-1951648d48a3","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:06.677587Z","iopub.execute_input":"2022-08-07T17:43:06.678079Z","iopub.status.idle":"2022-08-07T17:43:06.696797Z","shell.execute_reply.started":"2022-08-07T17:43:06.678041Z","shell.execute_reply":"2022-08-07T17:43:06.695362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Let's view the fixed data","metadata":{"_cell_guid":"f9755716-8ff1-4060-a2f6-6e617fb1c4cd","_uuid":"1e19f2d8-9061-4d2f-8637-daea273df28c"}},{"cell_type":"code","source":"# Plot\np = df_oil_fixed.melt(id_vars=['date']+list(df_oil_proccesed.keys()[5:]), var_name='Legend')\npx.line(p.sort_values([\"Legend\", \"date\"], ascending = [False, True]), x='date', y='value', color='Legend',title = \"Daily Oil Price interpolated\" )","metadata":{"_cell_guid":"43a97f0f-e603-49a0-bcac-f28241194143","_uuid":"cdf01813-b723-4488-8eb2-878fd9bd4efa","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:08.011557Z","iopub.execute_input":"2022-08-07T17:43:08.012004Z","iopub.status.idle":"2022-08-07T17:43:08.165626Z","shell.execute_reply.started":"2022-08-07T17:43:08.011970Z","shell.execute_reply":"2022-08-07T17:43:08.164194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OIL DATAFRAME IS FIXED! :)","metadata":{"_cell_guid":"e4cef171-ec91-4a88-9be9-d93fb6c9e26c","_uuid":"4243c077-1f59-4f54-9509-67d6364cf149"}},{"cell_type":"markdown","source":"We'll view the rest the dataframes to make sure we didn't miss any more information","metadata":{"_cell_guid":"b04a796e-8d61-4b05-af29-dc5dd9264312","_uuid":"2ea43b39-23d8-44ae-b451-0bf205840045"}},{"cell_type":"code","source":"titles = [str(i) for i in range(0,10)]\nfig = make_subplots(rows=5, cols=2, subplot_titles = titles)\n\nfig.add_trace(go.Scatter(x=df_oil_fixed['date'], y=df_oil_fixed['dcoilwtico']),row=1, col=1)\nfig.add_trace(go.Scatter(x=df_stores['store_nbr'], y=df_stores['city']),row=1, col=2)\nfig.add_trace(go.Scatter(x=df_stores['store_nbr'], y=df_stores['state']),row=2, col=1)\nfig.add_trace(go.Scatter(x=df_stores['store_nbr'], y=df_stores['type']),row=2, col=2)\nfig.add_trace(go.Scatter(x=df_stores['store_nbr'], y=df_stores['cluster']),row=3, col=1)\nfig.add_trace(go.Bar(x=df_stores['state'], y=df_stores['type']),row=3, col=2)\nfig.add_trace(go.Histogram(x=df_stores['city'], y=df_stores['type']),row=4, col=1)\nfig.add_trace(go.Histogram(x=df_stores['cluster'], y=df_stores['type']),row=4, col=2)\n# fig = px.bar(long_df, x=\"nation\", y=\"count\", color=\"medal\", title=\"Long-Form Input\")","metadata":{"_cell_guid":"8a046709-9b41-481d-b1b6-b4c2e8306baa","_uuid":"2b6a8985-d8f7-4ad1-9482-4c9808e3e820","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:11.701020Z","iopub.execute_input":"2022-08-07T17:43:11.701762Z","iopub.status.idle":"2022-08-07T17:43:11.886877Z","shell.execute_reply.started":"2022-08-07T17:43:11.701720Z","shell.execute_reply":"2022-08-07T17:43:11.885875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We'll view some of the data to analyze it","metadata":{"_cell_guid":"ba393a08-873a-4018-af64-aacec6a170e0","_uuid":"c1fabd97-6660-41ee-82d7-95b4515e9e5b"}},{"cell_type":"code","source":"fig = px.histogram(df_stores, x=\"city\",color='type')\nfig.update_layout(title = \"Citys by Store Type\")\nfig.show()","metadata":{"_cell_guid":"aebe7ce2-aa98-4ba1-8e10-98fc1fe7848e","_uuid":"62e19b42-f75d-40e4-b627-d1726ef84a8d","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:13.378159Z","iopub.execute_input":"2022-08-07T17:43:13.378942Z","iopub.status.idle":"2022-08-07T17:43:13.457342Z","shell.execute_reply.started":"2022-08-07T17:43:13.378903Z","shell.execute_reply":"2022-08-07T17:43:13.456452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = make_subplots(rows=3, cols=2)\n\nfig.add_trace(go.Scatter(x=df_oil_fixed['date'], y=df_oil_fixed['dcoilwtico']),row=1, col=1)\nfig.add_trace(go.Scatter(x=df_stores['store_nbr'], y=df_stores['city']),row=1, col=2)","metadata":{"_cell_guid":"2da14247-a07e-41e0-bef6-8ada96dc7d63","_uuid":"1d14cecc-7c28-4e0b-94d5-eea415ab3032","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:13.702886Z","iopub.execute_input":"2022-08-07T17:43:13.703663Z","iopub.status.idle":"2022-08-07T17:43:13.818218Z","shell.execute_reply.started":"2022-08-07T17:43:13.703625Z","shell.execute_reply":"2022-08-07T17:43:13.817362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#change date datatype as datetime\ndf_train.date = pd.to_datetime(df_train.date)\ndf_test.date= pd.to_datetime(df_test.date)\n\ndf_train['year'] = df_train.date.dt.year\ndf_test['year'] = df_test.date.dt.year\n\ndf_train['month'] = df_train.date.dt.month\ndf_test['month'] = df_test.date.dt.month\n\ndf_train['dayofmonth'] = df_train.date.dt.day\ndf_test['dayofmonth'] = df_test.date.dt.day\n\ndf_train['dayofweek'] = df_train.date.dt.dayofweek\ndf_test['dayofweek'] = df_test.date.dt.dayofweek\n\ndf_train['dayname'] = df_train.date.dt.strftime('%A')\ndf_test['dayname'] = df_test.date.dt.strftime('%A')","metadata":{"_cell_guid":"4bcc7a21-e850-49d3-9120-f9e4870cf0cc","_uuid":"81f1510d-b14d-418f-8cf4-cd94a7bafd13","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:15.891115Z","iopub.execute_input":"2022-08-07T17:43:15.891865Z","iopub.status.idle":"2022-08-07T17:43:33.226392Z","shell.execute_reply.started":"2022-08-07T17:43:15.891827Z","shell.execute_reply":"2022-08-07T17:43:33.225031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.date[5200])\nprint(df_train.date[5200].year)\nprint(df_train.date[5200].month)\nprint(df_train.date[5200].day)\nprint(df_train.date[5200].dayofweek)\nprint(df_train.date[5200].strftime('%A'))\n\nprint(df_train.date[0])\nprint(df_train.date[0].year)\nprint(df_train.date[0].month)\nprint(df_train.date[0].day)\nprint(df_train.date[0].dayofweek)\nprint(df_train.date[0].strftime('%A'))","metadata":{"_cell_guid":"a1f0b970-7ba6-4573-ae98-77cb3bce3447","_uuid":"37134161-3850-41e8-a4e9-4bae0f36896f","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:33.228742Z","iopub.execute_input":"2022-08-07T17:43:33.229167Z","iopub.status.idle":"2022-08-07T17:43:33.239498Z","shell.execute_reply.started":"2022-08-07T17:43:33.229131Z","shell.execute_reply":"2022-08-07T17:43:33.237971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(10)","metadata":{"_cell_guid":"9afa8277-473f-4c50-bf0a-979ee298ab34","_uuid":"55529630-c8fd-4f6b-8c36-675d58ae9dcd","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:33.241253Z","iopub.execute_input":"2022-08-07T17:43:33.241696Z","iopub.status.idle":"2022-08-07T17:43:33.270261Z","shell.execute_reply.started":"2022-08-07T17:43:33.241660Z","shell.execute_reply":"2022-08-07T17:43:33.268799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"_cell_guid":"f7ed1e54-db43-4cff-99ca-f4378e7404eb","_uuid":"117b5969-fe0f-4a1f-a774-1b21454b0d6b"}},{"cell_type":"code","source":"df_train.columns\nfor i in df_train:\n    print(i)\n    print(df_train[i].dtypes)","metadata":{"_cell_guid":"253d32e6-0a04-48ef-be84-c93ff1e023d9","_uuid":"bc29aaba-ea8c-4747-9f47-d8159071b088","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:43:33.273913Z","iopub.execute_input":"2022-08-07T17:43:33.275402Z","iopub.status.idle":"2022-08-07T17:43:33.284008Z","shell.execute_reply.started":"2022-08-07T17:43:33.275358Z","shell.execute_reply":"2022-08-07T17:43:33.282736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing Holiday and Events data.\n\nSince we cannot tell which of the information is the most importent there, and we wouldn't want to drop any valueable data, we'll try to use most of it\n","metadata":{}},{"cell_type":"code","source":"# copy df holiday while we work on it\nholidays_events = df_holidays_events.copy()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:43:33.285970Z","iopub.execute_input":"2022-08-07T17:43:33.286372Z","iopub.status.idle":"2022-08-07T17:43:33.296929Z","shell.execute_reply.started":"2022-08-07T17:43:33.286337Z","shell.execute_reply":"2022-08-07T17:43:33.295538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# found this while reading other notebooks - \n# turns out there was an official transfer for new years eve holiday, but that didn't translate on a difference in sales.\n# so let's fix it\n\nholidays_events.loc[297, 'transferred'] = False\nholidays_events = holidays_events.loc[~(holidays_events.index == 298)]\nholidays_events.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:43:33.298974Z","iopub.execute_input":"2022-08-07T17:43:33.299721Z","iopub.status.idle":"2022-08-07T17:43:33.320882Z","shell.execute_reply.started":"2022-08-07T17:43:33.299675Z","shell.execute_reply":"2022-08-07T17:43:33.319561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that both \"decription\" and \"locale_name\" will provide more of less the same data. Therefore we decided to remove the description coloum","metadata":{}},{"cell_type":"code","source":"holidays_events = holidays_events.drop('description', axis = 1)\nholidays_events.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:43:33.322191Z","iopub.execute_input":"2022-08-07T17:43:33.323247Z","iopub.status.idle":"2022-08-07T17:43:33.341323Z","shell.execute_reply.started":"2022-08-07T17:43:33.323195Z","shell.execute_reply":"2022-08-07T17:43:33.340210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.merge(df_train,holidays_events,how='left',on=\"date\")\ndf_test = pd.merge(df_test,holidays_events,how='left',on=\"date\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:43:42.396349Z","iopub.execute_input":"2022-08-07T17:43:42.396855Z","iopub.status.idle":"2022-08-07T17:43:43.668723Z","shell.execute_reply.started":"2022-08-07T17:43:42.396817Z","shell.execute_reply":"2022-08-07T17:43:43.667312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['type'] = df_train['type'].fillna('Working-Day')\ndf_train['locale_name'] = df_train['locale_name'].fillna('Global')\ndf_train['transferred'] = df_train['transferred'].fillna(True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:43:59.579482Z","iopub.execute_input":"2022-08-07T17:43:59.580955Z","iopub.status.idle":"2022-08-07T17:44:00.329164Z","shell.execute_reply.started":"2022-08-07T17:43:59.580877Z","shell.execute_reply":"2022-08-07T17:44:00.327919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['type'] = df_test['type'].fillna('Working-Day')\ndf_test['locale_name'] = df_test['locale_name'].fillna('Global')\ndf_test['transferred'] = df_test['transferred'].fillna(True)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:44:04.053850Z","iopub.execute_input":"2022-08-07T17:44:04.054288Z","iopub.status.idle":"2022-08-07T17:44:04.086863Z","shell.execute_reply.started":"2022-08-07T17:44:04.054245Z","shell.execute_reply":"2022-08-07T17:44:04.085501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"categories = ['family','city','state','type']\ncategories = ['family', 'dayname', 'type', 'locale_name', 'locale']\nfor i in categories:\n    encoder = preprocessing.LabelEncoder()\n    df_train[i] = encoder.fit_transform(df_train[i])\n    df_test[i] =  encoder.transform(df_test[i])","metadata":{"_cell_guid":"e65a3580-5fdc-4e21-bbcc-086bd9e0f1f8","_uuid":"cbb0833c-254f-45e0-acda-13fa13a90ef7","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:44:17.083947Z","iopub.execute_input":"2022-08-07T17:44:17.085424Z","iopub.status.idle":"2022-08-07T17:44:20.969389Z","shell.execute_reply.started":"2022-08-07T17:44:17.085375Z","shell.execute_reply":"2022-08-07T17:44:20.968022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_oil = pd.merge(df_train,df_oil_fixed,how='left',on=\"date\")\ndf_test_oil = pd.merge(df_test,df_oil_fixed,how='left',on=\"date\")\ndf_train_oil.head(10)","metadata":{"_cell_guid":"5f275786-eb4e-4f7c-91dc-22d00b482cb8","_uuid":"71486c85-eaee-457b-86f8-61798076457a","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T18:06:22.226045Z","iopub.execute_input":"2022-08-07T18:06:22.226515Z","iopub.status.idle":"2022-08-07T18:06:23.031785Z","shell.execute_reply.started":"2022-08-07T18:06:22.226461Z","shell.execute_reply":"2022-08-07T18:06:23.030389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES = ['store_nbr','family','onpromotion','dcoilwtico_interpolate', \\\n           \"year\", \"month\", \"dayofmonth\", \"dayofweek\", \"dayname\", \"type\", \"locale\", \"locale_name\", \"transferred\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:44:33.283932Z","iopub.execute_input":"2022-08-07T17:44:33.284392Z","iopub.status.idle":"2022-08-07T17:44:33.291125Z","shell.execute_reply.started":"2022-08-07T17:44:33.284357Z","shell.execute_reply":"2022-08-07T17:44:33.289657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nTARGET =['sales']\nX_train,X_val,y_train,y_val = train_test_split(df_train_oil[FEATURES],df_train_oil[TARGET],test_size=0.05,shuffle=False)","metadata":{"_cell_guid":"335d6ae8-6d16-41f1-b514-45a834136e4b","_uuid":"cd5d9a69-8d64-4f31-a3d2-441a782476a0","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:44:34.206930Z","iopub.execute_input":"2022-08-07T17:44:34.207832Z","iopub.status.idle":"2022-08-07T17:44:35.380980Z","shell.execute_reply.started":"2022-08-07T17:44:34.207789Z","shell.execute_reply":"2022-08-07T17:44:35.379401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Linear Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression, HuberRegressor\nfrom sklearn.metrics import mean_squared_log_error\n\nlinear = LinearRegression()\nmodel = linear.fit(X_train[FEATURES],y_train)","metadata":{"_cell_guid":"636ee431-9227-4296-aad7-13770715d4ee","_uuid":"3bbe920d-1484-475f-93ba-0f21fc933388","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:44:37.010523Z","iopub.execute_input":"2022-08-07T17:44:37.011030Z","iopub.status.idle":"2022-08-07T17:44:42.589249Z","shell.execute_reply.started":"2022-08-07T17:44:37.010994Z","shell.execute_reply":"2022-08-07T17:44:42.587941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions= model.predict(X_val[FEATURES])\npredictions = [a if a>0 else 0 for a in predictions]\n# print('MSLE: ' + str(mean_squared_log_error(y_val,predictions)))","metadata":{"_cell_guid":"a61ee5b7-b56c-43f9-82d9-346644bfca1a","_uuid":"d35a4cf3-731a-4428-a78e-d321a7f4459b","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:44:42.591231Z","iopub.execute_input":"2022-08-07T17:44:42.591679Z","iopub.status.idle":"2022-08-07T17:44:43.149438Z","shell.execute_reply.started":"2022-08-07T17:44:42.591641Z","shell.execute_reply":"2022-08-07T17:44:43.148440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test = pd.merge(df_test,df_oil_fixed,how='left',on=\"date\")\n# df_test.head(10)","metadata":{"_cell_guid":"dc480918-b2cd-4b6b-b55f-425a9d627bd7","_uuid":"3e7bb0e9-ec6d-4bdd-8994-309786d93e71","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:38:23.719637Z","iopub.execute_input":"2022-08-07T17:38:23.720798Z","iopub.status.idle":"2022-08-07T17:38:23.726590Z","shell.execute_reply.started":"2022-08-07T17:38:23.720744Z","shell.execute_reply":"2022-08-07T17:38:23.725373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# predictions= model.predict(df_test[FEATURES])\n# predictions = [a if a>0 else 0 for a in predictions]","metadata":{"_cell_guid":"4b2cf474-85f3-4ca9-b503-7e30104f07dd","_uuid":"c4c8f4ca-141d-4842-8b36-ffa5e6008754","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:38:23.728206Z","iopub.execute_input":"2022-08-07T17:38:23.729707Z","iopub.status.idle":"2022-08-07T17:38:23.740124Z","shell.execute_reply.started":"2022-08-07T17:38:23.729659Z","shell.execute_reply":"2022-08-07T17:38:23.738800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.plotting.scatter_matrix(df_train_oil.sample(n=100000, random_state=42), alpha=0.2, figsize=(20,20));","metadata":{"_cell_guid":"2cdb959a-087d-4a1d-a35e-16905533f1ca","_uuid":"119526e8-a4d9-43f9-b321-ec94011c6eae","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2022-08-07T17:38:23.742048Z","iopub.execute_input":"2022-08-07T17:38:23.743260Z","iopub.status.idle":"2022-08-07T17:38:23.753136Z","shell.execute_reply.started":"2022-08-07T17:38:23.743214Z","shell.execute_reply":"2022-08-07T17:38:23.751649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Bagging And Random Forest","metadata":{"_cell_guid":"846971f3-8b3f-4052-8ee1-43fcc100c832","_uuid":"aedbcd06-1b1c-413c-8b63-4c851f98c6f0"}},{"cell_type":"markdown","source":"## Train test split","metadata":{}},{"cell_type":"code","source":"X = df_train_oil.drop(['sales','dcoilwtico','date'], axis=1,inplace=False)\ny = df_train_oil['sales']","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:44:46.390673Z","iopub.execute_input":"2022-08-07T17:44:46.391174Z","iopub.status.idle":"2022-08-07T17:44:46.591267Z","shell.execute_reply.started":"2022-08-07T17:44:46.391135Z","shell.execute_reply":"2022-08-07T17:44:46.589876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T17:44:47.788732Z","iopub.execute_input":"2022-08-07T17:44:47.789837Z","iopub.status.idle":"2022-08-07T17:44:47.807898Z","shell.execute_reply.started":"2022-08-07T17:44:47.789794Z","shell.execute_reply":"2022-08-07T17:44:47.806320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=SEED)","metadata":{"_cell_guid":"e0adeb76-a03b-40a1-8c1e-2112f488ab55","_uuid":"c1facc34-ba8a-4902-9c9d-93f895081e5f","execution":{"iopub.status.busy":"2022-08-07T17:44:49.655090Z","iopub.execute_input":"2022-08-07T17:44:49.655561Z","iopub.status.idle":"2022-08-07T17:44:51.303882Z","shell.execute_reply.started":"2022-08-07T17:44:49.655517Z","shell.execute_reply":"2022-08-07T17:44:51.302431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Instantiate br and rfr\nbr = BaggingRegressor(n_estimators=5, random_state=SEED,n_jobs=-1)\nrfr = RandomForestRegressor( n_estimators=5, random_state=SEED,n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:02:55.191442Z","iopub.execute_input":"2022-08-07T18:02:55.192785Z","iopub.status.idle":"2022-08-07T18:02:55.198414Z","shell.execute_reply.started":"2022-08-07T18:02:55.192692Z","shell.execute_reply":"2022-08-07T18:02:55.197566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit br to the training set\nbr.fit(X_train, y_train)\n\n# Predict sales for X_test\ny_pred = br.predict(X_test)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-08-07T18:02:58.549358Z","iopub.execute_input":"2022-08-07T18:02:58.550764Z","iopub.status.idle":"2022-08-07T18:04:18.111422Z","shell.execute_reply.started":"2022-08-07T18:02:58.550700Z","shell.execute_reply":"2022-08-07T18:04:18.109865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc_test = mean_squared_log_error(y_pred, y_test)\nprint('Test set RMSE of bagging regressor: {:.2f}'.format(acc_test)) ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:04:18.114270Z","iopub.execute_input":"2022-08-07T18:04:18.114713Z","iopub.status.idle":"2022-08-07T18:04:18.164643Z","shell.execute_reply.started":"2022-08-07T18:04:18.114674Z","shell.execute_reply":"2022-08-07T18:04:18.163076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit br to the training set\nrfr.fit(X_train, y_train)\n\n\n# Predict sales for X_test\ny_pred = rfr.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:09:21.077915Z","iopub.execute_input":"2022-08-07T18:09:21.078415Z","iopub.status.idle":"2022-08-07T18:10:20.296981Z","shell.execute_reply.started":"2022-08-07T18:09:21.078378Z","shell.execute_reply":"2022-08-07T18:10:20.295777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc_test_2 = mean_squared_log_error(y_pred, y_test)\nprint('Test set RMSE of rfr regressor: {:.2f}'.format(acc_test_2)) ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:05:39.148619Z","iopub.execute_input":"2022-08-07T18:05:39.148995Z","iopub.status.idle":"2022-08-07T18:05:39.202182Z","shell.execute_reply.started":"2022-08-07T18:05:39.148963Z","shell.execute_reply":"2022-08-07T18:05:39.200884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tuning Bagging and Random Forest","metadata":{}},{"cell_type":"code","source":"# n_estimators_list = range(1,21)\n# br_rmsle_list = []\n# rfr_rmsle_list= []\n# for n in n_estimators_list:\n#     br = BaggingRegressor(n_estimators=n, random_state=SEED,n_jobs=-1)\n#     rfr = RandomForestRegressor( n_estimators=n, random_state=SEED,n_jobs=-1)\n#     br.fit(X_train, y_train)\n#     rfr.fit(X_train, y_train)\n#     y_pred = br.predict(X_test)\n#     y_pred_2 = rfr.predict(X_test)\n#     acc_test=mean_squared_log_error(y_pred, y_test)\n#     br_rmsle_list.append(acc_test)\n#     acc_test_2=mean_squared_log_error(y_pred_2, y_test)\n#     rfr_rmsle_list.append(acc_test_2)\n# print(\"Bagging Regressor RMLSE for n=1,..,20:\", br_rmsle_list)\n# print(\"Random Forest Regressor RMLSE for n=1,..,20:\", rfr_rmsle_list)   ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:24:09.584517Z","iopub.execute_input":"2022-08-07T18:24:09.585136Z","iopub.status.idle":"2022-08-07T18:24:09.591869Z","shell.execute_reply.started":"2022-08-07T18:24:09.585093Z","shell.execute_reply":"2022-08-07T18:24:09.590349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = px.line(y=br_rmsle_list,x=range(1,21))\n# fig.update_layout(\n#     title=\"RMLSE Versus N for Bagging\",\n#     xaxis_title=\"Number of Estimators\",\n#     yaxis_title=\"RMLSE\")\n# fig.show()\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-08-07T18:24:10.893314Z","iopub.execute_input":"2022-08-07T18:24:10.894516Z","iopub.status.idle":"2022-08-07T18:24:10.900519Z","shell.execute_reply.started":"2022-08-07T18:24:10.894458Z","shell.execute_reply":"2022-08-07T18:24:10.898522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = px.line(y=rfr_rmsle_list,x=range(1,21))\n# fig.update_layout(\n#     title=\"RMLSE Versus N for Random Forest\",\n#     xaxis_title=\"Number of Estimators\",\n#     yaxis_title=\"RMLSE\")\n# fig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:24:11.812111Z","iopub.execute_input":"2022-08-07T18:24:11.813219Z","iopub.status.idle":"2022-08-07T18:24:11.819314Z","shell.execute_reply.started":"2022-08-07T18:24:11.813167Z","shell.execute_reply":"2022-08-07T18:24:11.817864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Values","metadata":{}},{"cell_type":"code","source":"X_submission = df_test_oil.drop(['dcoilwtico','date'], axis=1,inplace=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:11:22.242841Z","iopub.execute_input":"2022-08-07T18:11:22.243400Z","iopub.status.idle":"2022-08-07T18:11:22.259571Z","shell.execute_reply.started":"2022-08-07T18:11:22.243360Z","shell.execute_reply":"2022-08-07T18:11:22.258002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv('../input/store-sales-time-series-forecasting/sample_submission.csv')\ndf_sub_br  = df_sub.copy()\ndf_sub_rfr  = df_sub.copy()\ndf_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:22:22.338746Z","iopub.execute_input":"2022-08-07T18:22:22.340072Z","iopub.status.idle":"2022-08-07T18:22:22.361714Z","shell.execute_reply.started":"2022-08-07T18:22:22.340023Z","shell.execute_reply":"2022-08-07T18:22:22.360369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submission_br = pd.DataFrame(br.predict(X_submission), index = X_submission.index, columns = [\"sales\"]).clip(0.)\ny_submission_br.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:22:23.185965Z","iopub.execute_input":"2022-08-07T18:22:23.187405Z","iopub.status.idle":"2022-08-07T18:22:24.331360Z","shell.execute_reply.started":"2022-08-07T18:22:23.187346Z","shell.execute_reply":"2022-08-07T18:22:24.330389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submission_rfr = pd.DataFrame(rfr.predict(X_submission), index = X_submission.index, columns = [\"sales\"]).clip(0.)\ny_submission_rfr.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:22:24.475434Z","iopub.execute_input":"2022-08-07T18:22:24.476803Z","iopub.status.idle":"2022-08-07T18:22:24.626321Z","shell.execute_reply.started":"2022-08-07T18:22:24.476747Z","shell.execute_reply":"2022-08-07T18:22:24.624841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_br.sales = y_submission_br.values\ndf_sub_br.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:22:26.554975Z","iopub.execute_input":"2022-08-07T18:22:26.555572Z","iopub.status.idle":"2022-08-07T18:22:26.570621Z","shell.execute_reply.started":"2022-08-07T18:22:26.555524Z","shell.execute_reply":"2022-08-07T18:22:26.569211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_rfr.sales = y_submission_rfr.values\ndf_sub_rfr.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:22:31.892592Z","iopub.execute_input":"2022-08-07T18:22:31.893107Z","iopub.status.idle":"2022-08-07T18:22:31.905802Z","shell.execute_reply.started":"2022-08-07T18:22:31.893069Z","shell.execute_reply":"2022-08-07T18:22:31.904819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_rfr.to_csv('submission.csv', index=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T18:24:42.572466Z","iopub.execute_input":"2022-08-07T18:24:42.572935Z","iopub.status.idle":"2022-08-07T18:24:42.644359Z","shell.execute_reply.started":"2022-08-07T18:24:42.572892Z","shell.execute_reply":"2022-08-07T18:24:42.642972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}