{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T00:12:01.722975Z","iopub.execute_input":"2022-07-22T00:12:01.723358Z","iopub.status.idle":"2022-07-22T00:12:01.733156Z","shell.execute_reply.started":"2022-07-22T00:12:01.723327Z","shell.execute_reply":"2022-07-22T00:12:01.731973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--read all files---------------------------------------------","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/store-sales-time-series-forecasting/train.csv\")\ntest = pd.read_csv(\"../input/store-sales-time-series-forecasting/test.csv\")\nstores = pd.read_csv(\"../input/store-sales-time-series-forecasting/stores.csv\")\ntransactions = pd.read_csv(\"../input/store-sales-time-series-forecasting/transactions.csv\")\noil = pd.read_csv(\"../input/store-sales-time-series-forecasting/oil.csv\")\nholidays = pd.read_csv(\"../input/store-sales-time-series-forecasting/holidays_events.csv\")\nsubmission = pd.read_csv(\"../input/store-sales-time-series-forecasting/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:01.735633Z","iopub.execute_input":"2022-07-22T00:12:01.736454Z","iopub.status.idle":"2022-07-22T00:12:03.870124Z","shell.execute_reply.started":"2022-07-22T00:12:01.736409Z","shell.execute_reply":"2022-07-22T00:12:03.868783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--format dates-----------------------------------------------","metadata":{}},{"cell_type":"code","source":"train['date'] = pd.to_datetime(train['date'])\ntest['date'] = pd.to_datetime(test['date'])\ntransactions['date'] = pd.to_datetime(transactions['date'])\noil['date'] = pd.to_datetime(oil['date'])\nholidays['date'] = pd.to_datetime(holidays['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:03.873038Z","iopub.execute_input":"2022-07-22T00:12:03.873550Z","iopub.status.idle":"2022-07-22T00:12:04.366189Z","shell.execute_reply.started":"2022-07-22T00:12:03.873493Z","shell.execute_reply":"2022-07-22T00:12:04.365113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--show train and test data-----------------------------------","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.367639Z","iopub.execute_input":"2022-07-22T00:12:04.368080Z","iopub.status.idle":"2022-07-22T00:12:04.388422Z","shell.execute_reply.started":"2022-07-22T00:12:04.368050Z","shell.execute_reply":"2022-07-22T00:12:04.387312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.393184Z","iopub.execute_input":"2022-07-22T00:12:04.393931Z","iopub.status.idle":"2022-07-22T00:12:04.412407Z","shell.execute_reply.started":"2022-07-22T00:12:04.393886Z","shell.execute_reply":"2022-07-22T00:12:04.411406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.413570Z","iopub.execute_input":"2022-07-22T00:12:04.414183Z","iopub.status.idle":"2022-07-22T00:12:04.433671Z","shell.execute_reply.started":"2022-07-22T00:12:04.414139Z","shell.execute_reply":"2022-07-22T00:12:04.432433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--split train data for train and test------------------------","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\ntrain_train = train[train.date < datetime(2017, 1, 1)]\ntrain_test = train[train.date >= datetime(2017, 1, 1)]\ntrain_test = train_test.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.435242Z","iopub.execute_input":"2022-07-22T00:12:04.436173Z","iopub.status.idle":"2022-07-22T00:12:04.617007Z","shell.execute_reply.started":"2022-07-22T00:12:04.436137Z","shell.execute_reply":"2022-07-22T00:12:04.615887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_train","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.618503Z","iopub.execute_input":"2022-07-22T00:12:04.618955Z","iopub.status.idle":"2022-07-22T00:12:04.639976Z","shell.execute_reply.started":"2022-07-22T00:12:04.618913Z","shell.execute_reply":"2022-07-22T00:12:04.638910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.642139Z","iopub.execute_input":"2022-07-22T00:12:04.642581Z","iopub.status.idle":"2022-07-22T00:12:04.662357Z","shell.execute_reply.started":"2022-07-22T00:12:04.642539Z","shell.execute_reply":"2022-07-22T00:12:04.661015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_original = train_test.copy()\ntrain_test_original.drop(columns = ['index','date','store_nbr','family','onpromotion'] , inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.663942Z","iopub.execute_input":"2022-07-22T00:12:04.664281Z","iopub.status.idle":"2022-07-22T00:12:04.684971Z","shell.execute_reply.started":"2022-07-22T00:12:04.664252Z","shell.execute_reply":"2022-07-22T00:12:04.683797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_original","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.688903Z","iopub.execute_input":"2022-07-22T00:12:04.689216Z","iopub.status.idle":"2022-07-22T00:12:04.702420Z","shell.execute_reply.started":"2022-07-22T00:12:04.689188Z","shell.execute_reply":"2022-07-22T00:12:04.701513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_prediction = train_test_original.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.703658Z","iopub.execute_input":"2022-07-22T00:12:04.704038Z","iopub.status.idle":"2022-07-22T00:12:04.716423Z","shell.execute_reply.started":"2022-07-22T00:12:04.704007Z","shell.execute_reply":"2022-07-22T00:12:04.715426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--prepare train data-------------------","metadata":{}},{"cell_type":"code","source":"train_train_temp = train_train.copy()\ntrain_train_sales = train_train_temp['sales']\ntrain_train_temp.drop(columns = ['sales'] , inplace = True , axis = 1)\ntrain_train_category = pd.get_dummies(train_train_temp, columns=['store_nbr','family'], drop_first=True)\n\ntrain_test_temp = train_test.copy()\ntrain_test_temp.drop(columns = ['index','sales'] , inplace = True , axis = 1)\ntrain_test_category = pd.get_dummies(train_test_temp, columns=['store_nbr','family'], drop_first=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:04.717754Z","iopub.execute_input":"2022-07-22T00:12:04.718601Z","iopub.status.idle":"2022-07-22T00:12:06.751660Z","shell.execute_reply.started":"2022-07-22T00:12:04.718521Z","shell.execute_reply":"2022-07-22T00:12:06.750081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_train_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:06.753084Z","iopub.execute_input":"2022-07-22T00:12:06.753464Z","iopub.status.idle":"2022-07-22T00:12:06.772402Z","shell.execute_reply.started":"2022-07-22T00:12:06.753430Z","shell.execute_reply":"2022-07-22T00:12:06.771002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:06.774040Z","iopub.execute_input":"2022-07-22T00:12:06.774423Z","iopub.status.idle":"2022-07-22T00:12:06.798680Z","shell.execute_reply.started":"2022-07-22T00:12:06.774386Z","shell.execute_reply":"2022-07-22T00:12:06.797454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--preprocessing train and test data--------------------------","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\ntrain_standard = StandardScaler()\ntest_standard = StandardScaler()\n\ntrain_train_category_temp = train_train_category.copy()\ntrain_standard.fit(train_train_category_temp[['onpromotion']])\ntrain_standard_temp = pd.DataFrame(train_standard.transform(train_train_category_temp[['onpromotion']]))\ntrain_train_category [['onpromotion']] = train_standard_temp\n\ntrain_test_category_temp = train_test_category.copy()\ntest_standard.fit(train_test_temp[['onpromotion']])\ntest_standard_temp = pd.DataFrame(test_standard.transform(train_test_category_temp[['onpromotion']]))\ntrain_test_category [['onpromotion']] = test_standard_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:06.799987Z","iopub.execute_input":"2022-07-22T00:12:06.800295Z","iopub.status.idle":"2022-07-22T00:12:07.215992Z","shell.execute_reply.started":"2022-07-22T00:12:06.800267Z","shell.execute_reply":"2022-07-22T00:12:07.214881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--remove id and dates on category data-----------------------","metadata":{}},{"cell_type":"code","source":"train_train_category.drop(columns = ['id','date'], inplace = True , axis = 1)\ntrain_test_category.drop(columns = ['id','date'], inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:07.217691Z","iopub.execute_input":"2022-07-22T00:12:07.217997Z","iopub.status.idle":"2022-07-22T00:12:07.807251Z","shell.execute_reply.started":"2022-07-22T00:12:07.217970Z","shell.execute_reply":"2022-07-22T00:12:07.806198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--view data------------------------","metadata":{}},{"cell_type":"code","source":"train_train_sales","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:07.808622Z","iopub.execute_input":"2022-07-22T00:12:07.808923Z","iopub.status.idle":"2022-07-22T00:12:07.818153Z","shell.execute_reply.started":"2022-07-22T00:12:07.808896Z","shell.execute_reply":"2022-07-22T00:12:07.816990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_train_category","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:07.819989Z","iopub.execute_input":"2022-07-22T00:12:07.820763Z","iopub.status.idle":"2022-07-22T00:12:08.016806Z","shell.execute_reply.started":"2022-07-22T00:12:07.820719Z","shell.execute_reply":"2022-07-22T00:12:08.015620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_category","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:08.018289Z","iopub.execute_input":"2022-07-22T00:12:08.019302Z","iopub.status.idle":"2022-07-22T00:12:08.059289Z","shell.execute_reply.started":"2022-07-22T00:12:08.019258Z","shell.execute_reply":"2022-07-22T00:12:08.058416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--train model using sklearn LinearRegression-----------------","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\ntrain_model = LinearRegression()\ntrain_model.fit(train_train_category, train_train_sales)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:08.060476Z","iopub.execute_input":"2022-07-22T00:12:08.061216Z","iopub.status.idle":"2022-07-22T00:12:22.654511Z","shell.execute_reply.started":"2022-07-22T00:12:08.061184Z","shell.execute_reply":"2022-07-22T00:12:22.653338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--predict sales for test-------------------------------------","metadata":{}},{"cell_type":"code","source":"train_test_predict = train_model.predict(train_test_category)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:22.656081Z","iopub.execute_input":"2022-07-22T00:12:22.656772Z","iopub.status.idle":"2022-07-22T00:12:22.804396Z","shell.execute_reply.started":"2022-07-22T00:12:22.656730Z","shell.execute_reply":"2022-07-22T00:12:22.803028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_predict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:22.805889Z","iopub.execute_input":"2022-07-22T00:12:22.806309Z","iopub.status.idle":"2022-07-22T00:12:22.819332Z","shell.execute_reply.started":"2022-07-22T00:12:22.806272Z","shell.execute_reply":"2022-07-22T00:12:22.816096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nfor predict_sales in train_test_predict:\n if predict_sales<0 :\n    train_test_predict[i] = 0\n i = i+1\n        \ntrain_test_predict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:22.821006Z","iopub.execute_input":"2022-07-22T00:12:22.821555Z","iopub.status.idle":"2022-07-22T00:12:23.191063Z","shell.execute_reply.started":"2022-07-22T00:12:22.821514Z","shell.execute_reply":"2022-07-22T00:12:23.189026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--save test prediction in required format-----","metadata":{}},{"cell_type":"code","source":"train_test_prediction['sales']= train_test_predict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.192582Z","iopub.execute_input":"2022-07-22T00:12:23.193388Z","iopub.status.idle":"2022-07-22T00:12:23.200037Z","shell.execute_reply.started":"2022-07-22T00:12:23.193325Z","shell.execute_reply":"2022-07-22T00:12:23.198894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--compare training test data and prediction data----","metadata":{}},{"cell_type":"code","source":"train_test_original","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.201998Z","iopub.execute_input":"2022-07-22T00:12:23.202353Z","iopub.status.idle":"2022-07-22T00:12:23.218384Z","shell.execute_reply.started":"2022-07-22T00:12:23.202321Z","shell.execute_reply":"2022-07-22T00:12:23.217552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test_prediction","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.219799Z","iopub.execute_input":"2022-07-22T00:12:23.220404Z","iopub.status.idle":"2022-07-22T00:12:23.240597Z","shell.execute_reply.started":"2022-07-22T00:12:23.220347Z","shell.execute_reply":"2022-07-22T00:12:23.239225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\ntrain_test_original_sales = train_test_original['sales']\ntrain_test_prediction_sales = train_test_prediction['sales']","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.241884Z","iopub.execute_input":"2022-07-22T00:12:23.242689Z","iopub.status.idle":"2022-07-22T00:12:23.251323Z","shell.execute_reply.started":"2022-07-22T00:12:23.242654Z","shell.execute_reply":"2022-07-22T00:12:23.250552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\nevaluation_rmsle = metrics.mean_squared_log_error(train_test_original_sales, train_test_prediction_sales, squared=False)\nevaluation_rmsle","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.258476Z","iopub.execute_input":"2022-07-22T00:12:23.259064Z","iopub.status.idle":"2022-07-22T00:12:23.284346Z","shell.execute_reply.started":"2022-07-22T00:12:23.259028Z","shell.execute_reply":"2022-07-22T00:12:23.283316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"/////////////////////////////////////////////////////\ntrainデータのみの学習で、2.0812の評価値が得られた。\nさらに、oilデータを考慮した学習も試してみる。\n/////////////////////////////////////////////////////","metadata":{}},{"cell_type":"markdown","source":"--implement oil data------------------","metadata":{}},{"cell_type":"code","source":"oil","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.286147Z","iopub.execute_input":"2022-07-22T00:12:23.286635Z","iopub.status.idle":"2022-07-22T00:12:23.303701Z","shell.execute_reply.started":"2022-07-22T00:12:23.286592Z","shell.execute_reply":"2022-07-22T00:12:23.302837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil['dcoilwtico'].plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.305073Z","iopub.execute_input":"2022-07-22T00:12:23.306224Z","iopub.status.idle":"2022-07-22T00:12:23.490351Z","shell.execute_reply.started":"2022-07-22T00:12:23.306181Z","shell.execute_reply":"2022-07-22T00:12:23.489507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.491740Z","iopub.execute_input":"2022-07-22T00:12:23.492601Z","iopub.status.idle":"2022-07-22T00:12:23.500890Z","shell.execute_reply.started":"2022-07-22T00:12:23.492565Z","shell.execute_reply":"2022-07-22T00:12:23.499816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data = oil.copy()\noil_data['dcoilwtico'].fillna(oil_data['dcoilwtico'].median(skipna=True), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.502437Z","iopub.execute_input":"2022-07-22T00:12:23.502965Z","iopub.status.idle":"2022-07-22T00:12:23.511594Z","shell.execute_reply.started":"2022-07-22T00:12:23.502931Z","shell.execute_reply":"2022-07-22T00:12:23.510715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.512807Z","iopub.execute_input":"2022-07-22T00:12:23.513923Z","iopub.status.idle":"2022-07-22T00:12:23.530383Z","shell.execute_reply.started":"2022-07-22T00:12:23.513887Z","shell.execute_reply":"2022-07-22T00:12:23.529427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_temp = train.copy()\noil_temp = pd.DataFrame(oil_data,columns = ['date','dcoilwtico'])\noil_train = pd.merge(train_temp,oil_temp,left_on='date',right_on = 'date',how = 'left')\noil_train","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:23.531819Z","iopub.execute_input":"2022-07-22T00:12:23.532421Z","iopub.status.idle":"2022-07-22T00:12:24.123432Z","shell.execute_reply.started":"2022-07-22T00:12:23.532389Z","shell.execute_reply":"2022-07-22T00:12:24.122296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:24.125117Z","iopub.execute_input":"2022-07-22T00:12:24.125838Z","iopub.status.idle":"2022-07-22T00:12:24.497780Z","shell.execute_reply.started":"2022-07-22T00:12:24.125794Z","shell.execute_reply":"2022-07-22T00:12:24.496435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_fix = oil_train.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:24.500307Z","iopub.execute_input":"2022-07-22T00:12:24.500890Z","iopub.status.idle":"2022-07-22T00:12:25.046372Z","shell.execute_reply.started":"2022-07-22T00:12:24.500857Z","shell.execute_reply":"2022-07-22T00:12:25.045265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_fix.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.047765Z","iopub.execute_input":"2022-07-22T00:12:25.048108Z","iopub.status.idle":"2022-07-22T00:12:25.302716Z","shell.execute_reply.started":"2022-07-22T00:12:25.048069Z","shell.execute_reply":"2022-07-22T00:12:25.301666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\noil_train_train = oil_train_fix[oil_train_fix.date < datetime(2017, 1, 1)]\noil_train_test = oil_train_fix[oil_train_fix.date >= datetime(2017, 1, 1)]\noil_train_test = oil_train_test.reset_index()\noil_train_test.drop(columns = ['index'] , inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.306383Z","iopub.execute_input":"2022-07-22T00:12:25.306736Z","iopub.status.idle":"2022-07-22T00:12:25.418551Z","shell.execute_reply.started":"2022-07-22T00:12:25.306704Z","shell.execute_reply":"2022-07-22T00:12:25.417507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_train","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.419860Z","iopub.execute_input":"2022-07-22T00:12:25.420158Z","iopub.status.idle":"2022-07-22T00:12:25.440328Z","shell.execute_reply.started":"2022-07-22T00:12:25.420132Z","shell.execute_reply":"2022-07-22T00:12:25.439405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.441608Z","iopub.execute_input":"2022-07-22T00:12:25.442116Z","iopub.status.idle":"2022-07-22T00:12:25.460802Z","shell.execute_reply.started":"2022-07-22T00:12:25.442088Z","shell.execute_reply":"2022-07-22T00:12:25.459767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_original = oil_train_test.copy()\noil_train_test_original.drop(columns = ['date','store_nbr','family','onpromotion'] , inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.462233Z","iopub.execute_input":"2022-07-22T00:12:25.462867Z","iopub.status.idle":"2022-07-22T00:12:25.474702Z","shell.execute_reply.started":"2022-07-22T00:12:25.462832Z","shell.execute_reply":"2022-07-22T00:12:25.473454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_original","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.476426Z","iopub.execute_input":"2022-07-22T00:12:25.477253Z","iopub.status.idle":"2022-07-22T00:12:25.491580Z","shell.execute_reply.started":"2022-07-22T00:12:25.477221Z","shell.execute_reply":"2022-07-22T00:12:25.490455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_prediction = oil_train_test_original.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.493281Z","iopub.execute_input":"2022-07-22T00:12:25.493962Z","iopub.status.idle":"2022-07-22T00:12:25.501214Z","shell.execute_reply.started":"2022-07-22T00:12:25.493905Z","shell.execute_reply":"2022-07-22T00:12:25.499774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_train_temp = oil_train_train.copy()\noil_train_train_sales = oil_train_train_temp['sales']\noil_train_train_temp.drop(columns = ['sales'] , inplace = True , axis = 1)\noil_train_train_category = pd.get_dummies(oil_train_train_temp, columns=['store_nbr','family'], drop_first=True)\n\noil_train_test.drop(columns = ['sales'] , inplace = True , axis = 1)\noil_train_test_category = pd.get_dummies(oil_train_test, columns=['store_nbr','family'], drop_first=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:25.502608Z","iopub.execute_input":"2022-07-22T00:12:25.503479Z","iopub.status.idle":"2022-07-22T00:12:26.924386Z","shell.execute_reply.started":"2022-07-22T00:12:25.503435Z","shell.execute_reply":"2022-07-22T00:12:26.923190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_train_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:26.926069Z","iopub.execute_input":"2022-07-22T00:12:26.926626Z","iopub.status.idle":"2022-07-22T00:12:26.944311Z","shell.execute_reply.started":"2022-07-22T00:12:26.926594Z","shell.execute_reply":"2022-07-22T00:12:26.943542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:26.945645Z","iopub.execute_input":"2022-07-22T00:12:26.946194Z","iopub.status.idle":"2022-07-22T00:12:26.970718Z","shell.execute_reply.started":"2022-07-22T00:12:26.946161Z","shell.execute_reply":"2022-07-22T00:12:26.969401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\ntrain_standard = StandardScaler()\ntest_standard = StandardScaler()\n\noil_train_train_temp2 = oil_train_train_category.copy()\ntrain_standard.fit(oil_train_train_temp[['onpromotion']])\noil_train_standard_temp = pd.DataFrame(train_standard.transform(oil_train_train_temp2[['onpromotion']]))\noil_train_train_category [['onpromotion']] = train_standard_temp\n\noil_train_test_temp = oil_train_test_category.copy()\ntest_standard.fit(oil_train_test_temp[['onpromotion']])\noil_test_standard_temp = pd.DataFrame(test_standard.transform(oil_train_test_temp[['onpromotion']]))\noil_train_test_category [['onpromotion']] = test_standard_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:26.971929Z","iopub.execute_input":"2022-07-22T00:12:26.972776Z","iopub.status.idle":"2022-07-22T00:12:27.312284Z","shell.execute_reply.started":"2022-07-22T00:12:26.972735Z","shell.execute_reply":"2022-07-22T00:12:27.310969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_train_category.drop(columns = ['id','date'], inplace = True , axis = 1)\noil_train_test_category.drop(columns = ['id','date'], inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:27.314027Z","iopub.execute_input":"2022-07-22T00:12:27.314448Z","iopub.status.idle":"2022-07-22T00:12:27.755624Z","shell.execute_reply.started":"2022-07-22T00:12:27.314409Z","shell.execute_reply":"2022-07-22T00:12:27.754404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\ntrain_model = LinearRegression()\ntrain_model.fit(oil_train_train_category, oil_train_train_sales)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:27.757224Z","iopub.execute_input":"2022-07-22T00:12:27.757936Z","iopub.status.idle":"2022-07-22T00:12:37.946815Z","shell.execute_reply.started":"2022-07-22T00:12:27.757895Z","shell.execute_reply":"2022-07-22T00:12:37.945559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_category","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:37.948781Z","iopub.execute_input":"2022-07-22T00:12:37.949555Z","iopub.status.idle":"2022-07-22T00:12:38.007807Z","shell.execute_reply.started":"2022-07-22T00:12:37.949507Z","shell.execute_reply":"2022-07-22T00:12:38.006520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_predict = train_model.predict(oil_train_test_category)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.010231Z","iopub.execute_input":"2022-07-22T00:12:38.011051Z","iopub.status.idle":"2022-07-22T00:12:38.117774Z","shell.execute_reply.started":"2022-07-22T00:12:38.011000Z","shell.execute_reply":"2022-07-22T00:12:38.116334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_category","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.119930Z","iopub.execute_input":"2022-07-22T00:12:38.120785Z","iopub.status.idle":"2022-07-22T00:12:38.175203Z","shell.execute_reply.started":"2022-07-22T00:12:38.120734Z","shell.execute_reply":"2022-07-22T00:12:38.173967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--replace negative values--------------------------","metadata":{}},{"cell_type":"code","source":"i = 0\nfor predict_sales in oil_train_test_predict:\n if predict_sales<0 :\n    oil_train_test_predict[i] = 0\n i = i+1\n        \noil_train_test_predict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.177207Z","iopub.execute_input":"2022-07-22T00:12:38.178055Z","iopub.status.idle":"2022-07-22T00:12:38.431511Z","shell.execute_reply.started":"2022-07-22T00:12:38.178004Z","shell.execute_reply":"2022-07-22T00:12:38.430622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_train_test_prediction['sales']= oil_train_test_predict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.432888Z","iopub.execute_input":"2022-07-22T00:12:38.433435Z","iopub.status.idle":"2022-07-22T00:12:38.439123Z","shell.execute_reply.started":"2022-07-22T00:12:38.433404Z","shell.execute_reply":"2022-07-22T00:12:38.438071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn import metrics\n\noil_train_test_original_sales = oil_train_test_original['sales']\noil_train_test_prediction_sales = oil_train_test_prediction['sales']\n\nevaluation_rmsle = metrics.mean_squared_log_error(oil_train_test_original_sales, oil_train_test_prediction_sales, squared=False)\nevaluation_rmsle","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.440407Z","iopub.execute_input":"2022-07-22T00:12:38.440882Z","iopub.status.idle":"2022-07-22T00:12:38.468971Z","shell.execute_reply.started":"2022-07-22T00:12:38.440852Z","shell.execute_reply":"2022-07-22T00:12:38.467694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"////////////////////////////////////////////\noilデータを考慮した学習で、1.9848の評価値が得られた。\n前回より、精度が上がった。\nよって、oilデータを考慮したモデルで、実際のtestデータを予測する。\n/////////////////////////////////////////////","metadata":{}},{"cell_type":"markdown","source":"--use model on test for submission-------","metadata":{}},{"cell_type":"code","source":"submission_test = test.copy()\nsubmission_test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.471828Z","iopub.execute_input":"2022-07-22T00:12:38.472666Z","iopub.status.idle":"2022-07-22T00:12:38.491596Z","shell.execute_reply.started":"2022-07-22T00:12:38.472619Z","shell.execute_reply":"2022-07-22T00:12:38.490411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_test_category = pd.get_dummies(submission_test, columns=['store_nbr','family'], drop_first=True)\n\nsubmission_test_temp = submission_test_category.copy()\ntest_standard.fit(submission_test_temp[['onpromotion']])\ntest_standard_temp = pd.DataFrame(test_standard.transform(submission_test_temp[['onpromotion']]))\nsubmission_test_category [['onpromotion']] = test_standard_temp\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.493559Z","iopub.execute_input":"2022-07-22T00:12:38.494383Z","iopub.status.idle":"2022-07-22T00:12:38.531992Z","shell.execute_reply.started":"2022-07-22T00:12:38.494315Z","shell.execute_reply":"2022-07-22T00:12:38.531192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_test_category.drop(columns = ['id','date'], inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.533315Z","iopub.execute_input":"2022-07-22T00:12:38.533682Z","iopub.status.idle":"2022-07-22T00:12:38.542179Z","shell.execute_reply.started":"2022-07-22T00:12:38.533652Z","shell.execute_reply":"2022-07-22T00:12:38.541209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_test_category","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.543763Z","iopub.execute_input":"2022-07-22T00:12:38.544077Z","iopub.status.idle":"2022-07-22T00:12:38.574842Z","shell.execute_reply.started":"2022-07-22T00:12:38.544050Z","shell.execute_reply":"2022-07-22T00:12:38.573554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = submission_test_category.copy()\ntemp.insert(1, \"dcoilwtico\", 0)\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.576541Z","iopub.execute_input":"2022-07-22T00:12:38.577234Z","iopub.status.idle":"2022-07-22T00:12:38.606807Z","shell.execute_reply.started":"2022-07-22T00:12:38.577188Z","shell.execute_reply":"2022-07-22T00:12:38.605551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_test_predict = train_model.predict(temp)\nsubmission_test_predict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.608514Z","iopub.execute_input":"2022-07-22T00:12:38.608935Z","iopub.status.idle":"2022-07-22T00:12:38.631608Z","shell.execute_reply.started":"2022-07-22T00:12:38.608892Z","shell.execute_reply":"2022-07-22T00:12:38.630239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nfor predict_sales in submission_test_predict:\n if predict_sales<0 :\n    submission_test_predict[i] = 0\n i = i+1","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.633934Z","iopub.execute_input":"2022-07-22T00:12:38.634904Z","iopub.status.idle":"2022-07-22T00:12:38.681600Z","shell.execute_reply.started":"2022-07-22T00:12:38.634848Z","shell.execute_reply":"2022-07-22T00:12:38.680250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--save and show submission data------------------------------","metadata":{}},{"cell_type":"code","source":"submission['sales']= submission_test_predict\nsubmission.to_csv('submission.csv',index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-22T00:12:38.684007Z","iopub.execute_input":"2022-07-22T00:12:38.684966Z","iopub.status.idle":"2022-07-22T00:12:38.814200Z","shell.execute_reply.started":"2022-07-22T00:12:38.684913Z","shell.execute_reply":"2022-07-22T00:12:38.813111Z"},"trusted":true},"execution_count":null,"outputs":[]}]}