{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-10T23:03:20.135891Z","iopub.execute_input":"2022-08-10T23:03:20.136955Z","iopub.status.idle":"2022-08-10T23:03:20.170810Z","shell.execute_reply.started":"2022-08-10T23:03:20.136861Z","shell.execute_reply":"2022-08-10T23:03:20.169608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\n\nwarnings.filterwarnings(action='ignore')\n\ndata_path = '/kaggle/input/competitive-data-science-predict-future-sales/'\n\nsales_train = pd.read_csv(data_path + 'sales_train.csv')\nshops = pd.read_csv(data_path + 'shops.csv')\nitems = pd.read_csv(data_path + 'items.csv')\nitem_categories = pd.read_csv(data_path + 'item_categories.csv')\ntest = pd.read_csv(data_path + 'test.csv')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:20.190906Z","iopub.execute_input":"2022-08-10T23:03:20.191323Z","iopub.status.idle":"2022-08-10T23:03:22.897486Z","shell.execute_reply.started":"2022-08-10T23:03:20.191281Z","shell.execute_reply":"2022-08-10T23:03:22.896184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.1 피처 엔지니어링1 : 피처명 한글화","metadata":{}},{"cell_type":"code","source":"sales_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:22.900427Z","iopub.execute_input":"2022-08-10T23:03:22.901248Z","iopub.status.idle":"2022-08-10T23:03:22.934876Z","shell.execute_reply.started":"2022-08-10T23:03:22.901162Z","shell.execute_reply":"2022-08-10T23:03:22.933969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales_train = sales_train.rename(columns={'date':'날짜',\n                                          'date_block_num':'월ID',\n                                          'shop_id':'상점ID',\n                                          'item_id':'상품ID',\n                                          'item_price':'판매가',\n                                          'item_cnt_day':'판매량'})\nsales_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:22.936503Z","iopub.execute_input":"2022-08-10T23:03:22.937422Z","iopub.status.idle":"2022-08-10T23:03:23.033554Z","shell.execute_reply.started":"2022-08-10T23:03:22.937371Z","shell.execute_reply":"2022-08-10T23:03:23.032204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.037675Z","iopub.execute_input":"2022-08-10T23:03:23.038024Z","iopub.status.idle":"2022-08-10T23:03:23.051457Z","shell.execute_reply.started":"2022-08-10T23:03:23.037994Z","shell.execute_reply":"2022-08-10T23:03:23.049961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops = shops.rename(columns={'shop_name':'상점명',\n                              'shop_id':'상점ID'})\nshops.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.053137Z","iopub.execute_input":"2022-08-10T23:03:23.054322Z","iopub.status.idle":"2022-08-10T23:03:23.066154Z","shell.execute_reply.started":"2022-08-10T23:03:23.054217Z","shell.execute_reply":"2022-08-10T23:03:23.065011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.067839Z","iopub.execute_input":"2022-08-10T23:03:23.068312Z","iopub.status.idle":"2022-08-10T23:03:23.081798Z","shell.execute_reply.started":"2022-08-10T23:03:23.068253Z","shell.execute_reply":"2022-08-10T23:03:23.080156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items = items.rename(columns={'item_name':'상품명',\n                              'item_id':'상품ID',\n                              'item_category_id':'상품분류ID'})\n\nitems.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.083728Z","iopub.execute_input":"2022-08-10T23:03:23.084469Z","iopub.status.idle":"2022-08-10T23:03:23.097928Z","shell.execute_reply.started":"2022-08-10T23:03:23.084420Z","shell.execute_reply":"2022-08-10T23:03:23.097098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories = item_categories.rename(columns={'item_category_name':'상품분류명',\n                                                  'item_category_id':'상품분류ID'})\nitem_categories.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.099221Z","iopub.execute_input":"2022-08-10T23:03:23.099752Z","iopub.status.idle":"2022-08-10T23:03:23.119530Z","shell.execute_reply.started":"2022-08-10T23:03:23.099720Z","shell.execute_reply":"2022-08-10T23:03:23.118652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.rename(columns={'shop_id':'상점ID',\n                            'item_id':'상품ID'})\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.120905Z","iopub.execute_input":"2022-08-10T23:03:23.121299Z","iopub.status.idle":"2022-08-10T23:03:23.137049Z","shell.execute_reply.started":"2022-08-10T23:03:23.121242Z","shell.execute_reply":"2022-08-10T23:03:23.136100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.2 피처 엔지니어링 2 : 데이터 다운캐스팅","metadata":{}},{"cell_type":"code","source":"def downcast(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024 ** 2\n    for col in df.columns:\n        dtype_name = df[col].dtype.name\n        if dtype_name == 'object':\n            pass\n        elif dtype_name == 'bool':\n            df[col] = df[col].astype('int8')\n        elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n            df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            df[col] = pd.to_numeric(df[col], downcast='float')\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    if verbose:\n        print('{:.1f}% 압축됨'.format(100 * (start_mem - end_mem) / start_mem))\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.140792Z","iopub.execute_input":"2022-08-10T23:03:23.141360Z","iopub.status.idle":"2022-08-10T23:03:23.149151Z","shell.execute_reply.started":"2022-08-10T23:03:23.141325Z","shell.execute_reply":"2022-08-10T23:03:23.148332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df = [sales_train, shops, items, item_categories, test]\n\nfor df in all_df:\n    df = downcast(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.150516Z","iopub.execute_input":"2022-08-10T23:03:23.151010Z","iopub.status.idle":"2022-08-10T23:03:23.525611Z","shell.execute_reply.started":"2022-08-10T23:03:23.150980Z","shell.execute_reply":"2022-08-10T23:03:23.524330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.3 피처 엔지니어링 3 : 데이터 조합 생성","metadata":{}},{"cell_type":"code","source":"from itertools import product\n\ntrain = []\nfor i in sales_train['월ID'].unique():\n    all_shop = sales_train.loc[sales_train['월ID'] == i, '상점ID'].unique()\n    all_item = sales_train.loc[sales_train['월ID'] == i, '상품ID'].unique()\n    train.append(np.array(list(product([i], all_shop, all_item))))\n    \nidx_features = ['월ID', '상점ID', '상품ID']\n\ntrain = pd.DataFrame(np.vstack(train), columns = idx_features)\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:23.527443Z","iopub.execute_input":"2022-08-10T23:03:23.528188Z","iopub.status.idle":"2022-08-10T23:03:33.059253Z","shell.execute_reply.started":"2022-08-10T23:03:23.528132Z","shell.execute_reply":"2022-08-10T23:03:33.057986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.4 피처 엔지니어링 4 : 타깃값(월간 판매량) 추가","metadata":{}},{"cell_type":"code","source":"# idx_features를 기준으로 그룹화해 판매량 합 구하기\ngroup = sales_train.groupby(idx_features).agg({'판매량':'sum'})\n\n# 인덱스 재설정\ngroup = group.reset_index()\n\n# 피처명을 '판매량'에서 '월간 판매량'으로 변경\ngroup = group.rename(columns={'판매량':'월간 판매량'})\n\ngroup","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:33.062258Z","iopub.execute_input":"2022-08-10T23:03:33.062712Z","iopub.status.idle":"2022-08-10T23:03:33.963764Z","shell.execute_reply.started":"2022-08-10T23:03:33.062674Z","shell.execute_reply":"2022-08-10T23:03:33.962491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(group, on=idx_features, how='left')\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:33.965400Z","iopub.execute_input":"2022-08-10T23:03:33.965935Z","iopub.status.idle":"2022-08-10T23:03:38.710351Z","shell.execute_reply.started":"2022-08-10T23:03:33.965895Z","shell.execute_reply":"2022-08-10T23:03:38.709038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.5 피처 엔지니어링 5 : 테스트 데이터 이어 붙이기","metadata":{}},{"cell_type":"code","source":"import gc # garbage collector 불러오기\n\ndel group # 더는 사용하지 않는 변수 지정\ngc.collect(); # garbage collection 수행","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:38.712106Z","iopub.execute_input":"2022-08-10T23:03:38.712955Z","iopub.status.idle":"2022-08-10T23:03:38.824681Z","shell.execute_reply.started":"2022-08-10T23:03:38.712910Z","shell.execute_reply":"2022-08-10T23:03:38.823378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['월ID'] = 34","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:38.828673Z","iopub.execute_input":"2022-08-10T23:03:38.829025Z","iopub.status.idle":"2022-08-10T23:03:38.837762Z","shell.execute_reply.started":"2022-08-10T23:03:38.828994Z","shell.execute_reply":"2022-08-10T23:03:38.836589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.concat([train, test.drop('ID', axis=1)],\n                     ignore_index=True, # 기존 인덱스 무시\n                     keys=idx_features) # 이어 붙이는 기준이 되는 피처","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:38.839183Z","iopub.execute_input":"2022-08-10T23:03:38.839780Z","iopub.status.idle":"2022-08-10T23:03:38.923509Z","shell.execute_reply.started":"2022-08-10T23:03:38.839747Z","shell.execute_reply":"2022-08-10T23:03:38.922572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:38.924926Z","iopub.execute_input":"2022-08-10T23:03:38.925311Z","iopub.status.idle":"2022-08-10T23:03:38.937155Z","shell.execute_reply.started":"2022-08-10T23:03:38.925249Z","shell.execute_reply":"2022-08-10T23:03:38.935951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = all_data.fillna(0)\n\nall_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:38.940136Z","iopub.execute_input":"2022-08-10T23:03:38.941035Z","iopub.status.idle":"2022-08-10T23:03:39.125400Z","shell.execute_reply.started":"2022-08-10T23:03:38.940996Z","shell.execute_reply":"2022-08-10T23:03:39.124088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.6 피처 엔지니어링 6: 나머지 데이터 병합(최종 데이터 생성)","metadata":{}},{"cell_type":"code","source":"all_data = all_data.merge(shops, on='상점ID', how='left')\nall_data = all_data.merge(items, on='상품ID', how='left')\nall_data = all_data.merge(item_categories, on='상품분류ID', how='left')\n\n# 데이터 다운캐스팅\nall_data = downcast(all_data)\n\n# Garbage Collection\ndel shops, items, item_categories\n\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:39.126939Z","iopub.execute_input":"2022-08-10T23:03:39.127256Z","iopub.status.idle":"2022-08-10T23:03:47.236121Z","shell.execute_reply.started":"2022-08-10T23:03:39.127228Z","shell.execute_reply":"2022-08-10T23:03:47.234936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = all_data.drop(['상점명', '상품명', '상품분류명'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:47.238209Z","iopub.execute_input":"2022-08-10T23:03:47.238685Z","iopub.status.idle":"2022-08-10T23:03:48.806649Z","shell.execute_reply.started":"2022-08-10T23:03:47.238634Z","shell.execute_reply":"2022-08-10T23:03:48.805438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.7 피처 엔지니어링 7 : 마무리","metadata":{}},{"cell_type":"code","source":"X_train = all_data[all_data['월ID'] < 33]\nX_train = X_train.drop(['월간 판매량'], axis=1)\n\nX_valid = all_data[all_data['월ID'] == 33]\nX_valid = X_valid.drop(['월간 판매량'], axis=1)\n\nX_test = all_data[all_data['월ID'] == 34]\nX_test = X_test.drop(['월간 판매량'], axis=1)\n\ny_train = all_data[all_data['월ID'] < 33]['월간 판매량']\ny_train = y_train.clip(0,20) # 타깃값을 0~20으로 제한\n\ny_valid = all_data[all_data['월ID'] == 33]['월간 판매량']\ny_valid = y_valid.clip(0,20)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:48.808300Z","iopub.execute_input":"2022-08-10T23:03:48.808666Z","iopub.status.idle":"2022-08-10T23:03:49.661610Z","shell.execute_reply.started":"2022-08-10T23:03:48.808634Z","shell.execute_reply":"2022-08-10T23:03:49.660471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del all_data\n# gc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:49.663450Z","iopub.execute_input":"2022-08-10T23:03:49.663901Z","iopub.status.idle":"2022-08-10T23:03:49.668297Z","shell.execute_reply.started":"2022-08-10T23:03:49.663854Z","shell.execute_reply":"2022-08-10T23:03:49.667132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.8 모델 훈련 및 성능 검증","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\nparams = {'metric':'rmse',\n          'num_leaves':255,\n          'learning_rate':0.01,\n          'force_col_wise':True,\n          'random_state':10}\n\ncat_features = ['상점ID', '상품분류ID']\n\ndtrain = lgb.Dataset(X_train, y_train)\ndvalid = lgb.Dataset(X_valid, y_valid)\n\nlgb_model = lgb.train(params = params,\n                      train_set = dtrain,\n                      num_boost_round = 500,\n                      valid_sets = (dtrain, dvalid),\n                      categorical_feature = cat_features,\n                      verbose_eval = 50)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:03:49.669647Z","iopub.execute_input":"2022-08-10T23:03:49.669956Z","iopub.status.idle":"2022-08-10T23:07:04.168534Z","shell.execute_reply.started":"2022-08-10T23:03:49.669928Z","shell.execute_reply":"2022-08-10T23:07:04.167569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = ['상점ID', '상품분류ID']\nfor cat_feature in cat_features:\n    all_data[cat_feature] = all_data[cat_feature].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:07:04.169945Z","iopub.execute_input":"2022-08-10T23:07:04.171064Z","iopub.status.idle":"2022-08-10T23:07:04.536609Z","shell.execute_reply.started":"2022-08-10T23:07:04.171015Z","shell.execute_reply":"2022-08-10T23:07:04.535521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.9 예측 및 결과 제출","metadata":{}},{"cell_type":"code","source":"preds = lgb_model.predict(X_test).clip(0,20)\n\nsubmission['item_cnt_month'] = preds\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:07:04.537951Z","iopub.execute_input":"2022-08-10T23:07:04.538412Z","iopub.status.idle":"2022-08-10T23:07:11.129625Z","shell.execute_reply.started":"2022-08-10T23:07:04.538368Z","shell.execute_reply":"2022-08-10T23:07:11.128464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train, y_train, X_valid, y_valid, X_test, lgb_model, dtrain, dvalid\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-08-10T23:07:11.131110Z","iopub.execute_input":"2022-08-10T23:07:11.131599Z","iopub.status.idle":"2022-08-10T23:07:11.269802Z","shell.execute_reply.started":"2022-08-10T23:07:11.131552Z","shell.execute_reply":"2022-08-10T23:07:11.268687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}