{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\n\nwarnings.filterwarnings(action='ignore') # 경고 문구 생략\n\n# 데이터 경로\ndata_path = '/kaggle/input/competitive-data-science-predict-future-sales/'\n\nsales_train = pd.read_csv(data_path + 'sales_train.csv')\nshops = pd.read_csv(data_path + 'shops.csv')\nitems = pd.read_csv(data_path + 'items.csv')\nitem_categories = pd.read_csv(data_path + 'item_categories.csv')\ntest = pd.read_csv(data_path + 'test.csv')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-13T12:18:11.921916Z","iopub.execute_input":"2022-08-13T12:18:11.922681Z","iopub.status.idle":"2022-08-13T12:18:14.892107Z","shell.execute_reply.started":"2022-08-13T12:18:11.922562Z","shell.execute_reply":"2022-08-13T12:18:14.890543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales_train = sales_train.rename(columns={'date': '날짜', \n                                          'date_block_num': '월ID',\n                                          'shop_id': '상점ID',\n                                          'item_id': '상품ID',\n                                          'item_price': '판매가',\n                                          'item_cnt_day': '판매량'})\n\nshops = shops.rename(columns={'shop_name': '상점명',\n                              'shop_id': '상점ID'})\n\nitems = items.rename(columns={'item_name': '상품명',\n                              'item_id': '상품ID',\n                              'item_category_id': '상품분류ID'})\n\nitem_categories = item_categories.rename(columns=\n                                         {'item_category_name': '상품분류명',\n                                          'item_category_id': '상품분류ID'})\n\ntest = test.rename(columns={'shop_id': '상점ID',\n                            'item_id': '상품ID'})","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:19.417406Z","iopub.execute_input":"2022-08-13T12:18:19.417911Z","iopub.status.idle":"2022-08-13T12:18:19.522359Z","shell.execute_reply.started":"2022-08-13T12:18:19.417871Z","shell.execute_reply":"2022-08-13T12:18:19.521333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def downcast(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        dtype_name = df[col].dtype.name\n        if dtype_name == 'object':\n            pass\n        elif dtype_name == 'bool':\n            df[col] = df[col].astype('int8')\n        elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n            df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            df[col] = pd.to_numeric(df[col], downcast='float')\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print('{:.1f}% 압축됨'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\nall_df = [sales_train, shops, items, item_categories, test]\nfor df in all_df:\n    df = downcast(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:21.492379Z","iopub.execute_input":"2022-08-13T12:18:21.492869Z","iopub.status.idle":"2022-08-13T12:18:21.870157Z","shell.execute_reply.started":"2022-08-13T12:18:21.492829Z","shell.execute_reply":"2022-08-13T12:18:21.869125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 판매가가 0보다 큰 데이터 추출\nsales_train = sales_train[sales_train['판매가'] > 0]\n# 판매가가 50,000보다 작은 데이터 추출\nsales_train = sales_train[sales_train['판매가'] < 50000]\n\n# 판매량이 0보다 큰 데이터 추출\nsales_train = sales_train[sales_train['판매량'] > 0]\n# 판매량이 1,000보다 작은 데이터 추출\nsales_train = sales_train[sales_train['판매량'] < 1000]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:24.423144Z","iopub.execute_input":"2022-08-13T12:18:24.423622Z","iopub.status.idle":"2022-08-13T12:18:25.130500Z","shell.execute_reply.started":"2022-08-13T12:18:24.423579Z","shell.execute_reply":"2022-08-13T12:18:25.129440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(shops['상점명'][0], '||', shops['상점명'][57])\nprint(shops['상점명'][1], '||', shops['상점명'][58])\nprint(shops['상점명'][10], '||', shops['상점명'][11])\nprint(shops['상점명'][39], '||', shops['상점명'][40])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:26.920566Z","iopub.execute_input":"2022-08-13T12:18:26.921562Z","iopub.status.idle":"2022-08-13T12:18:26.929340Z","shell.execute_reply.started":"2022-08-13T12:18:26.921513Z","shell.execute_reply":"2022-08-13T12:18:26.927907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sales_train 데이터에서 상점ID 수정\nsales_train.loc[sales_train['상점ID'] == 0, '상점ID'] = 57\nsales_train.loc[sales_train['상점ID'] == 1, '상점ID'] = 58\nsales_train.loc[sales_train['상점ID'] == 10, '상점ID'] = 11\nsales_train.loc[sales_train['상점ID'] == 39, '상점ID'] = 40\n\n# test 데이터에서 상점ID 수정\ntest.loc[test['상점ID'] == 0, '상점ID'] = 57\ntest.loc[test['상점ID'] == 1, '상점ID'] = 58\ntest.loc[test['상점ID'] == 10, '상점ID'] = 11\ntest.loc[test['상점ID'] == 39, '상점ID'] = 40","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:29.785581Z","iopub.execute_input":"2022-08-13T12:18:29.785992Z","iopub.status.idle":"2022-08-13T12:18:29.814860Z","shell.execute_reply.started":"2022-08-13T12:18:29.785958Z","shell.execute_reply":"2022-08-13T12:18:29.813865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 테스트 데이터에 있는 상점ID만 추출\nunique_test_shop_id = test['상점ID'].unique()\nsales_train = sales_train[sales_train['상점ID'].isin(unique_test_shop_id)]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:36.695106Z","iopub.execute_input":"2022-08-13T12:18:36.695460Z","iopub.status.idle":"2022-08-13T12:18:36.814490Z","shell.execute_reply.started":"2022-08-13T12:18:36.695430Z","shell.execute_reply":"2022-08-13T12:18:36.813522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops['도시'] = shops['상점명'].apply(lambda x: x.split()[0])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:38.710992Z","iopub.execute_input":"2022-08-13T12:18:38.711342Z","iopub.status.idle":"2022-08-13T12:18:38.717854Z","shell.execute_reply.started":"2022-08-13T12:18:38.711313Z","shell.execute_reply":"2022-08-13T12:18:38.716746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops['도시'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:41.822446Z","iopub.execute_input":"2022-08-13T12:18:41.823085Z","iopub.status.idle":"2022-08-13T12:18:41.832284Z","shell.execute_reply.started":"2022-08-13T12:18:41.823052Z","shell.execute_reply":"2022-08-13T12:18:41.831184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops.loc[shops['도시'] =='!Якутск', '도시'] = 'Якутск'","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:43.791177Z","iopub.execute_input":"2022-08-13T12:18:43.792068Z","iopub.status.idle":"2022-08-13T12:18:43.798315Z","shell.execute_reply.started":"2022-08-13T12:18:43.792030Z","shell.execute_reply":"2022-08-13T12:18:43.797124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# 레이블 인코더 생성\nlabel_encoder = LabelEncoder()\n# 도시 피처 레이블 인코딩\nshops['도시'] = label_encoder.fit_transform(shops['도시'])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:47.939534Z","iopub.execute_input":"2022-08-13T12:18:47.940198Z","iopub.status.idle":"2022-08-13T12:18:48.378843Z","shell.execute_reply.started":"2022-08-13T12:18:47.940163Z","shell.execute_reply":"2022-08-13T12:18:48.377770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상점명 피처 제거\nshops = shops.drop('상점명', axis=1)\n\nshops.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:54.717370Z","iopub.execute_input":"2022-08-13T12:18:54.717861Z","iopub.status.idle":"2022-08-13T12:18:54.739513Z","shell.execute_reply.started":"2022-08-13T12:18:54.717813Z","shell.execute_reply":"2022-08-13T12:18:54.738511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품명 피처 제거\nitems = items.drop(['상품명'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:56.659237Z","iopub.execute_input":"2022-08-13T12:18:56.659677Z","iopub.status.idle":"2022-08-13T12:18:56.668088Z","shell.execute_reply.started":"2022-08-13T12:18:56.659643Z","shell.execute_reply":"2022-08-13T12:18:56.666685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품이 맨 처음 팔린 날을 피처로 추가\nitems['첫 판매월'] = sales_train.groupby('상품ID').agg({'월ID': 'min'})['월ID']\n\nitems.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:18:58.346230Z","iopub.execute_input":"2022-08-13T12:18:58.346909Z","iopub.status.idle":"2022-08-13T12:18:58.428137Z","shell.execute_reply.started":"2022-08-13T12:18:58.346870Z","shell.execute_reply":"2022-08-13T12:18:58.427007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items[items['첫 판매월'].isna()]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:00.427316Z","iopub.execute_input":"2022-08-13T12:19:00.428163Z","iopub.status.idle":"2022-08-13T12:19:00.442669Z","shell.execute_reply.started":"2022-08-13T12:19:00.428126Z","shell.execute_reply":"2022-08-13T12:19:00.441651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 첫 판매월 피처의 결측값을 34로 대체\nitems['첫 판매월'] = items['첫 판매월'].fillna(34)\n\nitems","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:02.546366Z","iopub.execute_input":"2022-08-13T12:19:02.546893Z","iopub.status.idle":"2022-08-13T12:19:02.571590Z","shell.execute_reply.started":"2022-08-13T12:19:02.546842Z","shell.execute_reply":"2022-08-13T12:19:02.570541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품분류명의 첫 단어를 대분류로 추출\nitem_categories['대분류'] = item_categories['상품분류명'].apply(lambda x: x.split()[0])  ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:04.494258Z","iopub.execute_input":"2022-08-13T12:19:04.494631Z","iopub.status.idle":"2022-08-13T12:19:04.500681Z","shell.execute_reply.started":"2022-08-13T12:19:04.494601Z","shell.execute_reply":"2022-08-13T12:19:04.499613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories['대분류'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:06.462652Z","iopub.execute_input":"2022-08-13T12:19:06.463011Z","iopub.status.idle":"2022-08-13T12:19:06.473360Z","shell.execute_reply.started":"2022-08-13T12:19:06.462982Z","shell.execute_reply":"2022-08-13T12:19:06.472033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_etc(x):\n    if len(item_categories[item_categories['대분류']==x]) >= 5:\n        return x\n    else:\n        return 'etc'\n\n# 대분류의 고윳값 개수가 5개 미만이면 'etc'로 바꾸기\nitem_categories['대분류'] = item_categories['대분류'].apply(make_etc)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:10.139777Z","iopub.execute_input":"2022-08-13T12:19:10.140146Z","iopub.status.idle":"2022-08-13T12:19:10.175714Z","shell.execute_reply.started":"2022-08-13T12:19:10.140115Z","shell.execute_reply":"2022-08-13T12:19:10.174527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:11.294519Z","iopub.execute_input":"2022-08-13T12:19:11.295620Z","iopub.status.idle":"2022-08-13T12:19:11.306623Z","shell.execute_reply.started":"2022-08-13T12:19:11.295575Z","shell.execute_reply":"2022-08-13T12:19:11.305508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 레이블 인코더 생성\nlabel_encoder = LabelEncoder()\n\n# 대분류 피처 레이블 인코딩\nitem_categories['대분류'] = label_encoder.fit_transform(item_categories['대분류'])\n\n# 상품분류명 피처 제거\nitem_categories = item_categories.drop('상품분류명', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:13.059976Z","iopub.execute_input":"2022-08-13T12:19:13.061065Z","iopub.status.idle":"2022-08-13T12:19:13.068413Z","shell.execute_reply.started":"2022-08-13T12:19:13.061020Z","shell.execute_reply":"2022-08-13T12:19:13.067409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import product\n\ntrain = []\n# 월ID, 상점ID, 상품ID 조합 생성\nfor i in sales_train['월ID'].unique():\n    all_shop = sales_train.loc[sales_train['월ID']==i, '상점ID'].unique()\n    all_item = sales_train.loc[sales_train['월ID']==i, '상품ID'].unique()\n    train.append(np.array(list(product([i], all_shop, all_item))))\n\nidx_features = ['월ID', '상점ID', '상품ID'] # 기준 피처\ntrain = pd.DataFrame(np.vstack(train), columns=idx_features)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:15.268739Z","iopub.execute_input":"2022-08-13T12:19:15.269369Z","iopub.status.idle":"2022-08-13T12:19:22.175165Z","shell.execute_reply.started":"2022-08-13T12:19:15.269332Z","shell.execute_reply":"2022-08-13T12:19:22.174181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group = sales_train.groupby(idx_features).agg({'판매량': 'sum',\n                                               '판매가': 'mean'})\ngroup = group.reset_index()\ngroup = group.rename(columns={'판매량': '월간 판매량', '판매가': '평균 판매가'})\n\ntrain = train.merge(group, on=idx_features, how='left')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:30.615335Z","iopub.execute_input":"2022-08-13T12:19:30.615931Z","iopub.status.idle":"2022-08-13T12:19:34.968396Z","shell.execute_reply.started":"2022-08-13T12:19:30.615894Z","shell.execute_reply":"2022-08-13T12:19:34.967321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\n# group 변수 가비지 컬렉션\ndel group\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:36.810941Z","iopub.execute_input":"2022-08-13T12:19:36.811300Z","iopub.status.idle":"2022-08-13T12:19:36.915399Z","shell.execute_reply.started":"2022-08-13T12:19:36.811269Z","shell.execute_reply":"2022-08-13T12:19:36.914324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품 판매건수 피처 추가\ngroup = sales_train.groupby(idx_features).agg({'판매량': 'count'})\ngroup = group.reset_index()\ngroup = group.rename(columns={'판매량': '판매건수'})\n\ntrain = train.merge(group, on=idx_features, how='left')\n\n# 가비지 컬렉션\ndel group, sales_train\ngc.collect()\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:41.236492Z","iopub.execute_input":"2022-08-13T12:19:41.237159Z","iopub.status.idle":"2022-08-13T12:19:45.646985Z","shell.execute_reply.started":"2022-08-13T12:19:41.237124Z","shell.execute_reply":"2022-08-13T12:19:45.645888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 테스트 데이터 월ID를 34로 설정\ntest['월ID'] = 34\n\n# train과 test 이어붙이기\nall_data = pd.concat([train, test.drop('ID', axis=1)],\n                     ignore_index=True,\n                     keys=idx_features)\n# 결측값을 0으로 대체\nall_data = all_data.fillna(0)\n\nall_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:48.926927Z","iopub.execute_input":"2022-08-13T12:19:48.927280Z","iopub.status.idle":"2022-08-13T12:19:49.352488Z","shell.execute_reply.started":"2022-08-13T12:19:48.927250Z","shell.execute_reply":"2022-08-13T12:19:49.351457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 나머지 데이터 병합\nall_data = all_data.merge(shops, on='상점ID', how='left')\nall_data = all_data.merge(items, on='상품ID', how='left')\nall_data = all_data.merge(item_categories, on='상품분류ID', how='left')\n\n# 데이터 다운캐스팅\nall_data = downcast(all_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:19:52.910196Z","iopub.execute_input":"2022-08-13T12:19:52.910577Z","iopub.status.idle":"2022-08-13T12:19:59.505559Z","shell.execute_reply.started":"2022-08-13T12:19:52.910544Z","shell.execute_reply":"2022-08-13T12:19:59.504492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 가비지 컬렉션\ndel shops, items, item_categories\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:05:53.963875Z","iopub.execute_input":"2022-08-13T11:05:53.964347Z","iopub.status.idle":"2022-08-13T11:05:54.132994Z","shell.execute_reply.started":"2022-08-13T11:05:53.964304Z","shell.execute_reply":"2022-08-13T11:05:54.131770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_mean_features(df, mean_features, idx_features):\n    # 기준 피처 확인 \n    assert (idx_features[0] == '월ID') and \\\n           len(idx_features) in [2, 3]\n    \n    # 파생 피처명 설정 \n    if len(idx_features) == 2:\n        feature_name = idx_features[1] + '별 평균 판매량'\n    else:\n        feature_name = idx_features[1] + ' ' + idx_features[2] + '별 평균 판매량'\n    \n    # 기준 피처를 토대로 그룹화해 월간 평균 판매량 구하기 \n    group = df.groupby(idx_features).agg({'월간 판매량': 'mean'})\n    group = group.reset_index()\n    group = group.rename(columns={'월간 판매량': feature_name})\n    \n    # df와 group 병합 \n    df = df.merge(group, on=idx_features, how='left')\n    # 데이터 다운캐스팅 \n    df = downcast(df, verbose=False)\n    # 새로 만든 feature_name 피처명을 mean_features 리스트에 추가 \n    mean_features.append(feature_name)\n    \n    # 가비지 컬렉션\n    del group\n    gc.collect()\n    \n    return df, mean_features","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:20:50.682080Z","iopub.execute_input":"2022-08-13T12:20:50.683160Z","iopub.status.idle":"2022-08-13T12:20:50.691296Z","shell.execute_reply.started":"2022-08-13T12:20:50.683115Z","shell.execute_reply":"2022-08-13T12:20:50.690200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 그룹화 기준 피처 중 '상품ID'가 포함된 파생 피처명을 담을 리스트\nitem_mean_features = []\n\n# ['월ID', '상품ID']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, item_mean_features = add_mean_features(df=all_data,\n                                                 mean_features=item_mean_features,\n                                                 idx_features=['월ID', '상품ID'])\n\n# ['월ID', '상품ID', '도시']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, item_mean_features = add_mean_features(df=all_data,\n                                                 mean_features=item_mean_features,\n                                                 idx_features=['월ID', '상품ID', '도시'])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:20:53.399325Z","iopub.execute_input":"2022-08-13T12:20:53.399920Z","iopub.status.idle":"2022-08-13T12:21:04.122358Z","shell.execute_reply.started":"2022-08-13T12:20:53.399886Z","shell.execute_reply":"2022-08-13T12:21:04.121252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_mean_features","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:06:15.085719Z","iopub.execute_input":"2022-08-13T11:06:15.086676Z","iopub.status.idle":"2022-08-13T11:06:15.095202Z","shell.execute_reply.started":"2022-08-13T11:06:15.086615Z","shell.execute_reply":"2022-08-13T11:06:15.093938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 그룹화 기준 피처 중 '상점ID'가 포함된 파생 피처명을 담을 리스트\nshop_mean_features = []\n\n# ['월ID', '상점ID', '상품분류ID']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, shop_mean_features = add_mean_features(df=all_data, \n                                                 mean_features=shop_mean_features,\n                                                 idx_features=['월ID', '상점ID', '상품분류ID'])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:21:06.673056Z","iopub.execute_input":"2022-08-13T12:21:06.673411Z","iopub.status.idle":"2022-08-13T12:21:09.417453Z","shell.execute_reply.started":"2022-08-13T12:21:06.673381Z","shell.execute_reply":"2022-08-13T12:21:09.416524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shop_mean_features","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:06:19.048705Z","iopub.execute_input":"2022-08-13T11:06:19.049592Z","iopub.status.idle":"2022-08-13T11:06:19.058385Z","shell.execute_reply.started":"2022-08-13T11:06:19.049533Z","shell.execute_reply":"2022-08-13T11:06:19.056909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_lag_features(df, lag_features_to_clip, idx_features, \n                     lag_feature, nlags=3, clip=False):\n    # 시차 피처 생성에 필요한 DataFrame 부분만 복사 \n    df_temp = df[idx_features + [lag_feature]].copy() \n\n    # 시차 피처 생성 \n    for i in range(1, nlags+1):\n        # 시차 피처명 \n        lag_feature_name = lag_feature +'_시차' + str(i)\n        # df_temp 열 이름 설정 \n        df_temp.columns = idx_features + [lag_feature_name]\n        # df_temp의 date_block_num 피처에 i 더하기 \n        df_temp['월ID'] += i\n        # idx_feature를 기준으로 df와 df_temp 병합하기 \n        df = df.merge(df_temp.drop_duplicates(), \n                      on=idx_features, \n                      how='left')\n        # 결측값 0으로 대체 \n        df[lag_feature_name] = df[lag_feature_name].fillna(0)\n        # 0 ~ 20 사이로 제한할 시차 피처명을 lag_features_to_clip에 추가 \n        if clip: \n            lag_features_to_clip.append(lag_feature_name)\n    \n    # 데이터 다운캐스팅\n    df = downcast(df, False)\n    # 가비지 컬렉션\n    del df_temp\n    gc.collect()\n    \n    return df, lag_features_to_clip","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:21:11.163603Z","iopub.execute_input":"2022-08-13T12:21:11.165767Z","iopub.status.idle":"2022-08-13T12:21:11.175100Z","shell.execute_reply.started":"2022-08-13T12:21:11.165721Z","shell.execute_reply":"2022-08-13T12:21:11.174002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_features_to_clip = [] # 0 ~ 20 사이로 제한할 시차 피처명을 담을 리스트\nidx_features = ['월ID', '상점ID', '상품ID'] # 기준 피처\n\n# idx_features를 기준으로 월간 판매량의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='월간 판매량', \n                                                  nlags=3,\n                                                  clip=True) # 값을 0 ~ 20 사이로 제한","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:21:15.629025Z","iopub.execute_input":"2022-08-13T12:21:15.630055Z","iopub.status.idle":"2022-08-13T12:21:40.982910Z","shell.execute_reply.started":"2022-08-13T12:21:15.630009Z","shell.execute_reply":"2022-08-13T12:21:40.981913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data=all_data.drop('도시', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:22:32.813704Z","iopub.execute_input":"2022-08-13T12:22:32.814062Z","iopub.status.idle":"2022-08-13T12:22:33.143855Z","shell.execute_reply.started":"2022-08-13T12:22:32.814033Z","shell.execute_reply":"2022-08-13T12:22:33.142844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head().T","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:06:57.121945Z","iopub.execute_input":"2022-08-13T11:06:57.122407Z","iopub.status.idle":"2022-08-13T11:06:57.141388Z","shell.execute_reply.started":"2022-08-13T11:06:57.122354Z","shell.execute_reply":"2022-08-13T11:06:57.140131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_features_to_clip","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:06:57.143123Z","iopub.execute_input":"2022-08-13T11:06:57.143469Z","iopub.status.idle":"2022-08-13T11:06:57.150586Z","shell.execute_reply.started":"2022-08-13T11:06:57.143437Z","shell.execute_reply":"2022-08-13T11:06:57.148992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx_features를 기준으로 판매건수 피처의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='판매건수', \n                                                  nlags=3)\n\n# idx_features를 기준으로 평균 판매가 피처의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='평균 판매가', \n                                                  nlags=3)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:22:40.269196Z","iopub.execute_input":"2022-08-13T12:22:40.269579Z","iopub.status.idle":"2022-08-13T12:23:33.037089Z","shell.execute_reply.started":"2022-08-13T12:22:40.269547Z","shell.execute_reply":"2022-08-13T12:23:33.035980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx_features를 기준으로 item_mean_features 요소별 시차 피처 생성\nfor item_mean_feature in item_mean_features:\n    all_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                      lag_features_to_clip=lag_features_to_clip, \n                                                      idx_features=idx_features, \n                                                      lag_feature=item_mean_feature, \n                                                      nlags=3,\n                                                      clip=True)\n# item_mean_features 피처 제거\nall_data = all_data.drop(item_mean_features, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:25:00.065962Z","iopub.execute_input":"2022-08-13T12:25:00.066321Z","iopub.status.idle":"2022-08-13T12:25:57.207816Z","shell.execute_reply.started":"2022-08-13T12:25:00.066291Z","shell.execute_reply":"2022-08-13T12:25:57.206843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ['월ID', '상점ID', '상품분류ID']를 기준으로 shop_mean_features 요소별 시차 피처 생성\nfor shop_mean_feature in shop_mean_features:\n    all_data, lag_features_to_clip = add_lag_features(df=all_data,\n                                                      lag_features_to_clip=lag_features_to_clip, \n                                                      idx_features=['월ID', '상점ID', '상품분류ID'], \n                                                      lag_feature=shop_mean_feature, \n                                                      nlags=3,\n                                                      clip=True)\n# shop_mean_features 피처 제거\nall_data = all_data.drop(shop_mean_features, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:25:57.209715Z","iopub.execute_input":"2022-08-13T12:25:57.210155Z","iopub.status.idle":"2022-08-13T12:26:08.749260Z","shell.execute_reply.started":"2022-08-13T12:25:57.210121Z","shell.execute_reply":"2022-08-13T12:26:08.748309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 월ID 3미만인 데이터 제거\nall_data = all_data.drop(all_data[all_data['월ID'] < 3].index)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:08.750885Z","iopub.execute_input":"2022-08-13T12:26:08.751259Z","iopub.status.idle":"2022-08-13T12:26:10.721841Z","shell.execute_reply.started":"2022-08-13T12:26:08.751219Z","shell.execute_reply":"2022-08-13T12:26:10.720847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['월간 판매량 시차평균'] = all_data[['월간 판매량_시차1',\n                                          '월간 판매량_시차2', \n                                          '월간 판매량_시차3']].mean(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:10.724519Z","iopub.execute_input":"2022-08-13T12:26:10.724897Z","iopub.status.idle":"2022-08-13T12:26:10.845812Z","shell.execute_reply.started":"2022-08-13T12:26:10.724860Z","shell.execute_reply":"2022-08-13T12:26:10.844836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0 ~ 20 사이로 값 제한\nall_data[lag_features_to_clip + ['월간 판매량', '월간 판매량 시차평균']] = all_data[lag_features_to_clip + ['월간 판매량', '월간 판매량 시차평균']].clip(0, 20)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:10.847246Z","iopub.execute_input":"2022-08-13T12:26:10.847705Z","iopub.status.idle":"2022-08-13T12:26:13.647437Z","shell.execute_reply.started":"2022-08-13T12:26:10.847666Z","shell.execute_reply":"2022-08-13T12:26:13.646441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['시차변화량1'] = all_data['월간 판매량_시차1']/all_data['월간 판매량_시차2']\nall_data['시차변화량1'] = all_data['시차변화량1'].replace([np.inf, -np.inf], \n                                                        np.nan).fillna(0)\n\nall_data['시차변화량2'] = all_data['월간 판매량_시차2']/all_data['월간 판매량_시차3']\nall_data['시차변화량2'] = all_data['시차변화량2'].replace([np.inf, -np.inf], \n                                                        np.nan).fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:13.648880Z","iopub.execute_input":"2022-08-13T12:26:13.649367Z","iopub.status.idle":"2022-08-13T12:26:14.043254Z","shell.execute_reply.started":"2022-08-13T12:26:13.649326Z","shell.execute_reply":"2022-08-13T12:26:14.042201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['신상여부'] = all_data['첫 판매월'] == all_data['월ID']","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:14.044573Z","iopub.execute_input":"2022-08-13T12:26:14.045642Z","iopub.status.idle":"2022-08-13T12:26:14.055957Z","shell.execute_reply.started":"2022-08-13T12:26:14.045597Z","shell.execute_reply":"2022-08-13T12:26:14.054880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['첫 판매 후 기간'] = all_data['월ID'] - all_data['첫 판매월']","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:14.057669Z","iopub.execute_input":"2022-08-13T12:26:14.058418Z","iopub.status.idle":"2022-08-13T12:26:14.067828Z","shell.execute_reply.started":"2022-08-13T12:26:14.058381Z","shell.execute_reply":"2022-08-13T12:26:14.066872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['월'] = all_data['월ID'] % 12","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:14.069145Z","iopub.execute_input":"2022-08-13T12:26:14.070007Z","iopub.status.idle":"2022-08-13T12:26:14.112768Z","shell.execute_reply.started":"2022-08-13T12:26:14.069973Z","shell.execute_reply":"2022-08-13T12:26:14.111934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 첫 판매월, 평균 판매가, 판매건수 피처 제거\nall_data = all_data.drop(['첫 판매월', '평균 판매가', '판매건수'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:14.115752Z","iopub.execute_input":"2022-08-13T12:26:14.116667Z","iopub.status.idle":"2022-08-13T12:26:14.659235Z","shell.execute_reply.started":"2022-08-13T12:26:14.116632Z","shell.execute_reply":"2022-08-13T12:26:14.658271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = downcast(all_data, False) # 데이터 다운캐스팅","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:14.660667Z","iopub.execute_input":"2022-08-13T12:26:14.661115Z","iopub.status.idle":"2022-08-13T12:26:15.796265Z","shell.execute_reply.started":"2022-08-13T12:26:14.661075Z","shell.execute_reply":"2022-08-13T12:26:15.795299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:10:08.153621Z","iopub.execute_input":"2022-08-13T11:10:08.154464Z","iopub.status.idle":"2022-08-13T11:10:08.171371Z","shell.execute_reply.started":"2022-08-13T11:10:08.154419Z","shell.execute_reply":"2022-08-13T11:10:08.170138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 훈련 데이터 (피처)\nX_train = all_data[all_data['월ID'] < 33]\nX_train = X_train.drop(['월간 판매량'], axis=1)\n# 검증 데이터 (피처)\nX_valid = all_data[all_data['월ID'] == 33]\nX_valid = X_valid.drop(['월간 판매량'], axis=1)\n# 테스트 데이터 (피처)\nX_test = all_data[all_data['월ID'] == 34]\nX_test = X_test.drop(['월간 판매량'], axis=1)\n\n# 훈련 데이터 (타깃값)\ny_train = all_data[all_data['월ID'] < 33]['월간 판매량']\n# 검증 데이터 (타깃값)\ny_valid = all_data[all_data['월ID'] == 33]['월간 판매량']\n\n# 가비지 컬렉션\ndel all_data\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:26:15.797581Z","iopub.execute_input":"2022-08-13T12:26:15.799856Z","iopub.status.idle":"2022-08-13T12:26:18.772786Z","shell.execute_reply.started":"2022-08-13T12:26:15.799827Z","shell.execute_reply":"2022-08-13T12:26:18.771719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\n\n# LightGBM 하이퍼파라미터\nparams = {'metric': 'rmse', \n          'num_leaves': 255,\n          'learning_rate': 0.005,\n          'feature_fraction': 0.75,\n          'bagging_fraction': 0.75,\n          'bagging_freq': 5,\n          'force_col_wise': True,\n          'random_state': 10}\n\ncat_features = ['상점ID', '상품분류ID', '대분류', '월']\n\n# LightGBM 훈련 및 검증 데이터셋\ndtrain = lgb.Dataset(X_train, y_train)\ndvalid = lgb.Dataset(X_valid, y_valid)\n \n# LightGBM 모델 훈련\nlgb_model = lgb.train(params=params,\n                      train_set=dtrain,\n                      num_boost_round=1500,\n                      valid_sets=(dtrain, dvalid),\n                      early_stopping_rounds=150,\n                      categorical_feature=cat_features,\n                      verbose_eval=100)      ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:27:48.071984Z","iopub.execute_input":"2022-08-13T12:27:48.072591Z","iopub.status.idle":"2022-08-13T12:47:10.019124Z","shell.execute_reply.started":"2022-08-13T12:27:48.072552Z","shell.execute_reply":"2022-08-13T12:47:10.017993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmse(y_true, y_pred):\n  \n    # RMSE 계산\n    output = np.sqrt(np.mean((y_true - y_pred)**2))\n    return output","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:51:34.754846Z","iopub.execute_input":"2022-08-13T12:51:34.755426Z","iopub.status.idle":"2022-08-13T12:51:34.760771Z","shell.execute_reply.started":"2022-08-13T12:51:34.755388Z","shell.execute_reply":"2022-08-13T12:51:34.759547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nxgb_model=xgb.XGBRegressor()\nxgb_model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:48:20.195814Z","iopub.execute_input":"2022-08-13T12:48:20.196753Z","iopub.status.idle":"2022-08-13T12:51:15.839565Z","shell.execute_reply.started":"2022-08-13T12:48:20.196706Z","shell.execute_reply":"2022-08-13T12:51:15.838631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=xgb_model.predict(X_valid)\nscore=rmse(y_valid, y_pred)\nprint(score)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:51:37.202349Z","iopub.execute_input":"2022-08-13T12:51:37.202935Z","iopub.status.idle":"2022-08-13T12:51:37.626654Z","shell.execute_reply.started":"2022-08-13T12:51:37.202899Z","shell.execute_reply":"2022-08-13T12:51:37.625424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 예측\npreds = lgb_model.predict(X_test).clip(0, 20)\n\n# 제출 파일 생성\nsubmission['item_cnt_month'] = preds\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:51:52.966634Z","iopub.execute_input":"2022-08-13T12:51:52.967219Z","iopub.status.idle":"2022-08-13T12:52:12.674954Z","shell.execute_reply.started":"2022-08-13T12:51:52.967183Z","shell.execute_reply":"2022-08-13T12:52:12.673961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.DataFrame(data=xgb_model.feature_importances_, index=X_valid.columns)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:52:12.676663Z","iopub.execute_input":"2022-08-13T12:52:12.677045Z","iopub.status.idle":"2022-08-13T12:52:12.691205Z","shell.execute_reply.started":"2022-08-13T12:52:12.677008Z","shell.execute_reply":"2022-08-13T12:52:12.689946Z"},"trusted":true},"execution_count":null,"outputs":[]}]}