{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#importing the libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n# import seaborn as sns\nimport seaborn as sns; sns.set(style=\"ticks\", color_codes=True)\n\nfrom datetime import datetime\nfrom scipy import stats\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T08:38:13.269032Z","iopub.execute_input":"2022-07-22T08:38:13.269546Z","iopub.status.idle":"2022-07-22T08:38:15.605665Z","shell.execute_reply.started":"2022-07-22T08:38:13.269414Z","shell.execute_reply":"2022-07-22T08:38:15.602488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:38:21.144748Z","iopub.execute_input":"2022-07-22T08:38:21.145154Z","iopub.status.idle":"2022-07-22T08:38:21.168001Z","shell.execute_reply.started":"2022-07-22T08:38:21.145122Z","shell.execute_reply":"2022-07-22T08:38:21.166552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')\ntest = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')\n\n# Idは不要なので、削除して別に変数化し、スコア提出時に使用\ntrain_Id = train.Id\ntest_Id = test.Id\n\n# Id列削除\ntrain.drop('Id', axis=1, inplace=True)\ntest.drop('Id', axis=1, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:38:35.505655Z","iopub.execute_input":"2022-07-22T08:38:35.506205Z","iopub.status.idle":"2022-07-22T08:38:36.070816Z","shell.execute_reply.started":"2022-07-22T08:38:35.506156Z","shell.execute_reply":"2022-07-22T08:38:36.069653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 最大カラム数を100に拡張(デフォルトだと省略されてしまうので)\n# 常に全ての列（カラム）を表示\npd.options.display.max_columns = None\npd.options.display.max_rows = 80\n\n# 小数点2桁で表示(指数表記しないように)\npd.options.display.float_format = '{:.2f}'.format\n%matplotlib inline\n#ワーニングを抑止\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:38:51.604726Z","iopub.execute_input":"2022-07-22T08:38:51.605136Z","iopub.status.idle":"2022-07-22T08:38:51.614015Z","shell.execute_reply.started":"2022-07-22T08:38:51.605103Z","shell.execute_reply":"2022-07-22T08:38:51.612897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of train data', train.shape)\nprint('Size of test data', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:39:00.846113Z","iopub.execute_input":"2022-07-22T08:39:00.846624Z","iopub.status.idle":"2022-07-22T08:39:00.852477Z","shell.execute_reply.started":"2022-07-22T08:39:00.846574Z","shell.execute_reply":"2022-07-22T08:39:00.851280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:39:13.944944Z","iopub.execute_input":"2022-07-22T08:39:13.945297Z","iopub.status.idle":"2022-07-22T08:39:13.951631Z","shell.execute_reply.started":"2022-07-22T08:39:13.945269Z","shell.execute_reply":"2022-07-22T08:39:13.950881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:39:21.704765Z","iopub.execute_input":"2022-07-22T08:39:21.705163Z","iopub.status.idle":"2022-07-22T08:39:21.727507Z","shell.execute_reply.started":"2022-07-22T08:39:21.705131Z","shell.execute_reply":"2022-07-22T08:39:21.726624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:39:36.404659Z","iopub.execute_input":"2022-07-22T08:39:36.405056Z","iopub.status.idle":"2022-07-22T08:39:36.504369Z","shell.execute_reply.started":"2022-07-22T08:39:36.405016Z","shell.execute_reply":"2022-07-22T08:39:36.503275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:39:46.744699Z","iopub.execute_input":"2022-07-22T08:39:46.745100Z","iopub.status.idle":"2022-07-22T08:39:46.764061Z","shell.execute_reply.started":"2022-07-22T08:39:46.745066Z","shell.execute_reply":"2022-07-22T08:39:46.762883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"revenue\"].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:39:58.004759Z","iopub.execute_input":"2022-07-22T08:39:58.005161Z","iopub.status.idle":"2022-07-22T08:39:58.015540Z","shell.execute_reply.started":"2022-07-22T08:39:58.005128Z","shell.execute_reply":"2022-07-22T08:39:58.014579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#目的変数であるrevenueのヒストグラムとQ-Qプロットを表示する\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(train['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(train['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(train['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % train['revenue'].skew())\nprint(\"尖度: %f\" % train['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:40:13.145903Z","iopub.execute_input":"2022-07-22T08:40:13.146354Z","iopub.status.idle":"2022-07-22T08:40:13.505696Z","shell.execute_reply.started":"2022-07-22T08:40:13.146322Z","shell.execute_reply":"2022-07-22T08:40:13.504529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データをコピーし、新たなdataframeで検証\ndf = train.copy()\n\n#目的変数の対数log(x+1)をとる\ndf['revenue'] = np.log1p(df['revenue'])\n\n# 標準化(平均0, 分散1)\nscaler=StandardScaler()\ndf['revenue']=scaler.fit_transform(df[['revenue']])\n\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(df['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(df['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(df['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % df['revenue'].skew())\nprint(\"尖度: %f\" % df['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:40:32.326296Z","iopub.execute_input":"2022-07-22T08:40:32.326782Z","iopub.status.idle":"2022-07-22T08:40:32.643898Z","shell.execute_reply.started":"2022-07-22T08:40:32.326740Z","shell.execute_reply":"2022-07-22T08:40:32.642808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データをコピーし、新たなdataframeで検証\ndf = train.copy()\n\n# 標準化(平均0, 分散1)\nscaler=StandardScaler()\ndf['revenue']=scaler.fit_transform(df[['revenue']])\n\n\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(df['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(df['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(df['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % df['revenue'].skew())\nprint(\"尖度: %f\" % df['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:40:44.445983Z","iopub.execute_input":"2022-07-22T08:40:44.446369Z","iopub.status.idle":"2022-07-22T08:40:44.781038Z","shell.execute_reply.started":"2022-07-22T08:40:44.446336Z","shell.execute_reply":"2022-07-22T08:40:44.779772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データをコピーし、新たなdataframeで検証\ndf = train.copy()\n\n# Min-Max変換(正規化(最大1, 最小0))\nscaler=MinMaxScaler()\ndf['revenue']=scaler.fit_transform(df[['revenue']])\n\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(df['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(df['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(df['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % df['revenue'].skew())\nprint(\"尖度: %f\" % df['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:41:01.466701Z","iopub.execute_input":"2022-07-22T08:41:01.467104Z","iopub.status.idle":"2022-07-22T08:41:01.890066Z","shell.execute_reply.started":"2022-07-22T08:41:01.467069Z","shell.execute_reply":"2022-07-22T08:41:01.888979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データ\n# Open Dateを日付型に変換\ntrain['pd_date'] = pd.to_datetime(train['Open Date'], format='%m/%d/%Y')\n# 年のみを抽出\ntrain['Open_Year'] = train['pd_date'].dt.strftime('%Y')\n# 月のみを抽出\ntrain['Open_Month'] = train['pd_date'].dt.strftime('%m')\n# 経過年数\ntrain[\"Open Date\"] = pd.to_datetime(train[\"Open Date\"])\ntrain[\"Day\"] = train[\"Open Date\"].apply(lambda x:x.day)\ntrain[\"kijun\"] = \"2022-07-10\"\ntrain[\"kijun\"] = pd.to_datetime(train[\"kijun\"])\ntrain[\"BusinessPeriod\"] = (train[\"kijun\"] - train[\"Open Date\"]).apply(lambda x: x.days)\n\ntrain = train.drop('kijun', axis=1)\n\ntrain = train.drop('pd_date',axis=1)\ntrain = train.drop('Open Date',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:41:16.201944Z","iopub.execute_input":"2022-07-22T08:41:16.202700Z","iopub.status.idle":"2022-07-22T08:41:16.232423Z","shell.execute_reply.started":"2022-07-22T08:41:16.202662Z","shell.execute_reply":"2022-07-22T08:41:16.231242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# テストデータ\n# Open Dateを日付型に変換\ntest['pd_date'] = pd.to_datetime(test['Open Date'], format='%m/%d/%Y')\n# 年のみを抽出\ntest['Open_Year'] = test['pd_date'].dt.strftime('%Y')\n# 月のみを抽出\ntest['Open_Month'] = test['pd_date'].dt.strftime('%m')\n# 経過年数\ntest[\"Open Date\"] = pd.to_datetime(test[\"Open Date\"])\ntest[\"Day\"] = test[\"Open Date\"].apply(lambda x:x.day)\ntest[\"kijun\"] = \"2022-07-10\"\ntest[\"kijun\"] = pd.to_datetime(test[\"kijun\"])\ntest[\"BusinessPeriod\"] = (test[\"kijun\"] - test[\"Open Date\"]).apply(lambda x: x.days)\n\ntest = test.drop('kijun', axis=1)\ntest = test.drop('pd_date',axis=1)\ntest = test.drop('Open Date',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:41:26.885804Z","iopub.execute_input":"2022-07-22T08:41:26.886212Z","iopub.status.idle":"2022-07-22T08:41:29.125266Z","shell.execute_reply.started":"2022-07-22T08:41:26.886181Z","shell.execute_reply":"2022-07-22T08:41:29.124192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:41:36.393639Z","iopub.execute_input":"2022-07-22T08:41:36.394043Z","iopub.status.idle":"2022-07-22T08:41:36.405325Z","shell.execute_reply.started":"2022-07-22T08:41:36.394000Z","shell.execute_reply":"2022-07-22T08:41:36.404455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#カテゴリ変数と数値変数に分ける\ncats = list(train.select_dtypes(include=['object']).columns)\nnums = list(train.select_dtypes(exclude=['object']).columns)\nprint(f'categorical variables:  {cats}')\nprint(f'numerical variables:  {nums}')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:41:51.684510Z","iopub.execute_input":"2022-07-22T08:41:51.685460Z","iopub.status.idle":"2022-07-22T08:41:51.694692Z","shell.execute_reply.started":"2022-07-22T08:41:51.685418Z","shell.execute_reply":"2022-07-22T08:41:51.693884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.nunique(axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:42:01.383674Z","iopub.execute_input":"2022-07-22T08:42:01.384895Z","iopub.status.idle":"2022-07-22T08:42:01.399086Z","shell.execute_reply.started":"2022-07-22T08:42:01.384857Z","shell.execute_reply":"2022-07-22T08:42:01.398175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'categorical variables:  {cats}')\nprint(f'numerical variables:  {nums}')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:42:12.407473Z","iopub.execute_input":"2022-07-22T08:42:12.407899Z","iopub.status.idle":"2022-07-22T08:42:12.413371Z","shell.execute_reply.started":"2022-07-22T08:42:12.407862Z","shell.execute_reply":"2022-07-22T08:42:12.412300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 名義変数\nnominal_list =cats\n               \n# 順序変数\n# ordinal_list = []\n\n# 数値変数\nnum_list = nums","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:42:45.688326Z","iopub.execute_input":"2022-07-22T08:42:45.689411Z","iopub.status.idle":"2022-07-22T08:42:45.694451Z","shell.execute_reply.started":"2022-07-22T08:42:45.689362Z","shell.execute_reply":"2022-07-22T08:42:45.693118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(nominal_list)/2+1)\n\nfig = plt.figure(figsize=(30, 20))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(nominal_list)):\n    ax = fig.add_subplot(columns, 2, i+1)\n    sns.countplot(x=nominal_list[i], data=train, ax=ax)\n    plt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:42:56.567993Z","iopub.execute_input":"2022-07-22T08:42:56.568348Z","iopub.status.idle":"2022-07-22T08:42:57.600432Z","shell.execute_reply.started":"2022-07-22T08:42:56.568319Z","shell.execute_reply":"2022-07-22T08:42:57.599564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(num_list)/3+1)\n\nfig = plt.figure(figsize=(30, 40))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(num_list)):\n    ax = fig.add_subplot(columns, 3, i+1)\n\n    train[num_list[i]].hist(ax=ax)\n    ax2 = train[num_list[i]].plot.kde(ax=ax, secondary_y=True,title=num_list[i])\n    ax2.set_ylim(0)\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:43:09.049767Z","iopub.execute_input":"2022-07-22T08:43:09.050212Z","iopub.status.idle":"2022-07-22T08:43:16.662732Z","shell.execute_reply.started":"2022-07-22T08:43:09.050178Z","shell.execute_reply":"2022-07-22T08:43:16.661895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(nominal_list)/2+1)\n\nfig = plt.figure(figsize=(20, 10))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(nominal_list)):\n    ax = fig.add_subplot(columns, 2, i+1)\n\n    # 回帰の場合    \n    sns.boxplot(x=nominal_list[i], y=train.revenue, data=train, ax=ax)\n    plt.xticks(rotation=45)\n    # 分類の場合\n#     sns.barplot(x = nominal_list[i], y = train.revenue, data=train, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:43:32.524658Z","iopub.execute_input":"2022-07-22T08:43:32.525030Z","iopub.status.idle":"2022-07-22T08:43:33.881456Z","shell.execute_reply.started":"2022-07-22T08:43:32.525000Z","shell.execute_reply":"2022-07-22T08:43:33.880348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop('Open_Month',axis=1)\ntest= test.drop('Open_Month',axis=1)\nnominal_list.remove('Open_Month')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:43:47.184705Z","iopub.execute_input":"2022-07-22T08:43:47.185116Z","iopub.status.idle":"2022-07-22T08:43:47.205748Z","shell.execute_reply.started":"2022-07-22T08:43:47.185082Z","shell.execute_reply":"2022-07-22T08:43:47.204602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(num_list)/4+1)\n\nfig = plt.figure(figsize=(30, 35))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(num_list)):\n    ax = fig.add_subplot(columns, 4, i+1)\n\n    # 回帰の場合    \n    sns.regplot(x=num_list[i],y='revenue',data=train, ax=ax)\n    plt.xticks(rotation=45)\n    # 分類の場合\n#     sns.barplot(x = nominal_list[i], y = train.revenue, data=train, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:43:57.342802Z","iopub.execute_input":"2022-07-22T08:43:57.343187Z","iopub.status.idle":"2022-07-22T08:44:05.306674Z","shell.execute_reply.started":"2022-07-22T08:43:57.343156Z","shell.execute_reply":"2022-07-22T08:44:05.305356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['City','revenue']].groupby('City').mean().plot(kind='bar')\nplt.title('Mean Revenue Generated vs City')\nplt.xlabel('City')\nplt.ylabel('Mean Revenue Generated')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:44:11.622539Z","iopub.execute_input":"2022-07-22T08:44:11.622911Z","iopub.status.idle":"2022-07-22T08:44:11.989703Z","shell.execute_reply.started":"2022-07-22T08:44:11.622879Z","shell.execute_reply":"2022-07-22T08:44:11.988903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cityごとのrevenue平均値を1000000単位とする\nmean_revenue_per_city = train[['City', 'revenue']].groupby('City', as_index=False).mean()\nmean_revenue_per_city.head()\nmean_revenue_per_city['revenue'] = mean_revenue_per_city['revenue'].apply(lambda x: int(x/1e6)) \n\nmean_revenue_per_city\n\nmean_dict = dict(zip(mean_revenue_per_city.City, mean_revenue_per_city.revenue))\nmean_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:44:22.342771Z","iopub.execute_input":"2022-07-22T08:44:22.343154Z","iopub.status.idle":"2022-07-22T08:44:22.358981Z","shell.execute_reply.started":"2022-07-22T08:44:22.343122Z","shell.execute_reply":"2022-07-22T08:44:22.357726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train['City'].sort_values().unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:44:56.145475Z","iopub.execute_input":"2022-07-22T08:44:56.145876Z","iopub.status.idle":"2022-07-22T08:44:56.152745Z","shell.execute_reply.started":"2022-07-22T08:44:56.145817Z","shell.execute_reply":"2022-07-22T08:44:56.151568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['City'].sort_values().unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:45:04.742796Z","iopub.execute_input":"2022-07-22T08:45:04.743203Z","iopub.status.idle":"2022-07-22T08:45:04.843904Z","shell.execute_reply.started":"2022-07-22T08:45:04.743169Z","shell.execute_reply":"2022-07-22T08:45:04.842702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cityについて、学習データとテストデータにて重複削除し、リスト化\ncity_train_list = list(train['City'].unique())\ncity_test_list = list(test['City'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:45:14.246630Z","iopub.execute_input":"2022-07-22T08:45:14.247545Z","iopub.status.idle":"2022-07-22T08:45:14.259973Z","shell.execute_reply.started":"2022-07-22T08:45:14.247491Z","shell.execute_reply":"2022-07-22T08:45:14.258872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1_l2_and = set(city_train_list) & set(city_test_list)\nprint(l1_l2_and)\nprint(len(l1_l2_and))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:45:22.384773Z","iopub.execute_input":"2022-07-22T08:45:22.385215Z","iopub.status.idle":"2022-07-22T08:45:22.391747Z","shell.execute_reply.started":"2022-07-22T08:45:22.385178Z","shell.execute_reply":"2022-07-22T08:45:22.390433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# どちらかにしかないCityを抽出\nl1_l2_sym_diff = set(city_test_list) ^ set(city_train_list)\nprint(l1_l2_sym_diff)\nprint(len(l1_l2_sym_diff))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:45:34.165360Z","iopub.execute_input":"2022-07-22T08:45:34.165710Z","iopub.status.idle":"2022-07-22T08:45:34.171289Z","shell.execute_reply.started":"2022-07-22T08:45:34.165681Z","shell.execute_reply":"2022-07-22T08:45:34.170371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# テストデータのみ存在するCityの件数\nlen(set(city_test_list).difference(city_train_list))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:45:42.592998Z","iopub.execute_input":"2022-07-22T08:45:42.593496Z","iopub.status.idle":"2022-07-22T08:45:42.601576Z","shell.execute_reply.started":"2022-07-22T08:45:42.593449Z","shell.execute_reply":"2022-07-22T08:45:42.600609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データのみ存在するCityの件数\nlen(set(city_train_list).difference(city_test_list))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:45:50.344513Z","iopub.execute_input":"2022-07-22T08:45:50.344865Z","iopub.status.idle":"2022-07-22T08:45:50.352218Z","shell.execute_reply.started":"2022-07-22T08:45:50.344821Z","shell.execute_reply":"2022-07-22T08:45:50.350987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# P変数の1つのクラスは地理的属性であると指定されているため\n# 各都市のP変数の平均をプロットすると、どのP変数が都市と関連性が高いかが分かる\ndistinct_cities = train.loc[:, \"City\"].unique()\n\n# P変数のcityごとの平均値を取得\nmeans = []\nfor i in range(len(num_list)):\n    temp = []\n    for city in distinct_cities:\n        temp.append(train.loc[train.City == city, num_list[i]].mean())  \n    means.append(temp)\n    \ncity_pvars = pd.DataFrame(columns=[\"city_var\", \"means\"])\nfor i in range(37):\n    for j in range(len(distinct_cities)):\n        city_pvars.loc[i+37*j] = [\"P\"+str(i+1), means[i][j]]\n\nprint(city_pvars)            \n# 箱ひげ図を表示\nplt.rcParams['figure.figsize'] = (18.0, 6.0)\nsns.boxplot(x=\"city_var\", y=\"means\", data=city_pvars)\n\n# From this we observe that P1, P2, P11, P19, P20, P23, and P30 are approximately a good\n# proxy for geographical location.","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:46:08.268570Z","iopub.execute_input":"2022-07-22T08:46:08.268943Z","iopub.status.idle":"2022-07-22T08:46:11.905665Z","shell.execute_reply.started":"2022-07-22T08:46:08.268911Z","shell.execute_reply":"2022-07-22T08:46:11.904889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import cluster\n\ndef adjust_cities(full_full_data, train, k):\n    \n    # As found by box plot of each city's mean over each p-var\n    relevant_pvars =  [\"P1\", \"P2\", \"P11\", \"P19\", \"P20\", \"P23\",\"P30\"]\n    train = train.loc[:, relevant_pvars]\n    \n    # Optimal k is 20 as found by DB-Index plot    \n    kmeans = cluster.KMeans(n_clusters=k)\n    kmeans.fit(train)\n    \n    # Get the cluster centers and classify city of each full_data instance to one of the centers\n    full_data['City_Cluster'] = kmeans.predict(full_data.loc[:, relevant_pvars])\n    \n    return full_data","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:46:22.708599Z","iopub.execute_input":"2022-07-22T08:46:22.708961Z","iopub.status.idle":"2022-07-22T08:46:22.950048Z","shell.execute_reply.started":"2022-07-22T08:46:22.708929Z","shell.execute_reply":"2022-07-22T08:46:22.948905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_train = train.shape[0]\nnum_test = test.shape[0]\nprint(num_train, num_test)\n\nfull_data = pd.concat([train, test], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:46:31.928728Z","iopub.execute_input":"2022-07-22T08:46:31.929976Z","iopub.status.idle":"2022-07-22T08:46:31.969582Z","shell.execute_reply.started":"2022-07-22T08:46:31.929925Z","shell.execute_reply":"2022-07-22T08:46:31.968430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データを使用しクラスタリングを行い、その学習結果を全データに適用させる\nfull_data = adjust_cities(full_data, train, 20)\nfull_data\n\n# City項目は不要なので削除\nfull_data = full_data.drop(['City'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:46:43.548647Z","iopub.execute_input":"2022-07-22T08:46:43.549798Z","iopub.status.idle":"2022-07-22T08:46:43.838664Z","shell.execute_reply.started":"2022-07-22T08:46:43.549755Z","shell.execute_reply":"2022-07-22T08:46:43.836946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into train and test datasets\ntrain = full_data[:num_train]\ntest = full_data[num_train:]\n# check the shapes \nprint(\"Train :\",train.shape)\nprint(\"Test:\",test.shape)\ntest","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:46:52.728929Z","iopub.execute_input":"2022-07-22T08:46:52.729374Z","iopub.status.idle":"2022-07-22T08:46:52.766432Z","shell.execute_reply.started":"2022-07-22T08:46:52.729338Z","shell.execute_reply":"2022-07-22T08:46:52.765292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['City_Cluster','revenue']].groupby('City_Cluster').mean().plot(kind='bar')\nplt.title('Mean Revenue Generated vs City Cluster')\nplt.xlabel('City Cluster')\nplt.ylabel('Mean Revenue Generated')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:47:02.888685Z","iopub.execute_input":"2022-07-22T08:47:02.889087Z","iopub.status.idle":"2022-07-22T08:47:03.185237Z","shell.execute_reply.started":"2022-07-22T08:47:02.889056Z","shell.execute_reply":"2022-07-22T08:47:03.184107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_revenue_per_city = train[['City_Cluster', 'revenue']].groupby('City_Cluster', as_index=False).mean()\nmean_revenue_per_city.head()\nmean_revenue_per_city['revenue'] = mean_revenue_per_city['revenue'].apply(lambda x: int(x/1e6)) \n\nmean_revenue_per_city\n\nmean_dict = dict(zip(mean_revenue_per_city.City_Cluster, mean_revenue_per_city.revenue))\nmean_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:47:14.248714Z","iopub.execute_input":"2022-07-22T08:47:14.249159Z","iopub.status.idle":"2022-07-22T08:47:14.264732Z","shell.execute_reply.started":"2022-07-22T08:47:14.249123Z","shell.execute_reply":"2022-07-22T08:47:14.263915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"city_rev = []\n\nfor i in full_data['City_Cluster']:\n    for key, value in mean_dict.items():\n        if i == key:\n            city_rev.append(value)\n            \ndf_city_rev = pd.DataFrame({'city_rev':city_rev})\nfull_data = pd.concat([full_data,df_city_rev],axis=1)\nfull_data.head\n\n# 値の追加\nnominal_list.extend(['City_Cluster'])\n# 値の削除\nnominal_list.remove('City')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:47:24.628736Z","iopub.execute_input":"2022-07-22T08:47:24.629448Z","iopub.status.idle":"2022-07-22T08:47:24.913575Z","shell.execute_reply.started":"2022-07-22T08:47:24.629405Z","shell.execute_reply":"2022-07-22T08:47:24.912519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nle_count = 0\n\n# Iterate through the columns\n# for col in application_full_data:\nfor i in range(len(nominal_list)):    \n    \n#     if application_full_data[col].dtype == 'object':\n        # If 2 or fewer unique categories\n        if len(list(full_data[nominal_list[i]].unique())) <= 2:\n            # full_data on the full_dataing data\n            le.fit(full_data[nominal_list[i]])\n            # Transform both full_dataing and testing data\n            full_data[nominal_list[i]] = le.transform(full_data[nominal_list[i]])\n            \n            # Keep track of how many columns were label encoded\n            le_count += 1\n            \nprint('%d columns were label encoded.' % le_count)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:47:39.385666Z","iopub.execute_input":"2022-07-22T08:47:39.386037Z","iopub.status.idle":"2022-07-22T08:47:39.438133Z","shell.execute_reply.started":"2022-07-22T08:47:39.386007Z","shell.execute_reply":"2022-07-22T08:47:39.437115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one-hot encoding of categorical variables\nfull_data = pd.get_dummies(full_data)\nprint('full_dataing Features shape: ', full_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:47:47.705604Z","iopub.execute_input":"2022-07-22T08:47:47.705969Z","iopub.status.idle":"2022-07-22T08:47:47.805816Z","shell.execute_reply.started":"2022-07-22T08:47:47.705939Z","shell.execute_reply":"2022-07-22T08:47:47.805007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tukey_outliers(x):\n    q1 = np.percentile(x,25)\n    q3 = np.percentile(x,75)\n    \n    iqr = q3-q1\n    \n    min_range = q1 - iqr*1.5\n    max_range = q3 + iqr*1.5\n    \n    outliers = x[(x<min_range) | (x>max_range)]\n    return outliers","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:48:12.365891Z","iopub.execute_input":"2022-07-22T08:48:12.366237Z","iopub.status.idle":"2022-07-22T08:48:12.372246Z","shell.execute_reply.started":"2022-07-22T08:48:12.366209Z","shell.execute_reply":"2022-07-22T08:48:12.371171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(num_list)/4+1)\n\n# boxplot\nfig = plt.figure(figsize=(15,20))\nplt.subplots_adjust(hspace=0.2, wspace=0.8)\nfor i in range(len(num_list)):\n    ax = fig.add_subplot(columns, 4, i+1)\n    sns.boxplot(y=full_data[num_list[i]], data=full_data, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:48:20.825437Z","iopub.execute_input":"2022-07-22T08:48:20.826673Z","iopub.status.idle":"2022-07-22T08:48:25.276698Z","shell.execute_reply.started":"2022-07-22T08:48:20.826614Z","shell.execute_reply":"2022-07-22T08:48:25.275452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewed_data = train[num_list].apply(lambda x: skew(x)).sort_values(ascending=False)\nskewed_data[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:48:30.477641Z","iopub.execute_input":"2022-07-22T08:48:30.478587Z","iopub.status.idle":"2022-07-22T08:48:30.499117Z","shell.execute_reply.started":"2022-07-22T08:48:30.478549Z","shell.execute_reply":"2022-07-22T08:48:30.497991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into train and test datasets\ntrain = full_data[:num_train]\ntest = full_data[num_train:]\n# check the shapes \nprint(\"Train :\",train.shape)\nprint(\"Test:\",test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:48:39.231057Z","iopub.execute_input":"2022-07-22T08:48:39.232138Z","iopub.status.idle":"2022-07-22T08:48:39.238998Z","shell.execute_reply.started":"2022-07-22T08:48:39.232099Z","shell.execute_reply":"2022-07-22T08:48:39.237612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(font_scale=1.1)\ncorrelation_train = train.corr()\nmask = np.triu(correlation_train.corr())\nfig = plt.figure(figsize=(50,50))\nsns.heatmap(correlation_train,\n            annot=True,\n            fmt='.1f',\n            cmap='coolwarm',\n            square=True,\n#             mask=mask,\n            linewidths=1)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:49:02.849624Z","iopub.execute_input":"2022-07-22T08:49:02.849991Z","iopub.status.idle":"2022-07-22T08:49:17.608162Z","shell.execute_reply.started":"2022-07-22T08:49:02.849961Z","shell.execute_reply":"2022-07-22T08:49:17.606927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find correlations with the target and sort\ncorrelations = train.corr()['revenue'].sort_values()\n\n# Display correlations\nprint('Most Positive Correlations:\\n', correlations.tail(15))\nprint('\\nMost Negative Correlations:\\n', correlations.head(15))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:49:22.511161Z","iopub.execute_input":"2022-07-22T08:49:22.511524Z","iopub.status.idle":"2022-07-22T08:49:22.523544Z","shell.execute_reply.started":"2022-07-22T08:49:22.511496Z","shell.execute_reply":"2022-07-22T08:49:22.522410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 相関が高い10項目のみ抽出\ncorrelations = train.corr()\n# 絶対値で取得\ncorrelations = abs(correlations)\n\ncols = correlations.nlargest(10,'revenue')['revenue'].index\ncols","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:49:30.570048Z","iopub.execute_input":"2022-07-22T08:49:30.571088Z","iopub.status.idle":"2022-07-22T08:49:30.583533Z","shell.execute_reply.started":"2022-07-22T08:49:30.571041Z","shell.execute_reply":"2022-07-22T08:49:30.582524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 相関が高い10項目のみ抽出\ntrain = train[cols]\n\n#学習データを目的変数とそれ以外に分ける\ntrain_X = train.drop(\"revenue\",axis=1)\ntrain_y = train[\"revenue\"]\n\n#revenueを対数変換する \ntrain_y = np.log1p(train_y)\n\n#テストデータを学習データのカラムのみにする \ntmp_cols = train_X.columns\ntest_X = test[tmp_cols]\n\n#それぞれのデータのサイズを確認\nprint(\"train_X: \"+str(train_X.shape))\nprint(\"train_y: \"+str(train_y.shape))\nprint(\"test_X: \"+str(test_X.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:49:41.124583Z","iopub.execute_input":"2022-07-22T08:49:41.125005Z","iopub.status.idle":"2022-07-22T08:49:41.165316Z","shell.execute_reply.started":"2022-07-22T08:49:41.124969Z","shell.execute_reply":"2022-07-22T08:49:41.163915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#訓練データとモデル評価用データに分けるライブラリ\nfrom sklearn.model_selection import train_test_split\n\n#フォールドアウト法により、学習データとテストデータに分割 \n(train_x, valid_x, train_y, valid_y) = train_test_split(train_X, train_y , test_size = 0.3 , random_state = 0)\n\nprint(\"X_train: \"+str(train_x.shape))\nprint(\"X_test: \"+str(valid_x.shape))\nprint(\"y_train: \"+str(train_y.shape))\nprint(\"y_test: \"+str(valid_y.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:49:49.652514Z","iopub.execute_input":"2022-07-22T08:49:49.652872Z","iopub.status.idle":"2022-07-22T08:49:49.664456Z","shell.execute_reply.started":"2022-07-22T08:49:49.652839Z","shell.execute_reply":"2022-07-22T08:49:49.663413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVR\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\n\n# lightGBMによる予測\nlgb_train = lgb.Dataset(train_x, train_y)\nlgb_eval = lgb.Dataset(valid_x, valid_y, reference=lgb_train)\n\n# LightGBM parameters\nparams = {\n        'task' : 'train',\n        'boosting_type' : 'gbdt',\n        'objective' : 'regression',\n        'metric' : {'l2'},\n        'num_leaves' : 31,\n        'learning_rate' : 0.2,\n        'feature_fraction' : 0.9,\n        'bagging_fraction' : 0.8,\n        'bagging_freq': 5,\n        'verbose' : 0,\n        'n_jobs': 2\n}\n\ngbm = lgb.train(params,\n            lgb_train,\n            num_boost_round=100,\n            valid_sets=lgb_eval,\n            early_stopping_rounds=10)\n\nprediction_lgb = np.exp(gbm.predict(test_X))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:55:18.728886Z","iopub.execute_input":"2022-07-22T08:55:18.729282Z","iopub.status.idle":"2022-07-22T08:55:18.876700Z","shell.execute_reply.started":"2022-07-22T08:55:18.729249Z","shell.execute_reply":"2022-07-22T08:55:18.875153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 予測した値を提出用CSVファイル(submissionファイル)に書き出し\nsubmission = pd.DataFrame({\"Id\":test_Id, \"Prediction\":prediction_lgb})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T08:50:37.985385Z","iopub.execute_input":"2022-07-22T08:50:37.985756Z","iopub.status.idle":"2022-07-22T08:50:38.211349Z","shell.execute_reply.started":"2022-07-22T08:50:37.985728Z","shell.execute_reply":"2022-07-22T08:50:38.210209Z"},"trusted":true},"execution_count":null,"outputs":[]}]}