{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#importing the libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n# import seaborn as sns\nimport seaborn as sns; sns.set(style=\"ticks\", color_codes=True)\n\nfrom datetime import datetime\nfrom scipy import stats\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:34.291375Z","iopub.execute_input":"2022-07-15T08:00:34.291944Z","iopub.status.idle":"2022-07-15T08:00:36.545070Z","shell.execute_reply.started":"2022-07-15T08:00:34.291814Z","shell.execute_reply":"2022-07-15T08:00:36.543482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:36.547436Z","iopub.execute_input":"2022-07-15T08:00:36.547865Z","iopub.status.idle":"2022-07-15T08:00:36.569422Z","shell.execute_reply.started":"2022-07-15T08:00:36.547823Z","shell.execute_reply":"2022-07-15T08:00:36.568244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')\ntest = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')\n\n# Idは不要なので、削除して別に変数化し、スコア提出時に使用\ntrain_Id = train.Id\ntest_Id = test.Id\n\n# Id列削除\ntrain.drop('Id', axis=1, inplace=True)\ntest.drop('Id', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:36.571842Z","iopub.execute_input":"2022-07-15T08:00:36.572568Z","iopub.status.idle":"2022-07-15T08:00:37.201221Z","shell.execute_reply.started":"2022-07-15T08:00:36.572517Z","shell.execute_reply":"2022-07-15T08:00:37.200051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 最大カラム数を100に拡張(デフォルトだと省略されてしまうので)\n# 常に全ての列（カラム）を表示\npd.options.display.max_columns = None\npd.options.display.max_rows = 80\n\n# 小数点2桁で表示(指数表記しないように)\npd.options.display.float_format = '{:.2f}'.format\n%matplotlib inline\n#ワーニングを抑止\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.203064Z","iopub.execute_input":"2022-07-15T08:00:37.203629Z","iopub.status.idle":"2022-07-15T08:00:37.216645Z","shell.execute_reply.started":"2022-07-15T08:00:37.203521Z","shell.execute_reply":"2022-07-15T08:00:37.215037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of train data', train.shape)\nprint('Size of test data', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.219609Z","iopub.execute_input":"2022-07-15T08:00:37.219993Z","iopub.status.idle":"2022-07-15T08:00:37.236509Z","shell.execute_reply.started":"2022-07-15T08:00:37.219960Z","shell.execute_reply":"2022-07-15T08:00:37.235249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.238249Z","iopub.execute_input":"2022-07-15T08:00:37.238926Z","iopub.status.idle":"2022-07-15T08:00:37.250162Z","shell.execute_reply.started":"2022-07-15T08:00:37.238880Z","shell.execute_reply":"2022-07-15T08:00:37.249222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.251413Z","iopub.execute_input":"2022-07-15T08:00:37.251876Z","iopub.status.idle":"2022-07-15T08:00:37.280526Z","shell.execute_reply.started":"2022-07-15T08:00:37.251847Z","shell.execute_reply":"2022-07-15T08:00:37.279658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.281908Z","iopub.execute_input":"2022-07-15T08:00:37.282265Z","iopub.status.idle":"2022-07-15T08:00:37.396583Z","shell.execute_reply.started":"2022-07-15T08:00:37.282230Z","shell.execute_reply":"2022-07-15T08:00:37.395446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.398244Z","iopub.execute_input":"2022-07-15T08:00:37.398908Z","iopub.status.idle":"2022-07-15T08:00:37.418073Z","shell.execute_reply.started":"2022-07-15T08:00:37.398867Z","shell.execute_reply":"2022-07-15T08:00:37.417306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"revenue\"].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.419595Z","iopub.execute_input":"2022-07-15T08:00:37.420268Z","iopub.status.idle":"2022-07-15T08:00:37.431003Z","shell.execute_reply.started":"2022-07-15T08:00:37.420224Z","shell.execute_reply":"2022-07-15T08:00:37.429856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#目的変数であるrevenueのヒストグラムとQ-Qプロットを表示する\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(train['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(train['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(train['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % train['revenue'].skew())\nprint(\"尖度: %f\" % train['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.432511Z","iopub.execute_input":"2022-07-15T08:00:37.433194Z","iopub.status.idle":"2022-07-15T08:00:37.828640Z","shell.execute_reply.started":"2022-07-15T08:00:37.433154Z","shell.execute_reply":"2022-07-15T08:00:37.827428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データをコピーし、新たなdataframeで検証\ndf = train.copy()\n\n#目的変数の対数log(x+1)をとる\ndf['revenue'] = np.log1p(df['revenue'])\n\n# 標準化(平均0, 分散1)\nscaler=StandardScaler()\ndf['revenue']=scaler.fit_transform(df[['revenue']])\n\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(df['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(df['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(df['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % df['revenue'].skew())\nprint(\"尖度: %f\" % df['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:37.830192Z","iopub.execute_input":"2022-07-15T08:00:37.830613Z","iopub.status.idle":"2022-07-15T08:00:38.169664Z","shell.execute_reply.started":"2022-07-15T08:00:37.830571Z","shell.execute_reply":"2022-07-15T08:00:38.168386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データをコピーし、新たなdataframeで検証\ndf = train.copy()\n\n# 標準化(平均0, 分散1)\nscaler=StandardScaler()\ndf['revenue']=scaler.fit_transform(df[['revenue']])\n\n\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(df['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(df['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(df['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % df['revenue'].skew())\nprint(\"尖度: %f\" % df['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:38.171265Z","iopub.execute_input":"2022-07-15T08:00:38.171683Z","iopub.status.idle":"2022-07-15T08:00:38.498128Z","shell.execute_reply.started":"2022-07-15T08:00:38.171644Z","shell.execute_reply":"2022-07-15T08:00:38.496717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データをコピーし、新たなdataframeで検証\ndf = train.copy()\n\n# Min-Max変換(正規化(最大1, 最小0))\nscaler=MinMaxScaler()\ndf['revenue']=scaler.fit_transform(df[['revenue']])\n\n# 分布確認\nfig = plt.figure(figsize=(10, 4))\nplt.subplots_adjust(wspace=0.4)\n\n# ヒストグラム\nax = fig.add_subplot(1, 2, 1)\nsns.distplot(df['revenue'], ax=ax)\n\n# QQプロット\nax2 = fig.add_subplot(1, 2, 2)\nstats.probplot(df['revenue'], plot=ax2)\n\nplt.show()\n\n# 変換後の要約統計量表示\nprint(df['revenue'].describe())\nprint(\"------------------------------\")\nprint(\"歪度: %f\" % df['revenue'].skew())\nprint(\"尖度: %f\" % df['revenue'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:38.507686Z","iopub.execute_input":"2022-07-15T08:00:38.508854Z","iopub.status.idle":"2022-07-15T08:00:38.986297Z","shell.execute_reply.started":"2022-07-15T08:00:38.508805Z","shell.execute_reply":"2022-07-15T08:00:38.984995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データ\n# Open Dateを日付型に変換\ntrain['pd_date'] = pd.to_datetime(train['Open Date'], format='%m/%d/%Y')\n# 年のみを抽出\ntrain['Open_Year'] = train['pd_date'].dt.strftime('%Y')\n# 月のみを抽出\ntrain['Open_Month'] = train['pd_date'].dt.strftime('%m')\n# 経過年数\ntrain[\"Open Date\"] = pd.to_datetime(train[\"Open Date\"])\ntrain[\"Day\"] = train[\"Open Date\"].apply(lambda x:x.day)\ntrain[\"kijun\"] = \"2022-07-10\"\ntrain[\"kijun\"] = pd.to_datetime(train[\"kijun\"])\ntrain[\"BusinessPeriod\"] = (train[\"kijun\"] - train[\"Open Date\"]).apply(lambda x: x.days)\n\ntrain = train.drop('kijun', axis=1)\n\ntrain = train.drop('pd_date',axis=1)\ntrain = train.drop('Open Date',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:38.988076Z","iopub.execute_input":"2022-07-15T08:00:38.988972Z","iopub.status.idle":"2022-07-15T08:00:39.023582Z","shell.execute_reply.started":"2022-07-15T08:00:38.988930Z","shell.execute_reply":"2022-07-15T08:00:39.022338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# テストデータ\n# Open Dateを日付型に変換\ntest['pd_date'] = pd.to_datetime(test['Open Date'], format='%m/%d/%Y')\n# 年のみを抽出\ntest['Open_Year'] = test['pd_date'].dt.strftime('%Y')\n# 月のみを抽出\ntest['Open_Month'] = test['pd_date'].dt.strftime('%m')\n# 経過年数\ntest[\"Open Date\"] = pd.to_datetime(test[\"Open Date\"])\ntest[\"Day\"] = test[\"Open Date\"].apply(lambda x:x.day)\ntest[\"kijun\"] = \"2022-07-10\"\ntest[\"kijun\"] = pd.to_datetime(test[\"kijun\"])\ntest[\"BusinessPeriod\"] = (test[\"kijun\"] - test[\"Open Date\"]).apply(lambda x: x.days)\n\ntest = test.drop('kijun', axis=1)\ntest = test.drop('pd_date',axis=1)\ntest = test.drop('Open Date',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:39.025123Z","iopub.execute_input":"2022-07-15T08:00:39.025812Z","iopub.status.idle":"2022-07-15T08:00:41.416533Z","shell.execute_reply.started":"2022-07-15T08:00:39.025773Z","shell.execute_reply":"2022-07-15T08:00:41.415405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:41.418134Z","iopub.execute_input":"2022-07-15T08:00:41.418481Z","iopub.status.idle":"2022-07-15T08:00:41.426647Z","shell.execute_reply.started":"2022-07-15T08:00:41.418451Z","shell.execute_reply":"2022-07-15T08:00:41.425506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#カテゴリ変数と数値変数に分ける\ncats = list(train.select_dtypes(include=['object']).columns)\nnums = list(train.select_dtypes(exclude=['object']).columns)\nprint(f'categorical variables:  {cats}')\nprint(f'numerical variables:  {nums}')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:41.428309Z","iopub.execute_input":"2022-07-15T08:00:41.428706Z","iopub.status.idle":"2022-07-15T08:00:41.443706Z","shell.execute_reply.started":"2022-07-15T08:00:41.428674Z","shell.execute_reply":"2022-07-15T08:00:41.442834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.nunique(axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:41.444732Z","iopub.execute_input":"2022-07-15T08:00:41.445428Z","iopub.status.idle":"2022-07-15T08:00:41.461439Z","shell.execute_reply.started":"2022-07-15T08:00:41.445390Z","shell.execute_reply":"2022-07-15T08:00:41.460627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'categorical variables:  {cats}')\nprint(f'numerical variables:  {nums}')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:41.462568Z","iopub.execute_input":"2022-07-15T08:00:41.463433Z","iopub.status.idle":"2022-07-15T08:00:41.468044Z","shell.execute_reply.started":"2022-07-15T08:00:41.463400Z","shell.execute_reply":"2022-07-15T08:00:41.467267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 名義変数\nnominal_list =cats\n               \n# 順序変数\n# ordinal_list = []\n\n# 数値変数\nnum_list = nums\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:41.469072Z","iopub.execute_input":"2022-07-15T08:00:41.469812Z","iopub.status.idle":"2022-07-15T08:00:41.481292Z","shell.execute_reply.started":"2022-07-15T08:00:41.469781Z","shell.execute_reply":"2022-07-15T08:00:41.480431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(nominal_list)/2+1)\n\nfig = plt.figure(figsize=(30, 20))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(nominal_list)):\n    ax = fig.add_subplot(columns, 2, i+1)\n    sns.countplot(x=nominal_list[i], data=train, ax=ax)\n    plt.xticks(rotation=45)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:41.482408Z","iopub.execute_input":"2022-07-15T08:00:41.482903Z","iopub.status.idle":"2022-07-15T08:00:42.484419Z","shell.execute_reply.started":"2022-07-15T08:00:41.482870Z","shell.execute_reply":"2022-07-15T08:00:42.483261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(num_list)/3+1)\n\nfig = plt.figure(figsize=(30, 40))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(num_list)):\n    ax = fig.add_subplot(columns, 3, i+1)\n\n    train[num_list[i]].hist(ax=ax)\n    ax2 = train[num_list[i]].plot.kde(ax=ax, secondary_y=True,title=num_list[i])\n    ax2.set_ylim(0)\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:42.485964Z","iopub.execute_input":"2022-07-15T08:00:42.486309Z","iopub.status.idle":"2022-07-15T08:00:49.781165Z","shell.execute_reply.started":"2022-07-15T08:00:42.486279Z","shell.execute_reply":"2022-07-15T08:00:49.780112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(nominal_list)/2+1)\n\nfig = plt.figure(figsize=(20, 10))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(nominal_list)):\n    ax = fig.add_subplot(columns, 2, i+1)\n\n    # 回帰の場合    \n    sns.boxplot(x=nominal_list[i], y=train.revenue, data=train, ax=ax)\n    plt.xticks(rotation=45)\n    # 分類の場合\n#     sns.barplot(x = nominal_list[i], y = train.revenue, data=train, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:49.782435Z","iopub.execute_input":"2022-07-15T08:00:49.782799Z","iopub.status.idle":"2022-07-15T08:00:51.175625Z","shell.execute_reply.started":"2022-07-15T08:00:49.782768Z","shell.execute_reply":"2022-07-15T08:00:51.174564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop('Open_Month',axis=1)\ntest= test.drop('Open_Month',axis=1)\nnominal_list.remove('Open_Month')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:51.177013Z","iopub.execute_input":"2022-07-15T08:00:51.177467Z","iopub.status.idle":"2022-07-15T08:00:51.199394Z","shell.execute_reply.started":"2022-07-15T08:00:51.177432Z","shell.execute_reply":"2022-07-15T08:00:51.198389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(num_list)/4+1)\n\nfig = plt.figure(figsize=(30, 35))\nplt.subplots_adjust(hspace=0.6, wspace=0.4)\n\nfor i in range(len(num_list)):\n    ax = fig.add_subplot(columns, 4, i+1)\n\n    # 回帰の場合    \n    sns.regplot(x=num_list[i],y='revenue',data=train, ax=ax)\n    plt.xticks(rotation=45)\n    # 分類の場合\n#     sns.barplot(x = nominal_list[i], y = train.revenue, data=train, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:51.200614Z","iopub.execute_input":"2022-07-15T08:00:51.201109Z","iopub.status.idle":"2022-07-15T08:00:59.117274Z","shell.execute_reply.started":"2022-07-15T08:00:51.201076Z","shell.execute_reply":"2022-07-15T08:00:59.116149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['City','revenue']].groupby('City').mean().plot(kind='bar')\nplt.title('Mean Revenue Generated vs City')\nplt.xlabel('City')\nplt.ylabel('Mean Revenue Generated')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.118765Z","iopub.execute_input":"2022-07-15T08:00:59.119056Z","iopub.status.idle":"2022-07-15T08:00:59.485463Z","shell.execute_reply.started":"2022-07-15T08:00:59.119029Z","shell.execute_reply":"2022-07-15T08:00:59.484280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cityごとのrevenue平均値を1000000単位とする\nmean_revenue_per_city = train[['City', 'revenue']].groupby('City', as_index=False).mean()\nmean_revenue_per_city.head()\nmean_revenue_per_city['revenue'] = mean_revenue_per_city['revenue'].apply(lambda x: int(x/1e6)) \n\nmean_revenue_per_city\n\nmean_dict = dict(zip(mean_revenue_per_city.City, mean_revenue_per_city.revenue))\nmean_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.486817Z","iopub.execute_input":"2022-07-15T08:00:59.487782Z","iopub.status.idle":"2022-07-15T08:00:59.501405Z","shell.execute_reply.started":"2022-07-15T08:00:59.487748Z","shell.execute_reply":"2022-07-15T08:00:59.500267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train['City'].sort_values().unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.502427Z","iopub.execute_input":"2022-07-15T08:00:59.502722Z","iopub.status.idle":"2022-07-15T08:00:59.511876Z","shell.execute_reply.started":"2022-07-15T08:00:59.502696Z","shell.execute_reply":"2022-07-15T08:00:59.510770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['City'].sort_values().unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.513041Z","iopub.execute_input":"2022-07-15T08:00:59.513650Z","iopub.status.idle":"2022-07-15T08:00:59.610815Z","shell.execute_reply.started":"2022-07-15T08:00:59.513618Z","shell.execute_reply":"2022-07-15T08:00:59.609957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cityについて、学習データとテストデータにて重複削除し、リスト化\ncity_train_list = list(train['City'].unique())\ncity_test_list = list(test['City'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.611913Z","iopub.execute_input":"2022-07-15T08:00:59.612772Z","iopub.status.idle":"2022-07-15T08:00:59.624549Z","shell.execute_reply.started":"2022-07-15T08:00:59.612738Z","shell.execute_reply":"2022-07-15T08:00:59.623372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1_l2_and = set(city_train_list) & set(city_test_list)\nprint(l1_l2_and)\nprint(len(l1_l2_and))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.626250Z","iopub.execute_input":"2022-07-15T08:00:59.626864Z","iopub.status.idle":"2022-07-15T08:00:59.637664Z","shell.execute_reply.started":"2022-07-15T08:00:59.626834Z","shell.execute_reply":"2022-07-15T08:00:59.636547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# どちらかにしかないCityを抽出\nl1_l2_sym_diff = set(city_test_list) ^ set(city_train_list)\nprint(l1_l2_sym_diff)\nprint(len(l1_l2_sym_diff))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.638778Z","iopub.execute_input":"2022-07-15T08:00:59.639609Z","iopub.status.idle":"2022-07-15T08:00:59.651062Z","shell.execute_reply.started":"2022-07-15T08:00:59.639575Z","shell.execute_reply":"2022-07-15T08:00:59.650281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# テストデータのみ存在するCityの件数\nlen(set(city_test_list).difference(city_train_list))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.652221Z","iopub.execute_input":"2022-07-15T08:00:59.652614Z","iopub.status.idle":"2022-07-15T08:00:59.670512Z","shell.execute_reply.started":"2022-07-15T08:00:59.652582Z","shell.execute_reply":"2022-07-15T08:00:59.669391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データのみ存在するCityの件数\nlen(set(city_train_list).difference(city_test_list))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.671718Z","iopub.execute_input":"2022-07-15T08:00:59.672442Z","iopub.status.idle":"2022-07-15T08:00:59.683287Z","shell.execute_reply.started":"2022-07-15T08:00:59.672409Z","shell.execute_reply":"2022-07-15T08:00:59.682422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# P変数の1つのクラスは地理的属性であると指定されているため\n# 各都市のP変数の平均をプロットすると、どのP変数が都市と関連性が高いかが分かる\ndistinct_cities = train.loc[:, \"City\"].unique()\n\n# P変数のcityごとの平均値を取得\nmeans = []\nfor i in range(len(num_list)):\n    temp = []\n    for city in distinct_cities:\n        temp.append(train.loc[train.City == city, num_list[i]].mean())  \n    means.append(temp)\n    \ncity_pvars = pd.DataFrame(columns=[\"city_var\", \"means\"])\nfor i in range(37):\n    for j in range(len(distinct_cities)):\n        city_pvars.loc[i+37*j] = [\"P\"+str(i+1), means[i][j]]\n\nprint(city_pvars)            \n# 箱ひげ図を表示\nplt.rcParams['figure.figsize'] = (18.0, 6.0)\nsns.boxplot(x=\"city_var\", y=\"means\", data=city_pvars)\n\n# From this we observe that P1, P2, P11, P19, P20, P23, and P30 are approximately a good\n# proxy for geographical location.\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:00:59.684932Z","iopub.execute_input":"2022-07-15T08:00:59.685484Z","iopub.status.idle":"2022-07-15T08:01:03.388180Z","shell.execute_reply.started":"2022-07-15T08:00:59.685441Z","shell.execute_reply":"2022-07-15T08:01:03.387289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import cluster\n\ndef adjust_cities(full_full_data, train, k):\n    \n    # As found by box plot of each city's mean over each p-var\n    relevant_pvars =  [\"P1\", \"P2\", \"P11\", \"P19\", \"P20\", \"P23\",\"P30\"]\n    train = train.loc[:, relevant_pvars]\n    \n    # Optimal k is 20 as found by DB-Index plot    \n    kmeans = cluster.KMeans(n_clusters=k)\n    kmeans.fit(train)\n    \n    # Get the cluster centers and classify city of each full_data instance to one of the centers\n    full_data['City_Cluster'] = kmeans.predict(full_data.loc[:, relevant_pvars])\n    \n    return full_data","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:03.389297Z","iopub.execute_input":"2022-07-15T08:01:03.390029Z","iopub.status.idle":"2022-07-15T08:01:03.605028Z","shell.execute_reply.started":"2022-07-15T08:01:03.389996Z","shell.execute_reply":"2022-07-15T08:01:03.603738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_train = train.shape[0]\nnum_test = test.shape[0]\nprint(num_train, num_test)\n\nfull_data = pd.concat([train, test], ignore_index=True)  ","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:03.606902Z","iopub.execute_input":"2022-07-15T08:01:03.607367Z","iopub.status.idle":"2022-07-15T08:01:03.637968Z","shell.execute_reply.started":"2022-07-15T08:01:03.607321Z","shell.execute_reply":"2022-07-15T08:01:03.637054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データを使用しクラスタリングを行い、その学習結果を全データに適用させる\nfull_data = adjust_cities(full_data, train, 20)\nfull_data\n\n# City項目は不要なので削除\nfull_data = full_data.drop(['City'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:03.639656Z","iopub.execute_input":"2022-07-15T08:01:03.639976Z","iopub.status.idle":"2022-07-15T08:01:03.925831Z","shell.execute_reply.started":"2022-07-15T08:01:03.639946Z","shell.execute_reply":"2022-07-15T08:01:03.924572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into train and test datasets\ntrain = full_data[:num_train]\ntest = full_data[num_train:]\n# check the shapes \nprint(\"Train :\",train.shape)\nprint(\"Test:\",test.shape)\ntest","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:03.927706Z","iopub.execute_input":"2022-07-15T08:01:03.929060Z","iopub.status.idle":"2022-07-15T08:01:03.968836Z","shell.execute_reply.started":"2022-07-15T08:01:03.929001Z","shell.execute_reply":"2022-07-15T08:01:03.967676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['City_Cluster','revenue']].groupby('City_Cluster').mean().plot(kind='bar')\nplt.title('Mean Revenue Generated vs City Cluster')\nplt.xlabel('City Cluster')\nplt.ylabel('Mean Revenue Generated')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:03.975774Z","iopub.execute_input":"2022-07-15T08:01:03.976135Z","iopub.status.idle":"2022-07-15T08:01:04.275330Z","shell.execute_reply.started":"2022-07-15T08:01:03.976104Z","shell.execute_reply":"2022-07-15T08:01:04.274476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_revenue_per_city = train[['City_Cluster', 'revenue']].groupby('City_Cluster', as_index=False).mean()\nmean_revenue_per_city.head()\nmean_revenue_per_city['revenue'] = mean_revenue_per_city['revenue'].apply(lambda x: int(x/1e6)) \n\nmean_revenue_per_city\n\nmean_dict = dict(zip(mean_revenue_per_city.City_Cluster, mean_revenue_per_city.revenue))\nmean_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:04.276784Z","iopub.execute_input":"2022-07-15T08:01:04.277122Z","iopub.status.idle":"2022-07-15T08:01:04.291628Z","shell.execute_reply.started":"2022-07-15T08:01:04.277090Z","shell.execute_reply":"2022-07-15T08:01:04.290455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"city_rev = []\n\nfor i in full_data['City_Cluster']:\n    for key, value in mean_dict.items():\n        if i == key:\n            city_rev.append(value)\n            \ndf_city_rev = pd.DataFrame({'city_rev':city_rev})\nfull_data = pd.concat([full_data,df_city_rev],axis=1)\nfull_data.head\n\n# 値の追加\nnominal_list.extend(['City_Cluster'])\n# 値の削除\nnominal_list.remove('City')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:04.292884Z","iopub.execute_input":"2022-07-15T08:01:04.293255Z","iopub.status.idle":"2022-07-15T08:01:04.582837Z","shell.execute_reply.started":"2022-07-15T08:01:04.293214Z","shell.execute_reply":"2022-07-15T08:01:04.581691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nle_count = 0\n\n# Iterate through the columns\n# for col in application_full_data:\nfor i in range(len(nominal_list)):    \n    \n#     if application_full_data[col].dtype == 'object':\n        # If 2 or fewer unique categories\n        if len(list(full_data[nominal_list[i]].unique())) <= 2:\n            # full_data on the full_dataing data\n            le.fit(full_data[nominal_list[i]])\n            # Transform both full_dataing and testing data\n            full_data[nominal_list[i]] = le.transform(full_data[nominal_list[i]])\n            \n            # Keep track of how many columns were label encoded\n            le_count += 1\n            \nprint('%d columns were label encoded.' % le_count)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:04.584988Z","iopub.execute_input":"2022-07-15T08:01:04.586033Z","iopub.status.idle":"2022-07-15T08:01:04.637694Z","shell.execute_reply.started":"2022-07-15T08:01:04.585982Z","shell.execute_reply":"2022-07-15T08:01:04.636314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one-hot encoding of categorical variables\nfull_data = pd.get_dummies(full_data)\nprint('full_dataing Features shape: ', full_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:04.640158Z","iopub.execute_input":"2022-07-15T08:01:04.640906Z","iopub.status.idle":"2022-07-15T08:01:04.729843Z","shell.execute_reply.started":"2022-07-15T08:01:04.640857Z","shell.execute_reply":"2022-07-15T08:01:04.728526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tukey_outliers(x):\n    q1 = np.percentile(x,25)\n    q3 = np.percentile(x,75)\n    \n    iqr = q3-q1\n    \n    min_range = q1 - iqr*1.5\n    max_range = q3 + iqr*1.5\n    \n    outliers = x[(x<min_range) | (x>max_range)]\n    return outliers","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:04.731268Z","iopub.execute_input":"2022-07-15T08:01:04.731587Z","iopub.status.idle":"2022-07-15T08:01:04.737759Z","shell.execute_reply.started":"2022-07-15T08:01:04.731558Z","shell.execute_reply":"2022-07-15T08:01:04.736600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = int(len(num_list)/4+1)\n\n# boxplot\nfig = plt.figure(figsize=(15,20))\nplt.subplots_adjust(hspace=0.2, wspace=0.8)\nfor i in range(len(num_list)):\n    ax = fig.add_subplot(columns, 4, i+1)\n    sns.boxplot(y=full_data[num_list[i]], data=full_data, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:04.739068Z","iopub.execute_input":"2022-07-15T08:01:04.739414Z","iopub.status.idle":"2022-07-15T08:01:08.971268Z","shell.execute_reply.started":"2022-07-15T08:01:04.739387Z","shell.execute_reply":"2022-07-15T08:01:08.969945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewed_data = train[num_list].apply(lambda x: skew(x)).sort_values(ascending=False)\nskewed_data[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:08.972893Z","iopub.execute_input":"2022-07-15T08:01:08.973379Z","iopub.status.idle":"2022-07-15T08:01:08.993999Z","shell.execute_reply.started":"2022-07-15T08:01:08.973332Z","shell.execute_reply":"2022-07-15T08:01:08.993026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into train and test datasets\ntrain = full_data[:num_train]\ntest = full_data[num_train:]\n# check the shapes \nprint(\"Train :\",train.shape)\nprint(\"Test:\",test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:08.995253Z","iopub.execute_input":"2022-07-15T08:01:08.995573Z","iopub.status.idle":"2022-07-15T08:01:09.022955Z","shell.execute_reply.started":"2022-07-15T08:01:08.995545Z","shell.execute_reply":"2022-07-15T08:01:09.021469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(font_scale=1.1)\ncorrelation_train = train.corr()\nmask = np.triu(correlation_train.corr())\nfig = plt.figure(figsize=(50,50))\nsns.heatmap(correlation_train,\n            annot=True,\n            fmt='.1f',\n            cmap='coolwarm',\n            square=True,\n#             mask=mask,\n            linewidths=1)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:09.024813Z","iopub.execute_input":"2022-07-15T08:01:09.025455Z","iopub.status.idle":"2022-07-15T08:01:24.222332Z","shell.execute_reply.started":"2022-07-15T08:01:09.025407Z","shell.execute_reply":"2022-07-15T08:01:24.220822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find correlations with the target and sort\ncorrelations = train.corr()['revenue'].sort_values()\n\n# Display correlations\nprint('Most Positive Correlations:\\n', correlations.tail(15))\nprint('\\nMost Negative Correlations:\\n', correlations.head(15))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:24.223933Z","iopub.execute_input":"2022-07-15T08:01:24.224337Z","iopub.status.idle":"2022-07-15T08:01:24.236011Z","shell.execute_reply.started":"2022-07-15T08:01:24.224295Z","shell.execute_reply":"2022-07-15T08:01:24.234786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 相関が高い10項目のみ抽出\ncorrelations = train.corr()\n# 絶対値で取得\ncorrelations = abs(correlations)\n\ncols = correlations.nlargest(10,'revenue')['revenue'].index\ncols","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:24.237706Z","iopub.execute_input":"2022-07-15T08:01:24.238872Z","iopub.status.idle":"2022-07-15T08:01:24.258145Z","shell.execute_reply.started":"2022-07-15T08:01:24.238829Z","shell.execute_reply":"2022-07-15T08:01:24.256628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 相関が高い10項目のみ抽出\ntrain = train[cols]\n\n#学習データを目的変数とそれ以外に分ける\ntrain_X = train.drop(\"revenue\",axis=1)\ntrain_y = train[\"revenue\"]\n\n#revenueを対数変換する \ntrain_y = np.log1p(train_y)\n\n#テストデータを学習データのカラムのみにする \ntmp_cols = train_X.columns\ntest_X = test[tmp_cols]\n\n#それぞれのデータのサイズを確認\nprint(\"train_X: \"+str(train_X.shape))\nprint(\"train_y: \"+str(train_y.shape))\nprint(\"test_X: \"+str(test_X.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:24.259838Z","iopub.execute_input":"2022-07-15T08:01:24.260421Z","iopub.status.idle":"2022-07-15T08:01:24.294486Z","shell.execute_reply.started":"2022-07-15T08:01:24.260387Z","shell.execute_reply":"2022-07-15T08:01:24.293262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#訓練データとモデル評価用データに分けるライブラリ\nfrom sklearn.model_selection import train_test_split\n\n#フォールドアウト法により、学習データとテストデータに分割 \n(train_x, valid_x, train_y, valid_y) = train_test_split(train_X, train_y , test_size = 0.3 , random_state = 0)\n\nprint(\"X_train: \"+str(train_x.shape))\nprint(\"X_test: \"+str(valid_x.shape))\nprint(\"y_train: \"+str(train_y.shape))\nprint(\"y_test: \"+str(valid_y.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:24.296128Z","iopub.execute_input":"2022-07-15T08:01:24.296531Z","iopub.status.idle":"2022-07-15T08:01:24.307309Z","shell.execute_reply.started":"2022-07-15T08:01:24.296498Z","shell.execute_reply":"2022-07-15T08:01:24.306274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVR\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\n\n# lightGBMによる予測\nlgb_train = lgb.Dataset(train_x, train_y)\nlgb_eval = lgb.Dataset(valid_x, valid_y, reference=lgb_train)\n\n# LightGBM parameters\nparams = {\n        'task' : 'train',\n        'boosting_type' : 'gbdt',\n        'objective' : 'regression',\n        'metric' : {'l2'},\n        'num_leaves' : 31,\n        'learning_rate' : 0.1,\n        'feature_fraction' : 0.9,\n        'bagging_fraction' : 0.8,\n        'bagging_freq': 5,\n        'verbose' : 0,\n        'n_jobs': 2\n}\n\ngbm = lgb.train(params,\n            lgb_train,\n            num_boost_round=100,\n            valid_sets=lgb_eval,\n            early_stopping_rounds=10)\n\nprediction_lgb = np.exp(gbm.predict(test_X))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:24.308747Z","iopub.execute_input":"2022-07-15T08:01:24.309048Z","iopub.status.idle":"2022-07-15T08:01:24.555139Z","shell.execute_reply.started":"2022-07-15T08:01:24.309022Z","shell.execute_reply":"2022-07-15T08:01:24.553991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 予測した値を提出用CSVファイル(submissionファイル)に書き出し\nsubmission = pd.DataFrame({\"Id\":test_Id, \"Prediction\":prediction_lgb})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:01:24.556575Z","iopub.execute_input":"2022-07-15T08:01:24.556904Z","iopub.status.idle":"2022-07-15T08:01:24.781072Z","shell.execute_reply.started":"2022-07-15T08:01:24.556874Z","shell.execute_reply":"2022-07-15T08:01:24.780232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}