{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"97dc55b4-cb6e-63c2-52b2-c48531e940c9","_uuid":"6b11cf62b45a22b0f620f223bb5850ba1dfccd70"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\n\nfrom heamy.dataset import Dataset\nfrom heamy.estimator import Regressor\nfrom heamy.pipeline import ModelsPipeline\n\nID = 'id'\nTARGET = 'loss'\n\nDATA_DIR = \"../input\"\nSUBMISSION_FILE = \"{0}/sample_submission.csv\".format(DATA_DIR)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"75cdea55-d064-3a70-4b18-e5c541bd0133","_uuid":"0e9d80b60425d80eeb6799e1974807407cadee82"},"outputs":[],"source":"def first_dataset():\n    train = pd.read_csv(\"{0}/train.csv\".format(DATA_DIR))\n    test = pd.read_csv(\"{0}/test.csv\".format(DATA_DIR))\n\n    y_train = train[TARGET].ravel()\n\n    train.drop([ID, TARGET], axis=1, inplace=True)\n    test.drop([ID], axis=1, inplace=True)\n\n    train_test = pd.concat((train, test)).reset_index(drop=True)\n\n    ntrain = train.shape[0]\n\n    features = train.columns\n    cats = [feat for feat in features if 'cat' in feat]\n    for feat in cats:\n        train_test[feat] = pd.factorize(train_test[feat], sort=True)[0]\n    x_train = np.array(train_test.iloc[:ntrain, :])\n    x_test = np.array(train_test.iloc[ntrain:, :])\n\n    return {'X_train': x_train, 'X_test': x_test, 'y_train': y_train}\n\n\ndef xgb_first(X_train, y_train, X_test, y_test=None):\n    params = {\n        'seed': 1111,\n        'colsample_bytree': 0.7,\n        'silent': 1,\n        'subsample': 0.8,\n        'learning_rate': 0.01,\n        'objective': 'reg:linear',\n        'max_depth': 8,\n        'num_estimators': 550,\n        'min_child_weight': 1,\n    }\n\n    X_train = xgb.DMatrix(X_train, label=y_train)\n    model = xgb.train(params, X_train, params['num_estimators'], )\n    return model.predict(xgb.DMatrix(X_test))\n\n\ndef xgb_stack(X_train, y_train, X_test, y_test=None):\n    params = {\n        'seed': 3333,\n        'colsample_bytree': 0.6,\n        'silent': 1,\n        'subsample': 0.85,\n        'learning_rate': 0.005,\n        'objective': 'reg:linear',\n        'max_depth': 10,\n        'num_estimators': 750,\n        'gamma': 0.005,\n    }\n    X_train = xgb.DMatrix(X_train, label=y_train, missing=np.nan)\n    model = xgb.train(params, X_train, params['num_estimators'], )\n    return model.predict(xgb.DMatrix(X_test, missing=np.nan))\n\n\net_params = {'n_estimators': 100, 'max_features': 0.5,\n             'max_depth': 18, 'min_samples_leaf': 4,\n             'n_jobs': -1}\nrf_params = {'n_estimators': 125, 'max_features': 0.2,\n             'max_depth': 25, 'min_samples_leaf': 4,\n             'n_jobs': -1}\n\nds = Dataset(preprocessor=first_dataset, use_cache=False)\npipeline = ModelsPipeline(\n    Regressor(estimator=xgb_first, dataset=ds, use_cache=False),\n    Regressor(estimator=ExtraTreesRegressor, dataset=ds, use_cache=False,\n              parameters=et_params),\n    Regressor(estimator=RandomForestRegressor, dataset=ds, use_cache=False,\n              parameters=rf_params),\n    Regressor(estimator=LinearRegression, dataset=ds, use_cache=False),\n\n)\n# 4 folds\nstack_ds = pipeline.stack(k=4, seed=111, add_diff=False, full_test=True)\n\n# One model approach\n# stacker = Regressor(dataset=stack_ds, estimator=xgb_stack, use_cache=False)\n# Uncomment for valdation\n# stacker.validate(k=2, scorer=mean_absolute_error)\n# predictions = stacker.predict()\n\n# Two models on the second layer\npipe2 = ModelsPipeline(\n    Regressor(estimator=xgb_stack, dataset=stack_ds, use_cache=False),\n    Regressor(estimator=ExtraTreesRegressor, dataset=stack_ds, use_cache=False,\n              parameters={'n_estimators': 100, 'max_depth': 15, 'max_features': 3}),\n\n)\n# pipe2.weight([0.75, 0.25]).validate(k=2, scorer=mean_absolute_error)\n# xgb*0.75+rf*0.25\npredictions = pipe2.weight([0.75, 0.25]).execute()\n\nsubmission = pd.read_csv(SUBMISSION_FILE)\nsubmission.iloc[:, 1] = predictions\nsubmission.to_csv('xgstacker_starter.sub.csv', index=None)\n\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f3ec0c60-35ae-8d33-b742-84f7f13374f5","_uuid":"5d6c41eca213332aa1aacfe5edcd11a2fea6c144"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}