{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9e1c07fb-5c01-43d3-6c1b-42f624089474" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "from subprocess import check_output\n", "#types={'Semana':np.uint8,'Agencia_ID':np.uint16,'Canal_ID':np.uint8,\n", " # 'Ruta_SAK':np.uint16,'Cliente_ID':np.uint32,'Producto_ID':np.uint16,\n", "# 'Demanda_uni_equil':np.uint32}\n", "types = {'Semana':np.uint8, 'Agencia_ID':np.uint16, 'Canal_ID':np.uint8,\n", " 'Ruta_SAK':np.uint16, 'Cliente_ID':np.uint32, 'Producto_ID':np.uint16,\n", " 'Demanda_uni_equil':np.uint32}\n", "#train=pd.read_csv('../input/train.csv',usecols=types.keys(),dtype=types)\n", "train=pd.read_csv('../input/train.csv',usecols=types.keys(),dtype=types,nrows=1000)\n", "print(train.dtype)\n", "print(train.info(memery_usage=True))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9edc3a3f-e828-f2eb-0b68-2d3d7d0273d9" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "from subprocess import check_output\n", "#types={'Semana':np.uint8,'Agencia_ID':np.uint16,'Canal_ID':np.uint8,\n", " # 'Ruta_SAK':np.uint16,'Cliente_ID':np.uint32,'Producto_ID':np.uint16,\n", "# 'Demanda_uni_equil':np.uint32}\n", "types = {'Semana':np.uint8, 'Agencia_ID':np.uint16, 'Canal_ID':np.uint8,\n", " 'Ruta_SAK':np.uint16, 'Cliente_ID':np.uint32, 'Producto_ID':np.uint16,\n", " 'Demanda_uni_equil':np.uint32}\n", "#train=pd.read_csv('../input/train.csv',usecols=types.keys(),dtypes=types)\n", "train=pd.read_csv('../input/train.csv',usecols=types.keys(),dtypes=types,nrows=1000)\n", "print(train.dtype)\n", "print(train.info(memery_usage=True))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "88b7798b-269b-4b56-0fe6-624d2e515fa8" }, "outputs": [], "source": [ "from subprocess import check_output\n", "import pandas as pd\n", "print(check_output(['ls','.']).decode('utf8'))\n", "submission=pd.read_csv('../input/sample_submission.csv')\n", "print(submission.shape)\n", "print(submission.columns)\n", "print(submission.head(20))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d4d6a265-0563-716a-b7eb-7d2b6e9ab0fd" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import gc\n", "import xgboost as xgb\n", "import math\n", "from sklearn.cross_validation import train_test_split\n", "from ml_metrics import rmsle\n", "def evalerror(preds, dtrain):\n", "\n", " labels = dtrain.get_label()\n", " assert len(preds) == len(labels)\n", " labels = labels.tolist()\n", " preds = preds.tolist()\n", " terms_to_sum = [(math.log(labels[i] + 1) - math.log(max(0,preds[i]) + 1)) ** 2.0 for i,pred in enumerate(labels)]\n", " return 'error', (sum(terms_to_sum) * (1.0/len(preds))) ** 0.5\n", "\n", "nrows=10000\n", "train=pd.read_csv('../input/train.csv',nrows=nrows)\n", "test=pd.read_csv('../input/test.csv',nrows=nrows)\n", "\n", "print(train.columns)\n", "print(test.columns)\n", "print(test.columns.values)\n", "ids=test['id']\n", "test=test.drop(['id'],axis=1)\n", "y=train['Demanda_uni_equil']\n", "X=train[test.columns.values]\n", "\n", "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1729)\n", "X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=1000)\n", "\n", "\n", "\n", "params = {}\n", "params['objective'] = \"reg:linear\"\n", "params['eta'] = 0.05\n", "params['max_depth'] = 5\n", "params['subsample'] = 0.8\n", "params['colsample_bytree'] = 0.6\n", "params['silent'] = True\n", "\n", "print ('')\n", "\n", "test_preds = np.zeros(test.shape[0])\n", "xg_train = xgb.DMatrix(X_train, label=y_train)\n", "xg_test = xgb.DMatrix(X_test)\n", "\n", "watchlist = [(xg_train, 'train')]\n", "num_rounds = 100\n", "\n", "xgclassifier = xgb.train(params, xg_train, num_rounds, watchlist, feval = evalerror, early_stopping_rounds= 20, verbose_eval = 10)\n", "preds = xgclassifier.predict(xg_test, ntree_limit=xgclassifier.best_iteration)\n", "\n", "print ('RMSLE Score:', rmsle(y_test, preds))\n", "\n", "fxg_test = xgb.DMatrix(test)\n", "fold_preds = np.around(xgclassifier.predict(fxg_test, ntree_limit=xgclassifier.best_iteration), decimals = 1)\n", "test_preds += fold_preds\n", "\n", "submission = pd.DataFrame({'id':ids, 'Demanda_uni_equil': test_preds})\n", "submission.to_csv('submission.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0bfcf487-13da-fe0b-49b8-49a54cd909e5" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import os\n", "import gc\n", "nrows=100000\n", "train=pd.read_csv('../input/train.csv',nrows=nrows)\n", "print(train.shape)\n", "print(train.columns)\n", "data=train.copy()\n", "data['target']=data['Demanda_uni_equil']\n", "data.drop(['Demanda_uni_equil'],axis=1,inplace=True)\n", "\n", "nCliente_ID = pd.DataFrame(pd.groupby(data,['Cliente_ID','Semana'])['target'].count())\n", "print(nCliente_ID.shape)\n", "print(nCliente_ID.columns)\n", "print(nCliente_ID.head(2))\n", "nCliente_ID = nCliente_ID.reset_index()\n", "print(nCliente_ID.shape)\n", "print(nCliente_ID.columns)\n", "print(nCliente_ID.head(2))\n", "nCliente_ID.rename(columns={'target': 'nCliente_ID'}, inplace=True)\n", "print(nCliente_ID.shape)\n", "print(nCliente_ID.columns)\n", "print(nCliente_ID.head(2))\n", "nCliente_ID = pd.DataFrame(pd.groupby(nCliente_ID,['Cliente_ID'])['nCliente_ID'].mean())\n", "print(nCliente_ID.shape)\n", "print(nCliente_ID.columns)\n", "print(nCliente_ID.head(2))\n", "nCliente_ID = nCliente_ID.reset_index()\n", "print(nCliente_ID.shape)\n", "print(nCliente_ID.columns)\n", "print(nCliente_ID.head(2))\n", " \n", "\n", "data = pd.merge(data, nCliente_ID, \n", " how='left',\n", " left_on=['Cliente_ID'], \n", " right_on=['Cliente_ID'],\n", " left_index=False, right_index=False, sort=True,\n", " suffixes=('_x', '_y'), copy=False) \n", "print(data.columns)\n", "print(data.head(50))\n", "\n", "del nCliente_ID\n", "gc.collect()\n", "print('merge completo nCliente_ID')\n", "print(data.shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3d3e69e1-06e5-2242-bcf2-43c5029e1125" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output.\n", "import datetime\n", "import time\n", "from collections import defaultdict\n", "import gc\n", "\n", "def run_solution():\n", " print('Preparing arrays...')\n", " f = open(\"../input/train.csv\", \"r\")\n", " f.readline()\n", " total = 0\n", "\n", " client_product_arr = defaultdict(int)\n", " client_product_arr_count = defaultdict(int)\n", " client_arr = defaultdict(int)\n", " client_arr_count = defaultdict(int)\n", " product_arr = defaultdict(int)\n", " product_arr_count = defaultdict(int)\n", "\n", " # Calc counts\n", " avg_target = 0.0\n", " while 1:\n", " line = f.readline().strip()\n", " total += 1\n", "\n", " if total % 10000000 == 0:\n", " print('Read {} lines...'.format(total))\n", "\n", " if line == '':\n", " break\n", "\n", " arr = line.split(\",\")\n", " week = int(arr[0])\n", " agency = arr[1]\n", " canal_id = arr[2]\n", " ruta_sak = arr[3]\n", " cliente_id = int(arr[4])\n", " producto_id = int(arr[5])\n", " vuh = arr[6]\n", " vh = arr[7]\n", " dup = arr[8]\n", " dp = arr[9]\n", " target = int(arr[10])\n", " avg_target += target\n", "\n", " client_product_arr[(cliente_id, producto_id)] += target\n", " client_product_arr_count[(cliente_id, producto_id)] += 1\n", " client_arr[cliente_id] += target\n", " client_arr_count[cliente_id] += 1\n", " product_arr[producto_id] += target\n", " product_arr_count[producto_id] += 1\n", "\n", " f.close()\n", " avg_target /= total\n", " print('Average target: ', avg_target)\n", " gc.collect()\n", " \n", " print('Generate submission...')\n", " now = datetime.datetime.now()\n", " path = 'submission_' + str(now.strftime(\"%Y-%m-%d-%H-%M\")) + '.csv'\n", " out = open(path, \"w\")\n", " f = open(\"../input/test.csv\", \"r\")\n", " f.readline()\n", " total = 0\n", " out.write(\"id,Demanda_uni_equil\\n\")\n", "\n", " index_both = 0\n", " index_client = 0\n", " index_product = 0\n", " index_empty = 0\n", "\n", " while 1:\n", " line = f.readline().strip()\n", " total += 1\n", "\n", " if total % 10000000 == 0:\n", " print('Write {} lines...'.format(total))\n", "\n", " if line == '':\n", " break\n", "\n", " arr = line.split(\",\")\n", " id = arr[0]\n", " week = int(arr[1])\n", " agency = arr[2]\n", " canal_id = arr[3]\n", " ruta_sak = arr[4]\n", " cliente_id = int(arr[5])\n", " producto_id = int(arr[6])\n", "\n", " out.write(str(id) + ',')\n", " if (cliente_id, producto_id) in client_product_arr:\n", " val = client_product_arr[(cliente_id, producto_id)]/client_product_arr_count[(cliente_id, producto_id)]\n", " out.write(str(val))\n", " index_both += 1\n", " elif cliente_id in client_arr:\n", " val = client_arr[cliente_id]/client_arr_count[cliente_id]\n", " out.write(str(val))\n", " index_client += 1\n", " elif producto_id in product_arr:\n", " val = product_arr[producto_id]/product_arr_count[producto_id]\n", " out.write(str(val))\n", " index_product += 1\n", " else:\n", " out.write(str(avg_target))\n", " index_empty += 1\n", " out.write(\"\\n\")\n", "\n", " print('Both: {}'.format(index_both))\n", " print('Client: {}'.format(index_client))\n", " print('Product: {}'.format(index_product))\n", " print('Empty: {}'.format(index_empty))\n", "\n", " out.close()\n", " f.close()\n", "\n", "start_time = time.time()\n", "#run_solution()\n", "print(\"Elapsed time overall: %s seconds\" % (time.time() - start_time))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6d902edb-3da7-b83e-30fc-1b1d7aa4a961" }, "outputs": [], "source": [ "print(check_output([\"ls\", \".\"]).decode(\"utf8\"))" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }