{"cells":[{"metadata":{"_uuid":"d749b95e0c0150e307850e26d4502573e53d9599"},"cell_type":"markdown","source":"Since there is no obvious quantative relationship of each feature, except click time, I think the best way to use those features is to  as category and one-hot those variables. Huge traffic records with such a sparse feature matrix will make computation hard, but it should be the direction of this competition once we have enough computing power. "},{"metadata":{"collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport time\nfrom sklearn.preprocessing import LabelBinarizer\nfrom scipy.sparse import hstack\nimport gc\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\n\npath = '../input/'\n\n\ndef dataPreProcessTime(df):\n    df['click_time'] = pd.to_datetime(df['click_time']).dt.date\n    df['click_time'] = df['click_time'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"start_time = time.time()\n\ntrain = pd.read_csv(path+\"train.csv\", nrows=20000000)\ntrain.columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'attributed_time', 'is_attributed']\ntrain = dataPreProcessTime(train)\n\ny = train['is_attributed'].astype(float)  \nnrow_train = train.shape[0] \n\ntrain.drop(['is_attributed', 'attributed_time'], axis=1, inplace=True)\n\ntest = pd.read_csv(path+\"test.csv\")\ntest = dataPreProcessTime(test)\n\nsub = pd.DataFrame()\nsub['click_id'] = test['click_id']\ntest.drop('click_id', axis=1, inplace=True)\n\nmerge: pd.DataFrame = pd.concat([train, test])\n    \nlb = LabelBinarizer(sparse_output=True)\nmerge_ip = lb.fit_transform(merge['ip'])\nmerge_app = lb.fit_transform(merge['app'])\nmerge_device = lb.fit_transform(merge['device'])\nmerge_os = lb.fit_transform(merge['os'])\nmerge_channel = lb.fit_transform(merge['channel'])\n\nsparse_merge = hstack((merge_ip, merge_app, merge_device, merge_os, merge_channel)).tocsr()\nsparse_merge = sparse_merge.astype(float)\n\nX = sparse_merge[:nrow_train]\nX_test = sparse_merge[nrow_train:]\n\ntrain_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.3, random_state=100)\n\nprint('[{}] Data completed.'.format(time.time() - start_time))\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"5eee3723-926a-49eb-bf9a-682445130e6b","_uuid":"daf32bdcdd86774c3a8de0045fab5f3e41ea66cc","trusted":false},"cell_type":"code","source":"start_time = time.time()\nprint(\"LGB startting\")\n\nparams = {\n        'learning_rate': 0.6,\n        'objective': 'binary',\n        'max_depth': 4,\n        'num_leaves': 31,\n        'verbosity': -1,\n        'metric': 'auc', \n        'data_random_seed': 1,\n        'bagging_fraction': 0.6,\n        'bagging_freq': 5,\n        'feature_fraction': 0.65,\n        'nthread': 4,\n        'min_data_in_leaf': 100,\n        'max_bin': 31\n    }\n\n\nd_train = lgb.Dataset(train_X, label=train_y)\nd_val = lgb.Dataset(valid_X, label=valid_y)\nwatchlist = [d_train, d_val]\nmodel = lgb.train(params, train_set=d_train, num_boost_round=7000, valid_sets=watchlist, verbose_eval=1000)\n\nprint('[{}] Finish LGB Training'.format(time.time() - start_time))\n\nsub['is_attributed'] = model.predict(X_test)\nsub.to_csv('lgb_sub.csv',index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}