{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n#import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n#import os\n#print(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":false},"cell_type":"code","source":"# Import the libraries\n\n# Author: kbv71\n\n# Many useful insights : https://www.kaggle.com/c/talkingdata-adtracking-fraud-detection/discussion/51411\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport dask.dataframe as dd\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import svm\nfrom sklearn.metrics import accuracy_score,roc_curve,recall_score,classification_report,mean_squared_error,confusion_matrix\nimport os\nprint(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6958710b-8d2a-4b2a-b34f-7920d7a909a0","_uuid":"b3d10524e1f09eb997d8654b131c9a5941480b68","collapsed":true,"trusted":false},"cell_type":"code","source":"# It is a bulky dataset. There are multiple options to deal with it. \n\n##A. We retrieve only min(nPos,nNeg) rows from both classes and get a smaller yet more balenced data\n\n##B. We use efficient data tools like Dask \n\n##Skip the attribute_time because if it is filled, it will be anyways denoted by 1 in label data.\n\n# Set a random state\n\nrandom_state = np.random.RandomState(2)\n\n# preset the data types\n\ndtyp = {'ip': np.int64, 'app': np.int16,'device': np.int16,'os': np.int16,'channel': np.int16,'is_attributed' : np.int16}\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e3a38515-f91a-41ce-bdec-8ebd4e3e20ec","_uuid":"3aa175bcc5fc68c84fbff4604162c676e11ac275","collapsed":true,"trusted":false},"cell_type":"code","source":"print(\"LOADING DATA..........................\")\n\n# TRAINING DATA\n\nprint(\"TRAINING DATA\")\n\ndfTrain = dd.read_csv(\"../input/train_sample.csv\",blocksize=1e05)\n\nprint(\"original dataframe\")\n\nprint(dfTrain.head())\n\ndel dfTrain['attributed_time']\n\n#nRows = dfTrain[dfTrain.ip > 0].ip.value_counts().compute()\n\nnRows = len(dfTrain)\n\nprint(\"nRows = \", nRows)\n\nprint(dfTrain.astype(dtyp))\n\n# Create new features out of time. Year and month are skipped.\n\ndfTrain['click_time'] =  dd.to_datetime(dfTrain['click_time'])\n\n# the given data is of 4 days. So useful data is day and hours\n\ndfTrain['click_time_day'] = dfTrain['click_time'].dt.day\n\ndfTrain['click_time_hour'] = dfTrain['click_time'].dt.hour\n\ndel dfTrain['click_time']\n\ndfTrain.columns = ['ip', 'app', 'device', 'os','channel','is_attributed','click_time_day','click_time_hour']\n\nprint(\"dfTrain.columns\",dfTrain.columns)\n\ndfTrain.astype(dtyp)\nprint(\"---------\")\nprint(dfTrain.info())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0c9acccc-3a6b-42fe-ac75-90c47219e2ff","_uuid":"f5a52b4541bff5e950f912205edc7cacf7abe698","collapsed":true,"trusted":false},"cell_type":"code","source":"# Find the ratio of positive / negetive to check for imbalance.\n\nnPos = dfTrain.is_attributed.sum().compute()\n\nnNeg = nRows - nPos\n\nr = np.longdouble(nPos/nRows)\n\nprint(\"positive cases in training set: \", 100.0*r, \"%\")\n\nprint(nPos)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c9eaaf1e-908a-4a33-ac75-33a24d94a62b","_uuid":"d1828e412d24634ee67cd257f9deb085df453441","collapsed":true,"trusted":false},"cell_type":"code","source":"# Create a balenced dataset\n\nposEx = dfTrain [ (dfTrain['is_attributed'] == 1) ]\n\nsampledNegEx =  dfTrain [ (dfTrain['is_attributed'] == 0) ].sample(frac=r,random_state=random_state)\n\nnewTrainsubs = [posEx, sampledNegEx]\n\ndfTrainBal = dd.concat(newTrainsubs)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2093a1ee-413d-4b01-a8f8-e8cbf9b973ed","_uuid":"d25e40acfacf5d90bdbde5f8ffc8cad1bc6c0254","collapsed":true,"trusted":false},"cell_type":"code","source":"# Split the balanced dataset to create cross validation set\n\n\n#print(XTrainBal.head())\n\n# create a randomly selected cross validation set\n#\n#train_test_split(XTrainBal[features], XTrainBal['is_attributed'], test_size=0.33, random_state=random_state)\n\ndTrain = pd.DataFrame()\n\ndCV = pd.DataFrame()\n\ndTrain, dCV = dfTrainBal.random_split([0.70,0.30], random_state=random_state)\n\n#print(dTrain.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"848a79c15982b531a6107e14d12c9a55dd0bf12f"},"cell_type":"code","source":"#Get X and y\n\nyTrain = dTrain['is_attributed']\n\nXTrain = dTrain.drop('is_attributed',axis=1).compute()\n\nyCV = dCV['is_attributed']\nXCV = dCV.drop('is_attributed',axis=1).compute()\n\nprint(yTrain.head())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4d1ee652-7e84-4f11-b071-a0c1b3a919c4","_uuid":"c41d6da5cf360ed32e35ecbf379f513019eed18c","collapsed":true,"trusted":false},"cell_type":"code","source":"# Create classifiers\n\nclfSVM = svm.SVC(kernel='linear', probability=True,random_state=random_state)\n\nclfLR = LogisticRegression()\n\nclfRF = RandomForestClassifier(n_estimators=100,random_state=random_state)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b75ebe2c-cf74-400c-a372-2cd86fb5c0e0","_uuid":"f9ea23721c8658e828e518917a0c4d466147a15a","collapsed":true,"trusted":false},"cell_type":"code","source":"# Train the classifier on training set\n\nclfSVM.fit(XTrain, yTrain)\n\nclfLR.fit(XTrain, yTrain)\n\nclfRF.fit(XTrain, yTrain)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c7223e8c-d8d1-4978-8219-8a155522d898","_uuid":"2be8f25d6e0111d20da7e4c87b731c3f1e45c5ca","collapsed":true,"trusted":false},"cell_type":"code","source":"# Apply on Cross validation set\n\nyCVPredSVM = clfSVM.predict(XCV)\n\nyCVPredLR = clfLR.predict(XCV)\n\nyCVPredRF = clfRF.predict(XCV)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"11fbbbf4-7ea8-42d6-a0e8-deb5392b810a","_uuid":"1212e877a6960591074de001bef26fb11303b303","collapsed":true,"trusted":false},"cell_type":"code","source":"# Check error matrix\n\n\nCMSVM = confusion_matrix(yCV,yCVPredSVM) \n\nCMLR = confusion_matrix(yCV,yCVPredLR)\n\nCMRF = confusion_matrix(yCV,yCVPredRF)\n\nprint(\"0,0 : true nagetive \\n 0,1 : False positive \\n 1,0 : False negetive \\n 1,1 : True positive \")\n\nprint(CMSVM)\n\nprint(CMLR)\n\nprint(CMRF)\n\n# random samlping helps evaluate the algorithms ","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}