{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this Kernel, Deep Net algorithm is applied for fraud detection.\nFor sack of simplicity, all naccassary models and function are written using Python function which makes working with data easy and chance for making mistake way less.\n\nSeveral reading functions were written which inlcuding reading data base on two classes. It was observed that the data is pretty imbalance and therefore it will affect the ML algorithm performance. The class 1 was read (from train samples) and then the same number of class 0 was read. two classes were merged to come up with 50-50 balance data. \nTo run codes effecintly, those data which are not naccassary for the next step in program was delete and garbace collactor fucntion was run.\nThe data was feathered based on the this Kernel. (https://www.kaggle.com/pranav84/lightgbm-fixing-unbalanced-data-lb-0-9680)"},{"metadata":{},"cell_type":"markdown","source":"# Essential Libraries"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport time\nimport csv\nimport os\nimport seaborn as sns\nimport random\nimport gc\nfrom sklearn import preprocessing\nfrom matplotlib import pyplot as plt\nimport matplotlib as mpl\nimport scipy.stats as st\nimport missingno as msno\nimport math\nimport copy\nfrom matplotlib import pyplot\nfrom sklearn.metrics import make_scorer, accuracy_score\nfrom sklearn.model_selection import train_test_split \nimport gc\nimport time\n%matplotlib inline\n\nbefore = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Essential Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def null_table(df):\n    print(\"Training Data\\n\\n\")\n    print(pd.isnull(df).sum()) \n\ndef some_useful_data_insight(df):\n    print(df.head(5))\n    print(df.dtypes)\n    print(null_table(df))\n    print('data length=', len(df))\n    \ndef Plot_Hist_column(df, x):\n    pyplot.hist(df[x], log = True)\n    pyplot.title(x)\n    pyplot.show()\n    \ndef Plot_Hist_columns(df, xlist):\n    [Plot_Hist_column(df, x) for x in xlist]  \n    pyplot.show()\n    \ndef Make_X_Y(df):\n    Y = pd.DataFrame()\n    Y['is_attributed'] = df['is_attributed']\n    X = df.copy()\n    X.drop(labels = [\"is_attributed\"], axis = 1, inplace = True)\n    return X, Y\n\ndef Train_Test_training_valid(X, Y, ratio):\n    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=ratio)\n    X_training, X_valid, y_training, y_valid = \\\n    train_test_split(X_train, y_train, test_size=ratio, random_state=0)\n    return X_training, y_training, X_valid, y_valid\n\ndef Drop_cols(df, x):\n    df.drop(labels = x, axis = 1, inplace = True)\n    return df\n\ndef Normalized(df):\n    df_col_names = df.columns\n    x = df.values \n    min_max_scaler = preprocessing.MinMaxScaler()\n    x_scaled = min_max_scaler.fit_transform(x)\n    df = pd.DataFrame(x_scaled)\n    df.columns = df_col_names\n    return df\n\n\ndef Parse_time(df):\n    df['day'] = df['click_time'].dt.day.astype('uint8')\n    df['hour'] = df['click_time'].dt.hour.astype('uint8')\n    df['minute'] = df['click_time'].dt.minute.astype('uint8')\n    df['second'] = df['click_time'].dt.second.astype('uint8')\n    \ndef Merge_data(df1, df2):\n    frames = [df1, df2]\n    df = pd.concat(frames)\n    return df\n\n\n\n\ndef read_train_test_data(address_train, train_nrows, address_test, test_nrows, Skip_range_low, Skip_range_Up, nrows):\n    df_train = pd.read_csv(address_train, parse_dates=['click_time'], skiprows=range(Skip_range_low,Skip_range_Up), nrows = nrows)\n    df_test = pd.read_csv(address_test, parse_dates=['click_time'])#, nrows = 100)#, nrows = test_nrows)\n    return df_train, df_test\n\ndef read_train_test_data2(address_train, address_test):\n    df_train = pd.read_csv(address_train, parse_dates=['click_time'])\n    df_test = pd.read_csv(address_test, parse_dates=['click_time'])\n    return df_train, df_test\n\ndef read_train_test_data_balanced(address_train, address_test):\n    #Read Training data, all class 1 and add same amount 0\n    iter_csv = pd.read_csv(address_train, iterator=True, chunksize=10000000, parse_dates=['click_time'])\n    df_train_1 = pd.concat([chunk[chunk['is_attributed'] > 0] for chunk in iter_csv])\n    iter_csv = pd.read_csv(address_train, iterator=True, chunksize=10000000, parse_dates=['click_time'], nrows=2000000)\n    df_train_0 = pd.concat([chunk[chunk['is_attributed'] == 0] for chunk in iter_csv])\n    #seperate same number values as train data with class 1\n    df_train_0 = df_train_0.head(len(df_train_1))\n    #Merge 0 and 1 data\n    df_train = Merge_data(df_train_1, df_train_0)\n    #Read Test data\n    df_test = pd.read_csv(address_test, parse_dates=['click_time'])\n    return df_train, df_test\n\ndef read_train_test_data_balanced_oversample1(address_train, address_test):\n    #Read Training data, all class 1 and add same amount 0\n    iter_csv = pd.read_csv(address_train, iterator=True, chunksize=10000000, parse_dates=['click_time'])\n    df_train_1 = pd.concat([chunk[chunk['is_attributed'] > 0] for chunk in iter_csv])\n    iter_csv = pd.read_csv(address_train, iterator=True, chunksize=10000000, parse_dates=['click_time'], skiprows=range(1,120000000), nrows=14000000)\n    df_train_0 = pd.concat([chunk[chunk['is_attributed'] == 0] for chunk in iter_csv])\n    count_class_0 = len(df_train_0)\n    df_train_1_over = df_train_1.sample(count_class_0, replace=True)\n    df_train_over = pd.concat([df_train_1_over, df_train_0], axis=0)\n    print(df_train_over.is_attributed.value_counts())\n    #Read Test data\n    df_test = pd.read_csv(address_test, parse_dates=['click_time'])\n    return df_train_over, df_test\n\ndef check_memory():\n    mem=str(os.popen('free -t -m').readlines())\n    T_ind=mem.index('T')\n    mem_G=mem[T_ind+14:-4]\n    S1_ind=mem_G.index(' ')\n    mem_T=mem_G[0:S1_ind]\n    mem_G1=mem_G[S1_ind+8:]\n    S2_ind=mem_G1.index(' ')\n    mem_U=mem_G1[0:S2_ind]\n    mem_F=mem_G1[S2_ind+8:]\n    print('Free Memory = ' + mem_F +' MB')\n\n\ndef Feature_engineering(df, ip_count):\n    df = pd.merge(df, ip_count, on='ip', how='left', sort=False)\n    df['clicks_by_ip'] = df['clicks_by_ip'].astype('uint16')\n    return df\n\n\ndef predict_And_Submit_using_xgb(df, Trained_Model):\n    data_to_submit = pd.DataFrame()\n    data_to_submit['click_id'] = range(0, len(df))\n    dtest = xgb.DMatrix(df)\n    del df\n    predict = Trained_Model.predict(dtest, ntree_limit=Trained_Model.best_ntree_limit)\n    data_to_submit['is_attributed'] = predict\n    pyplot.hist(data_to_submit['is_attributed'], log = True)\n    return data_to_submit\n\n\ndef predict_And_Submit(df, Trained_Model):\n    pred = Trained_Model.predict(df)\n    print('pred Done.')\n    predict = pd.DataFrame(pred)\n    data_to_submit = pd.DataFrame()\n    data_to_submit['click_id'] = range(0, len(df))\n    data_to_submit['is_attributed'] = predict\n    print(Num_of_line*'=')\n    print('data_to_submit = \\n', data_to_submit.head(5))\n    pyplot.hist(data_to_submit['is_attributed'], log = True)\n    return data_to_submit\n\n\ndef generate_ip_count(df_train, df_test):\n    \n    \n    df_train2 = df_train.copy()\n    df_test2 = df_test.copy()\n    # Drop the IP and the columns from target\n    y = df_train2['is_attributed']\n    df_train2.drop(['is_attributed'], axis=1, inplace=True)\n    # Drop IP and ID from test rows\n    sub = pd.DataFrame()\n    #sub['click_id'] = test['click_id'].astype('int')\n    df_test2.drop(['click_id'], axis=1, inplace=True)\n    gc.collect()\n    nrow_df_train2 = df_train2.shape[0]\n    merge = pd.concat([df_train2, df_test2])\n\n    del df_train2, df_test2\n    gc.collect()\n    \n    # Count the number of clicks by ip\n    ip_count = merge.groupby(['ip'])['channel'].count().reset_index()\n    ip_count.columns = ['ip', 'clicks_by_ip']\n    merge = pd.merge(merge, ip_count, on='ip', how='left', sort=False)\n    merge['clicks_by_ip'] = merge['clicks_by_ip'].astype('uint16')\n    merge.drop('ip', axis=1, inplace=True)\n\n    df_train2 = merge[:nrow_df_train2]\n    df_test2 = merge[nrow_df_train2:]\n    del df_test2, merge\n    gc.collect()\n    \n    return ip_count","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"Start_time = time.time()\n#Address to data\naddress_train = '../input/talkingdata-adtracking-fraud-detection/train.csv'\naddress_test = '../input/talkingdata-adtracking-fraud-detection/test.csv'\naddress_train_sample = '../input/talkingdata-adtracking-fraud-detection/train_sample.csv'\naddress_test_supplement = '../input/talkingdata-adtracking-fraud-detection/test_supplement.csv'\nprint('Reading data...!'); check_memory()\n#df_train, df_test = read_train_test_data_balanced(address_train, address_test)    \n#df_train, df_test = read_train_test_data_balanced_oversample1(address_train, address_test) \ndf_train, df_test = read_train_test_data2(address_train_sample, address_test)\nprint(len(df_train))\nprint('Reading Done!')\ncheck_memory()\n\n\n\n#Parse time\nprint('Parse, training data...'); check_memory(); Parse_time(df_train); print('Parse, training data, Done!'); \ncheck_memory()\n    \n#Feature_engineering data\nip_count = generate_ip_count(df_train, df_test)\ndf_train = Feature_engineering(df_train, ip_count); df_train.head(); null_table(df_train);  df_train.head(); #df_train = df_train.dropna()\n    \n#Drop and normalize \nprint('Drop colum and normalize, training data...!'); check_memory()\ncolmn_names = ['attributed_time','click_time', 'ip']; df_train = Drop_cols(df_train, colmn_names)\n#df_train = Normalized(df_train)\nprint('Drop colum and normalize, training data, Done!'); check_memory()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''xlist = ['app', 'device', 'os', 'channel', \\\n         'is_attributed', 'day', 'hour', \\\n         'minute', 'second', 'clicks_by_ip']\nPlot_Hist_columns(df_train, xlist)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"df_columns_name = []\nfor col in df_train.columns: \n    df_columns_name.append(col)\n    #print(col)\n    \ndf_columns_name.remove('is_attributed')\nprint(df_columns_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''for col1 in df_columns_name:\n    for col in df_columns_name:\n        name = col1 + '*' + col\n        df_train[name] = df_train[col1]*df_train[col]'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''for col2 in df_columns_name:\n    for col1 in df_columns_name:\n        for col in df_columns_name:\n            name = col2 + '*' + col1 + '*' + col\n            df_train[name] = df_train[col2]*df_train[col1]*df_train[col]'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Devide training data, X-Y\nprint('Begin devide training data, X_Y...'); check_memory()\nX, Y = Make_X_Y(df_train); X_training, y_training, X_valid, y_valid = Train_Test_training_valid(X, Y, 0.25)\nprint('Begin devide training data, X_Y, Done!'); check_memory()\nprint('Cleaning before training'); del df_train; gc.collect(); check_memory()\nprint('Begin training...'); check_memory()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Insert Deep net essential librarians "},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.utils import to_categorical\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_training.to_numpy(); y_training.to_numpy(); \nX_valid.to_numpy(); y_valid.to_numpy()\n\nX_training = preprocessing.scale(X_training)\ny_training = to_categorical(y_training)\nX_valid = preprocessing.scale(X_valid)\ny_valid = to_categorical(y_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_training","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#model\nmodel = Sequential()\nmodel.add(Dense(5, input_dim=9, activation='relu'))\n#model.add(Dense(4, input_dim=5, activation='relu'))\nmodel.add(Dense(2, activation='softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''#model\nmodel = Sequential()\nmodel.add(Dense(125, activation='relu', input_shape=(9,)))\nmodel.add(Dense(75, activation='relu', input_shape=(5,)))\n#model.add(Dropout(0.5, input_shape=(2,)))\n#model.add(Dense(50, activation='relu', input_shape=(5,)))\n#model.add(Dense(25, activation='relu', input_shape=(5,)))\nmodel.add(Dense(10, activation='relu', input_shape=(5,)))\nmodel.add(Dense(2, activation='softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fitting the model\nhistory = model.fit(X_training, y_training, validation_data=(X_valid, y_valid), epochs=70, batch_size=1000000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# list all data in history\n#print(history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read Test data\ndf_test = pd.read_csv(address_test, parse_dates=['click_time'])\n\n#Parse time\nprint('Parse, test data...'); \ncheck_memory(); \nParse_time(df_test); \ndf_test.head(); \nnull_table(df_test); \nprint('Parse, test data, Done!'); \ncheck_memory()\n\n\n\n\n#Feature_engineering data\ndf_test = pd.merge(df_test, ip_count, on='ip', how='left', sort=False)\n\n\n#Drop and normalize\nprint('Drop colum and normalize, test data...!'); check_memory()\ncolmn_names = [\"click_time\", \"click_id\", \"ip\"]; \ndf_test = Drop_cols(df_test, colmn_names); \ndf_test.head(); null_table(df_test);\ndf_test.head()\n\n\ndf_test_columns_name = []\nfor col in df_test.columns: \n    df_test_columns_name.append(col)\n\n    \ndf_test_columns_name\n\n\n\n'''for col1 in df_test_columns_name:\n    for col in df_test_columns_name:\n        name = col1 + '*' + col\n        df_test[name] = df_test[col1]*df_test[col]'''\n\n\n'''for col2 in df_test_columns_name:\n    for col1 in df_test_columns_name:\n        for col in df_test_columns_name:\n            name = col2 + '*' + col1 + '*' + col\n            df_test[name] = df_test[col2]*df_test[col1]*df_test[col]'''\n\ncheck_memory()\ndf_test.head()\n\n\n\n\n\n#df_test = Normalized(df_test)\ndf_test = preprocessing.scale(df_test)\n\nprint('Drop colum and normalize, test data, Done!'); check_memory()\nprint('Cleaning before prediction'); del X, Y, X_training, y_training, X_valid, y_valid, ip_count; gc.collect(); check_memory()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df_test.to_numpy()\ny_pred = model.predict_classes(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = pd.DataFrame(y_pred)\ny_pred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_to_submit = pd.DataFrame()\ndata_to_submit['click_id'] = range(0, len(y_pred))\ndata_to_submit['is_attributed'] = y_pred\npyplot.hist(data_to_submit['is_attributed'], log = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_to_submit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_to_submit.to_csv('Amin_csv_to_submit.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"After = time.time()\nAfter - before","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"}},"nbformat":4,"nbformat_minor":1}