{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"# standard libs\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n# import json\nimport datetime as dt\n# import warnings\nimport pandas_profiling\n\n# plotting libs\nimport seaborn as sns\n\n# warnings.filterwarnings(\"ignore\")\n\nex_date = dt.datetime.now() - dt.timedelta(hours=5)\nprint('   >>>Exectuted at '+str(ex_date))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Funciones"},{"metadata":{"trusted":true},"cell_type":"code","source":"def output_guides(op_type='prt_ex_at',df_list=None, df_rows=1):\n    \"\"\"Display the chosen type of output guide\n\n    Args:\n        op_type(str,optional): name of the output type to display\n        df_list(list,optional): list of Dataframes\n        df_rows(int or str,optional): number of rows to be displayed from all DataFrame\n    \"\"\"\n    def ex_at_str():\n        ex_date = dt.datetime.now() - dt.timedelta(hours=5)\n        return '   >>>Exectuted at '+str(ex_date)\n    \n    def prt_ex_at():\n        print(ex_at_str())\n    \n    def get_name(df):\n        for x in globals():\n            if globals()[x] is df:\n                name = x\n        return name\n                  \n    def created_df(df_list,df_rows):\n        names_list = []\n        for df in df_list:\n            name = get_name(df)\n            names_list.append(name)\n        for (name,df) in zip(names_list,df_list):\n            if df_rows == 'all':\n                df_rows = df.shape[0]\n            print(name)\n            print(str(df.shape)+ex_at_str())\n            display(df.head(df_rows))\n    \n    if op_type == 'prt_ex_at':\n        prt_ex_at()\n    elif op_type == 'created_df':\n        created_df(df_list,df_rows)\n    else:\n        print('A correct optput type has not been specified')        \n\noutput_guides()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data understanding"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read all csv and excel files of a working directory and create a shapes dataframe\nnames_list = []\ncolumns_list = []\nrows_list = []\nextentions_list = []\n#foo is a function that will execute some repetitive tasks in below lines\ndef foo(names_list,extentions_list,name,extention,program):\n    names_list.append(name)\n    extentions_list.append(extention)\n    exec(program)\n# test_dir = '/kaggle/input/submission-rld'\nmy_dir = '/kaggle/input/interbank20'\nfor root, dirs, files in os.walk(my_dir):\n    for file in files:\n        #'_' is added because name starting with number causses an error\n        name = '_'+str(os.path.splitext(file)[0])\n        #replace characters that can not be part of a name\n        charlist = [' ','.','+','-','*','/','=','&','^','>','<','!','~','|']\n        for char in charlist:\n            name = name.replace(char,'')\n        extention = os.path.splitext(file)[1]\n        shared = 'columns_list.append({}.shape[1])\\n'.format(name)\\\n            +'rows_list.append({}.shape[0])'.format(name)\n        if extention == '.csv':\n            program = 'global {}\\n{} = pd.read_csv(os.path.join(root, file))\\n'.format(name,name)+shared\n            foo(names_list,extentions_list,name,extention,program)\n        if extention == '.xlsx':\n            program = '{} = pd.read_excel(os.path.join(root, file))\\n'.format(name)+shared\n            foo(names_list,extentions_list,name,extention,program)\n#Creating shapes DataFrame\nshapes_dict = {'DataFrames': names_list, 'Rows': rows_list, 'Columns': columns_list, 'Extentions':extentions_list}\nshapes_df = pd.DataFrame(shapes_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shapes_df = shapes_df.sort_values(by=['Rows'], ascending = False)\noutput_guides('created_df',[shapes_df],df_rows='all')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### _censo_train"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(_censo_train,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### productos"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df = pd.DataFrame(_productos,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### rcc_train"},{"metadata":{},"cell_type":"markdown","source":"* Eliminar las filas duplicadas\n* Hacer join con y_test usando key_value\n* Hacer join con _productos usando PRODUCTO"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df = pd.DataFrame(_rcc_train,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"_rcc_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### sample_submission"},{"metadata":{},"cell_type":"markdown","source":"Se envían predicciones de target = 1 o 0"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df = pd.DataFrame(_sample_submission,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### _se_train"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(_se_train,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### _sunat_test"},{"metadata":{},"cell_type":"markdown","source":"* eliminar filas duplicadas"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df = pd.DataFrame(_sunat_test,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### _sunat_train"},{"metadata":{},"cell_type":"markdown","source":"* eliminar filas duplicadas"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df = pd.DataFrame(_sunat_train,copy=True)\npd.options.display.float_format = '{:.2f}'.format\noutput_guides('created_df',[df])\ndisplay(df.describe())\nprint('filas duplicadas: {}'.format(df.duplicated().sum()))\nprint('valores nulos:')\ndisplay(df.isnull().sum())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}