{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1><strong>8 of 25 Pandas Tricks Annotations</strong></h1><br>\n\n*w/ Arelius*","metadata":{}},{"cell_type":"markdown","source":"<h2>Source:</h2>\n\n> [My top 25 pandas tricks](https://www.youtube.com/watch?v=RlIiVeig3hc&t=148s&ab_channel=DataSchool) by [Data School](https://www.youtube.com/channel/UCnVzApLJE2ljPZSeQylSEyg)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-25T10:38:17.056262Z","iopub.execute_input":"2021-07-25T10:38:17.056974Z","iopub.status.idle":"2021-07-25T10:38:17.062322Z","shell.execute_reply.started":"2021-07-25T10:38:17.056904Z","shell.execute_reply":"2021-07-25T10:38:17.061033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1. Showing Installed Version**","metadata":{}},{"cell_type":"code","source":"pd.__version__","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.064867Z","iopub.execute_input":"2021-07-25T10:38:17.065464Z","iopub.status.idle":"2021-07-25T10:38:17.079328Z","shell.execute_reply.started":"2021-07-25T10:38:17.065419Z","shell.execute_reply":"2021-07-25T10:38:17.078592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.show_versions()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.080625Z","iopub.execute_input":"2021-07-25T10:38:17.081095Z","iopub.status.idle":"2021-07-25T10:38:17.108414Z","shell.execute_reply.started":"2021-07-25T10:38:17.081055Z","shell.execute_reply":"2021-07-25T10:38:17.107198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2. Create an example DataFrame**","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(np.random.rand(3,10), columns=list('helloworld'))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.10967Z","iopub.execute_input":"2021-07-25T10:38:17.110022Z","iopub.status.idle":"2021-07-25T10:38:17.129315Z","shell.execute_reply.started":"2021-07-25T10:38:17.109982Z","shell.execute_reply":"2021-07-25T10:38:17.128099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***3. Rename Columns***","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(np.random.rand(2,2), columns=(['issue 1', 'issue 2']))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.13071Z","iopub.execute_input":"2021-07-25T10:38:17.131106Z","iopub.status.idle":"2021-07-25T10:38:17.148041Z","shell.execute_reply.started":"2021-07-25T10:38:17.131066Z","shell.execute_reply":"2021-07-25T10:38:17.146938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> goal is to remove and/or add spacing within the column name values succinctly","metadata":{}},{"cell_type":"markdown","source":"*solution #1*","metadata":{}},{"cell_type":"code","source":"df.columns = ['col_one', 'col_two']\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.151642Z","iopub.execute_input":"2021-07-25T10:38:17.152159Z","iopub.status.idle":"2021-07-25T10:38:17.166678Z","shell.execute_reply.started":"2021-07-25T10:38:17.152109Z","shell.execute_reply":"2021-07-25T10:38:17.165796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*solution #2*","metadata":{}},{"cell_type":"code","source":"df = df.rename({'col_one':'column_won'}, axis='columns')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.168424Z","iopub.execute_input":"2021-07-25T10:38:17.168764Z","iopub.status.idle":"2021-07-25T10:38:17.18755Z","shell.execute_reply.started":"2021-07-25T10:38:17.168729Z","shell.execute_reply":"2021-07-25T10:38:17.186477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*solution #3*","metadata":{}},{"cell_type":"code","source":"df.columns = df.columns.str.replace('_', ' ')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.189395Z","iopub.execute_input":"2021-07-25T10:38:17.189866Z","iopub.status.idle":"2021-07-25T10:38:17.206582Z","shell.execute_reply.started":"2021-07-25T10:38:17.189817Z","shell.execute_reply":"2021-07-25T10:38:17.205483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns = df.columns.str.replace(' ', '_')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.207586Z","iopub.execute_input":"2021-07-25T10:38:17.20785Z","iopub.status.idle":"2021-07-25T10:38:17.21997Z","shell.execute_reply.started":"2021-07-25T10:38:17.207824Z","shell.execute_reply":"2021-07-25T10:38:17.218774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*bonus: add prefix and add suffix*","metadata":{}},{"cell_type":"code","source":"df = df.add_prefix('X_')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.223029Z","iopub.execute_input":"2021-07-25T10:38:17.22381Z","iopub.status.idle":"2021-07-25T10:38:17.235333Z","shell.execute_reply.started":"2021-07-25T10:38:17.223758Z","shell.execute_reply":"2021-07-25T10:38:17.234055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.add_prefix('two-')\ndf = df.add_suffix('-am-hustle')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.237343Z","iopub.execute_input":"2021-07-25T10:38:17.237785Z","iopub.status.idle":"2021-07-25T10:38:17.254981Z","shell.execute_reply.started":"2021-07-25T10:38:17.237742Z","shell.execute_reply":"2021-07-25T10:38:17.253783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***4. Reverse Row Order***","metadata":{}},{"cell_type":"code","source":"df.loc[::-1]","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.256718Z","iopub.execute_input":"2021-07-25T10:38:17.25705Z","iopub.status.idle":"2021-07-25T10:38:17.277183Z","shell.execute_reply.started":"2021-07-25T10:38:17.257021Z","shell.execute_reply":"2021-07-25T10:38:17.27597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*resetting index of the dataframe to zero*","metadata":{}},{"cell_type":"code","source":"df.loc[::-1].reset_index(drop=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.278371Z","iopub.execute_input":"2021-07-25T10:38:17.278677Z","iopub.status.idle":"2021-07-25T10:38:17.294058Z","shell.execute_reply.started":"2021-07-25T10:38:17.278645Z","shell.execute_reply":"2021-07-25T10:38:17.293254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***5. Reverse Column Order***","metadata":{}},{"cell_type":"code","source":"df.loc[:, ::-1]","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.295409Z","iopub.execute_input":"2021-07-25T10:38:17.295803Z","iopub.status.idle":"2021-07-25T10:38:17.311563Z","shell.execute_reply.started":"2021-07-25T10:38:17.295774Z","shell.execute_reply":"2021-07-25T10:38:17.31059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***6. Select columns by data type***","metadata":{}},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.312756Z","iopub.execute_input":"2021-07-25T10:38:17.313176Z","iopub.status.idle":"2021-07-25T10:38:17.319969Z","shell.execute_reply.started":"2021-07-25T10:38:17.313147Z","shell.execute_reply":"2021-07-25T10:38:17.319061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select_dtypes(include='number').head()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.321083Z","iopub.execute_input":"2021-07-25T10:38:17.321553Z","iopub.status.idle":"2021-07-25T10:38:17.337475Z","shell.execute_reply.started":"2021-07-25T10:38:17.321524Z","shell.execute_reply":"2021-07-25T10:38:17.336607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select_dtypes(include=['number','category', 'datetime','object'] ).head()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.340217Z","iopub.execute_input":"2021-07-25T10:38:17.340685Z","iopub.status.idle":"2021-07-25T10:38:17.357577Z","shell.execute_reply.started":"2021-07-25T10:38:17.340655Z","shell.execute_reply":"2021-07-25T10:38:17.356109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> *side note: ctrl-sfhit- - splits a cell into two at cursor*","metadata":{}},{"cell_type":"code","source":"df.select_dtypes(exclude='number').head()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:17.359753Z","iopub.execute_input":"2021-07-25T10:38:17.360439Z","iopub.status.idle":"2021-07-25T10:38:17.3756Z","shell.execute_reply.started":"2021-07-25T10:38:17.360388Z","shell.execute_reply":"2021-07-25T10:38:17.374383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***7. Convert Strings to Numbers***","metadata":{}},{"cell_type":"markdown","source":"*solution #1*","metadata":{}},{"cell_type":"code","source":"df = df.astype({'two-X_column_won-am-hustle':'object'}).dtypes\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:38:48.335192Z","iopub.execute_input":"2021-07-25T10:38:48.335847Z","iopub.status.idle":"2021-07-25T10:38:48.34542Z","shell.execute_reply.started":"2021-07-25T10:38:48.33579Z","shell.execute_reply":"2021-07-25T10:38:48.344213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*solution #2*","metadata":{}},{"cell_type":"code","source":"df = df.apply(pd.to_numeric, errors='coerce').fillna(0)\n#we intentionally change values to NaN w/ errors='coerce'\n\n#to_numeric gives us the ability to handle NaN errors with the errors param\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:40:54.357777Z","iopub.execute_input":"2021-07-25T10:40:54.358165Z","iopub.status.idle":"2021-07-25T10:40:54.367231Z","shell.execute_reply.started":"2021-07-25T10:40:54.35813Z","shell.execute_reply":"2021-07-25T10:40:54.366023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***8. Reduce DataFrame Size***","metadata":{}},{"cell_type":"code","source":"fresh_df = pd.DataFrame(np.random.rand(4,4), columns=list('ouch'))\nfresh_df","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:48:07.355852Z","iopub.execute_input":"2021-07-25T10:48:07.356468Z","iopub.status.idle":"2021-07-25T10:48:07.369236Z","shell.execute_reply.started":"2021-07-25T10:48:07.356421Z","shell.execute_reply":"2021-07-25T10:48:07.36832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fresh_df.info(memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:48:31.557221Z","iopub.execute_input":"2021-07-25T10:48:31.557599Z","iopub.status.idle":"2021-07-25T10:48:31.569917Z","shell.execute_reply.started":"2021-07-25T10:48:31.557566Z","shell.execute_reply":"2021-07-25T10:48:31.568821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*solution in comments, pretending our homemade df, fresh_df, was imported*","metadata":{}},{"cell_type":"code","source":"cols = ['o','u']\n#example_df = pd.read_csv('file path', usecols=cols)\ndtypes = {'o':'double'}\n#example_df = pd.read_csv('file path', usecols=cols, dtype=dtypes)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:51:52.324195Z","iopub.execute_input":"2021-07-25T10:51:52.324556Z","iopub.status.idle":"2021-07-25T10:51:52.330902Z","shell.execute_reply.started":"2021-07-25T10:51:52.324525Z","shell.execute_reply":"2021-07-25T10:51:52.329672Z"},"trusted":true},"execution_count":null,"outputs":[]}]}