{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write down here the exact problem that you are trying to solve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Descriptive Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing the basic libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>name</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22</td>\n",
       "      <td>Ahmed</td>\n",
       "      <td>1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33</td>\n",
       "      <td>Saleh</td>\n",
       "      <td>22000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44</td>\n",
       "      <td>Hashir</td>\n",
       "      <td>25000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Qasim</td>\n",
       "      <td>35000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42</td>\n",
       "      <td>Junaid</td>\n",
       "      <td>25000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>32</td>\n",
       "      <td>Ammaar</td>\n",
       "      <td>15000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>52</td>\n",
       "      <td>Bill Gates</td>\n",
       "      <td>300000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age        name  salary\n",
       "0   22       Ahmed    1200\n",
       "1   33       Saleh   22000\n",
       "2   44      Hashir   25000\n",
       "3   23       Qasim   35000\n",
       "4   42      Junaid   25000\n",
       "5   32      Ammaar   15000\n",
       "6   52  Bill Gates  300000"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating a dataframe\n",
    "df = pd.DataFrame([\n",
    " { 'name': 'Ahmed', 'salary': 1200, 'age': 22} ,\n",
    " { 'name': 'Saleh', 'salary': 22000, 'age': 33},\n",
    " { 'name': 'Hashir', 'salary': 25000, 'age': 44 },\n",
    " { 'name': 'Qasim', 'salary': 35000, 'age': 23 },\n",
    " { 'name': 'Junaid', 'salary': 25000, 'age': 42 },\n",
    " { 'name': 'Ammaar', 'salary': 15000, 'age': 32 },\n",
    " { 'name': 'Bill Gates', 'salary': 300000, 'age': 52 }    \n",
    "])\n",
    "\n",
    "#print it\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Taha</td>\n",
       "      <td>12000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mahad</td>\n",
       "      <td>14000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Name  Salary\n",
       "0   Taha   12000\n",
       "1  Mahad   14000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#another way of creating a data frame\n",
    "df2 = pd.DataFrame({'Name':['Taha', 'Mahad'],\n",
    "                     'Salary':[12000,14000]})\n",
    "\n",
    "#print it\n",
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Central Tendency (Mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean:  60457.142857142855\n",
      "Integerized Mean:  60457\n",
      "Standardized Mean:  60,457\n"
     ]
    }
   ],
   "source": [
    "#calculate the mean\n",
    "print(\"Mean: \",df['salary'].mean())\n",
    "\n",
    "#convert to integer\n",
    "print(\"Integerized Mean: \", int(df['salary'].mean()))\n",
    "\n",
    "#in the standard format\n",
    "print(\"Standardized Mean: \",\"{:,}\".format(int(df['salary'].mean())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Central Tendency (Median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      1200\n",
      "5     15000\n",
      "1     22000\n",
      "2     25000\n",
      "4     25000\n",
      "3     35000\n",
      "6    300000\n",
      "Name: salary, dtype: int64\n",
      "0      1200\n",
      "5     15000\n",
      "1     22000\n",
      "2     25000\n",
      "4     25000\n",
      "3     35000\n",
      "6    300000\n",
      "Name: salary, dtype: int64\n",
      "Median:  25000.0\n"
     ]
    }
   ],
   "source": [
    "# Mean is on the higher side due to Bill Gates Salary (an outlier)\n",
    "#Let's calculate the Median (central value)\n",
    "print(df['salary'].sort_values())\n",
    "print(df.salary.sort_values())\n",
    "\n",
    "print(\"Median: \", df['salary'].median())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Central Tendency (Mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>restaurant</th>\n",
       "      <th>reviewer</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Ahmed</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Saleh</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Hashir</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Qasim</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Junaid</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Ammaar</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Student Bryani</td>\n",
       "      <td>Bill Gates</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       restaurant    reviewer  stars\n",
       "0  Student Bryani       Ahmed      2\n",
       "1  Student Bryani       Saleh      2\n",
       "2  Student Bryani      Hashir      4\n",
       "3  Student Bryani       Qasim      4\n",
       "4  Student Bryani      Junaid      4\n",
       "5  Student Bryani      Ammaar      3\n",
       "6  Student Bryani  Bill Gates      2"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's build a data set\n",
    "reviews_df = pd.DataFrame([\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Ahmed', 'stars': 2 } ,\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Saleh', 'stars': 2 },\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Hashir', 'stars': 4 },\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Qasim', 'stars': 4 },\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Junaid', 'stars': 4 },\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Ammaar', 'stars': 3 },\n",
    " { 'restaurant': 'Student Bryani', 'reviewer': 'Bill Gates', 'stars': 2 }    \n",
    "])\n",
    "reviews_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['restaurant', 'reviewer', 'stars'], dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "restaurant    object\n",
       "reviewer      object\n",
       "stars          int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2\n",
       "1    4\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calling the mode function\n",
    "reviews_df['stars'].mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"data:image/png;base64,[encoded data, 135512 characters]\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Import HTML display from Amazon\n",
    "from IPython.display import display, HTML\n",
    "display(HTML('''<img src=\"data:image/png;base64,[encoded data, 135512 characters]\">'''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"data:image/png;base64,[encoded data, 48812 characters]\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#checking the mode\n",
    "display(HTML('''<img src=\"data:image/png;base64,[encoded data, 48812 characters]\">'''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>name</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44</td>\n",
       "      <td>Hashir</td>\n",
       "      <td>25000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Qasim</td>\n",
       "      <td>35000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42</td>\n",
       "      <td>Junaid</td>\n",
       "      <td>25000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>32</td>\n",
       "      <td>Ammaar</td>\n",
       "      <td>15000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>52</td>\n",
       "      <td>Bill Gates</td>\n",
       "      <td>300000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age        name  salary\n",
       "2   44      Hashir   25000\n",
       "3   23       Qasim   35000\n",
       "4   42      Junaid   25000\n",
       "5   32      Ammaar   15000\n",
       "6   52  Bill Gates  300000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7.000</td>\n",
       "      <td>7.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>35.429</td>\n",
       "      <td>60457.143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>11.133</td>\n",
       "      <td>106144.538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>22.000</td>\n",
       "      <td>1200.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>27.500</td>\n",
       "      <td>18500.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>33.000</td>\n",
       "      <td>25000.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>43.000</td>\n",
       "      <td>30000.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>52.000</td>\n",
       "      <td>300000.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         age     salary\n",
       "count  7.000      7.000\n",
       "mean  35.429  60457.143\n",
       "std   11.133 106144.538\n",
       "min   22.000   1200.000\n",
       "25%   27.500  18500.000\n",
       "50%   33.000  25000.000\n",
       "75%   43.000  30000.000\n",
       "max   52.000 300000.000"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make it so we only have three decimal places\n",
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Box and Whiskers Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9ae1fcef0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6772 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#df.boxplot(column='salary', sym='o', return_type='axes')\n",
    "df.boxplot(column='salary', sym='o', whis = 'range', return_type='axes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9ae5bfb00>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6848 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Let's try to remove Bill gates and see the plot\n",
    "mask = df['name'].isin(['Bill Gates'])\n",
    "df1 = df[~mask]\n",
    "\n",
    "df1.boxplot(column='salary', sym='o', whis = 'range', return_type='axes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9ae63f358>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6848 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#if we want to set whiskers at max and min (to identify the outlier range)\n",
    "df1.boxplot(column='salary', sym='o', whis = 'range', return_type='axes')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scatter Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.38690285 0.30888707 0.10462596 0.70277647 0.53152481 0.26632635\n",
      " 0.86630955 0.50050903 0.72466767 0.22137114 0.63355686 0.49388513\n",
      " 0.75910639 0.50638819 0.61812451 0.49622492 0.56633728 0.61925916\n",
      " 0.73388252 0.67247532]\n",
      "[0.77154222 0.26003137 0.12918799 0.69347609 0.56673114 0.84626787\n",
      " 0.19714426 0.01065766 0.11071273 0.66512512 0.76441634 0.04787095\n",
      " 0.66289938 0.40626008 0.55670699 0.89341369 0.36515358 0.58693842\n",
      " 0.3343323  0.56511311]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 9332 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create data\n",
    "N = 20\n",
    "x = np.random.rand(N)\n",
    "y = np.random.rand(N)\n",
    "print(x)\n",
    "print(y)\n",
    "colors = [[1,0,0]] #this means black. try out different values of RGB between 0 and 1\n",
    "area = np.pi*10 #this is the size of each dot. try out different values\n",
    "\n",
    "# Plot\n",
    "plt.scatter(x, y, s = area, c=colors, alpha=0.9) #alpha is the color intensity. try out different values\n",
    "plt.title('Scatter Plot Example')\n",
    "plt.xlabel('x label')\n",
    "plt.ylabel('y label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1c9ae756860>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7808 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(df1.age, df1.salary, s = area, c=colors, alpha=0.9) #alpha is the color intensity. try out different values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variance and Standard Deviation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list one is\n",
      "0   -10\n",
      "1     0\n",
      "2    10\n",
      "3    20\n",
      "4    30\n",
      "dtype: int64\n",
      "list two is\n",
      "0     8\n",
      "1     9\n",
      "2    10\n",
      "3    11\n",
      "4    12\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "list_one = pd.Series([-10, 0, 10, 20, 30])\n",
    "list_two = pd.Series([8, 9, 10, 11, 12])\n",
    "print(\"list one is\")\n",
    "print(list_one)\n",
    "print(\"list two is\")\n",
    "print(list_two)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean of list_one is 10.0\n",
      "The mean of list_two is 10.0\n",
      "The median of list_one is 10.0\n",
      "The median of list_two is 10.0\n"
     ]
    }
   ],
   "source": [
    "print(\"The mean of list_one is\", list_one.mean())\n",
    "print(\"The mean of list_two is\", list_two.mean())\n",
    "print(\"The median of list_one is\", list_one.median())\n",
    "print(\"The median of list_two is\", list_two.median())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250.0\n",
      "2.5\n"
     ]
    }
   ],
   "source": [
    "# ddof=0 has to do with sample variance\n",
    "print(list_one.var(ddof=1))\n",
    "print(list_two.var(ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15.811388300841896"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "# Data points [-10, 0, 10, 20, 30]\n",
    "# Variance: 200\n",
    "sd1 = math.sqrt(250)\n",
    "sd1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4142135623730951"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data points [8, 9, 10, 11, 12]\n",
    "# Variance: 2.0\n",
    "sd2 = math.sqrt(2)\n",
    "sd2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-39.37253892 -37.68689342 -36.00124792 -34.31560242 -32.62995692\n",
      " -30.94431142 -29.25866592 -27.57302042 -25.88737492 -24.20172941\n",
      " -22.51608391 -20.83043841 -19.14479291 -17.45914741 -15.77350191\n",
      " -14.08785641 -12.40221091 -10.71656541  -9.0309199   -7.3452744\n",
      "  -5.6596289   -3.9739834   -2.2883379   -0.6026924    1.0829531\n",
      "   2.7685986    4.45424411   6.13988961   7.82553511   9.51118061\n",
      "  11.19682611  12.88247161  14.56811711  16.25376261  17.93940811\n",
      "  19.62505362  21.31069912  22.99634462  24.68199012  26.36763562\n",
      "  28.05328112  29.73892662  31.42457212  33.11021762  34.79586313\n",
      "  36.48150863  38.16715413  39.85279963  41.53844513  43.22409063\n",
      "  44.90973613  46.59538163  48.28102713  49.96667264  51.65231814\n",
      "  53.33796364  55.02360914  56.70925464  58.39490014  60.08054564\n",
      "  61.76619114]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 7000 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#define a class to plot the normal distribution curve\n",
    "class norm1:\n",
    "    #constructor (mean, sd, density measure)\n",
    "    def __init__(self, a1, b1, c1):\n",
    "        self.a1 = a1\n",
    "        self.b1 = b1\n",
    "        self.c1 = c1\n",
    "    \n",
    "    #normal curve plotting function\n",
    "    def dist_curve(self):\n",
    "        plt.plot(self.c1, 1/(self.b1 * np.sqrt(2 * np.pi)) *\n",
    "            np.exp( - (self.c1 - self.a1)**2 / (2 * self.b1**2) ), linewidth=2, color='y')\n",
    "        plt.show()\n",
    "\n",
    "c = np.random.normal(list_one.mean(), sd1, 1000)\n",
    "w1, x1, z1 = plt.hist(c, 60, density=True) #hist\n",
    "print(x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 15932 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist1 = norm1(list_one.mean(), sd1, x1)\n",
    "plot1 = hist1.dist_curve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106144.53757562308\n",
      "60457.142857142855\n"
     ]
    }
   ],
   "source": [
    "#let's see the salary plots\n",
    "varsal = df['salary'].var(ddof=1)\n",
    "sdsal = math.sqrt(varsal)\n",
    "\n",
    "print(sdsal)\n",
    "print(df['salary'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9af8cb358>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5168 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.salary.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9af9463c8>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 8636 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df1.salary.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outlier Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['year', 'name', 'rank', 'citizenship', 'countrycode',\n",
       "       'networthusbillion', 'selfmade', 'typeofwealth', 'gender', 'age',\n",
       "       'industry', 'IndustryAggregates', 'region', 'north',\n",
       "       'politicalconnection', 'founder', 'generationofinheritance', 'sector',\n",
       "       'company', 'companytype', 'relationshiptocompany', 'foundingdate',\n",
       "       'gdpcurrentus', 'sourceofwealth', 'notes', 'notes2', 'source',\n",
       "       'source_2', 'source_3', 'source_4'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df = pd.read_excel(\"richpeople.xlsx\")\n",
    "rich_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1653, 30)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>countrycode</th>\n",
       "      <th>networthusbillion</th>\n",
       "      <th>selfmade</th>\n",
       "      <th>typeofwealth</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>...</th>\n",
       "      <th>relationshiptocompany</th>\n",
       "      <th>foundingdate</th>\n",
       "      <th>gdpcurrentus</th>\n",
       "      <th>sourceofwealth</th>\n",
       "      <th>notes</th>\n",
       "      <th>notes2</th>\n",
       "      <th>source</th>\n",
       "      <th>source_2</th>\n",
       "      <th>source_3</th>\n",
       "      <th>source_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014</td>\n",
       "      <td>A. Jerrold Perenchio</td>\n",
       "      <td>663</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>2.600</td>\n",
       "      <td>self-made</td>\n",
       "      <td>executive</td>\n",
       "      <td>male</td>\n",
       "      <td>83.000</td>\n",
       "      <td>...</td>\n",
       "      <td>former chairman and CEO</td>\n",
       "      <td>1955.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>television, Univision</td>\n",
       "      <td>represented Marlon Brando and Elizabeth Taylor</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Jerry_Perenchio</td>\n",
       "      <td>http://www.forbes.com/profile/a-jerrold-perenc...</td>\n",
       "      <td>COLUMN ONE; A Hollywood Player Who Owns the Ga...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014</td>\n",
       "      <td>Abdulla Al Futtaim</td>\n",
       "      <td>687</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>ARE</td>\n",
       "      <td>2.500</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>relation</td>\n",
       "      <td>1930.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>auto dealers, investments</td>\n",
       "      <td>company split between him and cousin in 2000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Al-Futtaim_Group</td>\n",
       "      <td>http://www.al-futtaim.ae/content/groupProfile.asp</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   year                  name  rank           citizenship countrycode  \\\n",
       "0  2014  A. Jerrold Perenchio   663         United States         USA   \n",
       "1  2014    Abdulla Al Futtaim   687  United Arab Emirates         ARE   \n",
       "\n",
       "   networthusbillion   selfmade typeofwealth gender    age  ...  \\\n",
       "0              2.600  self-made    executive   male 83.000  ...   \n",
       "1              2.500  inherited    inherited   male    nan  ...   \n",
       "\n",
       "     relationshiptocompany foundingdate gdpcurrentus  \\\n",
       "0  former chairman and CEO     1955.000          nan   \n",
       "1                 relation     1930.000          nan   \n",
       "\n",
       "              sourceofwealth                                           notes  \\\n",
       "0      television, Univision  represented Marlon Brando and Elizabeth Taylor   \n",
       "1  auto dealers, investments    company split between him and cousin in 2000   \n",
       "\n",
       "   notes2                                         source  \\\n",
       "0     NaN   http://en.wikipedia.org/wiki/Jerry_Perenchio   \n",
       "1     NaN  http://en.wikipedia.org/wiki/Al-Futtaim_Group   \n",
       "\n",
       "                                            source_2  \\\n",
       "0  http://www.forbes.com/profile/a-jerrold-perenc...   \n",
       "1  http://www.al-futtaim.ae/content/groupProfile.asp   \n",
       "\n",
       "                                            source_3 source_4  \n",
       "0  COLUMN ONE; A Hollywood Player Who Owns the Ga...      NaN  \n",
       "1                                                NaN      NaN  \n",
       "\n",
       "[2 rows x 30 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df = rich_df[rich_df['year'] == 2014]\n",
    "rich_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "year                          0\n",
       "name                          0\n",
       "rank                          0\n",
       "citizenship                   0\n",
       "countrycode                   0\n",
       "networthusbillion             0\n",
       "selfmade                      2\n",
       "typeofwealth                  2\n",
       "gender                        0\n",
       "age                          63\n",
       "industry                      1\n",
       "IndustryAggregates            1\n",
       "region                        0\n",
       "north                         0\n",
       "politicalconnection        1597\n",
       "founder                       0\n",
       "generationofinheritance       0\n",
       "sector                        0\n",
       "company                       8\n",
       "companytype                   7\n",
       "relationshiptocompany         2\n",
       "foundingdate                 12\n",
       "gdpcurrentus               1653\n",
       "sourceofwealth               18\n",
       "notes                       970\n",
       "notes2                     1542\n",
       "source                        0\n",
       "source_2                    237\n",
       "source_3                   1130\n",
       "source_4                   1636\n",
       "dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>rank</th>\n",
       "      <th>networthusbillion</th>\n",
       "      <th>age</th>\n",
       "      <th>north</th>\n",
       "      <th>politicalconnection</th>\n",
       "      <th>founder</th>\n",
       "      <th>foundingdate</th>\n",
       "      <th>gdpcurrentus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1578.000</td>\n",
       "      <td>1578.000</td>\n",
       "      <td>1578.000</td>\n",
       "      <td>1578.000</td>\n",
       "      <td>1578.000</td>\n",
       "      <td>54.000</td>\n",
       "      <td>1578.000</td>\n",
       "      <td>1578.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2014.000</td>\n",
       "      <td>805.160</td>\n",
       "      <td>3.958</td>\n",
       "      <td>63.393</td>\n",
       "      <td>0.577</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.519</td>\n",
       "      <td>1963.465</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000</td>\n",
       "      <td>463.300</td>\n",
       "      <td>5.857</td>\n",
       "      <td>13.154</td>\n",
       "      <td>0.494</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.500</td>\n",
       "      <td>38.329</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2014.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>24.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1615.000</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2014.000</td>\n",
       "      <td>408.000</td>\n",
       "      <td>1.400</td>\n",
       "      <td>53.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1948.000</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2014.000</td>\n",
       "      <td>796.000</td>\n",
       "      <td>2.200</td>\n",
       "      <td>63.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1973.000</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2014.000</td>\n",
       "      <td>1210.000</td>\n",
       "      <td>3.700</td>\n",
       "      <td>73.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1991.000</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2014.000</td>\n",
       "      <td>1565.000</td>\n",
       "      <td>76.000</td>\n",
       "      <td>98.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>2012.000</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          year     rank  networthusbillion      age    north  \\\n",
       "count 1578.000 1578.000           1578.000 1578.000 1578.000   \n",
       "mean  2014.000  805.160              3.958   63.393    0.577   \n",
       "std      0.000  463.300              5.857   13.154    0.494   \n",
       "min   2014.000    1.000              1.000   24.000    0.000   \n",
       "25%   2014.000  408.000              1.400   53.000    0.000   \n",
       "50%   2014.000  796.000              2.200   63.000    1.000   \n",
       "75%   2014.000 1210.000              3.700   73.000    1.000   \n",
       "max   2014.000 1565.000             76.000   98.000    1.000   \n",
       "\n",
       "       politicalconnection  founder  foundingdate  gdpcurrentus  \n",
       "count               54.000 1578.000      1578.000         0.000  \n",
       "mean                 1.000    0.519      1963.465           nan  \n",
       "std                  0.000    0.500        38.329           nan  \n",
       "min                  1.000    0.000      1615.000           nan  \n",
       "25%                  1.000    0.000      1948.000           nan  \n",
       "50%                  1.000    1.000      1973.000           nan  \n",
       "75%                  1.000    1.000      1991.000           nan  \n",
       "max                  1.000    1.000      2012.000           nan  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remove missing values and then describe\n",
    "rich_df = rich_df.dropna(subset=['age', 'networthusbillion', 'foundingdate'])\n",
    "rich_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9b171f588>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5620 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rich_df.boxplot(column='networthusbillion', whis='range', return_type='axes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9af946860>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5236 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rich_df.boxplot(column='age', whis='range', return_type='axes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "United States     482\n",
       "China             144\n",
       "Russia            109\n",
       "Germany            81\n",
       "Brazil             60\n",
       "India              54\n",
       "Hong Kong          44\n",
       "France             42\n",
       "United Kingdom     42\n",
       "Italy              32\n",
       "Canada             32\n",
       "Australia          27\n",
       "South Korea        26\n",
       "Japan              26\n",
       "Spain              25\n",
       "Turkey             24\n",
       "Taiwan             24\n",
       "Switzerland        19\n",
       "Indonesia          19\n",
       "Sweden             19\n",
       "Israel             18\n",
       "Mexico             15\n",
       "Singapore          14\n",
       "Malaysia           13\n",
       "Chile              12\n",
       "Philippines        10\n",
       "Thailand           10\n",
       "Austria            10\n",
       "Ukraine             9\n",
       "Norway              9\n",
       "                 ... \n",
       "Denmark             6\n",
       "Ireland             5\n",
       "Kazakhstan          5\n",
       "Poland              5\n",
       "Finland             4\n",
       "Morocco             4\n",
       "Colombia            4\n",
       "Cyprus              4\n",
       "Nigeria             4\n",
       "Argentina           4\n",
       "Monaco              3\n",
       "Portugal            3\n",
       "Venezuela           3\n",
       "Greece              3\n",
       "Belgium             3\n",
       "Kuwait              2\n",
       "Oman                2\n",
       "New Zealand         2\n",
       "Romania             1\n",
       "Algeria             1\n",
       "Angola              1\n",
       "Tanzania            1\n",
       "Uganda              1\n",
       "Lithuania           1\n",
       "Guernsey            1\n",
       "Swaziland           1\n",
       "Macau               1\n",
       "Vietnam             1\n",
       "Nepal               1\n",
       "Georgia             1\n",
       "Name: citizenship, Length: 67, dtype: int64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df['citizenship'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>countrycode</th>\n",
       "      <th>networthusbillion</th>\n",
       "      <th>selfmade</th>\n",
       "      <th>typeofwealth</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>...</th>\n",
       "      <th>relationshiptocompany</th>\n",
       "      <th>foundingdate</th>\n",
       "      <th>gdpcurrentus</th>\n",
       "      <th>sourceofwealth</th>\n",
       "      <th>notes</th>\n",
       "      <th>notes2</th>\n",
       "      <th>source</th>\n",
       "      <th>source_2</th>\n",
       "      <th>source_3</th>\n",
       "      <th>source_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>2014</td>\n",
       "      <td>David Rockefeller, Sr.</td>\n",
       "      <td>580</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>2.900</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>98.000</td>\n",
       "      <td>...</td>\n",
       "      <td>relation</td>\n",
       "      <td>1870.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>oil, banking</td>\n",
       "      <td>family made most of fortune in the late 19th a...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/David_Rockefeller</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Standard_Oil</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Rockefeller_family</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>2014</td>\n",
       "      <td>Karl Wlaschek</td>\n",
       "      <td>305</td>\n",
       "      <td>Austria</td>\n",
       "      <td>AUT</td>\n",
       "      <td>4.800</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>96.000</td>\n",
       "      <td>...</td>\n",
       "      <td>founder</td>\n",
       "      <td>1953.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>retail</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/BILLA</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Karl_Wlaschek</td>\n",
       "      <td>https://www.billa.at/Footer_Nav_Seiten/Geschic...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>2014</td>\n",
       "      <td>Kirk Kerkorian</td>\n",
       "      <td>328</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>4.500</td>\n",
       "      <td>self-made</td>\n",
       "      <td>self-made finance</td>\n",
       "      <td>male</td>\n",
       "      <td>96.000</td>\n",
       "      <td>...</td>\n",
       "      <td>investor</td>\n",
       "      <td>1924.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>casinos, investments</td>\n",
       "      <td>purchased in 1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Kirk_Kerkorian</td>\n",
       "      <td>http://www.forbes.com/profile/kirk-kerkorian/</td>\n",
       "      <td>PROFILE: Las Vegas billionaire amassed his wea...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                    name  rank    citizenship countrycode  \\\n",
       "343  2014  David Rockefeller, Sr.   580  United States         USA   \n",
       "816  2014           Karl Wlaschek   305        Austria         AUT   \n",
       "836  2014          Kirk Kerkorian   328  United States         USA   \n",
       "\n",
       "     networthusbillion   selfmade         typeofwealth gender    age  ...  \\\n",
       "343              2.900  inherited            inherited   male 98.000  ...   \n",
       "816              4.800  self-made  founder non-finance   male 96.000  ...   \n",
       "836              4.500  self-made    self-made finance   male 96.000  ...   \n",
       "\n",
       "    relationshiptocompany foundingdate gdpcurrentus        sourceofwealth  \\\n",
       "343              relation     1870.000          nan          oil, banking   \n",
       "816               founder     1953.000          nan                retail   \n",
       "836              investor     1924.000          nan  casinos, investments   \n",
       "\n",
       "                                                 notes  notes2  \\\n",
       "343  family made most of fortune in the late 19th a...     NaN   \n",
       "816                                                NaN     NaN   \n",
       "836                                  purchased in 1969     NaN   \n",
       "\n",
       "                                             source  \\\n",
       "343  http://en.wikipedia.org/wiki/David_Rockefeller   \n",
       "816              http://en.wikipedia.org/wiki/BILLA   \n",
       "836     http://en.wikipedia.org/wiki/Kirk_Kerkorian   \n",
       "\n",
       "                                          source_2  \\\n",
       "343      http://en.wikipedia.org/wiki/Standard_Oil   \n",
       "816     http://en.wikipedia.org/wiki/Karl_Wlaschek   \n",
       "836  http://www.forbes.com/profile/kirk-kerkorian/   \n",
       "\n",
       "                                              source_3 source_4  \n",
       "343    http://en.wikipedia.org/wiki/Rockefeller_family      NaN  \n",
       "816  https://www.billa.at/Footer_Nav_Seiten/Geschic...      NaN  \n",
       "836  PROFILE: Las Vegas billionaire amassed his wea...      NaN  \n",
       "\n",
       "[3 rows x 30 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df.sort_values(by='age', ascending=False).head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\JC\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>countrycode</th>\n",
       "      <th>networthusbillion</th>\n",
       "      <th>selfmade</th>\n",
       "      <th>typeofwealth</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>...</th>\n",
       "      <th>foundingdate</th>\n",
       "      <th>gdpcurrentus</th>\n",
       "      <th>sourceofwealth</th>\n",
       "      <th>notes</th>\n",
       "      <th>notes2</th>\n",
       "      <th>source</th>\n",
       "      <th>source_2</th>\n",
       "      <th>source_3</th>\n",
       "      <th>source_4</th>\n",
       "      <th>agestd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>2014</td>\n",
       "      <td>David Rockefeller, Sr.</td>\n",
       "      <td>580</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>2.900</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>98.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1870.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>oil, banking</td>\n",
       "      <td>family made most of fortune in the late 19th a...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/David_Rockefeller</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Standard_Oil</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Rockefeller_family</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>2014</td>\n",
       "      <td>Karl Wlaschek</td>\n",
       "      <td>305</td>\n",
       "      <td>Austria</td>\n",
       "      <td>AUT</td>\n",
       "      <td>4.800</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>96.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1953.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>retail</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/BILLA</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Karl_Wlaschek</td>\n",
       "      <td>https://www.billa.at/Footer_Nav_Seiten/Geschic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>2014</td>\n",
       "      <td>Kirk Kerkorian</td>\n",
       "      <td>328</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>4.500</td>\n",
       "      <td>self-made</td>\n",
       "      <td>self-made finance</td>\n",
       "      <td>male</td>\n",
       "      <td>96.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1924.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>casinos, investments</td>\n",
       "      <td>purchased in 1969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Kirk_Kerkorian</td>\n",
       "      <td>http://www.forbes.com/profile/kirk-kerkorian/</td>\n",
       "      <td>PROFILE: Las Vegas billionaire amassed his wea...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>2014</td>\n",
       "      <td>Erika Pohl-Stroher</td>\n",
       "      <td>1154</td>\n",
       "      <td>Germany</td>\n",
       "      <td>DEU</td>\n",
       "      <td>1.500</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>female</td>\n",
       "      <td>95.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1880.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>hair products</td>\n",
       "      <td>3rd generation</td>\n",
       "      <td>23% stake in the company</td>\n",
       "      <td>http://www.forbes.com/profile/erika-pohl-stroher/</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Wella</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>2014</td>\n",
       "      <td>Henry Hillman</td>\n",
       "      <td>687</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>2.500</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>95.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1942.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>investments</td>\n",
       "      <td>inherited from father</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/henry-hillman/</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Calgon_Carbon</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1428</th>\n",
       "      <td>2014</td>\n",
       "      <td>Sulaiman Al Rajhi</td>\n",
       "      <td>931</td>\n",
       "      <td>Saudi Arabia</td>\n",
       "      <td>SAU</td>\n",
       "      <td>1.900</td>\n",
       "      <td>self-made</td>\n",
       "      <td>self-made finance</td>\n",
       "      <td>male</td>\n",
       "      <td>94.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1957.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>banking</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Al-Rajhi_Bank</td>\n",
       "      <td>http://www.alrajhibank.com.sa/ar/investor-rela...</td>\n",
       "      <td>http://www.alrajhibank.com.sa/ar/about-us/page...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>2014</td>\n",
       "      <td>Karl Albrecht</td>\n",
       "      <td>23</td>\n",
       "      <td>Germany</td>\n",
       "      <td>DEU</td>\n",
       "      <td>25.000</td>\n",
       "      <td>self-made</td>\n",
       "      <td>executive</td>\n",
       "      <td>male</td>\n",
       "      <td>94.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1914.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>retail</td>\n",
       "      <td>(split from Aldi Nord in 1966, but both branch...</td>\n",
       "      <td>took over mother's single grocerty store</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Karl_Albrecht</td>\n",
       "      <td>http://www.bloomberg.com/news/articles/2014-07...</td>\n",
       "      <td>http://aldiuscareers.com/about-aldi/history</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>2014</td>\n",
       "      <td>Anne Cox Chambers</td>\n",
       "      <td>58</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>15.500</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>female</td>\n",
       "      <td>94.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1898.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>media</td>\n",
       "      <td>inherited from brother</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Anne_Cox_Chambers</td>\n",
       "      <td>http://www.forbes.com/lists/2010/10/billionair...</td>\n",
       "      <td>http://www.nytimes.com/2007/05/30/business/med...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1560</th>\n",
       "      <td>2014</td>\n",
       "      <td>Wang Yung-Tsai</td>\n",
       "      <td>520</td>\n",
       "      <td>Taiwan</td>\n",
       "      <td>Taiwan</td>\n",
       "      <td>3.100</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>93.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1954.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>plastics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/wang-yung-tsai/</td>\n",
       "      <td>What's good for the goose South China Morning ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980</th>\n",
       "      <td>2014</td>\n",
       "      <td>Marcel Adams</td>\n",
       "      <td>1465</td>\n",
       "      <td>Canada</td>\n",
       "      <td>CAN</td>\n",
       "      <td>1.100</td>\n",
       "      <td>self-made</td>\n",
       "      <td>self-made finance</td>\n",
       "      <td>male</td>\n",
       "      <td>93.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1958.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>real estate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/marcel-adams/</td>\n",
       "      <td>http://business.financialpost.com/news/93-year...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>2014</td>\n",
       "      <td>Aloysio de Andrade Faria</td>\n",
       "      <td>483</td>\n",
       "      <td>Brazil</td>\n",
       "      <td>BRA</td>\n",
       "      <td>3.300</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>93.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1925.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>banking</td>\n",
       "      <td>inherited from father</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Aloysio_de_Andrad...</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Banco_da_Lavoura_...</td>\n",
       "      <td>http://www.forbes.com/profile/aloysio-de-andra...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1585</th>\n",
       "      <td>2014</td>\n",
       "      <td>William Moncrief, Jr.</td>\n",
       "      <td>1565</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>1.000</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>93.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1929.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>oil</td>\n",
       "      <td>joined father's business following WWII</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/William_Moncrief</td>\n",
       "      <td>http://www.moncriefoil.com/history.htm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>665</th>\n",
       "      <td>2014</td>\n",
       "      <td>Jacques Servier</td>\n",
       "      <td>173</td>\n",
       "      <td>France</td>\n",
       "      <td>FRA</td>\n",
       "      <td>7.600</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>92.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1954.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>pharmaceuticals</td>\n",
       "      <td>died in April 2014</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/Jacques_Servier</td>\n",
       "      <td>http://www.forbes.com/profile/jacques-servier/</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1323</th>\n",
       "      <td>2014</td>\n",
       "      <td>S. Truett Cathy</td>\n",
       "      <td>224</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>6.200</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>92.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1946.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>Chick-fil-A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://en.wikipedia.org/wiki/S._Truett_Cathy</td>\n",
       "      <td>http://www.usatoday.com/story/news/nation/2014...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638</th>\n",
       "      <td>2014</td>\n",
       "      <td>Ingeburg Herz</td>\n",
       "      <td>731</td>\n",
       "      <td>Germany</td>\n",
       "      <td>DEU</td>\n",
       "      <td>2.400</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>female</td>\n",
       "      <td>92.000</td>\n",
       "      <td>...</td>\n",
       "      <td>1949.000</td>\n",
       "      <td>nan</td>\n",
       "      <td>coffee</td>\n",
       "      <td>widow</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://de.wikipedia.org/wiki/Maxingvest</td>\n",
       "      <td>http://www.tchibo.com/content/309018/-/de/ber-...</td>\n",
       "      <td>http://www.forbes.com/profile/daniela-herz/</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      year                      name  rank    citizenship countrycode  \\\n",
       "343   2014    David Rockefeller, Sr.   580  United States         USA   \n",
       "816   2014             Karl Wlaschek   305        Austria         AUT   \n",
       "836   2014            Kirk Kerkorian   328  United States         USA   \n",
       "435   2014        Erika Pohl-Stroher  1154        Germany         DEU   \n",
       "588   2014             Henry Hillman   687  United States         USA   \n",
       "1428  2014         Sulaiman Al Rajhi   931   Saudi Arabia         SAU   \n",
       "815   2014             Karl Albrecht    23        Germany         DEU   \n",
       "122   2014         Anne Cox Chambers    58  United States         USA   \n",
       "1560  2014            Wang Yung-Tsai   520         Taiwan      Taiwan   \n",
       "980   2014              Marcel Adams  1465         Canada         CAN   \n",
       "77    2014  Aloysio de Andrade Faria   483         Brazil         BRA   \n",
       "1585  2014     William Moncrief, Jr.  1565  United States         USA   \n",
       "665   2014           Jacques Servier   173         France         FRA   \n",
       "1323  2014           S. Truett Cathy   224  United States         USA   \n",
       "638   2014             Ingeburg Herz   731        Germany         DEU   \n",
       "\n",
       "      networthusbillion   selfmade         typeofwealth  gender    age  ...  \\\n",
       "343               2.900  inherited            inherited    male 98.000  ...   \n",
       "816               4.800  self-made  founder non-finance    male 96.000  ...   \n",
       "836               4.500  self-made    self-made finance    male 96.000  ...   \n",
       "435               1.500  inherited            inherited  female 95.000  ...   \n",
       "588               2.500  inherited            inherited    male 95.000  ...   \n",
       "1428              1.900  self-made    self-made finance    male 94.000  ...   \n",
       "815              25.000  self-made            executive    male 94.000  ...   \n",
       "122              15.500  inherited            inherited  female 94.000  ...   \n",
       "1560              3.100  self-made  founder non-finance    male 93.000  ...   \n",
       "980               1.100  self-made    self-made finance    male 93.000  ...   \n",
       "77                3.300  inherited            inherited    male 93.000  ...   \n",
       "1585              1.000  inherited            inherited    male 93.000  ...   \n",
       "665               7.600  self-made  founder non-finance    male 92.000  ...   \n",
       "1323              6.200  self-made  founder non-finance    male 92.000  ...   \n",
       "638               2.400  inherited            inherited  female 92.000  ...   \n",
       "\n",
       "     foundingdate gdpcurrentus        sourceofwealth  \\\n",
       "343      1870.000          nan          oil, banking   \n",
       "816      1953.000          nan                retail   \n",
       "836      1924.000          nan  casinos, investments   \n",
       "435      1880.000          nan         hair products   \n",
       "588      1942.000          nan           investments   \n",
       "1428     1957.000          nan               banking   \n",
       "815      1914.000          nan                retail   \n",
       "122      1898.000          nan                 media   \n",
       "1560     1954.000          nan              plastics   \n",
       "980      1958.000          nan           real estate   \n",
       "77       1925.000          nan               banking   \n",
       "1585     1929.000          nan                   oil   \n",
       "665      1954.000          nan       pharmaceuticals   \n",
       "1323     1946.000          nan           Chick-fil-A   \n",
       "638      1949.000          nan                coffee   \n",
       "\n",
       "                                                  notes  \\\n",
       "343   family made most of fortune in the late 19th a...   \n",
       "816                                                 NaN   \n",
       "836                                   purchased in 1969   \n",
       "435                                      3rd generation   \n",
       "588                               inherited from father   \n",
       "1428                                                NaN   \n",
       "815   (split from Aldi Nord in 1966, but both branch...   \n",
       "122                              inherited from brother   \n",
       "1560                                                NaN   \n",
       "980                                                 NaN   \n",
       "77                                inherited from father   \n",
       "1585            joined father's business following WWII   \n",
       "665                                  died in April 2014   \n",
       "1323                                                NaN   \n",
       "638                                               widow   \n",
       "\n",
       "                                        notes2  \\\n",
       "343                                        NaN   \n",
       "816                                        NaN   \n",
       "836                                        NaN   \n",
       "435                   23% stake in the company   \n",
       "588                                        NaN   \n",
       "1428                                       NaN   \n",
       "815   took over mother's single grocerty store   \n",
       "122                                        NaN   \n",
       "1560                                       NaN   \n",
       "980                                        NaN   \n",
       "77                                         NaN   \n",
       "1585                                       NaN   \n",
       "665                                        NaN   \n",
       "1323                                       NaN   \n",
       "638                                        NaN   \n",
       "\n",
       "                                                 source  \\\n",
       "343      http://en.wikipedia.org/wiki/David_Rockefeller   \n",
       "816                  http://en.wikipedia.org/wiki/BILLA   \n",
       "836         http://en.wikipedia.org/wiki/Kirk_Kerkorian   \n",
       "435   http://www.forbes.com/profile/erika-pohl-stroher/   \n",
       "588        http://www.forbes.com/profile/henry-hillman/   \n",
       "1428         http://en.wikipedia.org/wiki/Al-Rajhi_Bank   \n",
       "815          http://en.wikipedia.org/wiki/Karl_Albrecht   \n",
       "122      http://en.wikipedia.org/wiki/Anne_Cox_Chambers   \n",
       "1560      http://www.forbes.com/profile/wang-yung-tsai/   \n",
       "980         http://www.forbes.com/profile/marcel-adams/   \n",
       "77    http://en.wikipedia.org/wiki/Aloysio_de_Andrad...   \n",
       "1585      http://en.wikipedia.org/wiki/William_Moncrief   \n",
       "665        http://en.wikipedia.org/wiki/Jacques_Servier   \n",
       "1323       http://en.wikipedia.org/wiki/S._Truett_Cathy   \n",
       "638             http://de.wikipedia.org/wiki/Maxingvest   \n",
       "\n",
       "                                               source_2  \\\n",
       "343           http://en.wikipedia.org/wiki/Standard_Oil   \n",
       "816          http://en.wikipedia.org/wiki/Karl_Wlaschek   \n",
       "836       http://www.forbes.com/profile/kirk-kerkorian/   \n",
       "435                  http://en.wikipedia.org/wiki/Wella   \n",
       "588          http://en.wikipedia.org/wiki/Calgon_Carbon   \n",
       "1428  http://www.alrajhibank.com.sa/ar/investor-rela...   \n",
       "815   http://www.bloomberg.com/news/articles/2014-07...   \n",
       "122   http://www.forbes.com/lists/2010/10/billionair...   \n",
       "1560  What's good for the goose South China Morning ...   \n",
       "980   http://business.financialpost.com/news/93-year...   \n",
       "77    http://en.wikipedia.org/wiki/Banco_da_Lavoura_...   \n",
       "1585             http://www.moncriefoil.com/history.htm   \n",
       "665      http://www.forbes.com/profile/jacques-servier/   \n",
       "1323  http://www.usatoday.com/story/news/nation/2014...   \n",
       "638   http://www.tchibo.com/content/309018/-/de/ber-...   \n",
       "\n",
       "                                               source_3 source_4 agestd  \n",
       "343     http://en.wikipedia.org/wiki/Rockefeller_family      NaN  2.631  \n",
       "816   https://www.billa.at/Footer_Nav_Seiten/Geschic...      NaN  2.479  \n",
       "836   PROFILE: Las Vegas billionaire amassed his wea...      NaN  2.479  \n",
       "435                                                 NaN      NaN  2.403  \n",
       "588                                                 NaN      NaN  2.403  \n",
       "1428  http://www.alrajhibank.com.sa/ar/about-us/page...      NaN  2.327  \n",
       "815         http://aldiuscareers.com/about-aldi/history      NaN  2.327  \n",
       "122   http://www.nytimes.com/2007/05/30/business/med...      NaN  2.327  \n",
       "1560                                                NaN      NaN  2.251  \n",
       "980                                                 NaN      NaN  2.251  \n",
       "77    http://www.forbes.com/profile/aloysio-de-andra...      NaN  2.251  \n",
       "1585                                                NaN      NaN  2.251  \n",
       "665                                                 NaN      NaN  2.175  \n",
       "1323                                                NaN      NaN  2.175  \n",
       "638         http://www.forbes.com/profile/daniela-herz/      NaN  2.175  \n",
       "\n",
       "[15 rows x 31 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#some of the old people are rich.. let's check their standard deviation of ages\n",
    "rich_df['agestd'] = ((rich_df['age'] - rich_df['age'].mean()).apply(abs) / rich_df['age'].std())\n",
    "rich_df.sort_values(by='age', ascending=False).head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\JC\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>countrycode</th>\n",
       "      <th>networthusbillion</th>\n",
       "      <th>selfmade</th>\n",
       "      <th>typeofwealth</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>...</th>\n",
       "      <th>gdpcurrentus</th>\n",
       "      <th>sourceofwealth</th>\n",
       "      <th>notes</th>\n",
       "      <th>notes2</th>\n",
       "      <th>source</th>\n",
       "      <th>source_2</th>\n",
       "      <th>source_3</th>\n",
       "      <th>source_4</th>\n",
       "      <th>agestd</th>\n",
       "      <th>wealth_stdev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>2014</td>\n",
       "      <td>Bill Gates</td>\n",
       "      <td>1</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>76.000</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>58.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>Microsoft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/bill-gates/</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.410</td>\n",
       "      <td>12.301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>2014</td>\n",
       "      <td>Carlos Slim Helu</td>\n",
       "      <td>2</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>MEX</td>\n",
       "      <td>72.000</td>\n",
       "      <td>self-made</td>\n",
       "      <td>privatized and resources</td>\n",
       "      <td>male</td>\n",
       "      <td>74.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>telecom</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.ozy.com/provocateurs/carlos-slims-w...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.806</td>\n",
       "      <td>11.618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>2014</td>\n",
       "      <td>Amancio Ortega</td>\n",
       "      <td>3</td>\n",
       "      <td>Spain</td>\n",
       "      <td>ESP</td>\n",
       "      <td>64.000</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>77.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>retail</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/amancio-ortega/</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.034</td>\n",
       "      <td>10.252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1562</th>\n",
       "      <td>2014</td>\n",
       "      <td>Warren Buffett</td>\n",
       "      <td>4</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>58.200</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>83.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>Berkshire Hathaway</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/lists/2009/10/billionair...</td>\n",
       "      <td>http://www.forbes.com/companies/berkshire-hath...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.491</td>\n",
       "      <td>9.262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>2014</td>\n",
       "      <td>Larry Ellison</td>\n",
       "      <td>5</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>48.000</td>\n",
       "      <td>self-made</td>\n",
       "      <td>founder non-finance</td>\n",
       "      <td>male</td>\n",
       "      <td>69.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>Oracle</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/larry-ellison/</td>\n",
       "      <td>http://www.businessinsider.com/how-larry-ellis...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.426</td>\n",
       "      <td>7.520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>2014</td>\n",
       "      <td>Charles Koch</td>\n",
       "      <td>6</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>40.000</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>78.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>diversified</td>\n",
       "      <td>inherited from father</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.kochind.com/About_Koch/History_Time...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.110</td>\n",
       "      <td>6.154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>2014</td>\n",
       "      <td>David Koch</td>\n",
       "      <td>6</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>40.000</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>73.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>diversified</td>\n",
       "      <td>inherited from father</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.kochind.com/About_Koch/History_Time...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.730</td>\n",
       "      <td>6.154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1364</th>\n",
       "      <td>2014</td>\n",
       "      <td>Sheldon Adelson</td>\n",
       "      <td>8</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>38.000</td>\n",
       "      <td>self-made</td>\n",
       "      <td>self-made finance</td>\n",
       "      <td>male</td>\n",
       "      <td>80.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>casinos</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/sheldon-adelson/</td>\n",
       "      <td>http://lasvegassun.com/news/1996/nov/26/rat-pa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.262</td>\n",
       "      <td>5.813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>2014</td>\n",
       "      <td>Christy Walton</td>\n",
       "      <td>9</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>36.700</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>female</td>\n",
       "      <td>59.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>Wal-Mart</td>\n",
       "      <td>widow</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/christy-walton/</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.334</td>\n",
       "      <td>5.591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>724</th>\n",
       "      <td>2014</td>\n",
       "      <td>Jim Walton</td>\n",
       "      <td>10</td>\n",
       "      <td>United States</td>\n",
       "      <td>USA</td>\n",
       "      <td>34.700</td>\n",
       "      <td>inherited</td>\n",
       "      <td>inherited</td>\n",
       "      <td>male</td>\n",
       "      <td>66.000</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>Wal-Mart</td>\n",
       "      <td>inherited from father</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.forbes.com/profile/jim-walton/</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.198</td>\n",
       "      <td>5.249</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      year              name  rank    citizenship countrycode  \\\n",
       "184   2014        Bill Gates     1  United States         USA   \n",
       "226   2014  Carlos Slim Helu     2         Mexico         MEX   \n",
       "79    2014    Amancio Ortega     3          Spain         ESP   \n",
       "1562  2014    Warren Buffett     4  United States         USA   \n",
       "862   2014     Larry Ellison     5  United States         USA   \n",
       "247   2014      Charles Koch     6  United States         USA   \n",
       "339   2014        David Koch     6  United States         USA   \n",
       "1364  2014   Sheldon Adelson     8  United States         USA   \n",
       "280   2014    Christy Walton     9  United States         USA   \n",
       "724   2014        Jim Walton    10  United States         USA   \n",
       "\n",
       "      networthusbillion   selfmade              typeofwealth  gender    age  \\\n",
       "184              76.000  self-made       founder non-finance    male 58.000   \n",
       "226              72.000  self-made  privatized and resources    male 74.000   \n",
       "79               64.000  self-made       founder non-finance    male 77.000   \n",
       "1562             58.200  self-made       founder non-finance    male 83.000   \n",
       "862              48.000  self-made       founder non-finance    male 69.000   \n",
       "247              40.000  inherited                 inherited    male 78.000   \n",
       "339              40.000  inherited                 inherited    male 73.000   \n",
       "1364             38.000  self-made         self-made finance    male 80.000   \n",
       "280              36.700  inherited                 inherited  female 59.000   \n",
       "724              34.700  inherited                 inherited    male 66.000   \n",
       "\n",
       "      ... gdpcurrentus      sourceofwealth                  notes  notes2  \\\n",
       "184   ...          nan           Microsoft                    NaN     NaN   \n",
       "226   ...          nan             telecom                    NaN     NaN   \n",
       "79    ...          nan              retail                    NaN     NaN   \n",
       "1562  ...          nan  Berkshire Hathaway                    NaN     NaN   \n",
       "862   ...          nan              Oracle                    NaN     NaN   \n",
       "247   ...          nan         diversified  inherited from father     NaN   \n",
       "339   ...          nan         diversified  inherited from father     NaN   \n",
       "1364  ...          nan             casinos                    NaN     NaN   \n",
       "280   ...          nan            Wal-Mart                  widow     NaN   \n",
       "724   ...          nan            Wal-Mart  inherited from father     NaN   \n",
       "\n",
       "                                                 source  \\\n",
       "184           http://www.forbes.com/profile/bill-gates/   \n",
       "226   http://www.ozy.com/provocateurs/carlos-slims-w...   \n",
       "79        http://www.forbes.com/profile/amancio-ortega/   \n",
       "1562  http://www.forbes.com/lists/2009/10/billionair...   \n",
       "862        http://www.forbes.com/profile/larry-ellison/   \n",
       "247   http://www.kochind.com/About_Koch/History_Time...   \n",
       "339   http://www.kochind.com/About_Koch/History_Time...   \n",
       "1364     http://www.forbes.com/profile/sheldon-adelson/   \n",
       "280       http://www.forbes.com/profile/christy-walton/   \n",
       "724           http://www.forbes.com/profile/jim-walton/   \n",
       "\n",
       "                                               source_2 source_3 source_4  \\\n",
       "184                                                 NaN      NaN      NaN   \n",
       "226                                                 NaN      NaN      NaN   \n",
       "79                                                  NaN      NaN      NaN   \n",
       "1562  http://www.forbes.com/companies/berkshire-hath...      NaN      NaN   \n",
       "862   http://www.businessinsider.com/how-larry-ellis...      NaN      NaN   \n",
       "247                                                 NaN      NaN      NaN   \n",
       "339                                                 NaN      NaN      NaN   \n",
       "1364  http://lasvegassun.com/news/1996/nov/26/rat-pa...      NaN      NaN   \n",
       "280                                                 NaN      NaN      NaN   \n",
       "724                                                 NaN      NaN      NaN   \n",
       "\n",
       "     agestd wealth_stdev  \n",
       "184   0.410       12.301  \n",
       "226   0.806       11.618  \n",
       "79    1.034       10.252  \n",
       "1562  1.491        9.262  \n",
       "862   0.426        7.520  \n",
       "247   1.110        6.154  \n",
       "339   0.730        6.154  \n",
       "1364  1.262        5.813  \n",
       "280   0.334        5.591  \n",
       "724   0.198        5.249  \n",
       "\n",
       "[10 rows x 32 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#wealth standard dev\n",
    "rich_df['wealth_stdev'] = ((rich_df['networthusbillion'] - rich_df['networthusbillion'].mean()).apply(abs) / rich_df['networthusbillion'].std())\n",
    "rich_df.sort_values(by='networthusbillion', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       True\n",
       "4       True\n",
       "5       True\n",
       "6       True\n",
       "7       True\n",
       "8       True\n",
       "9       True\n",
       "10      True\n",
       "11      True\n",
       "12      True\n",
       "13      True\n",
       "14      True\n",
       "15      True\n",
       "16      True\n",
       "17      True\n",
       "18      True\n",
       "19      True\n",
       "20      True\n",
       "21      True\n",
       "22      True\n",
       "23      True\n",
       "24      True\n",
       "25      True\n",
       "26      True\n",
       "27      True\n",
       "28      True\n",
       "29      True\n",
       "30      True\n",
       "31      True\n",
       "32      True\n",
       "        ... \n",
       "1622    True\n",
       "1623    True\n",
       "1624    True\n",
       "1625    True\n",
       "1626    True\n",
       "1627    True\n",
       "1628    True\n",
       "1630    True\n",
       "1631    True\n",
       "1632    True\n",
       "1633    True\n",
       "1634    True\n",
       "1635    True\n",
       "1636    True\n",
       "1637    True\n",
       "1638    True\n",
       "1639    True\n",
       "1640    True\n",
       "1641    True\n",
       "1642    True\n",
       "1643    True\n",
       "1644    True\n",
       "1645    True\n",
       "1646    True\n",
       "1647    True\n",
       "1648    True\n",
       "1649    True\n",
       "1650    True\n",
       "1651    True\n",
       "1652    True\n",
       "Name: networthusbillion, Length: 1578, dtype: bool"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df2 = rich_df['networthusbillion'].apply(lambda x: x>50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9b39a2898>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7536 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plot the histogram to find out\n",
    "rich_df['networthusbillion'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9b3a9f128>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7016 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#some more bins\n",
    "rich_df['networthusbillion'].hist(bins=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count   1578.000\n",
       "mean       3.958\n",
       "std        5.857\n",
       "min        1.000\n",
       "25%        1.400\n",
       "50%        2.200\n",
       "75%        3.700\n",
       "max       76.000\n",
       "Name: networthusbillion, dtype: float64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rich_df['networthusbillion'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c9b3b48470>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6928 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rich_df['age'].hist(bins=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inferential Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T-Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.04934281 -0.78862407  1.76908632 -1.2126506   0.79187618  1.13749811\n",
      "  1.0828672  -0.1588928   2.04727668 -0.83962034]\n",
      "[ 0.80468112 -0.17467232  1.05223553  0.27756269 -0.36473561 -0.37077359\n",
      " -0.01783268  0.38818161  1.30169439  0.52400109]\n"
     ]
    }
   ],
   "source": [
    "from scipy import stats\n",
    "\n",
    "## Define 2 random distributions\n",
    "#Sample Size\n",
    "N = 10\n",
    "\n",
    "#Gaussian distributed data with mean = 2 and var = 1\n",
    "a = np.random.randn(N)+2\n",
    "print(a)\n",
    "\n",
    "#Gaussian distributed data with with mean = 0 and var = 1\n",
    "b = np.random.randn(N)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t = 4.6461240859511665\n",
      "p = 0.00020083151216132578\n"
     ]
    }
   ],
   "source": [
    "## Calculate the Standard Deviation\n",
    "#Calculate the variance to get the standard deviation\n",
    "\n",
    "#For unbiased max likelihood estimate we have to divide the var by N-1, and therefore the parameter ddof = 1\n",
    "var_a = a.var(ddof=1)\n",
    "var_b = b.var(ddof=1)\n",
    "\n",
    "#std deviation\n",
    "s = np.sqrt((var_a + var_b)/2)\n",
    "\n",
    "## Calculate the t-statistics\n",
    "t = (a.mean() - b.mean())/(s*np.sqrt(2/N))\n",
    "\n",
    "## Compare with the critical t-value\n",
    "#Degrees of freedom\n",
    "df = 2*N - 2\n",
    "\n",
    "#p-value after comparison with the t \n",
    "p = 1 - stats.t.cdf(t,df=df)\n",
    "\n",
    "print(\"t = \" + str(t))\n",
    "print(\"p = \" + str(2*p))\n",
    "### You can see that after comparing the t statistic with the critical t value (computed internally) we get a good p value of 0.0005 and thus we reject the null hypothesis and thus it proves that the mean of the two distributions are different and statistically significant.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t = 0.11282638711829754\n",
      "p = 0.9114169885274938\n"
     ]
    }
   ],
   "source": [
    "## Cross Checking with the internal scipy function\n",
    "t2, p2 = stats.ttest_ind(a,b)\n",
    "print(\"t = \" + str(t2))\n",
    "print(\"p = \" + str(p2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1-Way ANOVA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    A   B   C   D\n",
      "0  25  45  30  54\n",
      "1  30  55  29  60\n",
      "2  28  29  33  51\n",
      "3  36  56  37  62\n",
      "4  29  40  27  73\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x12bb35f9518>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5320 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "\n",
    "# load data file\n",
    "d = pd.read_csv(\"https://reneshbedre.github.io/myfiles/anova/onewayanova.txt\", sep=\"\\t\")\n",
    "print(d)\n",
    "# generate a boxplot to see the data distribution by treatments. Using boxplot, we can easily detect the differences \n",
    "# between different treatments\n",
    "d.boxplot(column=['A', 'B', 'C', 'D'], grid=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17.492810457516338 2.639241146210922e-05\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sum_sq</th>\n",
       "      <th>df</th>\n",
       "      <th>F</th>\n",
       "      <th>PR(&gt;F)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C(treatments)</th>\n",
       "      <td>3010.95</td>\n",
       "      <td>3.0</td>\n",
       "      <td>17.49281</td>\n",
       "      <td>0.000026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Residual</th>\n",
       "      <td>918.00</td>\n",
       "      <td>16.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                sum_sq    df         F    PR(>F)\n",
       "C(treatments)  3010.95   3.0  17.49281  0.000026\n",
       "Residual        918.00  16.0       NaN       NaN"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load packages\n",
    "import scipy.stats as stats\n",
    "# stats f_oneway functions takes the groups as input and returns F and P-value\n",
    "fvalue, pvalue = stats.f_oneway(d['A'], d['B'], d['C'], d['D'])\n",
    "print(fvalue, pvalue)\n",
    "# 17.492810457516338 2.639241146210922e-05\n",
    "\n",
    "# get ANOVA table as R like output\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.formula.api import ols\n",
    "# reshape the d dataframe suitable for statsmodels package \n",
    "d_melt = pd.melt(d.reset_index(), id_vars=['index'], value_vars=['A', 'B', 'C', 'D'])\n",
    "# replace column names\n",
    "d_melt.columns = ['index', 'treatments', 'value']\n",
    "# Ordinary Least Squares (OLS) model\n",
    "model = ols('value ~ C(treatments)', data=d_melt).fit()\n",
    "anova_table = sm.stats.anova_lm(model, typ=2)\n",
    "anova_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
      "=====================================================\n",
      "group1 group2 meandiff p-adj   lower    upper  reject\n",
      "-----------------------------------------------------\n",
      "     A      B     15.4 0.0251   1.6929 29.1071   True\n",
      "     A      C      1.6    0.9 -12.1071 15.3071  False\n",
      "     A      D     30.4  0.001  16.6929 44.1071   True\n",
      "     B      C    -13.8 0.0482 -27.5071 -0.0929   True\n",
      "     B      D     15.0 0.0296   1.2929 28.7071   True\n",
      "     C      D     28.8  0.001  15.0929 42.5071   True\n",
      "-----------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# load packages\n",
    "from statsmodels.stats.multicomp import pairwise_tukeyhsd\n",
    "# perform multiple pairwise comparison (Tukey HSD)\n",
    "m_comp = pairwise_tukeyhsd(endog=d_melt['value'], groups=d_melt['treatments'], alpha=0.05)\n",
    "print(m_comp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assumption 1: Homogeneity of variances (variances are equal between treatment groups) (Levene or Bartlett Test)\n",
    "import scipy.stats as stats\n",
    "w, pvalue = stats.levene(d['A'], d['B'], d['C'], d['D'])\n",
    "print(w, pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assumption 2: Residuals (experimental error) are normally distributed (Shapiro Wilks Test)\n",
    "import scipy.stats as stats\n",
    "w, pvalue = stats.shapiro(model.resid)\n",
    "print(w, pvalue)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-Way ANOVA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load packages\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "# load data file\n",
    "d = pd.read_csv(\"https://reneshbedre.github.io/myfiles/anova/twowayanova.txt\", sep=\"\\t\")\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reshape the d dataframe suitable for statsmodels package \n",
    "# you do not need to reshape if your data is already in stacked format. Compare d and d_melt tables for detail \n",
    "# understanding \n",
    "d_melt = pd.melt(d, id_vars=['Genotype'], value_vars=['1_year', '2_year', '3_year'])\n",
    "# replace column names\n",
    "d_melt.columns = ['Genotype', 'years', 'value']\n",
    "# generate a boxplot to see the data distribution by genotypes and years. Using boxplot, we can easily detect the \n",
    "# differences between different groups\n",
    "sns.boxplot(x=\"Genotype\", y=\"value\", hue=\"years\", data=d_melt, palette=\"Set3\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load packages\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.formula.api import ols\n",
    "# Ordinary Least Squares (OLS) model\n",
    "# C(Genotype):C(years) represent interaction term\n",
    "model = ols('value ~ C(Genotype) + C(years) + C(Genotype):C(years)', data=d_melt).fit()\n",
    "anova_table = sm.stats.anova_lm(model, typ=2)\n",
    "anova_table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
