{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot\n",
    "%matplotlib inline\n",
    "\n",
    "from numpy.random import seed\n",
    "from numpy.random import randn\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "\n",
    "from statsmodels.graphics.gofplots import qqplot\n",
    "from scipy.stats import shapiro\n",
    "from scipy.stats import normaltest\n",
    "from scipy.stats import anderson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read Sales.Dirty.xls\n",
    "salesdf = pd.read_excel('Sales.xls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row ID                           int64\n",
       "Order ID                         int64\n",
       "Order Date              datetime64[ns]\n",
       "Order Priority                  object\n",
       "Order Quantity                   int64\n",
       "Sales                          float64\n",
       "Discount                       float64\n",
       "Ship Mode                       object\n",
       "Profit                         float64\n",
       "Unit Price                     float64\n",
       "Shipping Cost                  float64\n",
       "Customer Name                   object\n",
       "Province                        object\n",
       "Region                          object\n",
       "Customer Segment                object\n",
       "Product Category                object\n",
       "Product Sub-Category            object\n",
       "Product Name                    object\n",
       "Product Container               object\n",
       "Product Base Margin            float64\n",
       "Ship Date               datetime64[ns]\n",
       "dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salesdf.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We need to decide whether data is gaussian for model learning, model evaluation and residual errors evaluation (from a regression problem)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We can only use parameteric ML models once Gaussian or Gaussian-enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean=50.318 stdv=5.143\n"
     ]
    }
   ],
   "source": [
    "seed(32)\n",
    "\n",
    "data = 5 * randn(100) + 50\n",
    "\n",
    "print('mean=%.3f stdv=%.3f' % (mean(data), std(data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAK/ElEQVR4nO3dUYild3nH8d9Tt71JLY1kDMGYTivB1hujLGlLIGiDoqY08UJooBKKsF5osVAo297Uy23B2hZK6KqpoVWL2AZDV6whLUihiJtWNDaVSNhqzDa7IqW2NxJ9ejFnYZnMZGbnnJmzz+bzgeGc887ZfZ8//90vJ+/OOanuDgDz/Ni6BwDgYAQcYCgBBxhKwAGGEnCAoY4d5cluuOGG3tzcPMpTAoz3+OOPf7e7N7YfP9KAb25u5uzZs0d5SoDxquo/dzruEgrAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUEf6TkzYy+bJM2s577lTd6/lvLAMr8ABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGCoPQNeVa+uqn+qqier6utV9YHF8VdU1aNV9dTi9vrDHxeAS/bzCvz5JL/T3b+Q5JeSvK+qXpfkZJLHuvvWJI8tHgNwRPYMeHef7+5/Xdz/fpInk7wqyT1JHlo87aEk9x7WkAC80LEreXJVbSZ5Q5IvJbmxu88nW5Gvqlfu8mtOJDmRJLfccssys3JENk+eWfcIwD7s+x8xq+onk/xtkt/u7v/Z76/r7tPdfby7j29sbBxkRgB2sK+AV9WPZyven+juv1scfq6qblp8/6YkFw5nRAB2sp+fQqkkH0vyZHf/8WXfeiTJ/Yv79yf57OrHA2A3+7kGfkeSdyf5WlV9ZXHs95OcSvLpqnpPkm8ledfhjAjATvYMeHf/c5La5dt3rXYcAPbLOzEBhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKGOrXsAdrd58sy6RwCuYl6BAwwl4ABDCTjAUAIOMJSAAwy1Z8Cr6sGqulBVT1x27INV9Z2q+sri6x2HOyYA2+3nFfjHk7xth+Mf7u7bFl+fW+1YAOxlz4B39xeTfO8IZgHgCixzDfz9VfXVxSWW63d7UlWdqKqzVXX24sWLS5wOgMsdNOAPJHlNktuSnE/yod2e2N2nu/t4dx/f2Ng44OkA2O5AAe/u57r7h939oyQfSXL7ascCYC8HCnhV3XTZw3cmeWK35wJwOPb8MKuq+lSSNyW5oaqeSfIHSd5UVbcl6STnkrz3EGcEYAd7Bry779vh8McOYRYAroB3YgIMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjDUnh9mBRyuzZNn1nLec6fuXst5WR2vwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxjKG3kg63szDSzDK3CAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYKg9A15VD1bVhap64rJjr6iqR6vqqcXt9Yc7JgDb7ecV+MeTvG3bsZNJHuvuW5M8tngMwBHaM+Dd/cUk39t2+J4kDy3uP5Tk3hXPBcAeDnoN/MbuPp8ki9tX7vbEqjpRVWer6uzFixcPeDoAtjv0f8Ts7tPdfby7j29sbBz26QBeMg4a8Oeq6qYkWdxeWN1IAOzHQQP+SJL7F/fvT/LZ1YwDwH7t58cIP5XkX5K8tqqeqar3JDmV5C1V9VSStyweA3CEju31hO6+b5dv3bXiWQC4At6JCTCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFB7fpgVyebJM+seAeAFvAIHGErAAYYScIChBBxgKAEHGErAAYYScIChBBxgKAEHGErAAYYScIChBBxgKAEHGErAAYYScIChBBxgKAEHGErAAYYScIChBBxgKAEHGErAAYYScIChji3zi6vqXJLvJ/lhkue7+/gqhgJgb0sFfOHN3f3dFfw+AFwBl1AAhlr2FXgn+UJVdZK/6O7T259QVSeSnEiSW265ZcnTAauyefLM2s597tTdazv3tWTZV+B3dPcbk7w9yfuq6s7tT+ju0919vLuPb2xsLHk6AC5ZKuDd/ezi9kKSh5PcvoqhANjbgQNeVddV1csv3U/y1iRPrGowAF7cMtfAb0zycFVd+n0+2d2fX8lUAOzpwAHv7qeTvH6FswBwBfwYIcBQAg4wlIADDCXgAEMJOMBQAg4wlIADDCXgAEMJOMBQAg4wlIADDCXgAEMJOMBQAg4wlIADDCXgAEMJOMBQAg4wlIADDCXgAEMJOMBQB/6/0h+1zZNn1j0CwFXFK3CAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGGvNGHoBlrfMNgedO3b3y39MrcIChBBxgKAEHGErAAYYScIChlgp4Vb2tqr5RVd+sqpOrGgqAvR044FX1siR/nuTtSV6X5L6qet2qBgPgxS3zCvz2JN/s7qe7+wdJ/ibJPasZC4C9LPNGnlcl+fZlj59J8ovbn1RVJ5KcWDz836r6xhLnXKUbknx33UOsmDXN8JJfU/3hIU6yOivdpyXX/DM7HVwm4LXDsX7Bge7TSU4vcZ5DUVVnu/v4uudYJWuawZpmmLCmZS6hPJPk1Zc9vjnJs8uNA8B+LRPwLye5tap+tqp+IsmvJ3lkNWMBsJcDX0Lp7uer6v1J/iHJy5I82N1fX9lkh++qu6yzAtY0gzXNcNWvqbpfcNkagAG8ExNgKAEHGOolE/CqellV/VtV/f3i8Suq6tGqempxe/26Z7xSO6zpg1X1nar6yuLrHeue8UpU1bmq+tpi9rOLY6P3aZc1Td+nn66qz1TVf1TVk1X1y9P3Kdl1XVf1Xr1kAp7kA0mevOzxySSPdfetSR5bPJ5m+5qS5MPdfdvi63PrGGpJb17Mfunnb6+Ffdq+pmT2Pv1pks93988neX22/gxeC/u007qSq3ivXhIBr6qbk9yd5KOXHb4nyUOL+w8lufeo51rGLmu6Fo3ep2tNVf1UkjuTfCxJuvsH3f3fGb5PL7Kuq9pLIuBJ/iTJ7yb50WXHbuzu80myuH3lOgZbwk5rSpL3V9VXq+rBgf8Z20m+UFWPLz6CIZm/TzutKZm7Tz+X5GKSv1xcvvtoVV2X+fu027qSq3ivrvmAV9WvJrnQ3Y+ve5ZVeZE1PZDkNUluS3I+yYeOerYl3dHdb8zWJ1y+r6ruXPdAK7DTmibv07Ekb0zyQHe/Icn/Zeblku12W9dVvVfXfMCT3JHk16rqXLY+MfFXquqvkzxXVTclyeL2wvpGvGI7rqm7n+vuH3b3j5J8JFufGDlGdz+7uL2Q5OFszT95n3Zc0/B9eibJM939pcXjz2QrfKP3Kbus62rfq2s+4N39e919c3dvZuvt/v/Y3b+Rrbf937942v1JPrumEa/Ybmu69Bdo4Z1JnljLgAdQVddV1csv3U/y1mzNP3afdlvT5H3q7v9K8u2qeu3i0F1J/j2D9ynZfV1X+14t82mE051K8umqek+SbyV515rnWYU/qqrbsnXd9VyS9653nCtyY5KHqyrZ+nP5ye7+fFV9OXP3abc1/dXgfUqS30ryicVnID2d5Dez9WJw6j5dstO6/uxq3itvpQcY6pq/hAJwrRJwgKEEHGAoAQcYSsABhhJwgKEEHGCo/wf+ber5wll25gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.hist(data)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14968 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "qqplot(data, line='s')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shapiro Wilk Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W=0.9874, p=0.4667\n"
     ]
    }
   ],
   "source": [
    "# considered to be reliable for smaller datasets\n",
    "# Null hypothesis=data is normal\n",
    "# Alternate hypothesis= data is not normal\n",
    "W, pvalue = shapiro(data)\n",
    "print('W=%.4f, p=%.4f' % (W, pvalue))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K2 test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K2stat=0.9570, p=0.6197\n"
     ]
    }
   ],
   "source": [
    "#checks gaussianness based on skewness and kurtosis\n",
    "# Null hypothesis=data is normal\n",
    "# Alternate hypothesis= data is not normal\n",
    "k2stat, pvalue = normaltest(data)\n",
    "print('K2stat=%.4f, p=%.4f' % (k2stat, pvalue))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kolmogorov-Smimov Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output: 0.3488\n",
      "15.0000: 0.5550, data looks normal (cant reject H0)\n",
      "10.0000: 0.6320, data looks normal (cant reject H0)\n",
      "5.0000: 0.7590, data looks normal (cant reject H0)\n",
      "2.5000: 0.8850, data looks normal (cant reject H0)\n",
      "1.0000: 1.0530, data looks normal (cant reject H0)\n"
     ]
    }
   ],
   "source": [
    "output = anderson(data)\n",
    "print('Output: %.4f' % output.statistic)\n",
    "\n",
    "p = 0\n",
    "for i in range(len(output.critical_values)):\n",
    "    sl, cv = output.significance_level[i], output.critical_values[i]\n",
    "    if output.statistic < output.critical_values[i]:\n",
    "        print('%.4f: %.4f, data looks normal (cant reject H0)' % (sl, cv))\n",
    "    else:\n",
    "        print('%.4f: %.4f, data not look normal (reject H0)' % (sl, cv))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 7284 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.hist(salesdf['Sales'])\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12504 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "qqplot(salesdf['Sales'], line='s')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W=0.4985, p=0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tmahmood\\Anaconda3\\lib\\site-packages\\scipy\\stats\\morestats.py:1681: UserWarning: p-value may not be accurate for N > 5000.\n",
      "  warnings.warn(\"p-value may not be accurate for N > 5000.\")\n"
     ]
    }
   ],
   "source": [
    "W, pvalue = shapiro(salesdf['Sales'])\n",
    "print('W=%.4f, p=%.4f' % (W, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K2stat=9206.9868, p=0.0000\n"
     ]
    }
   ],
   "source": [
    "k2stat, pvalue = normaltest(salesdf['Sales'])\n",
    "print('K2stat=%.4f, p=%.4f' % (k2stat, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output: 1296.3058\n",
      "15.0000: 0.5760, data not look normal (reject H0)\n",
      "10.0000: 0.6560, data not look normal (reject H0)\n",
      "5.0000: 0.7870, data not look normal (reject H0)\n",
      "2.5000: 0.9180, data not look normal (reject H0)\n",
      "1.0000: 1.0910, data not look normal (reject H0)\n"
     ]
    }
   ],
   "source": [
    "output = anderson(salesdf['Sales'])\n",
    "print('Output: %.4f' % output.statistic)\n",
    "\n",
    "p = 0\n",
    "for i in range(len(output.critical_values)):\n",
    "    sl, cv = output.significance_level[i], output.critical_values[i]\n",
    "    if output.statistic < output.critical_values[i]:\n",
    "        print('%.4f: %.4f, data looks normal (cant reject H0)' % (sl, cv))\n",
    "    else:\n",
    "        print('%.4f: %.4f, data not look normal (reject H0)' % (sl, cv))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
