{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot\n",
    "%matplotlib inline\n",
    "\n",
    "from numpy.random import seed\n",
    "from numpy.random import randn\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "\n",
    "from statsmodels.graphics.gofplots import qqplot\n",
    "from scipy.stats import shapiro\n",
    "from scipy.stats import normaltest\n",
    "from scipy.stats import anderson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read Sales.Dirty.xls\n",
    "salesdf = pd.read_excel('Sales.xls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "salesdf.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We need to decide whether data is gaussian for model learning, model evaluation and residual errors evaluation (from a regression problem)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We can only use parameteric ML models once Gaussian or Gaussian-enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed(32)\n",
    "\n",
    "data = 5 * randn(100) + 50\n",
    "\n",
    "print('mean=%.3f stdv=%.3f' % (mean(data), std(data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyplot.hist(data)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qqplot(data, line='s')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shapiro Wilk Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# considered to be reliable for smaller datasets\n",
    "# Null hypothesis=data is normal\n",
    "# Alternate hypothesis= data is not normal\n",
    "W, pvalue = shapiro(data)\n",
    "print('W=%.4f, p=%.4f' % (W, pvalue))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K2 test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#checks gaussianness based on skewness and kurtosis\n",
    "# Null hypothesis=data is normal\n",
    "# Alternate hypothesis= data is not normal\n",
    "k2stat, pvalue = normaltest(data)\n",
    "print('K2stat=%.4f, p=%.4f' % (k2stat, pvalue))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kolmogorov-Smimov Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = anderson(data)\n",
    "print('Output: %.4f' % output.statistic)\n",
    "\n",
    "p = 0\n",
    "for i in range(len(output.critical_values)):\n",
    "    sl, cv = output.significance_level[i], output.critical_values[i]\n",
    "    if output.statistic < output.critical_values[i]:\n",
    "        print('%.4f: %.4f, data looks normal (cant reject H0)' % (sl, cv))\n",
    "    else:\n",
    "        print('%.4f: %.4f, data not look normal (reject H0)' % (sl, cv))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyplot.hist(salesdf['Sales'])\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qqplot(salesdf['Sales'], line='s')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W, pvalue = shapiro(salesdf['Sales'])\n",
    "print('W=%.4f, p=%.4f' % (W, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k2stat, pvalue = normaltest(salesdf['Sales'])\n",
    "print('K2stat=%.4f, p=%.4f' % (k2stat, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = anderson(salesdf['Sales'])\n",
    "print('Output: %.4f' % output.statistic)\n",
    "\n",
    "p = 0\n",
    "for i in range(len(output.critical_values)):\n",
    "    sl, cv = output.significance_level[i], output.critical_values[i]\n",
    "    if output.statistic < output.critical_values[i]:\n",
    "        print('%.4f: %.4f, data looks normal (cant reject H0)' % (sl, cv))\n",
    "    else:\n",
    "        print('%.4f: %.4f, data not look normal (reject H0)' % (sl, cv))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
