{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation Analysis (basic stuff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# seed random number generator\n",
    "np.random.seed(49)\n",
    "\n",
    "# generate normally distributed data with STD 13 and Mean 250\n",
    "series1 = 13 * np.random.randn(1000) + 250\n",
    "\n",
    "#print(series1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate another series based on data 1 with some added noise\n",
    "series2 = series1 + (10 * np.random.randn(1000) + 50)\n",
    "\n",
    "#len(series2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize\n",
    "print('data1: mean=%.2f stdv=%.2f' % (np.mean(series1), np.std(series1)))\n",
    "print('data2: mean=%.2f stdv=%.2f' % (np.mean(series2), np.std(series2)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "plt.scatter(series1, series2, c='green')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All About Co-Variances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cov(X, Y) = (sum (x - mean(X)) * (y - mean(Y)) ) * 1/(n-1)\n",
    "\n",
    "# seed random number generator\n",
    "np.random.seed(49)\n",
    "\n",
    "# generate normally distributed data with STD 13 and Mean 250\n",
    "series1 = 13 * np.random.randn(1000) + 250\n",
    "\n",
    "#generate another series based on data 1 with some added noise\n",
    "series2 = series1 + (10 * np.random.randn(1000) + 50)\n",
    "\n",
    "# calculate covariance matrix\n",
    "covariance = np.cov(series1, series2)\n",
    "covariance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pearson Correlation Coefficient (Gaussian, Linear)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pearson coeff = covariance(X, Y) / (stdv(X) * stdv(Y))\n",
    "from scipy.stats import pearsonr\n",
    "# seed random number generator\n",
    "np.random.seed(49)\n",
    "\n",
    "# generate normally distributed data with STD 13 and Mean 250\n",
    "series1 = 23 * np.random.randn(500) + 290\n",
    "\n",
    "#generate another series based on data 1 with some added noise\n",
    "series2 = series1 + (52 * np.random.randn(500) + 30)\n",
    "\n",
    "# plot\n",
    "plt.scatter(series1, series2, c='green')\n",
    "plt.show()\n",
    "\n",
    "# calculate Pearson's correlation\n",
    "corr, p = pearsonr(series1, series2)\n",
    "print('Pearsons correlation: %.2f' % corr)\n",
    "print('P value: %.6f' % p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spearman Correlation Coefficient (Non-Gaussian, Non-Linear)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Spearman's coefficient = covariance(rank(X), rank(Y)) / (stdv(rank(X)) * stdv(rank(Y)))\n",
    "#could also work for linar relationship but the results can be less than Pearson\n",
    "#to be used when not sure about the true relationship between the variables\n",
    "\n",
    "from scipy.stats import spearmanr\n",
    "# seed random number generator\n",
    "np.random.seed(49)\n",
    "\n",
    "# generate normally distributed data with STD 13 and Mean 250\n",
    "series1 = 13 * np.random.randn(1000) + 250\n",
    "\n",
    "#generate another series based on data 1 with some added noise\n",
    "series2 = series1 + (10 * np.random.randn(1000) + 50)\n",
    "\n",
    "# calculate Pearson's correlation\n",
    "corr, _ = spearmanr(series1, series2)\n",
    "print('Spearman correlation: %.2f' % corr)\n",
    "\n",
    "# calculate Pearson's correlation\n",
    "corr, _ = pearsonr(series1, series2)\n",
    "print('Pearsons correlation: %.2f' % corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with House Prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "housedf = pd.read_csv(\"House.Price.csv\")\n",
    "housedf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "housedf.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#so there are many numerical columns (overall 81 columns)\n",
    "housedf.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#of more interest is the selling price\n",
    "housedf['SalePrice'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "housedf['SalePrice'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets expand it a bit\n",
    "plt.figure(figsize=(10,5))\n",
    "housedf['SalePrice'].plot(kind=\"hist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let us check the correlation of all numerical columns with each other\n",
    "corrmatrix = housedf.corr()\n",
    "f, axis = plt.subplots(figsize =(15, 10)) \n",
    "sns.heatmap(corrmatrix, ax = axis, linewidths = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#changing the color\n",
    "corrmatrix = housedf.corr()\n",
    "f, axis = plt.subplots(figsize =(15, 10)) \n",
    "sns.heatmap(corrmatrix, ax = axis, cmap =\"YlGnBu\", linewidths = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we can see that saleprice has some good correlation (more blue color) with several variables (e.g garage area)\n",
    "#let us plot a hierarchically clustered map to give an idea of which attributes can be clustered at the basic level\n",
    "  \n",
    "clmap = sns.clustermap(corrmatrix, cmap =\"YlGnBu\", linewidths = 0.2); \n",
    "plt.setp(clmap.ax_heatmap.yaxis.get_majorticklabels(), rotation = 0) \n",
    "  \n",
    "clmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let us plot a correlation matrix only for saleprice (v vars with largest correlation with sales price)\n",
    "\n",
    "v = 7 #number of variables\n",
    "  \n",
    "cols = corrmatrix.nlargest(v, 'SalePrice')['SalePrice'].index \n",
    "  \n",
    "coeffs = np.corrcoef(housedf[cols].values.T) \n",
    "f, axis = plt.subplots(figsize =(12, 10)) \n",
    "  \n",
    "sns.heatmap(coeffs, ax = axis, cmap =\"YlGnBu\", \n",
    "            linewidths = 0.2, yticklabels = cols.values,  \n",
    "                              xticklabels = cols.values) \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
