{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "419ef436",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "from sklearn.linear_model import LinearRegression\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bdc6641",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "\n",
    "x = np.arange(0, 10)\n",
    "y = 3 * x + 3 * np.random.randn(len(x)) + 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dede5c6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(x, y)  # rough linear trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "266c8998",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = LinearRegression()\n",
    "X = x[:, np.newaxis]  # need to tell that X is (n_samples, 1)  (could be (1, n_features))\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3608eb2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2e6c8b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(x, y, label='training data')\n",
    "plt.plot(x, reg.predict(X), label='fit', c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c69f3054",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(reg.predict(X))\n",
    "print(x * reg.coef_ + reg.intercept_)  # they are equal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "569f6255",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = LinearRegression()\n",
    "reg.coef_  # not fitted yet, attribute does not exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "419e9a46",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = LinearRegression(fit_intercept=False).fit(X, y)\n",
    "reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1408e30",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c59e7c4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b353e79",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e2daf18",
   "metadata": {},
   "outputs": [],
   "source": [
    "KNeighborsClassifier?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78b08d27",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dcebf8e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2eb2b91a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb01bac7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96872c1b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4777d107",
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_ols(x, y, theta):\n",
    "    plt.scatter(x, y, label='training data')\n",
    "    plt.plot(x, x * theta, label='linear fit')\n",
    "    loss = 0\n",
    "    for idx, (x_, y_) in enumerate(zip(x, y)):\n",
    "        plt.vlines(x=x_, ymin=min(y_, x_ * theta), ymax=max(y_, x_ * theta), color='k')\n",
    "        loss += 0.5 * (y_ - x_ * theta) ** 2\n",
    "    plt.title(f'$\\\\theta={theta:.2f}$, loss = {loss:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6278acef",
   "metadata": {},
   "outputs": [],
   "source": [
    "thetas = []\n",
    "theta = 0 \n",
    "\n",
    "# eta = 1.5 / np.linalg.norm(x) ** 2\n",
    "eta = 0.005 \n",
    "\n",
    "for _ in range(10):\n",
    "    theta = theta - eta * x.T @ (x * theta - y)\n",
    "    thetas.append(theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67b70e71",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(thetas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da056c89",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg.coef_  # Gradient descent indeed converges towards the minimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40c37aaf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
