{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "rng = np.random.default_rng()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aufgabe 1\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0, x))\n",
    "\n",
    "sigmoid = np.vectorize(lambda x : 1 / (1 + np.exp(-x)))\n",
    "\n",
    "def samples(n: int) -> list:\n",
    "    inputs  = rng.integers(0, 1, size=(n, 2), endpoint=True)\n",
    "    outputs = inputs.T[0] ^ inputs.T[1]\n",
    "    return (inputs, outputs)\n",
    "\n",
    "# Aufgabe 3\n",
    "\n",
    "# Binary Cross Entropy Loss\n",
    "bcel = np.vectorize(lambda y, ŷ : -(y * np.log(ŷ) + (1 - y) * np.log(1 - ŷ)))\n",
    "\n",
    "# Derivatives\n",
    "derivatives = {\n",
    "    bcel    : np.vectorize(lambda y, ŷ: (1 / (1 - ŷ)) if y == 0 else -(1 / ŷ)),\n",
    "    sigmoid : np.vectorize(lambda x: (1 / (1 + np.exp(-x))) * (1 - (1 / (1 + np.exp(-x))))),\n",
    "    relu    : np.vectorize(lambda x: 0 if x < 0 else 1)\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aufgabe 2\n",
    "from typing import Callable\n",
    "\n",
    "\n",
    "class NeuralNet:\n",
    "    def __init__(self, inputs: int = 2, hidden_layers: list[tuple[int, Callable]] = None):\n",
    "        \"\"\"\n",
    "        Initializes the neural network.\n",
    "        Hidden layers can be specified with the 'hidden_layers' parameter,\n",
    "        which takes list of the format [(layer1_size, layer1_activation_function), (layer2...), ...].\n",
    "        The output layer will always consist of a single neuron and use the sigmoid\n",
    "        activation function.\n",
    "        \"\"\"\n",
    "\n",
    "        self.input_shape = (-1, inputs)\n",
    "        self.layers = [] if hidden_layers is None else hidden_layers\n",
    "        self.layers.append((1, sigmoid))  # Add output layer\n",
    "\n",
    "        # Construct weights for hidden layer\n",
    "        self.activation_functions = []\n",
    "        self.weights = []\n",
    "        for index, (num_neurons, activation_function) in enumerate(self.layers):\n",
    "            self.activation_functions.append(activation_function)\n",
    "\n",
    "            num_layer_inputs = inputs if index == 0 else self.layers[index - 1][0]\n",
    "            self.weights.append(rng.uniform(low=-1.0, high=1.0, size=(num_layer_inputs, num_neurons)))\n",
    "\n",
    "    def forward_pass(self, x: np.array) -> tuple:\n",
    "        \"\"\"\n",
    "        Do a forward pass through the neural net.\n",
    "        Returns the linear and activation function results for each layer.\n",
    "        For the final output, see the last input in the F list.\n",
    "        \"\"\"\n",
    "\n",
    "        x = np.reshape(x, self.input_shape)\n",
    "\n",
    "        Z = []  # linear values for each layer\n",
    "        F = [x]  # activation function values for each layer\n",
    "        for weights, activation_function in zip(self.weights, self.activation_functions):\n",
    "            Z.append(np.matmul(F[-1], weights))  # linear\n",
    "            F.append(activation_function(Z[-1]))\n",
    "\n",
    "        return (Z, F)\n",
    "\n",
    "    def classify(self, x: np.array) -> float:\n",
    "        \"\"\"\n",
    "        Executes a forward pass,\n",
    "        and returns the classification result ŷ.\n",
    "        \"\"\"\n",
    "        \n",
    "        _, F = self.forward_pass(x)\n",
    "        ŷ = F[-1].reshape(x.shape[0])\n",
    "        return ŷ\n",
    "\n",
    "    # Aufgabe 4\n",
    "    def backward_pass(self, x, y: float):\n",
    "        \"\"\"\n",
    "        Do a backward pass through the neural net.\n",
    "        Returns the calculated weight difference.\n",
    "        \"\"\"\n",
    "        x = np.reshape(x, self.input_shape)\n",
    "        batch_size = x.shape[0]\n",
    "\n",
    "        Z, F = self.forward_pass(x)\n",
    "\n",
    "        layer_errors = [None for _ in range(len(self.layers))]\n",
    "        layer_errors[-1] = F[-1] - y.reshape((-1, 1))  # ŷ - y\n",
    "\n",
    "        # Backpropagation\n",
    "        for i in reversed(range(len(self.layers) - 1)):\n",
    "            layer_errors[i] = np.multiply(\n",
    "                self.weights[i + 1].T,\n",
    "                np.multiply(layer_errors[i + 1], derivatives[self.activation_functions[i]](Z[i])),\n",
    "            )\n",
    "\n",
    "        # Final weight updates, normalized\n",
    "        Δweights = [np.dot(F[i].T, error) / batch_size for i, error in enumerate(layer_errors)]\n",
    "        return Δweights\n",
    "\n",
    "    # Aufgabe 5\n",
    "    def train(self, data: list, batch_size: int = 10, epochs: int = 50, learning_rate: float = 0.5):\n",
    "        \"\"\"\n",
    "        Train the neural network with the given input data.\n",
    "        \"\"\"\n",
    "\n",
    "        x, y = data\n",
    "        for epoch in tqdm(range(epochs)):\n",
    "            for i in range(0, len(data), batch_size):\n",
    "                x_batch = x[i : i + batch_size]\n",
    "                y_batch = y[i : i + batch_size]\n",
    "                Δweights = self.backward_pass(x_batch, y_batch)\n",
    "\n",
    "                self.weights = [w - learning_rate * Δw for w, Δw in zip(self.weights, Δweights, strict=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unit test to make sure neural net behaves as expected\n",
    "# Uses example values from https://towardsdatascience.com/how-does-back-propagation-work-in-neural-networks-with-worked-example-bc59dfb97f48.\n",
    "\n",
    "nn = NeuralNet(inputs=3, hidden_layers=[(4, sigmoid)])\n",
    "\n",
    "nn.weights = [\n",
    "    np.array([[0.179, -0.186, -0.008, -0.048], [0.044, -0.028, -0.063, -0.131], [0.01, -0.035, -0.004, 0.088]]),\n",
    "    np.array([[0.088], [0.171], [0.005], [-0.04]]),\n",
    "]\n",
    "\n",
    "# Make sure forward pass is correct\n",
    "Z, F = nn.forward_pass(np.array([[7, 8, 10]]))\n",
    "assert np.allclose(F[1], np.array([[0.845, 0.132, 0.354, 0.377]]), atol=0.01)\n",
    "assert np.allclose(F[2], np.array([[0.521]]), atol=0.01)\n",
    "\n",
    "# Make sure backward pass is correct\n",
    "Δweights = nn.backward_pass(np.array([[7, 8, 10], [7,8,10]]), np.array([[1], [1]]))\n",
    "assert np.allclose(\n",
    "    Δweights[0],\n",
    "    np.array([[-0.039, -0.066, -0.004, 0.032], [-0.044, -0.075, -0.004, 0.036], [-0.055, -0.094, -0.005, 0.046]]),\n",
    "    atol=0.01,\n",
    ")\n",
    "assert np.allclose(Δweights[1], np.array([[-0.405], [-0.063], [-0.169], [-0.181]]), atol=0.01)\n",
    "\n",
    "# No output means success"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct classifications before training: 0.474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 80/80 [00:00<00:00, 3998.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct classifications after training: 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Aufgabe 4\n",
    "# To update the weights after each backward pass, set the batch size to 1 (Aufgabe 4)\n",
    "\n",
    "test_data = samples(1_000)\n",
    "training_data = samples(100)\n",
    "\n",
    "# Updating weights after every backward pass\n",
    "nn = NeuralNet(hidden_layers=[(8, relu)])\n",
    "\n",
    "correct = sum(np.around(nn.classify(test_data[0])) == test_data[1])\n",
    "print(f\"Correct classifications before training: {correct/test_data[0].shape[0]}\")\n",
    "\n",
    "nn.train(training_data, batch_size=20, learning_rate=1, epochs=80)\n",
    "\n",
    "correct = sum(np.around(nn.classify(test_data[0])) == test_data[1])\n",
    "print(f\"Correct classifications after training: {correct/test_data[0].shape[0]}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8000 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize learned separation\n",
    "\n",
    "def visualize(nn: NeuralNet):\n",
    "    # left, right, bottom, top  \n",
    "    dim = (0, 1, 0, 1)\n",
    "    resolution = 50\n",
    "\n",
    "    data = []\n",
    "    for px in np.linspace(dim[0], dim[1], resolution):\n",
    "        col = []\n",
    "        for py in np.linspace(dim[2], dim[3], resolution):\n",
    "            col.append(nn.classify(np.array([[px, py]])))\n",
    "        data.append(col)\n",
    "    data = np.array(data)\n",
    "\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    _, ax = plt.subplots()\n",
    "    ax.imshow(data, cmap=\"hot\", interpolation=\"nearest\", extent=dim)\n",
    "visualize(nn)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
