{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aufgabenblatt 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from random import sample\n",
    "from typing import Literal\n",
    "\n",
    "rng = np.random.default_rng()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aufgabe 1\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0, x))\n",
    "\n",
    "sigmoid = np.vectorize(lambda x : 1 / (1 + np.exp(-x)))\n",
    "\n",
    "def samples(n: int) -> list:\n",
    "    inputs  = rng.integers(0, 1, size=(n, 2), endpoint=True)\n",
    "    outputs = inputs.T[0] ^ inputs.T[1]\n",
    "    return (inputs, outputs)\n",
    "\n",
    "# Aufgabe 3\n",
    "\n",
    "# Binary Cross Entropy Loss\n",
    "bcel = np.vectorize(lambda y, ŷ : -(y * np.log(ŷ) + (1 - y) * np.log(1 - ŷ)))\n",
    "\n",
    "# Derivatives\n",
    "derivatives = {\n",
    "    bcel    : np.vectorize(lambda y, ŷ: (1 / (1 - ŷ)) if y == 0 else -(1 / ŷ)),\n",
    "    sigmoid : np.vectorize(lambda x: (1 / (1 + np.exp(-x))) * (1 - (1 / (1 + np.exp(-x))))),\n",
    "    relu    : np.vectorize(lambda x: 0 if x < 0 else 1)\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aufgabe 2\n",
    "from typing import Callable\n",
    "\n",
    "\n",
    "class NeuralNet:\n",
    "    def __init__(self, inputs: int = 2, hidden_layers: list[tuple[int, Callable]] = None, weights: list = None):\n",
    "        \"\"\"\n",
    "        Initializes the neural network.\n",
    "        Hidden layers can be specified with the 'hidden_layers' parameter,\n",
    "        which takes list of the format [(layer1_size, layer1_activation_function), (layer2...), ...].\n",
    "        The output layer will always consist of a single neuron and use the sigmoid\n",
    "        activation function.\n",
    "        \"\"\"\n",
    "\n",
    "        self.input_shape = (-1, inputs)\n",
    "        self.layers = [] if hidden_layers is None else hidden_layers\n",
    "        self.layers.append((1, sigmoid))  # Add output layer\n",
    "\n",
    "        self.activation_functions = [activation_function for _, activation_function in self.layers]\n",
    "\n",
    "        # Randomly create weights if not supplied\n",
    "        if weights is None:\n",
    "            self.weights = []\n",
    "            for index, (num_neurons, _) in enumerate(self.layers):\n",
    "                num_layer_inputs = inputs if index == 0 else self.layers[index - 1][0]\n",
    "                self.weights.append(rng.uniform(low=-1.0, high=1.0, size=(num_layer_inputs, num_neurons)))\n",
    "\n",
    "        # Check if weight shapes are correct\n",
    "        else:\n",
    "            assert len(weights) == len(self.layers)\n",
    "            for index, (weight, layer) in enumerate(zip(weights, self.layers)):\n",
    "                num_neurons, _ = layer\n",
    "                num_layer_inputs = inputs if index == 0 else self.layers[index - 1][0]\n",
    "                assert weight.shape == (num_layer_inputs, num_neurons)\n",
    "            self.weights = weights\n",
    "\n",
    "    def forward_pass(self, x: np.array) -> tuple:\n",
    "        \"\"\"\n",
    "        Do a forward pass through the neural net.\n",
    "        Returns the linear and activation function results for each layer.\n",
    "        For the final output, see the last input in the F list.\n",
    "        \"\"\"\n",
    "\n",
    "        x = np.reshape(x, self.input_shape)\n",
    "\n",
    "        Z = []  # linear values for each layer\n",
    "        F = [x]  # activation function values for each layer\n",
    "        for weights, activation_function in zip(self.weights, self.activation_functions):\n",
    "            Z.append(np.matmul(F[-1], weights))  # linear\n",
    "            F.append(activation_function(Z[-1]))\n",
    "\n",
    "        return (Z, F)\n",
    "\n",
    "    def classify(self, x: np.array) -> float:\n",
    "        \"\"\"\n",
    "        Executes a forward pass,\n",
    "        and returns the classification result ŷ.\n",
    "        \"\"\"\n",
    "\n",
    "        _, F = self.forward_pass(x)\n",
    "        ŷ = F[-1].reshape(x.shape[0])\n",
    "        return ŷ\n",
    "\n",
    "    # Aufgabe 4\n",
    "    def backward_pass(self, x, y: float):\n",
    "        \"\"\"\n",
    "        Do a backward pass through the neural net.\n",
    "        Returns the calculated weight difference.\n",
    "        \"\"\"\n",
    "        x = np.reshape(x, self.input_shape)\n",
    "        batch_size = x.shape[0]\n",
    "\n",
    "        Z, F = self.forward_pass(x)\n",
    "\n",
    "        layer_errors = [None for _ in range(len(self.layers))]\n",
    "        layer_errors[-1] = F[-1] - y.reshape((-1, 1))  # ŷ - y\n",
    "\n",
    "        # Backpropagation\n",
    "        for i in reversed(range(len(self.layers) - 1)):\n",
    "            layer_errors[i] = np.multiply(\n",
    "                self.weights[i + 1].T,\n",
    "                np.multiply(layer_errors[i + 1], derivatives[self.activation_functions[i]](Z[i])),\n",
    "            )\n",
    "\n",
    "        # Final weight updates, normalized\n",
    "        Δweights = [np.dot(F[i].T, error) / batch_size for i, error in enumerate(layer_errors)]\n",
    "        return Δweights\n",
    "\n",
    "    # Aufgabe 5\n",
    "    def train(self, data: list, batch_size: int = 10, epochs: int = 50, learning_rate: float = 0.5, verbose: bool = True):\n",
    "        \"\"\"\n",
    "        Train the neural network with the given input data.\n",
    "        \"\"\"\n",
    "\n",
    "        x, y = data\n",
    "        for epoch in tqdm(range(epochs), disable=not verbose):\n",
    "            for i in range(0, len(data), batch_size):\n",
    "                x_batch = x[i : i + batch_size]\n",
    "                y_batch = y[i : i + batch_size]\n",
    "                Δweights = self.backward_pass(x_batch, y_batch)\n",
    "\n",
    "                self.weights = [w - learning_rate * Δw for w, Δw in zip(self.weights, Δweights, strict=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unit test to make sure neural net behaves as expected\n",
    "# Uses example values from https://towardsdatascience.com/how-does-back-propagation-work-in-neural-networks-with-worked-example-bc59dfb97f48.\n",
    "\n",
    "nn = NeuralNet(inputs=3, hidden_layers=[(4, sigmoid)], weights=[\n",
    "    np.array([[0.179, -0.186, -0.008, -0.048], [0.044, -0.028, -0.063, -0.131], [0.01, -0.035, -0.004, 0.088]]),\n",
    "    np.array([[0.088], [0.171], [0.005], [-0.04]]),\n",
    "])\n",
    "\n",
    "\n",
    "# Make sure forward pass is correct\n",
    "Z, F = nn.forward_pass(np.array([[7, 8, 10]]))\n",
    "assert np.allclose(F[1], np.array([[0.845, 0.132, 0.354, 0.377]]), atol=0.01)\n",
    "assert np.allclose(F[2], np.array([[0.521]]), atol=0.01)\n",
    "\n",
    "# Make sure backward pass is correct\n",
    "Δweights = nn.backward_pass(np.array([[7, 8, 10], [7,8,10]]), np.array([[1], [1]]))\n",
    "assert np.allclose(\n",
    "    Δweights[0],\n",
    "    np.array([[-0.039, -0.066, -0.004, 0.032], [-0.044, -0.075, -0.004, 0.036], [-0.055, -0.094, -0.005, 0.046]]),\n",
    "    atol=0.01,\n",
    ")\n",
    "assert np.allclose(Δweights[1], np.array([[-0.405], [-0.063], [-0.169], [-0.181]]), atol=0.01)\n",
    "\n",
    "# No output means success"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct classifications before training: 0.479\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 80/80 [00:00<00:00, 3999.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct classifications after training: 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Aufgabe 4\n",
    "# To update the weights after each backward pass, set the batch size to 1 (Aufgabe 4)\n",
    "\n",
    "test_data = samples(1_000)\n",
    "training_data = samples(100)\n",
    "\n",
    "# Updating weights after every backward pass\n",
    "nn = NeuralNet(hidden_layers=[(8, relu)])\n",
    "\n",
    "correct = sum(np.around(nn.classify(test_data[0])) == test_data[1])\n",
    "print(f\"Correct classifications before training: {correct/test_data[0].shape[0]}\")\n",
    "\n",
    "nn.train(training_data, batch_size=20, learning_rate=1, epochs=80)\n",
    "\n",
    "correct = sum(np.around(nn.classify(test_data[0])) == test_data[1])\n",
    "print(f\"Correct classifications after training: {correct/test_data[0].shape[0]}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8532 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize learned separation\n",
    "\n",
    "def visualize(nn: NeuralNet):\n",
    "    # left, right, bottom, top  \n",
    "    dim = (0, 1, 0, 1)\n",
    "    resolution = 50\n",
    "\n",
    "    data = []\n",
    "    for px in np.linspace(dim[0], dim[1], resolution):\n",
    "        col = []\n",
    "        for py in np.linspace(dim[2], dim[3], resolution):\n",
    "            col.append(nn.classify(np.array([[px, py]])))\n",
    "        data.append(col)\n",
    "    data = np.array(data)\n",
    "\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    _, ax = plt.subplots()\n",
    "    ax.imshow(data, cmap=\"hot\", interpolation=\"nearest\", extent=dim)\n",
    "visualize(nn)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aufgabenblatt 4"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Probleme beim Crossover\n",
    "\n",
    "Das _Austauschen einzelner Gewichte_ macht hier bei neuronalen Netzwerken wenig Sinn, da die Gewichte nur in der Gesamtheit Sinn ergeben. Ersetze ich ein einzelnen Gewicht in einem Elternteil, wird dies höchstwahrscheinlich die Fitness nicht verbessern, da dieses Gewicht nur im Zusammenspiel der Gewichte des Elternteil von dem es kommt gut funktioniert."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mutate(nn: NeuralNet) -> None:\n",
    "    nn.weights = [w + rng.normal(scale=0.1, size=w.shape) for w in nn.weights]\n",
    "\n",
    "def recombine(mother: NeuralNet, father: NeuralNet) -> NeuralNet:\n",
    "    weights = [(wm + wf) / 2.0 for wm, wf in zip(mother.weights, father.weights)]\n",
    "    return NeuralNet(hidden_layers=[(8, relu)], weights=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fitness(nn: NeuralNet) -> int:\n",
    "    test_data = samples(100)\n",
    "\n",
    "    correct = sum(np.around(nn.classify(test_data[0])) == test_data[1])\n",
    "    return correct\n",
    "\n",
    "\n",
    "def evolution(\n",
    "    generations: int = 50,\n",
    "    population_size: int = 20,\n",
    "    selection_size: int = 5,\n",
    "    selection_type: Literal[\"elitist\"] | Literal[\"proportional\"] = \"elitist\",\n",
    "):\n",
    "    assert selection_type in (\"elitist\", \"proportional\")\n",
    "    population = [NeuralNet(hidden_layers=[(8, relu)]) for _ in range(population_size)]\n",
    "\n",
    "    for _ in range(generations):\n",
    "        # Select individuals with highest fitness for reproduction\n",
    "        population = sorted(population, key=lambda nn: fitness(nn), reverse=True)\n",
    "\n",
    "        if selection_type == \"elitist\":\n",
    "            selection = population[:selection_size]\n",
    "        elif selection_type == \"proportional\":\n",
    "            population_fitness = [fitness(nn) for nn in population]\n",
    "            selection = np.random.choice(\n",
    "                population,\n",
    "                selection_size,\n",
    "                p=[f / sum(population_fitness) for f in population_fitness],\n",
    "            ).tolist()\n",
    "\n",
    "        # Reproduce\n",
    "        offsprings = []\n",
    "        for _ in range(population_size - selection_size):\n",
    "            mother, father = sample(selection, 2)\n",
    "\n",
    "            offspring = recombine(mother, father)\n",
    "            mutate(offspring)\n",
    "\n",
    "            offsprings.append(offspring)\n",
    "\n",
    "        # Create new population\n",
    "        population = selection + offsprings\n",
    "\n",
    "    # Return best individual of final population\n",
    "    return max(population, key=lambda nn: fitness(nn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:51<00:00,  2.58s/it]\n"
     ]
    }
   ],
   "source": [
    "test_runs = 10\n",
    "\n",
    "tests = {}\n",
    "\n",
    "with tqdm(total=2 * test_runs) as progressbar:\n",
    "    for test in range(test_runs):\n",
    "        test_data = samples(500)\n",
    "\n",
    "        for selection_type in (\"elitist\", \"proportional\"):\n",
    "            nn = evolution(generations=100, selection_type=selection_type)\n",
    "            correct = sum(np.around(nn.classify(test_data[0])) == test_data[1])\n",
    "\n",
    "            test_results = tests.setdefault(selection_type, [])\n",
    "            test_results.append(correct / test_data[0].shape[0])\n",
    "\n",
    "            progressbar.update(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selection_type='elitist' : 0.8282\n",
      "selection_type='proportional' : 0.6898000000000001\n"
     ]
    }
   ],
   "source": [
    "for selection_type, results in tests.items():\n",
    "    print(f\"{selection_type=} : {sum(results) / test_runs}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Beobachtungen und finale Lösung\n",
    "\n",
    "In diesem Fall scheint die _Elitist selection_ (bei uns meist 80-90% korrekt gelöst über mehrere Durchläufe vs 70%) besser zu funktionieren als die _Fitness proportional selection_ . Im ersten Fall kann man sagen, dass hier das XOR Problem fast immer gelöst werden konnte, im zweiten Fall leider weniger.\n",
    "Mit zusätzlichem Bias würde das ganze vielleicht konsistenter gelöst werden."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow: 3.5656441999599338, Backpropagation: 0.09983029961585999, Evolutionary: 4.410684499889612\n"
     ]
    }
   ],
   "source": [
    "# Aufgabe 3 von Aufgabenblatt 4\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import timeit\n",
    "\n",
    "training_data = samples(100)\n",
    "test_data = samples(500)\n",
    "\n",
    "# Tensorflow\n",
    "model = tf.keras.models.Sequential()\n",
    "model.add(tf.keras.Input(shape=(2,)))\n",
    "model.add(tf.keras.layers.Dense(2, activation=tf.keras.activations.relu, kernel_initializer=tf.initializers.Constant(0.5)))\n",
    "model.add(tf.keras.layers.Dense(1, activation=tf.keras.activations.sigmoid))\n",
    "model.compile(optimizer=tf.keras.optimizers.legacy.Adam(learning_rate=0.05), loss=tf.keras.losses.BinaryCrossentropy(), metrics=['binary_accuracy'])\n",
    "tf_time = timeit.timeit(lambda: model.fit(training_data[0], training_data[1], batch_size=20, epochs=80, verbose=False), number=5)\n",
    "\n",
    "# Backpropagation\n",
    "nn = NeuralNet(hidden_layers=[(8, relu)])\n",
    "bp_time = timeit.timeit(lambda: nn.train(training_data, batch_size=20, epochs=80, verbose=False), number=5)\n",
    "\n",
    "# Evolutionary\n",
    "ev_time = timeit.timeit(lambda: evolution(selection_type=\"elitist\"), number=5)\n",
    "\n",
    "print(f\"TensorFlow: {tf_time}, Backpropagation: {bp_time}, Evolutionary: {ev_time}\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Geschwindigkeiten\n",
    "\n",
    "Nach unserem Test läuft Backpropagation am schnellsten ab, dann TensorFlow, und am langsamsten der evolutionäre Algorithmus.\n",
    "Um die Geschwindigkeiten zu verbessern, könnte man die Anzahl an Epochen / Generationen minimieren, sodass das neuronale Netz\n",
    "gerade so konvergiert. Die hier sichtbaren groben Geschwindigkeitsunterschiede sind in unserem Fall aber kaum durch leichte\n",
    "Veränderungen auszugleichen."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
