diff --git a/Aufgabe 5/aufgabe05.ipynb b/Aufgabe 5/aufgabe05.ipynb new file mode 100644 index 0000000..5b6984e --- /dev/null +++ b/Aufgabe 5/aufgabe05.ipynb @@ -0,0 +1,1571 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from tensorflow import keras\n", + "import numpy as np\n", + "import gym" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class ReplayBuffer(object):\n", + " def __init__(self, max_size, input_shape, n_actions, discrete = False):\n", + " self.mem_size = max_size\n", + " self.mem_cntr = 0\n", + " self.discrete = discrete\n", + " self.state_memory = np.zeros((self.mem_size, input_shape))\n", + " self.new_state_memory = np.zeros((self.mem_size, input_shape))\n", + " dtype = np.int8 if discrete else np.float32\n", + " self.action_memory = np.zeros((self.mem_size, n_actions), dtype=dtype)\n", + " self.reward_memory = np.zeros(self.mem_size)\n", + " self.terminal_memory = np.zeros(self.mem_size, dtype=np.float32)\n", + "\n", + " def store_transition(self, state, action, reward, state_, done):\n", + " index = self.mem_cntr % self.mem_size\n", + " self.state_memory[index] = state\n", + " self.new_state_memory[index] = state_\n", + " self.reward_memory[index] = reward\n", + " self.terminal_memory[index] = 1 - int(done)\n", + " if self.discrete:\n", + " actions = np.zeros(self.action_memory.shape[1])\n", + " actions[action] = 1.0\n", + " self.action_memory[index] = actions\n", + " else:\n", + " self.action_memory[index] = action\n", + " self.mem_cntr += 1\n", + "\n", + " def sample_buffer(self, batch_size):\n", + " max_mem = min(self.mem_cntr, self.mem_size)\n", + " batch = np.random.choice(max_mem, batch_size)\n", + " states = self.state_memory[batch]\n", + " states_ = self.new_state_memory[batch]\n", + " rewards = self.reward_memory[batch]\n", + " actions = self.action_memory[batch]\n", + " terminal = self.terminal_memory[batch]\n", + "\n", + " return states, actions, rewards, states_, terminal\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def build_dqn(lr, n_actions, input_dims, fc1_dims, fc2_dims):\n", + " model = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(fc1_dims, activation=tf.keras.activations.relu, input_shape=(input_dims,)),\n", + " tf.keras.layers.Dense(fc2_dims, activation=tf.keras.activations.relu),\n", + " tf.keras.layers.Dense(n_actions)\n", + " ])\n", + " model.compile(optimizer=tf.keras.optimizers.legacy.Adam(learning_rate=lr), loss=\"mse\")\n", + "\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "class Agent(object):\n", + " def __init__(self, alpha, gamma, n_actions, epsilon, batch_size, input_dims, epsilon_dec=0.996, epsilon_end=0.01, mem_size=1000000, fname=\"dqn_model.keras\"):\n", + " self.action_space = [i for i in range(n_actions)]\n", + " self.n_actions = n_actions\n", + " self.gamma = gamma\n", + " self.epsilon = epsilon\n", + " self.epsilon_dec = epsilon_dec\n", + " self.epsilon_min = epsilon_end\n", + " self.batch_size = batch_size\n", + " self.model_file = fname\n", + "\n", + " self.memory = ReplayBuffer(mem_size, input_dims, n_actions, discrete=True)\n", + " self.q_eval = build_dqn(alpha, n_actions, input_dims, 256, 256)\n", + "\n", + " def remember(self, state, action, reward, new_state, done):\n", + " self.memory.store_transition(state, action, reward, new_state, done)\n", + "\n", + " def choose_action(self, state):\n", + " state = state[np.newaxis, :]\n", + " rand = np.random.random()\n", + " if rand < self.epsilon:\n", + " action = np.random.choice(self.action_space)\n", + " else:\n", + " actions = self.q_eval.predict(state)\n", + " action = np.argmax(actions)\n", + "\n", + " return action\n", + " \n", + " def learn(self):\n", + " if self.memory.mem_cntr < self.batch_size:\n", + " return\n", + " state, action, reward, new_state, done = self.memory.sample_buffer(self.batch_size)\n", + "\n", + " action_values = np.array(self.action_space, dtype=np.int8)\n", + " action_indices = np.dot(action, action_values)\n", + "\n", + " q_eval = self.q_eval.predict(state)\n", + " q_next = self.q_eval.predict(new_state)\n", + "\n", + " q_target = q_eval.copy()\n", + "\n", + " batch_index = np.arange(self.batch_size, dtype=np.int32)\n", + "\n", + " q_target[batch_index, action_indices] = reward + self.gamma*np.max(q_next, axis=1)*done\n", + "\n", + " _ = self.q_eval.fit(state, q_target, verbose=0)\n", + "\n", + " self.epsilon = self.epsilon * self.epsilon_dec if self.epsilon > self.epsilon_min else self.epsilon_min\n", + "\n", + " def save_model(self):\n", + " self.q_eval.save(self.model_file)\n", + " \n", + " def load_model(self):\n", + " self.q_eval = tf.keras.models.load_model(self.model_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "episode 0 score 24.00 average score 24.00\n", + "episode 1 score 16.00 average score 20.00\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 873us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 2 score 26.00 average score 22.00\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 980us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 25ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 944us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 965us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 922us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 923us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 901us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 948us/step\n", + "2/2 [==============================] - 0s 786us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 3 score 23.00 average score 22.25\n", + "2/2 [==============================] - 0s 872us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 917us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 854us/step\n", + "2/2 [==============================] - 0s 858us/step\n", + "2/2 [==============================] - 0s 846us/step\n", + "2/2 [==============================] - 0s 948us/step\n", + "2/2 [==============================] - 0s 870us/step\n", + "2/2 [==============================] - 0s 979us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 946us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 894us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 806us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 842us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 997us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 879us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 847us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 895us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 803us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 945us/step\n", + "2/2 [==============================] - 0s 973us/step\n", + "2/2 [==============================] - 0s 953us/step\n", + "2/2 [==============================] - 0s 959us/step\n", + "2/2 [==============================] - 0s 961us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 4 score 23.00 average score 22.40\n", + "2/2 [==============================] - 0s 864us/step\n", + "2/2 [==============================] - 0s 918us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 974us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 994us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 828us/step\n", + "2/2 [==============================] - 0s 967us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 912us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 936us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 946us/step\n", + "2/2 [==============================] - 0s 822us/step\n", + "2/2 [==============================] - 0s 769us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 965us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 5 score 12.00 average score 20.67\n", + "2/2 [==============================] - 0s 818us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 886us/step\n", + "2/2 [==============================] - 0s 947us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 904us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 895us/step\n", + "2/2 [==============================] - 0s 859us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 908us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 918us/step\n", + "2/2 [==============================] - 0s 972us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 851us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 931us/step\n", + "2/2 [==============================] - 0s 830us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 899us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 779us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 959us/step\n", + "2/2 [==============================] - 0s 865us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 938us/step\n", + "2/2 [==============================] - 0s 922us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 982us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 947us/step\n", + "2/2 [==============================] - 0s 950us/step\n", + "2/2 [==============================] - 0s 947us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 986us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 925us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 795us/step\n", + "2/2 [==============================] - 0s 977us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 6 score 27.00 average score 21.57\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 855us/step\n", + "2/2 [==============================] - 0s 862us/step\n", + "2/2 [==============================] - 0s 877us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 846us/step\n", + "2/2 [==============================] - 0s 948us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 861us/step\n", + "2/2 [==============================] - 0s 913us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 933us/step\n", + "2/2 [==============================] - 0s 968us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 907us/step\n", + "2/2 [==============================] - 0s 940us/step\n", + "2/2 [==============================] - 0s 919us/step\n", + "2/2 [==============================] - 0s 892us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 977us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 886us/step\n", + "2/2 [==============================] - 0s 855us/step\n", + "2/2 [==============================] - 0s 957us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 901us/step\n", + "2/2 [==============================] - 0s 3ms/step\n", + "2/2 [==============================] - 0s 731us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 858us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 946us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 778us/step\n", + "2/2 [==============================] - 0s 903us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 930us/step\n", + "2/2 [==============================] - 0s 892us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 811us/step\n", + "2/2 [==============================] - 0s 945us/step\n", + "2/2 [==============================] - 0s 854us/step\n", + "2/2 [==============================] - 0s 929us/step\n", + "2/2 [==============================] - 0s 922us/step\n", + "2/2 [==============================] - 0s 886us/step\n", + "2/2 [==============================] - 0s 869us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 858us/step\n", + "2/2 [==============================] - 0s 799us/step\n", + "2/2 [==============================] - 0s 870us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 977us/step\n", + "2/2 [==============================] - 0s 806us/step\n", + "2/2 [==============================] - 0s 992us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 973us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 869us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 998us/step\n", + "2/2 [==============================] - 0s 817us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 826us/step\n", + "2/2 [==============================] - 0s 926us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 870us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 901us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 988us/step\n", + "2/2 [==============================] - 0s 924us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 896us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 811us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 897us/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "2/2 [==============================] - 0s 794us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 859us/step\n", + "2/2 [==============================] - 0s 920us/step\n", + "episode 7 score 47.00 average score 24.75\n", + "2/2 [==============================] - 0s 981us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 989us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 960us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 976us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 842us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 818us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 836us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 865us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 8 score 9.00 average score 23.00\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 897us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 867us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 951us/step\n", + "2/2 [==============================] - 0s 977us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "2/2 [==============================] - 0s 992us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 891us/step\n", + "2/2 [==============================] - 0s 992us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 879us/step\n", + "2/2 [==============================] - 0s 870us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 953us/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 937us/step\n", + "2/2 [==============================] - 0s 913us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 900us/step\n", + "2/2 [==============================] - 0s 824us/step\n", + "2/2 [==============================] - 0s 928us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 888us/step\n", + "2/2 [==============================] - 0s 766us/step\n", + "2/2 [==============================] - 0s 881us/step\n", + "episode 9 score 19.00 average score 22.60\n", + "2/2 [==============================] - 0s 891us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 889us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 988us/step\n", + "2/2 [==============================] - 0s 926us/step\n", + "2/2 [==============================] - 0s 894us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 916us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 998us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 988us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 910us/step\n", + "2/2 [==============================] - 0s 920us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 897us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 890us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 863us/step\n", + "2/2 [==============================] - 0s 960us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 951us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 995us/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "episode 10 score 17.00 average score 22.09\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 937us/step\n", + "2/2 [==============================] - 0s 985us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 938us/step\n", + "2/2 [==============================] - 0s 978us/step\n", + "2/2 [==============================] - 0s 949us/step\n", + "2/2 [==============================] - 0s 783us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 890us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 11 score 9.00 average score 21.00\n", + "2/2 [==============================] - 0s 962us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 974us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 970us/step\n", + "2/2 [==============================] - 0s 934us/step\n", + "2/2 [==============================] - 0s 959us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 977us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 975us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 962us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 952us/step\n", + "2/2 [==============================] - 0s 861us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 12 score 13.00 average score 20.38\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 924us/step\n", + "2/2 [==============================] - 0s 960us/step\n", + "2/2 [==============================] - 0s 788us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 929us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 928us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 965us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 946us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 13 score 10.00 average score 19.64\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 814us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 859us/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 990us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 979us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 801us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 985us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 896us/step\n", + "episode 14 score 8.00 average score 18.87\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 876us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 969us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 970us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 775us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 817us/step\n", + "2/2 [==============================] - 0s 942us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 885us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 825us/step\n", + "2/2 [==============================] - 0s 962us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 914us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 920us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 997us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 15 score 12.00 average score 18.44\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 811us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 895us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 814us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 950us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 947us/step\n", + "2/2 [==============================] - 0s 855us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 826us/step\n", + "2/2 [==============================] - 0s 949us/step\n", + "2/2 [==============================] - 0s 880us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 16 score 9.00 average score 17.88\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 931us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 865us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 941us/step\n", + "2/2 [==============================] - 0s 926us/step\n", + "2/2 [==============================] - 0s 993us/step\n", + "2/2 [==============================] - 0s 816us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 871us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 973us/step\n", + "2/2 [==============================] - 0s 801us/step\n", + "2/2 [==============================] - 0s 950us/step\n", + "2/2 [==============================] - 0s 965us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 843us/step\n", + "2/2 [==============================] - 0s 968us/step\n", + "2/2 [==============================] - 0s 895us/step\n", + "2/2 [==============================] - 0s 943us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 3ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 998us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 813us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 900us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 838us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 930us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 943us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 935us/step\n", + "2/2 [==============================] - 0s 895us/step\n", + "2/2 [==============================] - 0s 846us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 835us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 941us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 838us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 897us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 17 score 31.00 average score 18.61\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 911us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 3ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 986us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 935us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 824us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 868us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 840us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 965us/step\n", + "2/2 [==============================] - 0s 817us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 918us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 997us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 950us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 962us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 885us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 830us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 846us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 976us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 838us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 885us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 851us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 791us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 878us/step\n", + "2/2 [==============================] - 0s 816us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 820us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 777us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 818us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 940us/step\n", + "2/2 [==============================] - 0s 827us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 850us/step\n", + "2/2 [==============================] - 0s 827us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 928us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 922us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 840us/step\n", + "2/2 [==============================] - 0s 854us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 900us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 854us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 861us/step\n", + "2/2 [==============================] - 0s 857us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 920us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 963us/step\n", + "2/2 [==============================] - 0s 968us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 821us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 973us/step\n", + "2/2 [==============================] - 0s 984us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 878us/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 984us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 18 score 58.00 average score 20.68\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 898us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 973us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 901us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 953us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 825us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 883us/step\n", + "2/2 [==============================] - 0s 960us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 918us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 992us/step\n", + "2/2 [==============================] - 0s 987us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 891us/step\n", + "2/2 [==============================] - 0s 941us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 834us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 947us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 952us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 812us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 804us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 990us/step\n", + "2/2 [==============================] - 0s 984us/step\n", + "2/2 [==============================] - 0s 926us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 950us/step\n", + "2/2 [==============================] - 0s 863us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 953us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 942us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 926us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 990us/step\n", + "2/2 [==============================] - 0s 946us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 798us/step\n", + "2/2 [==============================] - 0s 860us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 925us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 980us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 873us/step\n", + "2/2 [==============================] - 0s 898us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 952us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 19 score 39.00 average score 21.60\n", + "2/2 [==============================] - 0s 779us/step\n", + "2/2 [==============================] - 0s 826us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 946us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 932us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 822us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 847us/step\n", + "2/2 [==============================] - 0s 939us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 873us/step\n", + "2/2 [==============================] - 0s 917us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 995us/step\n", + "2/2 [==============================] - 0s 878us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 937us/step\n", + "2/2 [==============================] - 0s 828us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 985us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 917us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 793us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 826us/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 952us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 889us/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 910us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 934us/step\n", + "2/2 [==============================] - 0s 855us/step\n", + "2/2 [==============================] - 0s 808us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 927us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 813us/step\n", + "episode 20 score 29.00 average score 21.95\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 905us/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 957us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 986us/step\n", + "2/2 [==============================] - 0s 934us/step\n", + "2/2 [==============================] - 0s 837us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 928us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 825us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 990us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 912us/step\n", + "2/2 [==============================] - 0s 981us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 906us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 929us/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 941us/step\n", + "2/2 [==============================] - 0s 826us/step\n", + "2/2 [==============================] - 0s 848us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 815us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 920us/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 929us/step\n", + "2/2 [==============================] - 0s 917us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 904us/step\n", + "2/2 [==============================] - 0s 896us/step\n", + "2/2 [==============================] - 0s 800us/step\n", + "2/2 [==============================] - 0s 944us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 917us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 817us/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 980us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 988us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 994us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 832us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 920us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 956us/step\n", + "2/2 [==============================] - 0s 941us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 990us/step\n", + "2/2 [==============================] - 0s 962us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 942us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 794us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 910us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 860us/step\n", + "2/2 [==============================] - 0s 930us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 929us/step\n", + "2/2 [==============================] - 0s 857us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 803us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 966us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 959us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 947us/step\n", + "2/2 [==============================] - 0s 937us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 951us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 822us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 810us/step\n", + "2/2 [==============================] - 0s 904us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 882us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 985us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 3ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "episode 21 score 60.00 average score 23.68\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 963us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 970us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 817us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 948us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 996us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 942us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 868us/step\n", + "2/2 [==============================] - 0s 963us/step\n", + "2/2 [==============================] - 0s 916us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 938us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 839us/step\n", + "2/2 [==============================] - 0s 880us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 805us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 969us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 972us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 925us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 859us/step\n", + "2/2 [==============================] - 0s 916us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 807us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 881us/step\n", + "2/2 [==============================] - 0s 987us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 955us/step\n", + "2/2 [==============================] - 0s 968us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "2/2 [==============================] - 0s 850us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 950us/step\n", + "2/2 [==============================] - 0s 964us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 969us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 899us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 990us/step\n", + "2/2 [==============================] - 0s 970us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 872us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 942us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 986us/step\n", + "2/2 [==============================] - 0s 940us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 954us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 952us/step\n", + "2/2 [==============================] - 0s 892us/step\n", + "episode 22 score 52.00 average score 24.91\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 998us/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 866us/step\n", + "1/1 [==============================] - 0s 10ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 976us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 817us/step\n", + "2/2 [==============================] - 0s 824us/step\n", + "1/1 [==============================] - 0s 9ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "2/2 [==============================] - 0s 875us/step\n", + "2/2 [==============================] - 0s 875us/step\n" + ] + } + ], + "source": [ + "env = gym.make('CartPole-v1')\n", + "n_games = 30\n", + "agent = Agent(gamma=0.99, epsilon=1.0, alpha=0.0005, input_dims=4, n_actions=2, mem_size=1000000, batch_size=64, epsilon_end=0.01)\n", + "\n", + "scores = []\n", + "eps_history = []\n", + "\n", + "for i in range(n_games):\n", + " done = False\n", + " score = 0\n", + " observation = env.reset()\n", + " while not done:\n", + " action = agent.choose_action(observation)\n", + " observation_, reward, done, info = env.step(action)\n", + " score += reward\n", + " agent.remember(observation, action, reward, observation_, done)\n", + " observation = observation_\n", + " agent.learn()\n", + "\n", + " eps_history.append(agent.epsilon)\n", + " scores.append(score)\n", + "\n", + " avg_score = np.mean(scores[max(0, i-100):(i+1)])\n", + " print('episode ', i, 'score %.2f' % score, 'average score %.2f' % avg_score)\n", + "\n", + " if i % 10 == 0 and i > 0:\n", + " agent.save_model()\n", + "\n", + "\n", + "#10 episodes = 10s, 20 = 18s,\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "bd385fe162c5ca0c84973b7dd5c518456272446b2b64e67c2a69f949ca7a1754" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Aufgabe 5/dqn_model.h5 b/Aufgabe 5/dqn_model.h5 new file mode 100644 index 0000000..8f07f98 Binary files /dev/null and b/Aufgabe 5/dqn_model.h5 differ diff --git a/Aufgabe 5/dqn_model.keras b/Aufgabe 5/dqn_model.keras new file mode 100644 index 0000000..96ff95f Binary files /dev/null and b/Aufgabe 5/dqn_model.keras differ