Merge branch 'dev' into bugfix/2455-notebook_updates

src/primaite/notebooks/Training-an-SB3-Agent.ipynb

@@ -1,5 +1,21 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Training an SB3 Agent\n",
+    "\n",
+    "This notebook will demonstrate how to use primaite to create and train a PPO agent, using a pre-defined configuration file."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### First, we import the inital packages and read in our configuration file."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -27,7 +43,14 @@
    "outputs": [],
    "source": [
     "with open(data_manipulation_config_path(), 'r') as f:\n",
-    "    cfg = yaml.safe_load(f)\n"
+    "    cfg = yaml.safe_load(f)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Using the given configuration, we generate the environment our agent will train in."
    ]
   },
   {
@@ -39,6 +62,13 @@
     "gym = PrimaiteGymEnv(game_config=cfg)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Lets define training parameters for the agent."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -48,9 +78,9 @@
     "from stable_baselines3 import PPO\n",
     "\n",
     "EPISODE_LEN = 128\n",
-    "NO_STEPS = EPISODE_LEN * 10\n",
-    "BATCH_SIZE = EPISODE_LEN * 10\n",
-    "TOTAL_TIMESTEPS = 5e3 * EPISODE_LEN\n",
+    "NUM_EPISODES = 10\n",
+    "NO_STEPS = EPISODE_LEN * NUM_EPISODES\n",
+    "BATCH_SIZE = 32\n",
     "LEARNING_RATE = 3e-4"
    ]
   },
@@ -60,7 +90,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "model = PPO('MlpPolicy', gym, learning_rate=LEARNING_RATE,  n_steps=NO_STEPS, batch_size=BATCH_SIZE, verbose=0, tensorboard_log=\"./PPO_UC2/\")\n"
+    "model = PPO('MlpPolicy', gym, learning_rate=LEARNING_RATE,  n_steps=NO_STEPS, batch_size=BATCH_SIZE, verbose=0, tensorboard_log=\"./PPO_UC2/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "With the agent configured, let's train for our defined number of episodes."
    ]
   },
   {
@@ -69,7 +106,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "model.learn(total_timesteps=TOTAL_TIMESTEPS)\n"
+    "model.learn(total_timesteps=NO_STEPS)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Next, let's save the agent to a zip file that can be used in future evaluation."
    ]
   },
   {
@@ -78,7 +122,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "model.save(\"PrimAITE-v3.0.0b7-PPO\")"
+    "model.save(\"PrimAITE-PPO-example-agent\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we load the saved agent and run it in evaluation mode."
    ]
   },
   {
@@ -88,7 +139,14 @@
    "outputs": [],
    "source": [
     "eval_model = PPO(\"MlpPolicy\", gym)\n",
-    "eval_model = PPO.load(\"PrimAITE-v3.0.0b7-PPO\", gym)"
+    "eval_model = PPO.load(\"PrimAITE-PPO-example-agent\", gym)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally, evaluate the agent."
    ]
   },
   {
@@ -119,7 +177,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.10.11"
   }
  },
  "nbformat": 4,