From b7fa826d9512a109353b1bd2fd928b0b3e275bb2 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 1 May 2024 11:16:45 +0100 Subject: [PATCH 1/7] #2442: Initial commit of MP test script --- src/primaite/notebooks/mp.py | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/primaite/notebooks/mp.py diff --git a/src/primaite/notebooks/mp.py b/src/primaite/notebooks/mp.py new file mode 100644 index 00000000..ebed7122 --- /dev/null +++ b/src/primaite/notebooks/mp.py @@ -0,0 +1,42 @@ +import yaml +from stable_baselines3 import PPO +from stable_baselines3.common.utils import set_random_seed +from stable_baselines3.common.vec_env import SubprocVecEnv + +from primaite.session.environment import PrimaiteGymEnv + +EPISODE_LEN = 128 +NUM_EPISODES = 10 +NO_STEPS = EPISODE_LEN * NUM_EPISODES +BATCH_SIZE = 32 +LEARNING_RATE = 3e-4 + +with open("c:/projects/primaite/src/primaite/config/_package_data/data_manipulation.yaml", "r") as f: + cfg = yaml.safe_load(f) + + +def make_env(rank: int, seed: int = 0) -> callable: + """Wrapper script for _init function.""" + + def _init() -> PrimaiteGymEnv: + env = PrimaiteGymEnv(env_config=cfg) + env.reset(seed=seed + rank) + model = PPO( + "MlpPolicy", + env, + learning_rate=LEARNING_RATE, + n_steps=NO_STEPS, + batch_size=BATCH_SIZE, + verbose=0, + tensorboard_log="./PPO_UC2/", + ) + model.learn(total_timesteps=NO_STEPS) + return env + + set_random_seed(seed) + return _init + + +if __name__ == "__main__": + n_procs = 4 + train_env = SubprocVecEnv([make_env(i + n_procs) for i in range(n_procs)]) From db29d336290f75e27fa65937b9481a60cf9b192b Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 2 May 2024 15:44:18 +0100 Subject: [PATCH 2/7] #2442: Add multi-processing notebook example. --- src/primaite/notebooks/multi-processing.ipynb | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 src/primaite/notebooks/multi-processing.ipynb diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb new file mode 100644 index 00000000..83366b54 --- /dev/null +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple multi-processing demo using SubprocVecEnv from SB3\n", + "Based on a code example provided by Rachael Proctor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#!primaite setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import packages and read config file." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set up training data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import multiprocessing as mp\n", + "mp.get_all_start_methods()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "from stable_baselines3 import PPO\n", + "from stable_baselines3.common.utils import set_random_seed\n", + "from stable_baselines3.common.vec_env import SubprocVecEnv\n", + "\n", + "from primaite.session.environment import PrimaiteGymEnv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "EPISODE_LEN = 128\n", + "NUM_EPISODES = 10\n", + "NO_STEPS = EPISODE_LEN * NUM_EPISODES\n", + "BATCH_SIZE = 32\n", + "LEARNING_RATE = 3e-4\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "with open(\"c:/projects/primaite/src/primaite/config/_package_data/data_manipulation.yaml\", \"r\") as f:\n", + " cfg = yaml.safe_load(f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "def make_env(rank: int, seed: int = 0) -> callable:\n", + " \"\"\"Wrapper script for _init function.\"\"\"\n", + "\n", + " def _init() -> PrimaiteGymEnv:\n", + " env = PrimaiteGymEnv(env_config=cfg)\n", + " env.reset(seed=seed + rank)\n", + " model = PPO(\n", + " \"MlpPolicy\",\n", + " env,\n", + " learning_rate=LEARNING_RATE,\n", + " n_steps=NO_STEPS,\n", + " batch_size=BATCH_SIZE,\n", + " verbose=0,\n", + " tensorboard_log=\"./PPO_UC2/\",\n", + " )\n", + " model.learn(total_timesteps=NO_STEPS)\n", + " return env\n", + "\n", + " set_random_seed(seed)\n", + " return _init\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "n_procs = 4\n", + "train_env = SubprocVecEnv([make_env(i + n_procs) for i in range(n_procs)])\n", + "print(train_env)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From a37ed4e487d2f5f7ba716ac2eeaca0c8a05ccf53 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 2 May 2024 16:16:36 +0100 Subject: [PATCH 3/7] #2442: Fix config file path to be OS independent. --- src/primaite/notebooks/multi-processing.ipynb | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index 83366b54..fc3ef088 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -21,24 +21,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Import packages and read config file." + "Set up training data." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Set up training data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import multiprocessing as mp\n", - "mp.get_all_start_methods()" + "Import packages and read config file." ] }, { @@ -61,12 +51,17 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "EPISODE_LEN = 128\n", - "NUM_EPISODES = 10\n", - "NO_STEPS = EPISODE_LEN * NUM_EPISODES\n", - "BATCH_SIZE = 32\n", - "LEARNING_RATE = 3e-4\n" + "from primaite.config.load import data_manipulation_config_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(data_manipulation_config_path(), 'r') as f:\n", + " cfg = yaml.safe_load(f)" ] }, { @@ -76,8 +71,11 @@ "outputs": [], "source": [ "\n", - "with open(\"c:/projects/primaite/src/primaite/config/_package_data/data_manipulation.yaml\", \"r\") as f:\n", - " cfg = yaml.safe_load(f)\n" + "EPISODE_LEN = 128\n", + "NUM_EPISODES = 10\n", + "NO_STEPS = EPISODE_LEN * NUM_EPISODES\n", + "BATCH_SIZE = 32\n", + "LEARNING_RATE = 3e-4\n" ] }, { @@ -116,11 +114,8 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "\n", - "n_procs = 4\n", - "train_env = SubprocVecEnv([make_env(i + n_procs) for i in range(n_procs)])\n", - "print(train_env)\n" + "n_procs = 2\n", + "train_env = SubprocVecEnv([make_env(i + n_procs) for i in range(n_procs)])\n" ] } ], @@ -140,7 +135,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 7c689a0d35c7d477a0b064bcb4e4a6f001c9ffbb Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 2 May 2024 16:32:54 +0100 Subject: [PATCH 4/7] #2442: Markdown changes. --- src/primaite/notebooks/multi-processing.ipynb | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index fc3ef088..d5b56cbb 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -17,13 +17,6 @@ "#!primaite setup" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set up training data." - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -64,6 +57,13 @@ " cfg = yaml.safe_load(f)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set up training data." + ] + }, { "cell_type": "code", "execution_count": null, @@ -78,6 +78,13 @@ "LEARNING_RATE = 3e-4\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define an environment function." + ] + }, { "cell_type": "code", "execution_count": null, @@ -108,6 +115,13 @@ " return _init\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run experiment." + ] + }, { "cell_type": "code", "execution_count": null, @@ -135,7 +149,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.11" } }, "nbformat": 4, From 5a3b6ade2f46f4c0ae7bf00fb9734412ad36f547 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 2 May 2024 16:52:35 +0100 Subject: [PATCH 5/7] #2442: Updated changelog. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a96b27c..d7d88e5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the ability for a DatabaseService to terminate a connection. - Added active_connection to DatabaseClientConnection so that if the connection is terminated active_connection is set to False and the object can no longer be used. - Added additional show functions to enable connection inspection. - +- Added notebook to demonstrate use of SubprocVecEnv from SB3 to vectorise environments to speed up training. ## [Unreleased] - Made requests fail to reach their target if the node is off From 689400e95cfb8ee469fec12c1dd246a3cf2c6829 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 2 May 2024 17:04:43 +0100 Subject: [PATCH 6/7] Merge branch 'dev' into bugfix/2442-add_SubprocVecEnv_support --- src/primaite/notebooks/mp.py | 42 ------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 src/primaite/notebooks/mp.py diff --git a/src/primaite/notebooks/mp.py b/src/primaite/notebooks/mp.py deleted file mode 100644 index ebed7122..00000000 --- a/src/primaite/notebooks/mp.py +++ /dev/null @@ -1,42 +0,0 @@ -import yaml -from stable_baselines3 import PPO -from stable_baselines3.common.utils import set_random_seed -from stable_baselines3.common.vec_env import SubprocVecEnv - -from primaite.session.environment import PrimaiteGymEnv - -EPISODE_LEN = 128 -NUM_EPISODES = 10 -NO_STEPS = EPISODE_LEN * NUM_EPISODES -BATCH_SIZE = 32 -LEARNING_RATE = 3e-4 - -with open("c:/projects/primaite/src/primaite/config/_package_data/data_manipulation.yaml", "r") as f: - cfg = yaml.safe_load(f) - - -def make_env(rank: int, seed: int = 0) -> callable: - """Wrapper script for _init function.""" - - def _init() -> PrimaiteGymEnv: - env = PrimaiteGymEnv(env_config=cfg) - env.reset(seed=seed + rank) - model = PPO( - "MlpPolicy", - env, - learning_rate=LEARNING_RATE, - n_steps=NO_STEPS, - batch_size=BATCH_SIZE, - verbose=0, - tensorboard_log="./PPO_UC2/", - ) - model.learn(total_timesteps=NO_STEPS) - return env - - set_random_seed(seed) - return _init - - -if __name__ == "__main__": - n_procs = 4 - train_env = SubprocVecEnv([make_env(i + n_procs) for i in range(n_procs)]) From e510c3ceffaa0edeb6f81b15d0d53259fc332f84 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Fri, 3 May 2024 15:10:24 +0100 Subject: [PATCH 7/7] #2442: Removed 'primaite setup' call --- src/primaite/notebooks/multi-processing.ipynb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index d5b56cbb..71addce6 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -8,15 +8,6 @@ "Based on a code example provided by Rachael Proctor." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#!primaite setup" - ] - }, { "cell_type": "markdown", "metadata": {},