sp
/
tempestpy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

renaming and notebooks

refactoring
Thomas Knoll 2 years ago
parent
4e182a8e5b
commit
3dee543e24
10 changed files with 587 additions and 50 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 13
      examples/shields/rl/11_minigridrl.py
  2. 5
      examples/shields/rl/13_minigridsb.py
  3. 19
      examples/shields/rl/14_train_eval.py
  4. 22
      examples/shields/rl/15_train_eval_tune.py
  5. 116
      examples/shields/rl/dqn_rllib.ipynb
  6. 117
      examples/shields/rl/ppo_rllib.ipynb
  7. 337
      examples/shields/rl/ppo_sb.ipynb
  8. 0
      examples/shields/rl/shieldhandlers.py
  9. 0
      examples/shields/rl/torch_action_mask_model.py
  10. 2
      examples/shields/rl/wrappers.py

13
examples/shields/rl/11_minigridrl.py
View File

@ -1,11 +1,6 @@
import gymnasium as gym
import minigrid
# import numpy as np
# import ray
from ray.tune import register_env
from ray.rllib.algorithms.ppo import PPOConfig
from ray.rllib.algorithms.dqn.dqn import DQNConfig
@ -13,14 +8,12 @@ from ray.tune.logger import pretty_print
from ray.rllib.models import ModelCatalog
from TorchActionMaskModel import TorchActionMaskModel
from Wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper
from torch_action_mask_model import TorchActionMaskModel
from wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper
from helpers import parse_arguments, create_log_dir, ShieldingConfig
from ShieldHandlers import MiniGridShieldHandler, create_shield_query
from shieldhandlers import MiniGridShieldHandler, create_shield_query
from callbacks import MyCallbacks
import matplotlib.pyplot as plt
from ray.tune.logger import TBXLogger
def shielding_env_creater(config):

5
examples/shields/rl/13_minigridsb.py
View File

@ -8,12 +8,11 @@ import gymnasium as gym
from minigrid.core.actions import Actions
import numpy as np
import time
from helpers import parse_arguments, create_log_dir, ShieldingConfig
from ShieldHandlers import MiniGridShieldHandler, create_shield_query
from Wrappers import MiniGridSbShieldingWrapper
from shieldhandlers import MiniGridShieldHandler, create_shield_query
from wrappers import MiniGridSbShieldingWrapper
class CustomCallback(BaseCallback):
def __init__(self, verbose: int = 0, env=None):

19
examples/shields/rl/14_train_eval.py
View File

@ -1,26 +1,19 @@
import gymnasium as gym
import minigrid
# import numpy as np
# import ray
from ray.tune import register_env
from ray.rllib.algorithms.ppo import PPOConfig
from ray.rllib.algorithms.dqn.dqn import DQNConfig
# from ray.rllib.algorithms.callbacks import DefaultCallbacks
from ray.tune.logger import pretty_print, TBXLogger, TBXLoggerCallback, DEFAULT_LOGGERS, UnifiedLogger, CSVLogger
from ray.tune.logger import pretty_print, UnifiedLogger, CSVLogger
from ray.rllib.models import ModelCatalog
from TorchActionMaskModel import TorchActionMaskModel
from Wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper
from torch_action_mask_model import TorchActionMaskModel
from wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper
from helpers import parse_arguments, create_log_dir, ShieldingConfig
from ShieldHandlers import MiniGridShieldHandler, create_shield_query
from shieldhandlers import MiniGridShieldHandler, create_shield_query
from callbacks import MyCallbacks
import matplotlib.pyplot as plt
from torch.utils.tensorboard import SummaryWriter
@ -34,10 +27,6 @@ def shielding_env_creater(config):
args.prism_path = F"{args.prism_path}_{config.worker_index}.prism"
shielding = config.get("shielding", False)
# if shielding:
# assert(False)
shield_creator = MiniGridShieldHandler(args.grid_path, args.grid_to_prism_binary_path, args.prism_path, args.formula)
env = gym.make(name)

22
examples/shields/rl/15_train_eval_tune.py
View File

@ -1,31 +1,20 @@
import gymnasium as gym
import minigrid
# import numpy as np
# import ray
from ray.tune import register_env
from ray import tune, air
from ray.rllib.algorithms.ppo import PPOConfig
from ray.rllib.algorithms.dqn.dqn import DQNConfig
# from ray.rllib.algorithms.callbacks import DefaultCallbacks
from ray.tune.logger import pretty_print, TBXLogger, TBXLoggerCallback, DEFAULT_LOGGERS, UnifiedLogger
from ray.tune.logger import UnifiedLogger
from ray.rllib.models import ModelCatalog
from TorchActionMaskModel import TorchActionMaskModel
from Wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper
from torch_action_mask_model import TorchActionMaskModel
from wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper
from helpers import parse_arguments, create_log_dir, ShieldingConfig
from ShieldHandlers import MiniGridShieldHandler, create_shield_query
from shieldhandlers import MiniGridShieldHandler, create_shield_query
from callbacks import MyCallbacks
import matplotlib.pyplot as plt
from torch.utils.tensorboard import SummaryWriter
def shielding_env_creater(config):
name = config.get("name", "MiniGrid-LavaCrossingS9N1-v0")
@ -98,9 +87,6 @@ def ppo(args):
tuner.fit()
iterations = args.iterations
print(config.to_dict())
tune.run("PPO", config=config)
# print(epsiode_reward_mean)
# writer.add_scalar("evaluation/episode_reward", epsiode_reward_mean, i)

116
examples/shields/rl/dqn_rllib.ipynb
View File

@ -0,0 +1,116 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Example how to combine shielding with rllibs dqn algorithm."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import gymnasium as gym\n",
"\n",
"import minigrid\n",
"\n",
"from ray.tune import register_env\n",
"from ray.rllib.algorithms.dqn.dqn import DQNConfig\n",
"from ray.tune.logger import pretty_print\n",
"from ray.rllib.models import ModelCatalog\n",
"\n",
"\n",
"from torch_action_mask_model import TorchActionMaskModel\n",
"from wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper\n",
"from shieldhandlers import MiniGridShieldHandler, create_shield_query\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def shielding_env_creater(config):\n",
" name = config.get(\"name\", \"MiniGrid-LavaCrossingS9N1-v0\")\n",
" framestack = config.get(\"framestack\", 4)\n",
" \n",
" shield_creator = MiniGridShieldHandler(\"grid.txt\", \"./main\", \"grid.prism\", \"Pmax=? [G !\\\"AgentIsInLavaAndNotDone\\\"]\")\n",
" \n",
" env = gym.make(name)\n",
" env = MiniGridShieldingWrapper(env, shield_creator=shield_creator, shield_query_creator=create_shield_query ,mask_actions=True)\n",
" env = OneHotShieldingWrapper(env, config.vector_index if hasattr(config, \"vector_index\") else 0,\n",
" framestack=framestack)\n",
" \n",
" return env\n",
"\n",
"\n",
"def register_minigrid_shielding_env():\n",
" env_name = \"mini-grid-shielding\"\n",
" register_env(env_name, shielding_env_creater)\n",
" ModelCatalog.register_custom_model(\n",
" \"shielding_model\", \n",
" TorchActionMaskModel)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"register_minigrid_shielding_env()\n",
"\n",
" \n",
"config = DQNConfig()\n",
"config = config.resources(num_gpus=0)\n",
"config = config.rollouts(num_rollout_workers=1)\n",
"config = config.environment(env=\"mini-grid-shielding\", env_config={\"name\": \"MiniGrid-LavaCrossingS9N1-v0\" })\n",
"config = config.framework(\"torch\")\n",
"config = config.rl_module(_enable_rl_module_api = False)\n",
"config = config.training(hiddens=[], dueling=False, model={ \n",
" \"custom_model\": \"shielding_model\"\n",
"})\n",
" \n",
"algo = (\n",
" config.build()\n",
")\n",
" \n",
"for i in range(30):\n",
" result = algo.train()\n",
" print(pretty_print(result))\n",
"\n",
" if i % 5 == 0:\n",
" print(\"Saving checkpoint\")\n",
" checkpoint_dir = algo.save()\n",
" print(f\"Checkpoint saved in directory {checkpoint_dir}\")\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "env",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}

117
examples/shields/rl/ppo_rllib.ipynb
View File

@ -0,0 +1,117 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Example how to combine shielding with rllibs ppo algorithm."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import gymnasium as gym\n",
"\n",
"import minigrid\n",
"\n",
"from ray.tune import register_env\n",
"from ray.rllib.algorithms.ppo import PPOConfig\n",
"from ray.tune.logger import pretty_print\n",
"from ray.rllib.models import ModelCatalog\n",
"\n",
"\n",
"from torch_action_mask_model import TorchActionMaskModel\n",
"from wrappers import OneHotShieldingWrapper, MiniGridShieldingWrapper\n",
"from shieldhandlers import MiniGridShieldHandler, create_shield_query\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def shielding_env_creater(config):\n",
" name = config.get(\"name\", \"MiniGrid-LavaCrossingS9N1-v0\")\n",
" framestack = config.get(\"framestack\", 4)\n",
" \n",
" shield_creator = MiniGridShieldHandler(\"grid.txt\", \"./main\", \"grid.prism\", \"Pmax=? [G !\\\"AgentIsInLavaAndNotDone\\\"]\")\n",
" \n",
" env = gym.make(name)\n",
" env = MiniGridShieldingWrapper(env, shield_creator=shield_creator, shield_query_creator=create_shield_query ,mask_actions=True)\n",
" env = OneHotShieldingWrapper(env, config.vector_index if hasattr(config, \"vector_index\") else 0,\n",
" framestack=framestack)\n",
" \n",
" return env\n",
"\n",
"\n",
"def register_minigrid_shielding_env():\n",
" env_name = \"mini-grid-shielding\"\n",
" register_env(env_name, shielding_env_creater)\n",
" ModelCatalog.register_custom_model(\n",
" \"shielding_model\", \n",
" TorchActionMaskModel)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"register_minigrid_shielding_env()\n",
"\n",
"\n",
"config = (PPOConfig()\n",
" .rollouts(num_rollout_workers=1)\n",
" .resources(num_gpus=0)\n",
" .environment(env=\"mini-grid-shielding\", env_config={\"name\": \"MiniGrid-LavaCrossingS9N1-v0\"})\n",
" .framework(\"torch\")\n",
" .rl_module(_enable_rl_module_api = False)\n",
" .training(_enable_learner_api=False ,model={\n",
" \"custom_model\": \"shielding_model\"\n",
" }))\n",
"\n",
"\n",
"algo = (\n",
" config.build()\n",
")\n",
" \n",
"for i in range(30):\n",
" result = algo.train()\n",
" print(pretty_print(result))\n",
"\n",
" if i % 5 == 0:\n",
" print(\"Saving checkpoint\")\n",
" checkpoint_dir = algo.save()\n",
" print(f\"Checkpoint saved in directory {checkpoint_dir}\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "env",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}

337
examples/shields/rl/ppo_sb.ipynb
View File

@ -0,0 +1,337 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Example how to combine shielding with stable baselines contrib maskable ppo algorithm."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pygame 2.5.1 (SDL 2.28.2, Python 3.10.12)\n",
"Hello from the pygame community. https://www.pygame.org/contribute.html\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2023-09-08 10:00:46.717621: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
"To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
"2023-09-08 10:00:47.771352: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
]
},
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'examples'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[1], line 9\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mstable_baselines3\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcommon\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcallbacks\u001b[39;00m \u001b[39mimport\u001b[39;00m BaseCallback\n\u001b[1;32m 7\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mgymnasium\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mgym\u001b[39;00m\n\u001b[0;32m----> 9\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mexamples\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mshields\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mrl\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mshieldhandlers\u001b[39;00m \u001b[39mimport\u001b[39;00m MiniGridShieldHandler, create_shield_query\n\u001b[1;32m 10\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mexamples\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mshields\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mrl\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mwrappers\u001b[39;00m \u001b[39mimport\u001b[39;00m MiniGridSbShieldingWrapper\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'examples'"
]
}
],
"source": [
"from sb3_contrib import MaskablePPO\n",
"from sb3_contrib.common.maskable.evaluation import evaluate_policy\n",
"from sb3_contrib.common.maskable.policies import MaskableActorCriticPolicy\n",
"from sb3_contrib.common.wrappers import ActionMasker\n",
"from stable_baselines3.common.callbacks import BaseCallback\n",
"\n",
"import gymnasium as gym\n",
"\n",
"from shieldhandlers import MiniGridShieldHandler, create_shield_query\n",
"from wrappers import MiniGridSbShieldingWrapper"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def mask_fn(env: gym.Env):\n",
" return env.create_action_mask()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using cpu device\n",
"Wrapping the env with a `Monitor` wrapper\n",
"Wrapping the env in a DummyVecEnv.\n",
"Wrapping the env in a VecTransposeImage.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWGVR VRVRVRVRVRWG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VRGGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG VRVRVRVRVRVRWG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWGVRVRVR VRVRVRWG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWGVRVRVR VRVRVRWG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG VR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG VRGGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWGVRVRVRVRVRVR WG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n",
"Write to file Grid.shield.\n",
"---------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 283 |\n",
"| ep_rew_mean | 0.157 |\n",
"| time/ | |\n",
"| fps | 165 |\n",
"| iterations | 1 |\n",
"| time_elapsed | 12 |\n",
"| total_timesteps | 2048 |\n",
"---------------------------------\n",
"\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Reading :\tWGXR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWGVRVRVRVRVRVR WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG WG\n",
"Reading :\tWG GGWG\n",
"Reading :\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWG WG\n",
"Background:\tWGWGWGWGWGWGWGWGWG\n",
"\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[3], line 9\u001b[0m\n\u001b[1;32m 5\u001b[0m env \u001b[39m=\u001b[39m ActionMasker(env, mask_fn)\n\u001b[1;32m 6\u001b[0m model \u001b[39m=\u001b[39m MaskablePPO(MaskableActorCriticPolicy, env, verbose\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m model\u001b[39m.\u001b[39;49mlearn(\u001b[39m10_000\u001b[39;49m)\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/sb3_contrib/ppo_mask/ppo_mask.py:526\u001b[0m, in \u001b[0;36mMaskablePPO.learn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, use_masking, progress_bar)\u001b[0m\n\u001b[1;32m 523\u001b[0m callback\u001b[39m.\u001b[39mon_training_start(\u001b[39mlocals\u001b[39m(), \u001b[39mglobals\u001b[39m())\n\u001b[1;32m 525\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_timesteps \u001b[39m<\u001b[39m total_timesteps:\n\u001b[0;32m--> 526\u001b[0m continue_training \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcollect_rollouts(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49menv, callback, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrollout_buffer, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mn_steps, use_masking)\n\u001b[1;32m 528\u001b[0m \u001b[39mif\u001b[39;00m continue_training \u001b[39mis\u001b[39;00m \u001b[39mFalse\u001b[39;00m:\n\u001b[1;32m 529\u001b[0m \u001b[39mbreak\u001b[39;00m\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/sb3_contrib/ppo_mask/ppo_mask.py:306\u001b[0m, in \u001b[0;36mMaskablePPO.collect_rollouts\u001b[0;34m(self, env, callback, rollout_buffer, n_rollout_steps, use_masking)\u001b[0m\n\u001b[1;32m 303\u001b[0m actions, values, log_probs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpolicy(obs_tensor, action_masks\u001b[39m=\u001b[39maction_masks)\n\u001b[1;32m 305\u001b[0m actions \u001b[39m=\u001b[39m actions\u001b[39m.\u001b[39mcpu()\u001b[39m.\u001b[39mnumpy()\n\u001b[0;32m--> 306\u001b[0m new_obs, rewards, dones, infos \u001b[39m=\u001b[39m env\u001b[39m.\u001b[39;49mstep(actions)\n\u001b[1;32m 308\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_timesteps \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m env\u001b[39m.\u001b[39mnum_envs\n\u001b[1;32m 310\u001b[0m \u001b[39m# Give access to local variables\u001b[39;00m\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py:197\u001b[0m, in \u001b[0;36mVecEnv.step\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 191\u001b[0m \u001b[39mStep the environments with the given action\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[39m:param actions: the action\u001b[39;00m\n\u001b[1;32m 194\u001b[0m \u001b[39m:return: observation, reward, done, information\u001b[39;00m\n\u001b[1;32m 195\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 196\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstep_async(actions)\n\u001b[0;32m--> 197\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mstep_wait()\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/stable_baselines3/common/vec_env/vec_transpose.py:95\u001b[0m, in \u001b[0;36mVecTransposeImage.step_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mstep_wait\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m VecEnvStepReturn:\n\u001b[0;32m---> 95\u001b[0m observations, rewards, dones, infos \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvenv\u001b[39m.\u001b[39;49mstep_wait()\n\u001b[1;32m 97\u001b[0m \u001b[39m# Transpose the terminal observations\u001b[39;00m\n\u001b[1;32m 98\u001b[0m \u001b[39mfor\u001b[39;00m idx, done \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(dones):\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py:70\u001b[0m, in \u001b[0;36mDummyVecEnv.step_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuf_dones[env_idx]:\n\u001b[1;32m 68\u001b[0m \u001b[39m# save final observation where user can get it, then reset\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuf_infos[env_idx][\u001b[39m\"\u001b[39m\u001b[39mterminal_observation\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m obs\n\u001b[0;32m---> 70\u001b[0m obs, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mreset_infos[env_idx] \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49menvs[env_idx]\u001b[39m.\u001b[39;49mreset()\n\u001b[1;32m 71\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_save_obs(env_idx, obs)\n\u001b[1;32m 72\u001b[0m \u001b[39mreturn\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_obs_from_buf(), np\u001b[39m.\u001b[39mcopy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuf_rews), np\u001b[39m.\u001b[39mcopy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuf_dones), deepcopy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuf_infos))\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/stable_baselines3/common/monitor.py:83\u001b[0m, in \u001b[0;36mMonitor.reset\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mExpected you to pass keyword argument \u001b[39m\u001b[39m{\u001b[39;00mkey\u001b[39m}\u001b[39;00m\u001b[39m into reset\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 82\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcurrent_reset_info[key] \u001b[39m=\u001b[39m value\n\u001b[0;32m---> 83\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49menv\u001b[39m.\u001b[39;49mreset(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/env/lib/python3.10/site-packages/gymnasium/core.py:414\u001b[0m, in \u001b[0;36mWrapper.reset\u001b[0;34m(self, seed, options)\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mreset\u001b[39m(\n\u001b[1;32m 411\u001b[0m \u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m, seed: \u001b[39mint\u001b[39m \u001b[39m|\u001b[39m \u001b[39mNone\u001b[39;00m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m, options: \u001b[39mdict\u001b[39m[\u001b[39mstr\u001b[39m, Any] \u001b[39m|\u001b[39m \u001b[39mNone\u001b[39;00m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 412\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mtuple\u001b[39m[WrapperObsType, \u001b[39mdict\u001b[39m[\u001b[39mstr\u001b[39m, Any]]:\n\u001b[1;32m 413\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Uses the :meth:`reset` of the :attr:`env` that can be overwritten to change the returned data.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 414\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49menv\u001b[39m.\u001b[39;49mreset(seed\u001b[39m=\u001b[39;49mseed, options\u001b[39m=\u001b[39;49moptions)\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/examples/shields/rl/Wrappers.py:222\u001b[0m, in \u001b[0;36mMiniGridSbShieldingWrapper.reset\u001b[0;34m(self, seed, options)\u001b[0m\n\u001b[1;32m 219\u001b[0m obs, infos \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39menv\u001b[39m.\u001b[39mreset(seed\u001b[39m=\u001b[39mseed, options\u001b[39m=\u001b[39moptions)\n\u001b[1;32m 221\u001b[0m keys \u001b[39m=\u001b[39m extract_keys(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39menv)\n\u001b[0;32m--> 222\u001b[0m shield \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mshield_creator\u001b[39m.\u001b[39;49mcreate_shield(env\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49menv)\n\u001b[1;32m 224\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mkeys \u001b[39m=\u001b[39m keys\n\u001b[1;32m 225\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshield \u001b[39m=\u001b[39m shield\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/examples/shields/rl/ShieldHandlers.py:82\u001b[0m, in \u001b[0;36mMiniGridShieldHandler.create_shield\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__export_grid_to_text(env)\n\u001b[1;32m 80\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__create_prism()\n\u001b[0;32m---> 82\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__create_shield_dict()\n",
"File \u001b[0;32m~/Documents/Projects/tempestpy/examples/shields/rl/ShieldHandlers.py:66\u001b[0m, in \u001b[0;36mMiniGridShieldHandler.__create_shield_dict\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 64\u001b[0m choice \u001b[39m=\u001b[39m shield_scheduler\u001b[39m.\u001b[39mget_choice(stateID)\n\u001b[1;32m 65\u001b[0m choices \u001b[39m=\u001b[39m choice\u001b[39m.\u001b[39mchoice_map\n\u001b[0;32m---> 66\u001b[0m state_valuation \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mstate_valuations\u001b[39m.\u001b[39;49mget_string(stateID)\n\u001b[1;32m 68\u001b[0m actions_to_be_executed \u001b[39m=\u001b[39m [(choice[\u001b[39m1\u001b[39m] ,model\u001b[39m.\u001b[39mchoice_labeling\u001b[39m.\u001b[39mget_labels_of_choice(model\u001b[39m.\u001b[39mget_choice_index(stateID, choice[\u001b[39m1\u001b[39m]))) \u001b[39mfor\u001b[39;00m choice \u001b[39min\u001b[39;00m choices]\n\u001b[1;32m 70\u001b[0m action_dictionary[state_valuation] \u001b[39m=\u001b[39m actions_to_be_executed\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"shield_creator = MiniGridShieldHandler(\"grid.txt\", \"./main\", \"grid.prism\", \"Pmax=? [G !\\\"AgentIsInLavaAndNotDone\\\"]\")\n",
"\n",
"env = gym.make(\"MiniGrid-LavaCrossingS9N1-v0\", render_mode=\"rgb_array\")\n",
"env = MiniGridSbShieldingWrapper(env, shield_creator=shield_creator, shield_query_creator=create_shield_query, mask_actions=True)\n",
"env = ActionMasker(env, mask_fn)\n",
"model = MaskablePPO(MaskableActorCriticPolicy, env, verbose=1)\n",
"\n",
"\n",
"model.learn(10_000)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "env",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}

0
examples/shields/rl/ShieldHandlers.py → examples/shields/rl/shieldhandlers.py
View File

0
examples/shields/rl/TorchActionMaskModel.py → examples/shields/rl/torch_action_mask_model.py
View File

2
examples/shields/rl/Wrappers.py → examples/shields/rl/wrappers.py
View File

@ -8,7 +8,7 @@ from collections import deque
from ray.rllib.utils.numpy import one_hot
from helpers import get_action_index_mapping, extract_keys
from ShieldHandlers import ShieldHandler
from shieldhandlers import ShieldHandler
class OneHotShieldingWrapper(gym.core.ObservationWrapper):
Write Preview
Loading…
Cancel
Save