From cb19465b694eb29516619443266c6b1e496141bd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Jul 2026 21:30:55 +0000 Subject: [PATCH 4/5] Add --single_action mode: one action per tool call (batching-ab experiment) - SINGLE_ACTION_TOOL_SCHEMA: actions array constrained to minItems=maxItems=1 - SINGLE_ACTION_PROMPT replaces BATCHED_ACTION_PROMPT; chain-calls sentence stripped from base SYSTEM_PROMPT - Defensive truncation to first action if the model batches anyway (logged) Co-Authored-By: Claude Fable 5 --- mm_agents/anthropic/main.py | 26 ++++++++++++++++++++++++-- mm_agents/anthropic/utils.py | 21 +++++++++++++++++++++ scripts/python/run_multienv_claude.py | 3 +++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/mm_agents/anthropic/main.py b/mm_agents/anthropic/main.py index 734eba3c..24b11265 100644 --- a/mm_agents/anthropic/main.py +++ b/mm_agents/anthropic/main.py @@ -21,6 +21,9 @@ from anthropic.types.beta import ( from .utils import ( BATCHED_ACTION_PROMPT, BATCHED_TOOL_SCHEMA, + CHAIN_CALLS_SENTENCE, + SINGLE_ACTION_PROMPT, + SINGLE_ACTION_TOOL_SCHEMA, CLAUDE_47_PROMPT_ADDITIONS, PROMPT_CACHING_BETA_FLAG, SYSTEM_PROMPT, @@ -65,6 +68,7 @@ class AnthropicAgent: effort: str = "max", temperature: Optional[float] = None, top_p: Optional[float] = None, + action_batching: bool = True, context_policy: str = POLICY_KEEP_ALL, context_budget_tokens: int = 25000, keep_n_images: Optional[int] = None, @@ -88,6 +92,8 @@ class AnthropicAgent: self.screen_size = screen_size self.image_target_size = image_target_size self.no_thinking = no_thinking + self.action_batching = action_batching + self.single_action_violations = 0 self.use_isp = use_isp self.effort = effort self.temperature = temperature @@ -283,6 +289,16 @@ class AnthropicAgent: if batched_actions is not None: if not isinstance(batched_actions, list): raise ValueError(f"{batched_actions} must be a list") + if not self.action_batching and len(batched_actions) > 1: + # Schema says maxItems=1; if the model still batches, execute + # only the first action — the auto-screenshot after it lets the + # model observe true state on the next turn. + self.single_action_violations += 1 + logger.warning( + f"single-action mode: model returned {len(batched_actions)} actions " + f"(violation #{self.single_action_violations}); executing only the first" + ) + batched_actions = batched_actions[:1] for batched_action in batched_actions: if not isinstance(batched_action, dict): raise ValueError(f"{batched_action} must be a dict") @@ -517,14 +533,18 @@ class AnthropicAgent: self.current_step += 1 system_text = f"{SYSTEM_PROMPT_WINDOWS if self.platform == 'Windows' else SYSTEM_PROMPT}{' ' + self.system_prompt_suffix if self.system_prompt_suffix else ''}" + action_style_prompt = BATCHED_ACTION_PROMPT if self.action_batching else SINGLE_ACTION_PROMPT if self.runtime_profile.use_claude_47_prompt: system_text += f"\n\n{CLAUDE_47_PROMPT_ADDITIONS}" system_text += f"\n\nYou have a maximum of {self.max_steps} steps to complete this task. Use your steps wisely." - system_text += f"\n\n{BATCHED_ACTION_PROMPT}" + system_text += f"\n\n{action_style_prompt}" elif self.provider == APIProvider.OPENROUTER: system_text += f"\n\n{CLAUDE_47_PROMPT_ADDITIONS}" system_text += f"\n\nYou have a maximum of {self.max_steps} steps to complete this task. Use your steps wisely." - system_text += f"\n\n{BATCHED_ACTION_PROMPT}" + system_text += f"\n\n{action_style_prompt}" + if not self.action_batching: + # Strip the base prompt's chain-multiple-calls exhortation too. + system_text = system_text.replace(CHAIN_CALLS_SENTENCE, "") if self.context_policy == POLICY_SUMMARIZE: system_text += f"\n\n{context_policies.SUMMARIZE_PROMPT_ADDITION}" @@ -736,6 +756,8 @@ class AnthropicAgent: if self.provider == APIProvider.OPENROUTER: if self.context_policy == POLICY_FOCUS_CHAIN: tools = [self._focus_chain_tool_schema()] + elif not self.action_batching: + tools = [SINGLE_ACTION_TOOL_SCHEMA] else: tools = [BATCHED_TOOL_SCHEMA] else: diff --git a/mm_agents/anthropic/utils.py b/mm_agents/anthropic/utils.py index 2b2139ef..f3f90d03 100644 --- a/mm_agents/anthropic/utils.py +++ b/mm_agents/anthropic/utils.py @@ -1,6 +1,7 @@ """ Utility functions for the Anthropic API. """ +import copy from dataclasses import dataclass from typing import List, Union, cast from enum import Enum @@ -341,6 +342,26 @@ BATCHED_TOOL_SCHEMA = { } +# --- Single-action variant (batching-ab experiment) --- +# Sentence in SYSTEM_PROMPT that exhorts chaining calls; stripped in +# single-action mode so the prompt carries no batching pressure. +CHAIN_CALLS_SENTENCE = " Where possible/feasible, try to chain multiple of these calls all into one function calls request." + +SINGLE_ACTION_PROMPT = """Each computer tool call executes exactly ONE action. The tool result (with a screenshot when relevant) is returned after each action.""" + +SINGLE_ACTION_TOOL_SCHEMA = copy.deepcopy(BATCHED_TOOL_SCHEMA) +SINGLE_ACTION_TOOL_SCHEMA["description"] = ( + "Execute a single computer use action. Actions use the same parameters as " + "the Anthropic computer use tool: screenshot, left_click, right_click, middle_click, " + "double_click, triple_click, type, key, scroll, mouse_move, left_click_drag, " + "left_mouse_down, left_mouse_up, hold_key, and wait." +) +_single_actions_prop = SINGLE_ACTION_TOOL_SCHEMA["input_schema"]["properties"]["actions"] +_single_actions_prop["minItems"] = 1 +_single_actions_prop["maxItems"] = 1 +_single_actions_prop["description"] = "A list containing exactly ONE action to execute." + + def _make_api_tool_result( result: ToolResult, tool_use_id: str diff --git a/scripts/python/run_multienv_claude.py b/scripts/python/run_multienv_claude.py index 3ba9ff5b..76b40f68 100644 --- a/scripts/python/run_multienv_claude.py +++ b/scripts/python/run_multienv_claude.py @@ -101,6 +101,8 @@ def config() -> argparse.Namespace: help="Effort level (default: max)") parser.add_argument("--no_thinking", action="store_true", help="Disable Claude extended thinking when supported") parser.add_argument("--use_isp", action="store_true", help="Enable interleaved scratchpad beta when supported") + parser.add_argument("--single_action", action="store_true", + help="Restrict the computer tool to ONE action per call (batching-ab experiment)") # context-management policy config (cache-aware-context experiment) parser.add_argument( @@ -282,6 +284,7 @@ def run_env_tasks(task_queue, args, shared_scores): ), no_thinking=args.no_thinking, use_isp=args.use_isp, + action_batching=not args.single_action, effort=getattr(args, 'effort', 'max'), max_steps=args.max_steps, context_policy=args.context_policy, -- 2.43.0