你的智能体为每个从未调用的工具付出代价

mkdir my-toolbox-agent && cd my-toolbox-agent

azd ai agent init \
  -m "https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/04-foundry-toolbox/agent.manifest.yaml" \
  --src src/toolbox-agent

azd ai toolbox create my-toolbox --from-file ./src/toolbox-agent/toolbox.yaml

https://<account>.services.ai.azure.com/api/projects/<project>/toolboxes/my-toolbox/versions/1/mcp?api-version=v1

azd init --minimal
azd env set FOUNDRY_PROJECT_ENDPOINT https://<account>.services.ai.azure.com/api/projects/<project>

# toolbox.yaml — illustrative structure, verify against the current sample
name: my-toolbox
description: Order operations, customer lookup, and fulfilment tools

toolSearch:
  enabled: true
  # Pinned tools skip retrieval entirely and are always in context.
  # Keep this list short — every pin is a permanent token cost.
  pinned:
    - get_order_status
    - search_customers
  autoPin:
    enabled: true
    minCallsPerWindow: 25

tools:
  - name: get_order_status
    source: mcp
    server: orders-mcp
    # Retrieval context: describe the tool the way your team describes it,
    # including the vocabulary users actually type.
    searchContext: >
      Look up the current fulfilment state of a single order. Use when the
      user mentions an order number, tracking number, "where is my package",
      or asks whether something shipped.

  - name: issue_refund
    source: mcp
    server: billing-mcp
    searchContext: >
      Issue a full or partial refund against a completed order. Requires an
      order ID and an amount. Do not use for cancellations of unshipped
      orders — use cancel_order instead.

pip install azure-ai-projects azure-identity opentelemetry-sdk azure-core-tracing-opentelemetry

from azure.core.settings import settings
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, ConsoleSpanExporter
from azure.ai.projects.telemetry import AIProjectInstrumentor

settings.tracing_implementation = "opentelemetry"

span_exporter = ConsoleSpanExporter()
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
trace.set_tracer_provider(tracer_provider)

# Emits GenAI spans for every agent and model call in this process
AIProjectInstrumentor().instrument()

import os
import json
from dataclasses import dataclass, field
from statistics import mean

@dataclass
class RunResult:
    config: str
    input_tokens: list = field(default_factory=list)
    turns: list = field(default_factory=list)
    correct_tool: list = field(default_factory=list)

TASKS = [
    # (prompt, tool the model should end up calling)
    ("Where is order 88231?",                      "get_order_status"),
    ("Refund the second item on order 88231",      "issue_refund"),
    ("Cancel order 90114, it hasn't shipped",      "cancel_order"),
    ("Which customers in Ohio ordered twice?",     "search_customers"),
    ("Send the June invoice to billing@acme.com",  "email_invoice"),
]

def score_run(spans, expected_tool):
    """Pull token usage and the actually-invoked tool out of collected spans."""
    input_tokens = sum(
        s.attributes.get("gen_ai.usage.input_tokens", 0) for s in spans
    )
    invoked = [
        s.attributes.get("gen_ai.tool.name")
        for s in spans
        if s.attributes.get("gen_ai.tool.name")
    ]
    # With Tool Search the target arrives as a call_tool argument, so unwrap it.
    resolved = [
        json.loads(s.attributes["gen_ai.tool.arguments"]).get("name", n)
        if n == "call_tool" else n
        for s, n in zip(spans, invoked)
    ]
    return input_tokens, len(invoked), expected_tool in resolved

def report(results):
    for r in results:
        print(f"\n{r.config}")
        print(f"  mean input tokens/task : {mean(r.input_tokens):>8,.0f}")
        print(f"  mean turns/task        : {mean(r.turns):>8.1f}")
        print(f"  tool selection accuracy: {mean(r.correct_tool):>8.1%}")