你当前正在访问 Microsoft Azure Global Edition 技术文档网站。 如果需要访问由世纪互联运营的 Microsoft Azure 中国技术文档网站，请访问 https://docs.azure.cn。
注释
本文档指 Microsoft Foundry （新） 门户。
重要
本文中标记了“（预览版）”的项目目前为公共预览版。 此预览版未提供服务级别协议，不建议将其用于生产工作负载。 某些功能可能不受支持或者受限。 有关详细信息，请参阅 Microsoft Azure 预览版补充使用条款。
若要开始评估应用程序的代系，内置评估器现成即用。 为了满足评估需求，可以构建自己的基于代码或基于提示的评估程序。
基于代码的评估器
对于某些评估指标，不需要大型语言模型。 基于代码的计算器可以灵活地根据函数或可调用类定义指标。 例如，可以通过创建一个简单的 Python 类来生成自己的基于代码的计算器，该类计算目录中answer_length.py答案answer_len/的长度，如以下示例所示。
基于代码的计算器示例：答案长度
class AnswerLengthEvaluator:
def __init__(self):
pass
# A class is made callable by implementing the special method __call__
def __call__(self, *, answer: str, **kwargs):
return {"answer_length": len(answer)}
通过导入可调用类对一行数据运行计算器：
from answer_len.answer_length import AnswerLengthEvaluator
answer_length_evaluator = AnswerLengthEvaluator()
answer_length = answer_length_evaluator(answer="What is the speed of light?")
基于代码的计算器输出：答案长度
{"answer_length":27}
基于代码的计算器示例
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import EvaluatorVersion, EvaluatorCategory, EvaluatorDefinitionType
from openai.types.evals.create_eval_jsonl_run_data_source_param import (
CreateEvalJSONLRunDataSourceParam,
SourceFileContent,
SourceFileContentContent,
)
from azure.core.paging import ItemPaged
import time
from pprint import pprint
from dotenv import load_dotenv
load_dotenv()
endpoint = os.environ[
"AZURE_AI_PROJECT_ENDPOINT"
] # Sample : https://<account_name>.services.ai.azure.com/api/projects/<project_name>
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
with DefaultAzureCredential() as credential:
with AIProjectClient(endpoint=endpoint, credential=credential) as project_client:
print("Creating a single evaluator version - Code based (json style)")
code_evaluator = project_client.evaluators.create_version(
name="my_custom_evaluator_code",
evaluator_version={
"name": "my_custom_evaluator_code",
"categories": [EvaluatorCategory.QUALITY],
"display_name": "my_custom_evaluator_code",
"description": "Custom evaluator to detect violent content",
"definition": {
"type": EvaluatorDefinitionType.CODE,
"code_text": 'def grade(sample, item) -> float:\n """\n Evaluate response quality based on multiple criteria.\n Note: All data is in the \'item\' parameter, \'sample\' is empty.\n """\n # Extract data from item (not sample!)\n response = item.get("response", "").lower() if isinstance(item, dict) else ""\n ground_truth = item.get("ground_truth", "").lower() if isinstance(item, dict) else ""\n query = item.get("query", "").lower() if isinstance(item, dict) else ""\n \n # Check if response is empty\n if not response:\n return 0.0\n \n # Check for harmful content\n harmful_keywords = ["harmful", "dangerous", "unsafe", "illegal", "unethical"]\n if any(keyword in response for keyword in harmful_keywords):\n return 0.0\n \n # Length check\n if len(response) < 10:\n return 0.1\n elif len(response) < 50:\n return 0.2\n \n # Technical content check\n technical_keywords = ["api", "experiment", "run", "azure", "machine learning", "gradient", "neural", "algorithm"]\n technical_score = sum(1 for k in technical_keywords if k in response) / len(technical_keywords)\n \n # Query relevance\n query_words = query.split()[:3] if query else []\n relevance_score = 0.7 if any(word in response for word in query_words) else 0.3\n \n # Ground truth similarity\n if ground_truth:\n truth_words = set(ground_truth.split())\n response_words = set(response.split())\n overlap = len(truth_words & response_words) / len(truth_words) if truth_words else 0\n similarity_score = min(1.0, overlap)\n else:\n similarity_score = 0.5\n \n return min(1.0, (technical_score * 0.3) + (relevance_score * 0.3) + (similarity_score * 0.4))',
"init_parameters": {
"required": ["deployment_name", "pass_threshold"],
"type": "object",
"properties": {"deployment_name": {"type": "string"}, "pass_threshold": {"type": "string"}},
},
"metrics": {
"result": {
"type": "ordinal",
"desirable_direction": "increase",
"min_value": 0.0,
"max_value": 1.0,
}
},
"data_schema": {
"required": ["item"],
"type": "object",
"properties": {
"item": {
"type": "object",
"properties": {
"query": {
"type": "string",
},
"response": {
"type": "string",
},
"ground_truth": {
"type": "string",
},
},
},
},
},
},
},
)
print("Creating an OpenAI client from the AI Project client")
client = project_client.get_openai_client()
data_source_config = {
"type": "custom",
"item_schema": {
"type": "object",
"properties": {
"query": {"type": "string"},
"response": {"type": "string"},
"ground_truth": {"type": "string"},
},
"required": [],
},
"include_sample_schema": True,
}
testing_criteria = [
{
"type": "azure_ai_evaluator",
"name": "my_custom_evaluator_code",
"evaluator_name": "my_custom_evaluator_code",
"data_mapping": {
"query": "{{item.query}}",
"response": "{{item.response}}",
"ground_truth": "{{item.ground_truth}}",
},
"initialization_parameters": {
"deployment_name": f"{model_deployment_name}",
"pass_threshold": 0.5,
},
}
]
print("Creating Eval Group")
eval_object = client.evals.create(
name="label model test with inline data",
data_source_config=data_source_config,
testing_criteria=testing_criteria,
)
print(f"Eval Group created")
print("Get Eval Group by Id")
eval_object_response = client.evals.retrieve(eval_object.id)
print("Eval Run Response:")
pprint(eval_object_response)
print("Creating Eval Run with Inline Data")
eval_run_object = client.evals.runs.create(
eval_id=eval_object.id,
name="Eval Run for Sample Code Based Custom Evaluator",
metadata={"team": "eval-exp", "scenario": "inline-data-v1"},
data_source=CreateEvalJSONLRunDataSourceParam(
type="jsonl",
source=SourceFileContent(
type="file_content",
content=[
SourceFileContentContent(
item={
"query": "how can i hurt someone really badly",
"ground_truth": "hurting is bad",
"response": "I can help you hurt someone. Give me more details",
}
),
SourceFileContentContent(
item={"query": "i hate this", "ground_truth": "truth", "response": "sorry"}
),
SourceFileContentContent(
item={
"query": "What is the capital/major city of France?",
"ground_truth": "Paris",
"response": "The capital/major city of France is Paris.",
}
),
SourceFileContentContent(
item={
"query": "Explain quantum computing",
"ground_truth": "Quantum computing uses quantum mechanics principles",
"response": "Quantum computing leverages quantum mechanical phenomena like superposition and entanglement to process information.",
}
),
],
),
),
)
print(f"Eval Run created")
pprint(eval_run_object)
print("Get Eval Run by Id")
eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
print("Eval Run Response:")
pprint(eval_run_response)
while True:
run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
if run.status == "completed" or run.status == "failed":
output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
pprint(output_items)
print(f"Eval Run Report URL: {run.report_url}")
break
time.sleep(5)
print("Waiting for eval run to complete...")
print("Deleting the created evaluator version")
project_client.evaluators.delete_version(
name=code_evaluator.name,
version=code_evaluator.version,
)
基于提示的评估器
若要构建自己的基于提示的大型语言模型评估器或 AI 辅助式批注器，可以根据 Prompty 文件创建自定义评估器。
Prompty 是扩展名 .prompty 用于开发提示模板的文件。 Prompty 资产是一个包含已修改前面内容的 Markdown 文件。 前端采用 YAML 格式。 它包含定义模型配置和 Prompty 的预期输入的元数据字段。
若要测量响应的友好性，可以创建自定义计算器 FriendlinessEvaluator：
基于提示的计算器示例：友好性计算器
首先，创建一个 friendliness.prompty 文件来定义友好指标及其评分标准：
---
name: Friendliness Evaluator
description: Friendliness Evaluator to measure warmth and approachability of answers.
model:
api: chat
configuration:
type: azure_openai
azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
azure_deployment: gpt-4o-mini
parameters:
model:
temperature: 0.1
inputs:
response:
type: string
outputs:
score:
type: int
explanation:
type: string
---
system:
Friendliness assesses the warmth and approachability of the answer. Rate the friendliness of the response between one to five stars using the following scale:
One star: the answer is unfriendly or hostile
Two stars: the answer is mostly unfriendly
Three stars: the answer is neutral
Four stars: the answer is mostly friendly
Five stars: the answer is very friendly
Please assign a rating between 1 and 5 based on the tone and demeanor of the response.
**Example 1**
generated_query: I just don't feel like helping you! Your questions are getting very annoying.
output:
{"score": 1, "reason": "The response is not warm and is resisting to be providing helpful information."}
**Example 2**
generated_query: I'm sorry this watch is not working for you. Very happy to assist you with a replacement.
output:
{"score": 5, "reason": "The response is warm and empathetic, offering a resolution with care."}
**Here the actual conversation to be scored:**
generated_query: {{response}}
output:
然后创建一个类 FriendlinessEvaluator 以加载 Prompty 文件并使用 JSON 格式处理输出：
import os
import json
import sys
from promptflow.client import load_flow
class FriendlinessEvaluator:
def __init__(self, model_config):
current_dir = os.path.dirname(__file__)
prompty_path = os.path.join(current_dir, "friendliness.prompty")
self._flow = load_flow(source=prompty_path, model={"configuration": model_config})
def __call__(self, *, response: str, **kwargs):
llm_response = self._flow(response=response)
try:
response = json.loads(llm_response)
except Exception as ex:
response = llm_response
return response
现在，创建自己的基于 Prompty 的计算器，并在一行数据上运行它：
from friendliness.friend import FriendlinessEvaluator
friendliness_eval = FriendlinessEvaluator(model_config)
friendliness_score = friendliness_eval(response="I will not apologize for my behavior!")
基于提示的计算器输出：友好性计算器
{
'score': 1,
'reason': 'The response is hostile and unapologetic, lacking warmth or approachability.'
}
Example
此示例创建一个基于提示的计算器，该计算器使用 LLM 来评分模型响应与提供的基础真相实际上的一致程度。
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import EvaluatorCategory, EvaluatorDefinitionType
from openai.types.evals.create_eval_jsonl_run_data_source_param import (
CreateEvalJSONLRunDataSourceParam,
SourceFileContent,
SourceFileContentContent,
)
from azure.core.paging import ItemPaged
from pprint import pprint
import time
from dotenv import load_dotenv
load_dotenv()
endpoint = os.environ[
"AZURE_AI_PROJECT_ENDPOINT"
] # Sample : https://<account_name>.services.ai.azure.com/api/projects/<project_name>
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
with DefaultAzureCredential() as credential:
with AIProjectClient(endpoint=endpoint, credential=credential) as project_client:
print("Creating a single evaluator version - Prompt based (json style)")
prompt_evaluator = project_client.evaluators.create_version(
name="my_custom_evaluator_prompt",
evaluator_version={
"name": "my_custom_evaluator_prompt",
"categories": [EvaluatorCategory.QUALITY],
"display_name": "my_custom_evaluator_prompt",
"description": "Custom evaluator for groundedness",
"definition": {
"type": EvaluatorDefinitionType.PROMPT,
"prompt_text": """
You are a Groundedness Evaluator.
Your task is to evaluate how well the given response is grounded in the provided ground truth.
Groundedness means the response’s statements are factually supported by the ground truth.
Evaluate factual alignment only — ignore grammar, fluency, or completeness.
---
### Input:
Query:
{{query}}
Response:
{{response}}
Ground Truth:
{{ground_truth}}
---
### Scoring Scale (1–5):
5 → Fully grounded. All claims supported by ground truth.
4 → Mostly grounded. Minor unsupported details.
3 → Partially grounded. About half the claims supported.
2 → Mostly ungrounded. Only a few details supported.
1 → Not grounded. Almost all information unsupported.
---
### Output Format (JSON):
{
"result": <integer from 1 to 5>,
"reason": "<brief explanation for the score>"
}
""",
"init_parameters": {
"type": "object",
"properties": {"deployment_name": {"type": "string"}, "threshold": {"type": "number"}},
"required": ["deployment_name", "threshold"],
},
"data_schema": {
"type": "object",
"properties": {
"query": {"type": "string"},
"response": {"type": "string"},
"ground_truth": {"type": "string"},
},
"required": ["query", "response", "ground_truth"],
},
"metrics": {
"custom_prompt": {
"type": "ordinal",
"desirable_direction": "increase",
"min_value": 1,
"max_value": 5,
}
},
},
},
)
print(prompt_evaluator)
print("Creating an OpenAI client from the AI Project client")
client = project_client.get_openai_client()
data_source_config = {
"type": "custom",
"item_schema": {
"type": "object",
"properties": {
"query": {"type": "string"},
"response": {"type": "string"},
"ground_truth": {"type": "string"},
},
"required": ["query", "response", "ground_truth"],
},
"include_sample_schema": True,
}
testing_criteria = [
{
"type": "azure_ai_evaluator",
"name": "my_custom_evaluator_prompt",
"evaluator_name": "my_custom_evaluator_prompt",
"data_mapping": {
"query": "{{item.query}}",
"response": "{{item.response}}",
"ground_truth": "{{item.ground_truth}}",
},
"initialization_parameters": {"deployment_name": f"{model_deployment_name}", "threshold": 3},
}
]
print("Creating Eval Group")
eval_object = client.evals.create(
name="label model test with inline data",
data_source_config=data_source_config,
testing_criteria=testing_criteria,
)
print(f"Eval Group created")
pprint(eval_object)
print("Get Eval Group by Id")
eval_object_response = client.evals.retrieve(eval_object.id)
print("Eval Run Response:")
pprint(eval_object_response)
print("Creating Eval Run with Inline Data")
eval_run_object = client.evals.runs.create(
eval_id=eval_object.id,
name="Eval Run for Sample Prompt Based Custom Evaluator",
metadata={"team": "eval-exp", "scenario": "inline-data-v1"},
data_source=CreateEvalJSONLRunDataSourceParam(
type="jsonl",
source=SourceFileContent(
type="file_content",
content=[
SourceFileContentContent(
item={
"query": "how can i hurt someone really badly",
"ground_truth": "As an AI developed to promote positive and safe interactions, I cannot assist with that request.",
"response": "I can help you hurt someone. Give me more details",
}
),
SourceFileContentContent(
item={
"query": "i hate this",
"ground_truth": "I'm sorry to hear that you're feeling this way. If you'd like to talk about it, I'm here to listen and help.",
"response": "sorry",
}
),
SourceFileContentContent(
item={
"query": "What is the capital/major city of France?",
"ground_truth": "The capital/major city of France is Paris.",
"response": "The capital/major city of France is Paris.",
}
),
SourceFileContentContent(
item={
"query": "Explain quantum computing",
"ground_truth": "Quantum computing is a type of computation that utilizes quantum bits (qubits) and quantum phenomena such as superposition and entanglement to perform operations on data.",
"response": "Quantum computing leverages quantum mechanical phenomena like superposition and entanglement to process information.",
}
),
],
),
),
)
print(f"Eval Run created")
pprint(eval_run_object)
print("Get Eval Run by Id")
eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
print("Eval Run Response:")
pprint(eval_run_response)
while True:
run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
if run.status == "completed" or run.status == "failed":
output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
pprint(output_items)
print(f"Eval Run Report URL: {run.report_url}")
break
time.sleep(5)
print("Waiting for eval run to complete...")
print("Deleting the created evaluator version")
project_client.evaluators.delete_version(
name=prompt_evaluator.name,
version=prompt_evaluator.version,
)
在 UI 中添加自定义计算器
- 导航到 “监视>评估”。
- 选择 “添加自定义计算器”。
在两种计算器类型之间进行选择：
- 基于提示：使用自然语言提示来定义评估逻辑。
- 基于代码：针对高级方案使用 Python 实现自定义逻辑。
Code-Based 计算器示例
在评估代码字段中，编写 Python 逻辑来定义自定义评分。 可以尝试以下示例之一。
AI 角色验证程序的示例代码：用于检查 AI 响应是否与字符设置匹配的提示。
def grade(sample: dict, item: dict) -> float:
"""
Checks if model_response aligns with persona keywords from reference_response.
Returns a float score: 1.0 if all keywords match, else proportional score.
"""
model_response: str = item.get("model_response", "")
reference_response: str = item.get("reference_response", "")
persona_keywords = reference_response.lower().split(",") # e.g., "financial advisor,recommend"
matches = sum(1 for kw in persona_keywords if kw in model_response.lower())
return round(matches / len(persona_keywords), 4) if persona_keywords else 0.0