Port Prompty code

A port of the Prompty code from the Promtpflow repo. The focus was on expediency, rather than elegance. The core logic of the the code is similar to the original except for the following changes:
- Added type annotations
- Removed support for the now legacy OpenAI completions API
- Removed support for functions and tools. The former relied on an insecure implementation using eval. Since none of the evaluators use this feature right now, these were cut
- Reworked the way images were handled to simplify (now handled in one pass, and no more surprise calls out to the internet to unnecessarily load image bytes)
- Minor obvious tweaks to the code to improve readability, and trim unnecessary code paths

Author: ralph-msft

.vscode/cspell.json

@@ -1400,6 +1400,7 @@
         "upia",
         "xpia",
         "expirable",
+        "ralphe"
       ]
     },
     {

sdk/evaluation/azure-ai-evaluation/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/evaluation/azure-ai-evaluation",
-  "Tag": "python/evaluation/azure-ai-evaluation_ceeaf3cbb7"
+  "Tag": "python/evaluation/azure-ai-evaluation_bef898409a"
 }

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/legacy/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/legacy/prompty/__init__.py

@@ -0,0 +1,36 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation.legacy.prompty._prompty import AsyncPrompty
+from azure.ai.evaluation.legacy.prompty._connection import Connection, OpenAIConnection, AzureOpenAIConnection
+from azure.ai.evaluation.legacy.prompty._exceptions import (
+    PromptyException,
+    MissingRequiredInputError,
+    InvalidInputError,
+    JinjaTemplateError,
+    NotSupportedError,
+)
+
+# =========================================================================================================
+# NOTE: All of the code here is largely copy of code from Promptflow. Generally speaking, the following
+#       changes were made:
+#       - Added type annotations
+#       - Legacy or deprecated functionality has been removed (e.g. no more support for completions API)
+#       - Reworked the way images are handled to 1) Reduce the amount of code brought over, 2) remove
+#         the need to do two passes over the template to insert images, 3) remove the completely unnecessary
+#         loading of image data from the internet when it is not actually needed
+#       - Minor obvious tweaks to improve code readability, and removal of unused code paths
+# =========================================================================================================
+
+__all__ = [
+    "AsyncPrompty",
+    "Connection",
+    "AzureOpenAIConnection",
+    "OpenAIConnection",
+    "PromptyException",
+    "MissingRequiredInputError",
+    "InvalidInputError",
+    "JinjaTemplateError",
+    "NotSupportedError",
+]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/legacy/prompty/_connection.py

@@ -0,0 +1,182 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar, Mapping, Optional, Set, Union
+
+from azure.ai.evaluation.legacy.prompty._exceptions import MissingRequiredInputError
+from azure.ai.evaluation.legacy.prompty._utils import dataclass_from_dict
+
+
+ENV_VAR_PATTERN = re.compile(r"^\$\{env:(.*)\}$")
+
+
+def _parse_environment_variable(value: Union[str, Any]) -> Union[str, Any]:
+    """Get environment variable from ${env:ENV_NAME}. If not found, return original value.
+
+    :param value: The value to parse.
+    :type value: str | Any
+    :return: The parsed value
+    :rtype: str | Any"""
+    if not isinstance(value, str):
+        return value
+
+    result = re.match(ENV_VAR_PATTERN, value)
+    if result:
+        env_name = result.groups()[0]
+        return os.environ.get(env_name, value)
+
+    return value
+
+
+def _is_empty_connection_config(connection_dict: Mapping[str, Any]) -> bool:
+    ignored_fields = set(["azure_deployment", "model"])
+    keys = {k for k, v in connection_dict.items() if v}
+    return len(keys - ignored_fields) == 0
+
+
+@dataclass
+class Connection(ABC):
+    """Base class for all connection classes."""
+
+    @property
+    @abstractmethod
+    def type(self) -> str:
+        """Gets the type of the connection.
+
+        :return: The type of the connection.
+        :rtype: str"""
+        ...
+
+    @abstractmethod
+    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
+        """Check if the connection is valid.
+
+        :param missing_fields: If set, this will be populated with the missing required fields.
+        :type missing_fields: Set[str] | None
+        :return: True if the connection is valid, False otherwise.
+        :rtype: bool"""
+        ...
+
+    @staticmethod
+    def parse_from_config(model_configuration: Mapping[str, Any]) -> "Connection":
+        """Parse a connection from a model configuration.
+
+        :param model_configuration: The model configuration.
+        :type model_configuration: Mapping[str, Any]
+        :return: The connection.
+        :rtype: Connection
+        """
+        connection: Connection
+        connection_dict = {k: _parse_environment_variable(v) for k, v in model_configuration.items()}
+        connection_type = connection_dict.pop("type", "")
+
+        if connection_type in [AzureOpenAIConnection.TYPE, "azure_openai"]:
+            if _is_empty_connection_config(connection_dict):
+                connection = AzureOpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(AzureOpenAIConnection, connection_dict)
+
+        elif connection_type in [OpenAIConnection.TYPE, "openai"]:
+            if _is_empty_connection_config(connection_dict):
+                connection = OpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(OpenAIConnection, connection_dict)
+
+        else:
+            error_message = (
+                f"'{connection_type}' is not a supported connection type. Valid values are "
+                f"[{AzureOpenAIConnection.TYPE}, {OpenAIConnection.TYPE}]"
+            )
+            raise MissingRequiredInputError(error_message)
+
+        missing_fields: Set[str] = set()
+        if not connection.is_valid(missing_fields):
+            raise MissingRequiredInputError(
+                f"The following required fields are missing for connection {connection.type}: "
+                f"{', '.join(missing_fields)}"
+            )
+
+        return connection
+
+
+@dataclass
+class OpenAIConnection(Connection):
+    """Connection class for OpenAI endpoints."""
+
+    base_url: str
+    api_key: Optional[str] = None
+    organization: Optional[str] = None
+
+    TYPE: ClassVar[str] = "openai"
+
+    @property
+    def type(self) -> str:
+        return OpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "OpenAIConnection":
+        return cls(
+            base_url=os.environ.get("OPENAI_BASE_URL", ""),
+            api_key=os.environ.get("OPENAI_API_KEY"),
+            organization=os.environ.get("OPENAI_ORG_ID"),
+        )
+
+    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
+        if missing_fields is None:
+            missing_fields = set()
+        if not self.base_url:
+            missing_fields.add("base_url")
+        if not self.api_key:
+            missing_fields.add("api_key")
+        if not self.organization:
+            missing_fields.add("organization")
+        return not bool(missing_fields)
+
+
+@dataclass
+class AzureOpenAIConnection(Connection):
+    """Connection class for Azure OpenAI endpoints."""
+
+    azure_endpoint: str
+    api_key: Optional[str] = None  # TODO ralphe: Replace this TokenCredential to allow for more flexible authentication
+    azure_deployment: Optional[str] = None
+    api_version: Optional[str] = None
+    resource_id: Optional[str] = None
+
+    TYPE: ClassVar[str] = "azure_openai"
+
+    @property
+    def type(self) -> str:
+        return AzureOpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "AzureOpenAIConnection":
+        return cls(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
+            api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
+            azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+        )
+
+    def __post_init__(self):
+        # set default API version
+        if not self.api_version:
+            self.api_version = "2024-02-01"
+
+    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
+        if missing_fields is None:
+            missing_fields = set()
+        if not self.azure_endpoint:
+            missing_fields.add("azure_endpoint")
+        if not self.api_key:
+            missing_fields.add("api_key")
+        if not self.azure_deployment:
+            missing_fields.add("azure_deployment")
+        if not self.api_version:
+            missing_fields.add("api_version")
+        return not bool(missing_fields)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/legacy/prompty/_exceptions.py

@@ -0,0 +1,59 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation._exceptions import ErrorCategory, ErrorBlame, ErrorTarget, EvaluationException
+
+
+class PromptyException(EvaluationException):
+    """Exception class for Prompty related errors.
+
+    This exception is used to indicate that the error was caused by Prompty execution.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.UNKNOWN)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+
+        super().__init__(message, **kwargs)
+
+
+class MissingRequiredInputError(PromptyException):
+    """Exception raised when missing required input"""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.MISSING_FIELD)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class InvalidInputError(PromptyException):
+    """Exception raised when an input is invalid, could not be loaded, or is not the expected format."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class JinjaTemplateError(PromptyException):
+    """Exception raised when the Jinja template is invalid or could not be rendered."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class NotSupportedError(PromptyException):
+    """Exception raised when the operation is not supported."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.UNKNOWN)
+        kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+        super().__init__(message, **kwargs)