feat(checks): ENG-1390 judges directory for llm and prompt reliant checks

libs/giskard-checks/src/giskard/checks/__init__.py

@@ -5,19 +5,14 @@
 
 from giskard.agents import add_prompts_path
 
-from . import builtin
+from . import builtin, judges
 from .builtin import (
-    BaseLLMCheck,
-    Conformity,
     Equals,
     FnCheck,
     GreaterEquals,
     GreaterThan,
-    Groundedness,
     LesserThan,
     LesserThanEquals,
-    LLMCheckResult,
-    LLMJudge,
     NotEquals,
     SemanticSimilarity,
     StringMatching,
@@ -38,6 +33,13 @@
 from .core.interaction import BaseInteractionSpec
 from .generators.user import UserSimulator
 from .interaction import InteractionSpec
+from .judges import (
+    BaseLLMCheck,
+    Conformity,
+    Groundedness,
+    LLMCheckResult,
+    LLMJudge,
+)
 from .scenarios.builder import ScenarioBuilder, scenario
 from .scenarios.runner import ScenarioRunner
 from .settings import get_default_generator, set_default_generator
@@ -61,6 +63,7 @@
 __all__ = [
     # Modules
     "builtin",
+    "judges",
     # Core classes
     "Check",
     "CheckResult",

libs/giskard-checks/src/giskard/checks/builtin/__init__.py

@@ -1,6 +1,9 @@
 """Built-in check implementations and helpers."""
 
-from .base import BaseLLMCheck, LLMCheckResult
+# Import judge checks from new location and re-export for backward compatibility
+from ..judges import BaseLLMCheck, Conformity, Groundedness, LLMCheckResult, LLMJudge
+
+# Import comparison checks (staying in builtin)
 from .comparison import (
     Equals,
     GreaterEquals,
@@ -9,10 +12,9 @@
     LesserThanEquals,
     NotEquals,
 )
-from .conformity import Conformity
+
+# Import other builtin checks (staying in builtin)
 from .fn import FnCheck, from_fn
-from .groundedness import Groundedness
-from .judge import LLMJudge
 from .semantic_similarity import SemanticSimilarity
 from .string_matching import StringMatching
 

libs/giskard-checks/src/giskard/checks/judges/__init__.py

@@ -0,0 +1,14 @@
+"""LLM-based judge checks for evaluating interactions."""
+
+from .base import BaseLLMCheck, LLMCheckResult
+from .conformity import Conformity
+from .groundedness import Groundedness
+from .judge import LLMJudge
+
+__all__ = [
+    "BaseLLMCheck",
+    "LLMCheckResult",
+    "Conformity",
+    "Groundedness",
+    "LLMJudge",
+]

libs/giskard-checks/src/giskard/checks/builtin/conformity.py → libs/giskard-checks/src/giskard/checks/judges/conformity.py

@@ -51,7 +51,7 @@ class Conformity[InputType, OutputType, TraceType: Trace](  # pyright: ignore[re
     @override
     def get_prompt(self) -> TemplateReference:
         """Return the Jinja2 template name for conformity evaluation."""
-        return TemplateReference(template_name="giskard.checks::checks/conformity.j2")
+        return TemplateReference(template_name="giskard.checks::judges/conformity.j2")
 
     @override
     async def get_inputs(self, trace: Trace[InputType, OutputType]) -> dict[str, str]:

libs/giskard-checks/src/giskard/checks/builtin/groundedness.py → libs/giskard-checks/src/giskard/checks/judges/groundedness.py

@@ -65,7 +65,7 @@ class Groundedness[InputType, OutputType, TraceType: Trace](  # pyright: ignore[
 
     @override
     def get_prompt(self) -> TemplateReference:
-        return TemplateReference(template_name="giskard.checks::checks/groundedness.j2")
+        return TemplateReference(template_name="giskard.checks::judges/groundedness.j2")
 
     @override
     async def get_inputs(self, trace: Trace[InputType, OutputType]) -> dict[str, str]: