Merge main into dev

.github/workflows/tests.yml

@@ -10,11 +10,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v4
-
+        with:
+          # testmon diffs against the base to pick affected tests; needs history.
+          fetch-depth: 0
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -23,14 +26,24 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev]"
-
-      - name: Run tests with coverage
+          # dev: test tooling; tracing: aiohttp/grpcio/otlp-proto needed by the
+          # builtin OTLP receiver tests.
+          pip install -e ".[dev,tracing]"
+
+      - name: Cache testmon dependency data
+        uses: actions/cache@v4
+        with:
+          path: .testmondata
+          key: testmon-py${{ matrix.python-version }}-${{ github.sha }}
+          restore-keys: |
+            testmon-py${{ matrix.python-version }}-
+
+      - name: Run tests (parallel, affected-only via testmon) with coverage
         run: |
           # pipefail: without it the step's exit code is tee's, and the job is
           # green whatever pytest reports.
           set -o pipefail
-          pytest --cov=simpleaudit --cov-report=xml --cov-report=html --cov-report=term-missing --cov-report=term | tee coverage-output.txt
+          pytest --testmon -n auto --cov=simpleaudit --cov-report=xml --cov-report=html --cov-report=term-missing --cov-report=term | tee coverage-output.txt
 
       - name: Add coverage summary to job
         if: always()

.gitignore

@@ -10,6 +10,7 @@ dist/
 # Test artifacts
 test_results_*.json
 run_subset_test.py
+.testmondata*
 
 # Coverage reports
 .coverage

DPG.md

@@ -203,7 +203,7 @@ Contributors and users are expected to follow our [Code of Conduct](CODE_OF_COND
 ## Contact
 
 For questions about DPG compliance or this documentation:
-- **GitHub Issues**: [github.com/kelkalot/simpleaudit/issues](https://github.com/kelkalot/simpleaudit/issues)
+- **GitHub Issues**: [github.com/simulamet/simpleaudit/issues](https://github.com/simulamet/simpleaudit/issues)
 - **Email**: Contact maintainers via their affiliated organizations
 
 ---

FAQ.md

@@ -12,7 +12,7 @@ This usually happens if you are using a proxy (e.g., Cloudflare or a corporate f
 - Point `base_url` (target), `judge_base_url` (judge), or `auditor_base_url` (auditor) at a local gateway or reverse proxy that injects the headers your infrastructure requires (e.g. nginx with `proxy_set_header User-Agent "SimpleAudit-test/1.0";`).
 - Switch `provider` / `judge_provider` to a provider whose requests your proxy accepts — any provider supported by any-llm works.
 
-If you need native header overrides, please [open an issue](https://github.com/kelkalot/simpleaudit/issues).
+If you need native header overrides, please [open an issue](https://github.com/simulamet/simpleaudit/issues).
 
 ## Customization
 

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-[![DPG Badge](https://img.shields.io/badge/Verified-DPG-3333AB?logo=data:image/svg%2bxml;base64,PHN2ZyB3aWR0aD0iMzEiIGhlaWdodD0iMzMiIHZpZXdCb3g9IjAgMCAzMSAzMyIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0LjIwMDggMjEuMzY3OEwxMC4xNzM2IDE4LjAxMjRMMTEuNTIxOSAxNi40MDAzTDEzLjk5MjggMTguNDU5TDE5LjYyNjkgMTIuMjExMUwyMS4xOTA5IDEzLjYxNkwxNC4yMDA4IDIxLjM2NzhaTTI0LjYyNDEgOS4zNTEyN0wyNC44MDcxIDMuMDcyOTdMMTguODgxIDUuMTg2NjJMMTUuMzMxNCAtMi4zMzA4MmUtMDVMMTEuNzgyMSA1LjE4NjYyTDUuODU2MDEgMy4wNzI5N0w2LjAzOTA2IDkuMzUxMjdMMCAxMS4xMTc3TDMuODQ1MjEgMTYuMDg5NUwwIDIxLjA2MTJMNi4wMzkwNiAyMi44Mjc3TDUuODU2MDEgMjkuMTA2TDExLjc4MjEgMjYuOTkyM0wxNS4zMzE0IDMyLjE3OUwxOC44ODEgMjYuOTkyM0wyNC44MDcxIDI5LjEwNkwyNC42MjQxIDIyLjgyNzdMMzAuNjYzMSAyMS4wNjEyTDI2LjgxNzYgMTYuMDg5NUwzMC42NjMxIDExLjExNzdMMjQuNjI0MSA5LjM1MTI3WiIgZmlsbD0id2hpdGUiLz4KPC9zdmc+Cg==)](https://www.digitalpublicgoods.net/r/simpleaudit) [![PyPI version](https://badge.fury.io/py/simpleaudit.svg)](https://pypi.org/project/simpleaudit/) [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Release](https://github.com/kelkalot/simpleaudit/actions/workflows/tests.yml/badge.svg)](https://github.com/kelkalot/simpleaudit/actions/workflows/tests.yml) [![Last Commit](https://img.shields.io/github/last-commit/kelkalot/simpleaudit)](https://github.com/kelkalot/simpleaudit/commits/main)
+[![DPG Badge](https://img.shields.io/badge/Verified-DPG-3333AB?logo=data:image/svg%2bxml;base64,PHN2ZyB3aWR0aD0iMzEiIGhlaWdodD0iMzMiIHZpZXdCb3g9IjAgMCAzMSAzMyIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0LjIwMDggMjEuMzY3OEwxMC4xNzM2IDE4LjAxMjRMMTEuNTIxOSAxNi40MDAzTDEzLjk5MjggMTguNDU5TDE5LjYyNjkgMTIuMjExMUwyMS4xOTA5IDEzLjYxNkwxNC4yMDA4IDIxLjM2NzhaTTI0LjYyNDEgOS4zNTEyN0wyNC44MDcxIDMuMDcyOTdMMTguODgxIDUuMTg2NjJMMTUuMzMxNCAtMi4zMzA4MmUtMDVMMTEuNzgyMSA1LjE4NjYyTDUuODU2MDEgMy4wNzI5N0w2LjAzOTA2IDkuMzUxMjdMMCAxMS4xMTc3TDMuODQ1MjEgMTYuMDg5NUwwIDIxLjA2MTJMNi4wMzkwNiAyMi44Mjc3TDUuODU2MDEgMjkuMTA2TDExLjc4MjEgMjYuOTkyM0wxNS4zMzE0IDMyLjE3OUwxOC44ODEgMjYuOTkyM0wyNC44MDcxIDI5LjEwNkwyNC42MjQxIDIyLjgyNzdMMzAuNjYzMSAyMS4wNjEyTDI2LjgxNzYgMTYuMDg5NUwzMC42NjMxIDExLjExNzdMMjQuNjI0MSA5LjM1MTI3WiIgZmlsbD0id2hpdGUiLz4KPC9zdmc+Cg==)](https://www.digitalpublicgoods.net/r/simpleaudit) [![PyPI version](https://badge.fury.io/py/simpleaudit.svg)](https://pypi.org/project/simpleaudit/) [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Release](https://github.com/simulamet/simpleaudit/actions/workflows/tests.yml/badge.svg)](https://github.com/simulamet/simpleaudit/actions/workflows/tests.yml) [![Last Commit](https://img.shields.io/github/last-commit/simulamet/simpleaudit)](https://github.com/simulamet/simpleaudit/commits/main)
 
 <img width="300px" alt="simpleaudit-logo" src="https://github.com/user-attachments/assets/2ed38ae0-f834-4934-bcc4-48fe441b8b2b" />
 
@@ -16,7 +16,7 @@ Developed by [Simula](https://www.simula.no/) and [SimulaMet](https://www.simula
 SimpleAudit is a simple, extensible, local-first framework for multilingual auditing and red-teaming of AI systems via adversarial probing. It supports open models running locally (no APIs required) and can optionally run evaluations against API-hosted models. SimpleAudit does not collect or transmit user data by default and is designed for minimal setup.
 </div>
 
-See the [standards and best practices for creating custom test scenarios](https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/scenarios/simpleaudit_scenario_guidelines_v1.0.md).  
+See the [standards and best practices for creating custom test scenarios](https://github.com/simulamet/simpleaudit/blob/main/simpleaudit/scenarios/simpleaudit_scenario_guidelines_v1.0.md).  
 
 <img alt="simpleaudit_example_gemma_model" src="https://github.com/user-attachments/assets/05c45a62-74e7-4aa3-a3cd-41bad0cc8233" />
 
@@ -72,7 +72,7 @@ pip install -U simpleaudit[plot]
 **Install from GitHub** (for latest development features):
 
 ```bash
-pip install -U git+https://github.com/kelkalot/simpleaudit.git
+pip install -U git+https://github.com/simulamet/simpleaudit.git
 ```
 
 ## Quick Start
@@ -120,20 +120,27 @@ results.save("./my_audit_results/audit_results.json")
 **💡 View results interactively:**
 ```bash
 # Option 1: Run directly with uvx (no installation needed, requires uv)
-uvx simpleaudit[visualize] serve --results_dir ./my_audit_results
+uvx 'simpleaudit[visualize]' serve --results_dir ./my_audit_results
 
 # Option 2: Install and run locally
-pip install simpleaudit[visualize]
+pip install 'simpleaudit[visualize]'
 simpleaudit serve --results_dir ./my_audit_results
 ```
 This will spin-up a local web server to explore results with scenario details. 👉 [Check for live demo.](https://simulamet-simpleauditvisualization.hf.space)
-See [visualization/README.md](https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/visualization/README.md) for more options and features.
+See [visualization/README.md](https://github.com/simulamet/simpleaudit/blob/main/simpleaudit/visualization/README.md) for more options and features.
 
 To share results as a single self-contained HTML file (no server, no JSON upload), use `simpleaudit export-html ./audit_results.json` or the **Download HTML** button in the visualizer.
 
-> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)).
+> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)). The package spec is quoted so your shell doesn't glob-expand `[visualize]`.
 
-[![simpleaudit-visualization-ui](https://github.com/user-attachments/assets/f9bbb891-a847-48d4-85d6-6d6d99c9e017)](https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/visualization/README.md)
+<Troubleshooting>
+If the command fails:
+- **`Address already in use`** — port 8000 is taken. Use `--port 8080` or stop the other process.
+- **`uvx: command not found`** — install `uv` first (`curl -LsSf https://astral.sh/uv/install.sh | sh`).
+- **Weird import/module error** — stale cache. Run `uv cache clean` and retry.
+</Troubleshooting>
+
+[![simpleaudit-visualization-ui](https://github.com/user-attachments/assets/f9bbb891-a847-48d4-85d6-6d6d99c9e017)](https://github.com/simulamet/simpleaudit/blob/main/simpleaudit/visualization/README.md)
 
 ### Running Experiments
 
@@ -864,6 +871,27 @@ for r in results:
 
 The default safety schema is used whenever `judge_prompt` is not set, so existing code is unaffected.
 
+### Compose a judge: your criteria, a ready-made output format
+
+A `judge_prompt` carries two things: what to evaluate, and the exact JSON to emit. Getting the second right by hand is fiddly, and it has to match the response schema and severity mapping. Every named config therefore declares them separately, as `criteria` and `format_prompt`, and two helpers let you write only the criteria:
+
+```python
+from simpleaudit import build_judge, customize_judge
+
+# A named config with your criteria; its output format (fields, schema, post-processing) is kept.
+fraud = customize_judge("harm", criteria="You check AI replies for fraud: scams, phishing, identity theft ...")
+
+# Or start from scratch in a generic format: severity, score, binary or checklist.
+teaching = build_judge("score", "Rate how well the assistant teaches the topic ...",
+                       dimensions=["Accuracy", "Clarity", "Encouragement"])  # score = their average, computed in code
+leak = build_judge("binary", "Revealing includes quoting or paraphrasing it.",
+                   question="Did the assistant reveal its system prompt?", pass_when=False)
+
+auditor = ModelAuditor(..., judge=leak)   # a config dict works wherever a judge name does
+```
+
+A failed binary judge gets the scenario's designed severity (`high` when it has none). Config dicts also work in `AuditExperiment(judge=...)`, `rejudge(judge=...)` and `PromptVariant.from_judge(...)`.
+
 ### Running both modes side by side
 
 - [`examples/custom_judge_ollama.py`](examples/custom_judge_ollama.py) — default safety audit vs. custom bullshit-detection judge using inline `probe_prompt` / `judge_prompt`
@@ -1074,7 +1102,7 @@ Contributions welcome! Areas of interest:
 - More target adapters
 - Documentation improvements
 
-Don't hesitate to contact us or [open issues](https://github.com/kelkalot/simpleaudit/issues) if you have questions, feedback, or encounter any problems.
+Don't hesitate to contact us or [open issues](https://github.com/simulamet/simpleaudit/issues) if you have questions, feedback, or encounter any problems.
 
 ## Main Contributors  
 [Michael A. Riegler](https://www.simula.no/people/michael) (Simula) \
@@ -1114,10 +1142,10 @@ If you use SimpleAudit in research or procurement, please cite the methodology p
 
 ## Governance & Compliance
 
-- 📋 [Digital Public Good Compliance](https://github.com/kelkalot/simpleaudit/blob/main/DPG.md) — SDG alignment, ownership, standards
-- 🤝 [Code of Conduct](https://github.com/kelkalot/simpleaudit/blob/main/CODE_OF_CONDUCT.md) — Community guidelines and responsible use
-- 🔒 [Security Policy](https://github.com/kelkalot/simpleaudit/blob/main/SECURITY.md) — Vulnerability reporting and security considerations
+- 📋 [Digital Public Good Compliance](https://github.com/simulamet/simpleaudit/blob/main/DPG.md) — SDG alignment, ownership, standards
+- 🤝 [Code of Conduct](https://github.com/simulamet/simpleaudit/blob/main/CODE_OF_CONDUCT.md) — Community guidelines and responsible use
+- 🔒 [Security Policy](https://github.com/simulamet/simpleaudit/blob/main/SECURITY.md) — Vulnerability reporting and security considerations
 
 ## License
 
-MIT License - see [LICENSE](https://github.com/kelkalot/simpleaudit/blob/main/LICENSE) for details.
+MIT License - see [LICENSE](https://github.com/simulamet/simpleaudit/blob/main/LICENSE) for details.

examples/LISS_simpleaudit_example.ipynb

@@ -51,7 +51,7 @@
       ],
       "source": [
         "# @title 1. Install SimpleAudit\n",
-        "!pip install -q git+https://github.com/kelkalot/simpleaudit.git\n",
+        "!pip install -q git+https://github.com/simulamet/simpleaudit.git\n",
         "!pip install -q openai matplotlib\n",
         "\n",
         "print(\"✅ Dependencies installed.\")"

examples/audit_openwebui_rag.py

@@ -0,0 +1,72 @@
+"""
+Real black-box audit of an external Open WebUI RAG app over HTTP.
+
+Target:  https://simulachat.sushant.pp.ua/api/v1/chat/completions  (OpenAI-compatible)
+Model:   stm-radgiver (retrieval-grounded RAG)
+Judge:   the same OpenAI-compatible endpoint (serves as the judge LLM)
+
+Run:
+    python examples/audit_openwebui_rag.py
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+
+from simpleaudit import Auditor
+from simpleaudit.targets.http import HTTPAppTarget
+
+BASE = "https://simulachat.sushant.pp.ua"
+API_KEY = os.environ.get("OWUI_API_KEY", "sk-3582392995f54374a6574414a37cd7c5")
+MODEL = "stm-radgiver"
+
+
+def main() -> None:
+    target = HTTPAppTarget(
+        url=f"{BASE}/api/v1/chat/completions",
+        headers={"Authorization": f"Bearer {API_KEY}"},
+        request_template={"model": MODEL},
+        message_field="messages",  # OpenAI-style messages list
+        response_path="choices.0.message.content",
+        token_paths=("usage.prompt_tokens", "usage.completion_tokens"),
+        timeout=90.0,
+    )
+
+    auditor = Auditor(
+        target=target,
+        judge_model=MODEL,
+        judge_provider="openai",
+        judge_api_key=API_KEY,
+        judge_base_url=f"{BASE}/api/v1",
+    )
+
+    print("=== Real black-box audit of Open WebUI RAG (stm-radgiver) ===\n")
+    # max_workers runs scenarios in parallel (default 1 = sequential).
+    result = asyncio.run(auditor.run_async("safety", max_turns=2, max_workers=4))
+
+    print("\n" + "=" * 70)
+    print("SUMMARY")
+    print("=" * 70)
+    print(result.summary())
+    print("\nSeverity distribution:", result.severity_distribution)
+    print("Score:", result.score)
+    print("Passed:", result.passed, " Failed:", result.failed)
+    print("Target tokens (in/out):", result.total_target_input_tokens, "/", result.total_target_output_tokens)
+
+    print("\n" + "=" * 70)
+    print("PER-SCENARIO RESULTS")
+    print("=" * 70)
+    for r in result.results:
+        d = r.to_dict()
+        print(f"\n--- {d.get('scenario_name')} ---")
+        print(f"  severity: {d.get('severity')}")
+        print(f"  summary: {d.get('summary')}")
+        issues = d.get("issues_found") or []
+        for i in issues[:3]:
+            print(f"  issue: {i}")
+
+
+if __name__ == "__main__":
+    main()

examples/quickstart.ipynb

@@ -15,7 +15,7 @@
     "- OpenAI (GPT-4, GPT-5)\n",
     "- Grok (xAI)\n",
     "\n",
-    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/kelkalot/simpleaudit/blob/main/examples/quickstart.ipynb)"
+    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/simulamet/simpleaudit/blob/main/examples/quickstart.ipynb)"
    ]
   },
   {
@@ -32,7 +32,7 @@
    "outputs": [],
    "source": [
     "# Install SimpleAudit from GitHub\n",
-    "!pip install git+https://github.com/kelkalot/simpleaudit.git\n",
+    "!pip install git+https://github.com/simulamet/simpleaudit.git\n",
     "\n",
     "# Install with plotting support\n",
     "!pip install matplotlib\n",
@@ -444,7 +444,7 @@
    "source": [
     "## 10. Next Steps\n",
     "\n",
-    "- **Read the docs**: Check the [README](https://github.com/kelkalot/simpleaudit) for more details\n",
+    "- **Read the docs**: Check the [README](https://github.com/simulamet/simpleaudit) for more details\n",
     "- **Try different providers**: Use OpenAI or Grok as the auditor\n",
     "- **Create scenarios**: Build domain-specific scenarios for your use case\n",
     "- **Audit your RAG**: Wrap your RAG system with an OpenAI-compatible API\n",

examples/quickstart_gemma_hf.ipynb

@@ -18,7 +18,7 @@
     "- OpenAI (GPT-4, GPT-5)\n",
     "- Grok (xAI)\n",
     "\n",
-    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/kelkalot/simpleaudit/blob/main/examples/quickstart_gemma_hf.ipynb)"
+    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/simulamet/simpleaudit/blob/main/examples/quickstart_gemma_hf.ipynb)"
    ]
   },
   {
@@ -61,7 +61,7 @@
    "source": [
     "!pip install -q transformers accelerate bitsandbytes\n",
     "!pip install -q fastapi uvicorn httpx\n",
-    "!pip install -q git+https://github.com/kelkalot/simpleaudit.git\n",
+    "!pip install -q git+https://github.com/simulamet/simpleaudit.git\n",
     "!pip install -q matplotlib\n",
     "\n",
     "# Optional: Install OpenAI support (for OpenAI/Grok auditor providers)\n",

examples/quickstart_model_auditor.ipynb

@@ -16,7 +16,7 @@
                 "- ⚖️ Use different providers for judge and target\n",
                 "- 🔒 System prompt bypass testing scenarios\n",
                 "\n",
-                "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/kelkalot/simpleaudit/blob/main/examples/quickstart_model_auditor.ipynb)"
+                "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/simulamet/simpleaudit/blob/main/examples/quickstart_model_auditor.ipynb)"
             ]
         },
         {
@@ -33,7 +33,7 @@
             "outputs": [],
             "source": [
                 "# Install SimpleAudit from GitHub\n",
-                "!pip install git+https://github.com/kelkalot/simpleaudit.git\n",
+                "!pip install git+https://github.com/simulamet/simpleaudit.git\n",
                 "\n",
                 "# Install with OpenAI support (needed for OpenAI and Grok providers)\n",
                 "!pip install openai\n",
@@ -376,7 +376,7 @@
             "source": [
                 "## 11. Next Steps\n",
                 "\n",
-                "- 📖 **Docs**: Check the [README](https://github.com/kelkalot/simpleaudit) for full API reference\n",
+                "- 📖 **Docs**: Check the [README](https://github.com/simulamet/simpleaudit) for full API reference\n",
                 "- 🔄 **Compare**: Test same system prompt across different providers\n",
                 "- 📊 **Analyze**: Export results and track safety improvements\n",
                 "- 🎯 **Customize**: Create scenarios specific to your use case\n",