Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .ai_config_defaults.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"security-agent": {
"enabled": true,
"model": {
"name": "claude-3-5-haiku-20241022",
"name": "claude-haiku-4-5-20251001",
"parameters": {}
},
"provider": {
Expand All @@ -20,7 +20,7 @@
"supervisor-agent": {
"enabled": true,
"model": {
"name": "claude-3-7-sonnet-latest",
"name": "claude-sonnet-4-6",
"parameters": {}
},
"provider": {
Expand All @@ -31,7 +31,7 @@
"support-agent": {
"enabled": true,
"model": {
"name": "claude-3-5-haiku-20241022",
"name": "claude-haiku-4-5-20251001",
"parameters": {
"tools": [
{
Expand Down
298 changes: 180 additions & 118 deletions README.md

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions api/segmentation_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ def main():
"user_context": {"country": "DE", "plan": "paid", "user_id": "user_eu_paid_001", "region": "europe"},
"expected_config": {
"support_agent": {
"model": "claude-3-7-sonnet-latest",
"model": "claude-sonnet-4-6",
"variation_key": "eu-paid",
"tools": ["search_v1", "search_v2", "reranking", "arxiv_search", "semantic_scholar"]
}
Expand All @@ -227,7 +227,7 @@ def main():
"user_context": {"country": "DE", "plan": "free", "user_id": "user_eu_free_001", "region": "europe"},
"expected_config": {
"support_agent": {
"model": "claude-3-5-haiku-20241022",
"model": "claude-haiku-4-5-20251001",
"variation_key": "eu-free",
"tools": ["search_v1"]
}
Expand Down
10 changes: 5 additions & 5 deletions bootstrap/ai_config_manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ project:
- key: "supervisor-basic"
modelConfig:
provider: "anthropic"
modelId: "claude-3-5-sonnet-20241022"
modelId: "claude-sonnet-4-6"
# NOTE: After bootstrap, modify instructions in LaunchDarkly UI, not here
instructions: "You are an intelligent routing supervisor for a multi-agent system. Your primary job is to assess whether user input likely contains PII (personally identifiable information) to determine the most efficient processing route.\n\n**PII Assessment:**\nAnalyze the user input and provide:\n- likely_contains_pii: boolean assessment\n- confidence: confidence score (0.0 to 1.0)\n- reasoning: clear explanation of your decision\n- recommended_route: either 'security_agent' or 'support_agent'\n\n**Route to SECURITY_AGENT** if the text likely contains:\n- Email addresses, phone numbers, addresses\n- Names (first/last names, usernames)\n- Financial information (credit cards, SSNs, account numbers)\n- Sensitive personal data\n\n**Route to SUPPORT_AGENT** if the text appears to be:\n- General questions without personal details\n- Technical queries\n- Search requests\n- Educational content requests\n\nAnalyze this user input and recommend the optimal route:"
customParameters:
Expand All @@ -123,7 +123,7 @@ project:
- key: "basic-security"
modelConfig:
provider: "anthropic"
modelId: "claude-3-5-haiku-20241022"
modelId: "claude-haiku-4-5-20251001"
# NOTE: After bootstrap, modify instructions in LaunchDarkly UI, not here
instructions: "You are a privacy agent that REMOVES direct PII. Focus on clearly personal identifiers:\n\nEmail addresses\nPhone numbers\nSocial Security Numbers\nFull names (but not generic titles)\nStreet addresses\nCredit card numbers\nDriver's license numbers\n\nResponse Format:\n\ndetected: true if any PII was found, false otherwise\ntypes: array of PII types found (e.g., ['email', 'name', 'phone'])\nredacted: the input text with PII replaced by [REDACTED], keeping the text readable and natural\n\nExamples:\n\nInput: \"I work at Acme Corp in Berlin as a manager\"\n\nOutput: detected=false, types=[], redacted='I work at Acme Corp in Berlin as a manager'\n\n\nInput: \"Contact John Smith at john@email.com or 555-1234\"\n\nOutput: detected=true, types=['name', 'email', 'phone'], redacted='Contact [REDACTED] at [REDACTED] or [REDACTED]'\n\n\nInput: \"The CEO from Microsoft contacted me\"\n\nOutput: detected=false, types=[], redacted='The CEO from Microsoft contacted me'"
customParameters:
Expand Down Expand Up @@ -157,7 +157,7 @@ project:
- key: "eu-free"
modelConfig:
provider: "anthropic"
modelId: "claude-3-5-haiku-20241022"
modelId: "claude-haiku-4-5-20251001"
tools: ["search_v1"]
# NOTE: After bootstrap, modify instructions in LaunchDarkly UI, not here
instructions: "You are a helpful assistant with access to basic search (search_v1). Use your search tool to find relevant information from the knowledge base while following EU privacy requirements. Always search before responding to ensure accuracy."
Expand All @@ -170,7 +170,7 @@ project:
- key: "eu-paid"
modelConfig:
provider: "anthropic"
modelId: "claude-3-5-sonnet-20241022"
modelId: "claude-sonnet-4-6"
tools: ["search_v1", "search_v2", "reranking", "arxiv_search", "semantic_scholar"]
instructions: "You are a premium research specialist with access to comprehensive RAG tools: search_v1 (basic search), search_v2 (semantic vector search), reranking (BM25 relevance scoring), arxiv_search (academic papers), and semantic_scholar (citation database). When using semantic_scholar, always request as few as possible and less than 5 results (num_results: 5) to optimize performance and avoid rate limiting. When search results are available, prioritize information from those results over your general knowledge. EU privacy laws require strict data handling. Always cite sources and explain confidence levels. Use multiple search approaches for complex queries."
customParameters:
Expand Down Expand Up @@ -208,7 +208,7 @@ project:
- key: "international-standard"
modelConfig:
provider: "anthropic" # Default to privacy-friendly
modelId: "claude-3-5-haiku-20241022"
modelId: "claude-haiku-4-5-20251001"
tools: ["search_v1", "search_v2", "reranking"]
instructions: "You are a helpful assistant with access to RAG tools: search_v1 (basic search), search_v2 (semantic vector search), and reranking (BM25 relevance scoring). When search results are available, prioritize information from those results over your general knowledge. Provide balanced, well-researched responses for international users."
customParameters:
Expand Down
12 changes: 6 additions & 6 deletions bootstrap/create_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,9 @@ def create_variation(self, project_key, config_key, variation_data):

# Map to correct LaunchDarkly model config keys
model_config_key_map = {
"claude-3-5-sonnet-20241022": "Anthropic.claude-3-7-sonnet-latest",
"claude-3-5-haiku-20241022": "Anthropic.claude-3-5-haiku-20241022",
"claude-opus-4-20250514": "Anthropic.claude-opus-4-20250514",
"claude-sonnet-4-6": "Anthropic.claude-sonnet-4-6",
"claude-haiku-4-5-20251001": "Anthropic.claude-haiku-4-5-20251001",
"claude-opus-4-7": "Anthropic.claude-opus-4-7",
"gpt-4o": "OpenAI.gpt-4o",
"gpt-4o-mini": "OpenAI.gpt-4o-mini-2024-07-18",
"mistral-small-latest": "Mistral.mistral-small-latest"
Expand Down Expand Up @@ -269,9 +269,9 @@ def update_variation(self, project_key, config_key, variation_data):

# Map to correct LaunchDarkly model config keys
model_config_key_map = {
"claude-3-5-sonnet-20241022": "Anthropic.claude-3-7-sonnet-latest",
"claude-3-5-haiku-20241022": "Anthropic.claude-3-5-haiku-20241022",
"claude-opus-4-20250514": "Anthropic.claude-opus-4-20250514",
"claude-sonnet-4-6": "Anthropic.claude-sonnet-4-6",
"claude-haiku-4-5-20251001": "Anthropic.claude-haiku-4-5-20251001",
"claude-opus-4-7": "Anthropic.claude-opus-4-7",
"gpt-4o": "OpenAI.gpt-4o",
"gpt-4o-mini": "OpenAI.gpt-4o-mini-2024-07-18",
"mistral-small-latest": "Mistral.mistral-small-latest"
Expand Down
4 changes: 2 additions & 2 deletions bootstrap/tutorial_3_experiment_variations.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def create_premium_model_variations(self) -> bool:
"name": "Claude Opus 4 Treatment",
"instructions": "You are a helpful assistant that can search documentation and research papers. When search results are available, prioritize information from those results over your general knowledge to provide the most accurate and up-to-date responses. Use available tools to search the knowledge base and external research databases to answer questions accurately and comprehensively.",
"model": {
"name": "claude-opus-4-20250514",
"name": "claude-opus-4-7",
"provider": "anthropic"
},
"tools": ["search_v1", "search_v2", "reranking", "arxiv_search", "semantic_scholar"],
Expand All @@ -96,7 +96,7 @@ def _create_variations(self, ai_config_key: str, variations: List[Dict[str, Any]
# Map model name to LaunchDarkly modelConfigKey
model_name = variation["model"]["name"]
model_config_key_map = {
"claude-opus-4-20250514": "Anthropic.claude-opus-4-20250514",
"claude-opus-4-7": "Anthropic.claude-opus-4-7",
}

model_config_key = model_config_key_map.get(model_name)
Expand Down
2 changes: 1 addition & 1 deletion config_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from pathlib import Path
import ldclient
from ldclient import Context
from ldai import LDAIClient, AIAgentConfigRequest, AIAgentConfigDefault, ModelConfig, ProviderConfig
from ldai import LDAIClient, AIAgentConfigDefault, ModelConfig, ProviderConfig
from ldai.tracker import FeedbackKind
from dotenv import load_dotenv
from utils.logger import log_student, log_debug
Expand Down
6 changes: 3 additions & 3 deletions tools/concurrent_traffic_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def evaluate_response(self, query, response):

try:
ai_response = self.claude.messages.create(
model="claude-3-haiku-20240307",
model="claude-haiku-4-5-20251001",
max_tokens=10,
messages=[{"role": "user", "content": prompt}]
)
Expand All @@ -88,7 +88,7 @@ def generate_base_topics(self):
"""Generate base RL topics"""
try:
response = self.claude.messages.create(
model="claude-3-5-haiku-20241022",
model="claude-haiku-4-5-20251001",
max_tokens=800,
messages=[{
"role": "user",
Expand Down Expand Up @@ -145,7 +145,7 @@ def generate_query(self, topic, inject_pii=False):

try:
response = self.claude.messages.create(
model="claude-3-5-haiku-20241022",
model="claude-haiku-4-5-20251001",
max_tokens=100,
messages=[{"role": "user", "content": prompt}]
)
Expand Down
Loading