Observability for AWS Bedrock with Opik

1
import boto3
2
3
REGION = "us-east-1"
4
MODEL_ID = "us.meta.llama3-2-3b-instruct-v1:0"
5
6
bedrock_client = boto3.client(
7
    service_name="bedrock-runtime",
8
    region_name=REGION,
9
    # aws_access_key_id=ACCESS_KEY,
10
    # aws_secret_access_key=SECRET_KEY,
11
    # aws_session_token=SESSION_TOKEN,
12
)

1
import os
2
from opik.integrations.bedrock import track_bedrock
3
4
# Set project name via environment variable
5
os.environ["OPIK_PROJECT_NAME"] = "bedrock-integration-demo"
6
7
bedrock_client = track_bedrock(bedrock_client)

1
import os
2
import boto3
3
from opik.integrations.bedrock import track_bedrock
4
5
# Set project name via environment variable
6
os.environ["OPIK_PROJECT_NAME"] = "bedrock-integration-demo"
7
8
# Initialize and track the Bedrock client
9
bedrock_client = boto3.client("bedrock-runtime", region_name="us-east-1")
10
bedrock_client = track_bedrock(bedrock_client)
11
12
PROMPT = "Why is it important to use a LLM Monitoring like CometML Opik tool that allows you to log traces and spans when working with LLM Models hosted on AWS Bedrock?"
13
14
response = bedrock_client.converse(
15
    modelId="us.meta.llama3-2-3b-instruct-v1:0",
16
    messages=[{"role": "user", "content": [{"text": PROMPT}]}],
17
    inferenceConfig={"temperature": 0.5, "maxTokens": 512, "topP": 0.9},
18
)
19
print("Response", response["output"]["message"]["content"][0]["text"])

1
import os
2
import boto3
3
from opik.integrations.bedrock import track_bedrock
4
5
# Set project name via environment variable
6
os.environ["OPIK_PROJECT_NAME"] = "bedrock-integration-demo"
7
8
# Initialize and track the Bedrock client
9
bedrock_client = boto3.client("bedrock-runtime", region_name="us-east-1")
10
bedrock_client = track_bedrock(bedrock_client)
11
12
def stream_conversation(
13
    bedrock_client,
14
    model_id,
15
    messages,
16
    system_prompts,
17
    inference_config,
18
):
19
    """
20
    Sends messages to a model and streams the response using Converse API.
21
    Args:
22
        bedrock_client: The Boto3 Bedrock runtime client.
23
        model_id (str): The model ID to use.
24
        messages (JSON) : The messages to send.
25
        system_prompts (JSON) : The system prompts to send.
26
        inference_config (JSON) : The inference configuration to use.
27
28
    Returns:
29
        Nothing.
30
    """
31
    response = bedrock_client.converse_stream(
32
        modelId=model_id,
33
        messages=messages,
34
        system=system_prompts,
35
        inferenceConfig=inference_config,
36
    )
37
38
    stream = response.get("stream")
39
    if stream:
40
        for event in stream:
41
            if "messageStart" in event:
42
                print(f"\nRole: {event['messageStart']['role']}")
43
44
            if "contentBlockDelta" in event:
45
                print(event["contentBlockDelta"]["delta"]["text"], end="")
46
47
            if "messageStop" in event:
48
                print(f"\nStop reason: {event['messageStop']['stopReason']}")
49
50
            if "metadata" in event:
51
                metadata = event["metadata"]
52
                if "usage" in metadata:
53
                    print("\nToken usage")
54
                    print(f"Input tokens: {metadata['usage']['inputTokens']}")
55
                    print(f"Output tokens: {metadata['usage']['outputTokens']}")
56
                    print(f"Total tokens: {metadata['usage']['totalTokens']}")
57
58
# Example usage
59
system_prompt = """You are an app that creates playlists for a radio station
60
  that plays rock and pop music. Only return song names and the artist."""
61
62
input_text = "Create a list of 3 pop songs."
63
messages = [{"role": "user", "content": [{"text": input_text}]}]
64
system_prompts = [{"text": system_prompt}]
65
inference_config = {"temperature": 0.5, "topP": 0.9}
66
67
stream_conversation(
68
    bedrock_client,
69
    "us.meta.llama3-2-3b-instruct-v1:0",
70
    messages,
71
    system_prompts,
72
    inference_config,
73
)

1
import boto3
2
from opik import track
3
from opik.integrations.bedrock import track_bedrock
4
5
# Initialize and track the Bedrock client
6
bedrock_client = boto3.client("bedrock-runtime", region_name="us-east-1")
7
bedrock_client = track_bedrock(bedrock_client, project_name="bedrock-integration-demo")
8
9
MODEL_ID = "us.anthropic.claude-3-5-sonnet-20241022-v2:0"
10
11
@track
12
def generate_story(prompt):
13
    res = bedrock_client.converse(
14
        modelId=MODEL_ID, 
15
        messages=[{"role": "user", "content": [{"text": prompt}]}],
16
        inferenceConfig={"temperature": 0.7, "maxTokens": 1000}
17
    )
18
    return res["output"]["message"]["content"][0]["text"]
19
20
@track
21
def generate_topic():
22
    prompt = "Generate a topic for a story about Opik."
23
    res = bedrock_client.converse(
24
        modelId=MODEL_ID, 
25
        messages=[{"role": "user", "content": [{"text": prompt}]}],
26
        inferenceConfig={"temperature": 0.7, "maxTokens": 500}
27
    )
28
    return res["output"]["message"]["content"][0]["text"]
29
30
@track
31
def generate_opik_story():
32
    topic = generate_topic()
33
    story = generate_story(topic)
34
    return story
35
36
# Execute the multi-step pipeline
37
generate_opik_story()