Evaluating Opik's Moderation Metric

For this guide we will be evaluating the Moderation metric included in the LLM Evaluation SDK which will showcase both how to use the evaluation functionality in the platform as well as the quality of the Moderation metric included in the SDK.

Creating an account on Comet.com

Comet provides a hosted version of the Opik platform, simply create an account and grab you API Key.

You can also run the Opik platform locally, see the installation guide for more information.

1 %pip install --upgrade --quiet opik pandas

1 import opik
2 
3 opik.configure(use_local=False)

Preparing our environment

First, we will configure the OpenAI API key and download a reference moderation dataset.

1 import os
2 import getpass
3 
4 if "OPENAI_API_KEY" not in os.environ:
5     os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter your OpenAI API key: ")

We will be using the OpenAI Moderation API Release dataset which according to this blog post GPT-4o detects 60% of hallucinations. The first step will be to create a dataset in the platform so we can keep track of the results of the evaluation.

Since the insert methods in the SDK deduplicates items, we can insert 50 items and if the items already exist, Opik will automatically remove them.

1 # Create dataset
2 import opik
3 import pandas as pd
4 import requests
5 from io import BytesIO
6 
7 client = opik.Opik()
8 
9 # Create dataset
10 dataset = client.get_or_create_dataset(
11     name="OpenAIModerationDataset", description="OpenAI Moderation Dataset"
12 )
13 
14 # Insert items into dataset
15 url = "https://github.com/openai/moderation-api-release/raw/main/data/samples-1680.jsonl.gz"
16 response = requests.get(url)
17 df = pd.read_json(BytesIO(response.content), lines=True, compression="gzip")
18 
19 df = df.sample(n=50, random_state=42)
20 
21 dataset_records = []
22 for x in df.to_dict(orient="records"):
23     moderation_fields = ["S", "H", "V", "HR", "SH", "S3", "H2", "V2"]
24     moderated_fields = [field for field in moderation_fields if x[field] == 1.0]
25     expected_output = "moderated" if moderated_fields else "not_moderated"
26 
27     dataset_records.append(
28         {
29             "output": x["prompt"],
30             "expected_output": expected_output,
31             "moderated_fields": moderated_fields,
32         }
33     )
34 
35 dataset.insert(dataset_records)

Evaluating the moderation metric

In order to evaluate the performance of the Opik moderation metric, we will define:

Evaluation task: Our evaluation task will use the data in the Dataset to return a moderation score computed using the Opik moderation metric.
Scoring metric: We will use the Equals metric to check if the moderation score computed matches the expected output.

By defining the evaluation task in this way, we will be able to understand how well Opik’s moderation metric is able to detect moderation violations in the dataset.

We can use the Opik SDK to compute a moderation score for each item in the dataset:

1 from opik.evaluation.metrics import Moderation, Equals
2 from opik.evaluation import evaluate
3 from opik import Opik
4 from opik.evaluation.metrics.llm_judges.moderation.template import generate_query
5 from typing import Dict
6 
7 
8 # Define the evaluation task
9 def evaluation_task(x: Dict):
10     metric = Moderation()
11     try:
12         metric_score = metric.score(output=x["output"])
13         moderation_score = "moderated" if metric_score.value > 0.5 else "not_moderated"
14         moderation_reason = metric_score.reason
15     except Exception as e:
16         print(e)
17         moderation_score = None
18         moderation_reason = str(e)
19 
20     return {
21         "moderation_score": moderation_score,
22         "moderation_reason": moderation_reason,
23     }
24 
25 
26 # Get the dataset
27 client = Opik()
28 dataset = client.get_dataset(name="OpenAIModerationDataset")
29 
30 # Define the scoring metric
31 moderation_metric = Equals(name="Correct moderation score")
32 
33 # Add the prompt template as an experiment configuration
34 experiment_config = {
35     "prompt_template": generate_query(output="{output}", few_shot_examples=[])
36 }
37 
38 res = evaluate(
39     dataset=dataset,
40     task=evaluation_task,
41     scoring_metrics=[moderation_metric],
42     experiment_config=experiment_config,
43     scoring_key_mapping={"reference": "expected_output", "output": "moderation_score"},
44 )

We are able to detect ~85% of moderation violations, this can be improved further by providing some additional examples to the model. We can view a breakdown of the results in the Opik UI:

Moderation Evaluation

1	import os
2	import getpass
3
4	if "OPENAI_API_KEY" not in os.environ:
5	os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter your OpenAI API key: ")

1	# Create dataset
2	import opik
3	import pandas as pd
4	import requests
5	from io import BytesIO
6
7	client = opik.Opik()
8
9	# Create dataset
10	dataset = client.get_or_create_dataset(
11	name="OpenAIModerationDataset", description="OpenAI Moderation Dataset"
12	)
13
14	# Insert items into dataset
15	url = "https://github.com/openai/moderation-api-release/raw/main/data/samples-1680.jsonl.gz"
16	response = requests.get(url)
17	df = pd.read_json(BytesIO(response.content), lines=True, compression="gzip")
18
19	df = df.sample(n=50, random_state=42)
20
21	dataset_records = []
22	for x in df.to_dict(orient="records"):
23	moderation_fields = ["S", "H", "V", "HR", "SH", "S3", "H2", "V2"]
24	moderated_fields = [field for field in moderation_fields if x[field] == 1.0]
25	expected_output = "moderated" if moderated_fields else "not_moderated"
26
27	dataset_records.append(
28	{
29	"output": x["prompt"],
30	"expected_output": expected_output,
31	"moderated_fields": moderated_fields,
32	}
33	)
34
35	dataset.insert(dataset_records)

1	from opik.evaluation.metrics import Moderation, Equals
2	from opik.evaluation import evaluate
3	from opik import Opik
4	from opik.evaluation.metrics.llm_judges.moderation.template import generate_query
5	from typing import Dict
6
7
8	# Define the evaluation task
9	def evaluation_task(x: Dict):
10	metric = Moderation()
11	try:
12	metric_score = metric.score(output=x["output"])
13	moderation_score = "moderated" if metric_score.value > 0.5 else "not_moderated"
14	moderation_reason = metric_score.reason
15	except Exception as e:
16	print(e)
17	moderation_score = None
18	moderation_reason = str(e)
19
20	return {
21	"moderation_score": moderation_score,
22	"moderation_reason": moderation_reason,
23	}
24
25
26	# Get the dataset
27	client = Opik()
28	dataset = client.get_dataset(name="OpenAIModerationDataset")
29
30	# Define the scoring metric
31	moderation_metric = Equals(name="Correct moderation score")
32
33	# Add the prompt template as an experiment configuration
34	experiment_config = {
35	"prompt_template": generate_query(output="{output}", few_shot_examples=[])
36	}
37
38	res = evaluate(
39	dataset=dataset,
40	task=evaluation_task,
41	scoring_metrics=[moderation_metric],
42	experiment_config=experiment_config,
43	scoring_key_mapping={"reference": "expected_output", "output": "moderation_score"},
44	)