Building upon Google's research Rich Human Feedback for Text-to-Image Generation we have collected over 1.5 million responses from 152'684 individual humans using Rapidata via the Python API. Collection took roughly 5 days.

If you get value from this dataset and would like to see more in the future, please consider liking it.

Overview

We asked humans to evaluate AI-generated images in style, coherence and prompt alignment. For images that contained flaws, participants were asked to identify specific problematic areas. Additionally, for all images, participants identified words from the prompts that were not accurately represented in the generated images.

If you want to replicate the annotation setup, the steps are outlined at the bottom.

This dataset and the annotation process is described in further detail in our blog post Beyond Image Preferences.

Usage Examples

Accessing this data is easy with the Huggingface dataset library. For quick demos or previews, we recommend setting streaming=True as downloading the whole dataset can take a while.

  from datasets import load_dataset

  ds = load_dataset("Rapidata/text-2-image-Rich-Human-Feedback", split="train", streaming=True)

As an example, below we show how to replicate the figures below.

Methods

from PIL import Image
from datasets import load_dataset
import cv2 
import numpy as np

def get_colors(words):
    colors = []
    for item in words:
        intensity = item / max(words)
        value = np.uint8((1 - intensity) * 255)
        color = tuple(map(int, cv2.applyColorMap(np.array([[value]]), cv2.COLORMAP_AUTUMN)[0][0]))
        colors.append(color)
    return colors

def get_wrapped_text(text_color_pairs, font, font_scale, thickness, word_spacing, max_width):
    wrapped_text_color_pairs, current_line, line_width = [], [], 0
    for text, color in text_color_pairs:
        text_size = cv2.getTextSize(text, font, font_scale, thickness)[0]
        if line_width + text_size[0] > max_width:
            wrapped_text_color_pairs.append(current_line)
            current_line, line_width = [], 0
        current_line.append((text, color, text_size))
        line_width += text_size[0] + word_spacing
    wrapped_text_color_pairs.append(current_line)
    return wrapped_text_color_pairs

def add_multicolor_text(input, text_color_pairs, font_scale=1, thickness=2, word_spacing=20):
    image = cv2.cvtColor(np.array(input), cv2.COLOR_RGB2BGR)
    image_height, image_width, _ = image.shape

    font = cv2.FONT_HERSHEY_SIMPLEX
    wrapped_text = get_wrapped_text(text_color_pairs, font, font_scale, thickness, word_spacing, int(image_width*0.95))

    position = (int(0.025*image_width), int(word_spacing*2))
    
    overlay = image.copy()
    cv2.rectangle(overlay, (0, 0), (image_width, int((len(wrapped_text)+1)*word_spacing*2)), (100,100,100), -1)
    out_img = cv2.addWeighted(overlay, 0.75, image, 0.25, 0)

    for idx, text_line in enumerate(wrapped_text):
        current_x, current_y = position[0], position[1] + int(idx*word_spacing*2)
        for text, color, text_size in text_line:
            cv2.putText(out_img, text, (current_x, current_y), font, font_scale, color, thickness)
            current_x += text_size[0] + word_spacing

    return Image.fromarray(cv2.cvtColor(out_img, cv2.COLOR_BGR2RGB))

Create figures

ds_words = ds.select_columns(["image","prompt", "word_scores"])

for example in ds_words.take(5):
    image = example["image"]
    prompt = example["prompt"]
    word_scores = [s[1] for s in eval(example["word_scores"])]
    words = [s[0] for s in eval(example["word_scores"])]
    colors = get_colors(word_scores)
    display(add_multicolor_text(image, list(zip(words, colors)), font_scale=1, thickness=2, word_spacing=20))

Methods

import cv2
import numpy as np
from PIL import Image

def overlay_heatmap(image, heatmap, alpha=0.3):
    cv2_image =  cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)   
    heatmap_normalized = ((heatmap - heatmap.min()) / (heatmap.max() - heatmap.min()))
    heatmap_normalized = np.uint8(255 * (heatmap_normalized))
    heatmap_colored = cv2.applyColorMap(heatmap_normalized, cv2.COLORMAP_HOT)
    overlaid_image = cv2.addWeighted(cv2_image, 1 - alpha, heatmap_colored, alpha, 0)
    
    return Image.fromarray(cv2.cvtColor(overlaid_image, cv2.COLOR_BGR2RGB))

Create figures

  ds_heatmap = ds.select_columns(["image","prompt", "alignment_heatmap"])
  
  for example in ds_heatmap.take(5):
      image = example["image"]
      heatmap = example["alignment_heatmap"]
      if heatmap:
          display(overlay_heatmap(image, np.asarray(heatmap)))

Data Summary

Word Scores

Users identified words from the prompts that were NOT accurately depicted in the generated images. Higher word scores indicate poorer representation in the image. Participants also had the option to select "[No_mistakes]" for prompts where all elements were accurately depicted.

Examples Results:

Coherence

The coherence score measures whether the generated image is logically consistent and free from artifacts or visual glitches. Without seeing the original prompt, users were asked: "Look closely, does this image have weird errors, like senseless or malformed objects, incomprehensible details, or visual glitches?" Each image received at least 21 responses indicating the level of coherence on a scale of 1-5, which were then averaged to produce the final scores where 5 indicates the highest coherence.

Images scoring below 3.8 in coherence were further evaluated, with participants marking specific errors in the image.

Example Results:

Alignment

The alignment score quantifies how well an image matches its prompt. Users were asked: "How well does the image match the description?". Again, each image received at least 21 responses indicating the level of alignment on a scale of 1-5 (5 being the highest), which were then averaged.

For images with an alignment score below 3.2, additional users were asked to highlight areas where the image did not align with the prompt. These responses were then compiled into a heatmap.

As mentioned in the google paper, aligment is harder to annotate consistently, if e.g. an object is missing, it is unclear to the annotators what they need to highlight.

Example Results:

Style

The style score reflects how visually appealing participants found each image, independent of the prompt. Users were asked: "How much do you like the way this image looks?" Each image received 21 responses grading on a scale of 1-5, which were then averaged. In contrast to other prefrence collection methods, such as the huggingface image arena, the preferences were collected from humans from around the world (156 different countries) from all walks of life, creating a more representative score.

About Rapidata

Rapidata's technology makes collecting human feedback at scale faster and more accessible than ever before. Visit rapidata.ai to learn more about how we're revolutionizing human feedback collection for AI development.

Other Datasets

We run a benchmark of the major image generation models, the results can be found on our website. We rank the models according to their coherence/plausiblity, their aligment with the given prompt and style prefernce. The underlying 2M+ annotations can be found here:

• Link to the Coherence dataset
• Link to the Text-2-Image Alignment dataset
• Link to the Preference dataset

We have also started to run a video generation benchmark, it is still a work in progress and currently only covers 2 models. They are also analysed in coherence/plausiblity, alignment and style preference.

Replicating the Annotation Setup

For researchers interested in producing their own rich preference dataset, you can directly use the Rapidata API through python. The code snippets below show how to replicate the modalities used in the dataset. Additional information is available through the documentation

  pip install rapidata

from rapidata import RapidataClient

client = RapidataClient()

As example data we will just use images from the dataset. Make sure to set streaming=True as downloading the whole dataset might take a significant amount of time.

from datasets import load_dataset

ds = load_dataset("Rapidata/text-2-image-Rich-Human-Feedback", split="train", streaming=True)
ds = ds.select_columns(["image","prompt"])

Since we use streaming, we can extract the prompts and download the images we need like this:

import os
tmp_folder = "demo_images"


# make folder if it doesn't exist
if not os.path.exists(tmp_folder):
   os.makedirs(tmp_folder)


prompts = []
image_paths = []
for i, row in enumerate(ds.take(10)):
   prompts.append(row["prompt"])
   # save image to disk
   save_path = os.path.join(tmp_folder, f"{i}.jpg")
   row["image"].save(save_path)
   image_paths.append(save_path)

# Alignment Example 
instruction = "How well does the image match the description?"
answer_options = [
        "1: Not at all",
        "2: A little",
        "3: Moderately",
        "4: Very well",
        "5: Perfectly"
    ]

audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") # curated audience pre-trained on image-text alignment

job_definition = client.job.create_classification_job_definition(
    name="Alignment Example",
    instruction=instruction,
    answer_options=answer_options,
    datapoints=image_paths,
    contexts=prompts, # for alignment, prompts are required as context for the annotators.
    responses_per_datapoint=10,
)

job = audience.assign_job(job_definition) # This starts the job. Follow the printed link to see progress.
results = job.get_results()

# alignment heatmap
# Note that the selected images may not actually have severely misaligned elements, but this is just for demonstration purposes.

audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") # curated audience pre-trained on image-text alignment

job_definition = client.job.create_locate_job_definition(
    name="Alignment Heatmap Example",
    instruction="What part of the image does not match with the description? Tap to select.",
    datapoints=image_paths,
    contexts=prompts, # for alignment, prompts are required as context for the annotators.
    responses_per_datapoint=10,
)

job = audience.assign_job(job_definition) # This starts the job. Follow the printed link to see progress.
results = job.get_results()

# Select words example

from rapidata import LanguageFilter

select_words_prompts = [p + " [No_Mistake]" for p in prompts]
# the filter ensures only english speaking annotators from the curated alignment audience are selected
audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO").filter([LanguageFilter(["en"])])

job_definition = client.job.create_select_words_job_definition(
    name="Select Words Example",
    instruction = "The image is based on the text below. Select mistakes, i.e., words that are not aligned with the image.",
    datapoints=image_paths,
    sentences=select_words_prompts,
    responses_per_datapoint=10,
)

job = audience.assign_job(job_definition)
results = job.get_results()