【Japanese Stable CLIP】Stability AI開発の日本語特化型マルチモーダルAIを使ってみた！

pip install ftfy regex tqdm gradio transformers sentencepiece 'accelerate>=0.12.0' 'bitsandbytes>=0.31.5'

#@title Load Japanese Stable CLIP
from typing import Union, List
import ftfy, html, re, io
import requests
from PIL import Image
import torch
from transformers import AutoModel, AutoTokenizer, AutoImageProcessor, BatchFeature


device = "cuda" if torch.cuda.is_available() else "cpu"
model_path = "stabilityai/japanese-stable-clip-vit-l-16"
model = AutoModel.from_pretrained(model_path, trust_remote_code=True).eval().to(device)
tokenizer = AutoTokenizer.from_pretrained(model_path)
processor = AutoImageProcessor.from_pretrained(model_path)


# taken from https://github.com/mlfoundations/open_clip/blob/main/src/open_clip/tokenizer.py#L65C8-L65C8
def basic_clean(text):
    text = ftfy.fix_text(text)
    text = html.unescape(html.unescape(text))
    return text.strip()


def whitespace_clean(text):
    text = re.sub(r"\s+", " ", text)
    text = text.strip()
    return text


def tokenize(
    texts: Union[str, List[str]],
    max_seq_len: int = 77,
):
    """
    This is a function that have the original clip's code has.
    https://github.com/openai/CLIP/blob/main/clip/clip.py#L195
    """
    if isinstance(texts, str):
        texts = [texts]
    texts = [whitespace_clean(basic_clean(text)) for text in texts]

    inputs = tokenizer(
        texts,
        max_length=max_seq_len - 1,
        padding="max_length",
        truncation=True,
        add_special_tokens=False,
    )
    # add bos token at first place
    input_ids = [[tokenizer.bos_token_id] + ids for ids in inputs["input_ids"]]
    attention_mask = [[1] + am for am in inputs["attention_mask"]]
    position_ids = [list(range(0, len(input_ids[0])))] * len(texts)

    return BatchFeature(
        {
            "input_ids": torch.tensor(input_ids, dtype=torch.long),
            "attention_mask": torch.tensor(attention_mask, dtype=torch.long),
            "position_ids": torch.tensor(position_ids, dtype=torch.long),
        }
    )


def compute_text_embeddings(text):
  if isinstance(text, str):
    text = [text]
  text = tokenize(texts=text)
  text_features = model.get_text_features(**text.to(device))
  text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)
  del text
  return text_features.cpu().detach()

def compute_image_embeddings(image):
  image = processor(images=image, return_tensors="pt").to(device)
  image_features = model.get_image_features(**image)
  image_features = image_features / image_features.norm(p=2, dim=-1, keepdim=True)
  del image
  return image_features.cpu().detach()

#@title Prepare for the demo
#@markdown Please feel free to change `categories` for your usage.
categories = [
  任意のもの
]

# pre-compute text embeddings
text_embeds = compute_text_embeddings(categories)

# @title Launch the demo
import gradio as gr

num_categories = len(categories)
TOP_K = 3


def inference_fn(img):
  image_embeds = compute_image_embeddings(img)
  similarity = (100.0 * image_embeds @ text_embeds.T).softmax(dim=-1)
  similarity = similarity[0].numpy().tolist()
  output_dict = {categories[i]: float(similarity[i]) for i in range(num_categories)}
  del image_embeds
  return output_dict


with gr.Blocks() as demo:
    gr.Markdown("# Japanese Stable CLIP Demo")
    gr.Markdown(
        """[Japanese Stable CLIP](https://huggingface.co/stabilityai/japanese-stable-clip-vit-l-16) is a [CLIP](https://arxiv.org/abs/2103.00020) model by [Stability AI](https://ja.stability.ai/).
                - Blog: https://ja.stability.ai/blog/japanese-stable-clip
                - Twitter: https://twitter.com/StabilityAI_JP
                - Discord: https://discord.com/invite/StableJP"""
    )
    with gr.Row():
      with gr.Column():
        inp = gr.Image(type="pil")
      with gr.Column():
        out = gr.Label(num_top_classes=TOP_K)

    btn = gr.Button("Run")
    btn.click(fn=inference_fn, inputs=inp, outputs=out)

if __name__ == "__main__":
    demo.launch(debug=True, share=True)

田舎の風景
盆地
都会の風景
田園風景
晴れた田舎
曇った田舎
富士山
阿蘇山
剣岳
高尾山
犬が走っている
犬が歩いている
猫が走っている
猫が歩いている
柴犬が走っている
シュナウザーが走っている
犬が寝ている
犬が座っている
犬が飛んでいる