【Style-Bert-VITS2 JP-Extra】日本語の発音・イントネーションが完璧な次世代音声AIを使ってみた

#@title このセルを実行して環境構築してください。
#@markdown 最後に赤文字でエラーや警告が出ても何故かうまくいくみたいです。

!git clone https://github.com/litagin02/Style-Bert-VITS2.git
%cd Style-Bert-VITS2/
!pip install -r requirements.txt
!apt install libcublas11
!python initialize.py --skip_jvnv

from google.colab import drive
drive.mount("/content/drive")

# 学習に必要なファイルや途中経過が保存されるディレクトリ
dataset_root = "/content/drive/MyDrive/Style-Bert-VITS2/Data"

# 学習結果（音声合成に必要なファイルたち）が保存されるディレクトリ
assets_root = "/content/drive/MyDrive/Style-Bert-VITS2/model_assets"

import yaml


with open("configs/paths.yml", "w", encoding="utf-8") as f:
    yaml.dump({"dataset_root": dataset_root, "assets_root": assets_root}, f)

# 元となる音声ファイル（wav形式）を入れるディレクトリ
input_dir = "/content/drive/MyDrive/Style-Bert-VITS2/inputs"
# モデル名（話者名）を入力
model_name = "your_model_name"

!python slice.py -i {input_dir} -o {dataset_root}/{model_name}/raw
!python transcribe.py -i {dataset_root}/{model_name}/raw -o {dataset_root}/{model_name}/esd.list --speaker_name {model_name} --compute_type float16

# 上でつけたフォルダの名前`Data/{model_name}/`
model_name = "your_model_name"

# JP-Extra （日本語特化版）を使うかどうか。日本語の能力が向上する代わりに英語と中国語は使えなくなります。
use_jp_extra = True

# 学習のバッチサイズ。VRAMのはみ出具合に応じて調整してください。
batch_size = 4

# 学習のエポック数（データセットを合計何周するか）。
# 100ぐらいで十分かもしれませんが、もっと多くやると質が上がるのかもしれません。
epochs = 100

# 保存頻度。何ステップごとにモデルを保存するか。分からなければデフォルトのままで。
save_every_steps = 1000

# 音声ファイルの音量を正規化するかどうか
normalize = False

# 音声ファイルの開始・終了にある無音区間を削除するかどうか
trim = False

from webui_train import preprocess_all

preprocess_all(
    model_name=model_name,
    batch_size=batch_size,
    epochs=epochs,
    save_every_steps=save_every_steps,
    num_processes=2,
    normalize=normalize,
    trim=trim,
    freeze_EN_bert=False,
    freeze_JP_bert=False,
    freeze_ZH_bert=False,
    freeze_style=False,
    use_jp_extra=use_jp_extra,
    val_per_lang=0,
    log_interval=200,
)

# 上でつけたモデル名を入力。学習を途中からする場合はきちんとモデルが保存されているフォルダ名を入力。
model_name = "your_model_name"


import yaml
from webui_train import get_path

dataset_path, _, _, _, config_path = get_path(model_name)

with open("default_config.yml", "r", encoding="utf-8") as f:
    yml_data = yaml.safe_load(f)
yml_data["model_name"] = model_name
with open("config.yml", "w", encoding="utf-8") as f:
    yaml.dump(yml_data, f, allow_unicode=True)

# 日本語特化版を「使う」場合
!python train_ms_jp_extra.py --config {config_path} --model {dataset_path} --assets_root {assets_root}

#@title 学習結果を試すならここから
!python app.py --share --dir {assets_root}

#@title Setup

!nvidia-smi
!nvcc --version

# モデル名の設定
# Google driveのマウント
#@markdown ★Google driveの接続の許可を求められるので承認してください。

from google.colab import drive
drive.mount("/content/drive")

# @markdown ★model_name に音声合成のモデル名を設定してください。
model_name = "your_model_name" #@param {type:"string"}


# Install package for LLM (Swallow13B-AWQ)
!pip -q install --upgrade accelerate autoawq
!pip install torch==2.1.0+cu121 torchtext==0.16.0+cpu torchdata==0.7.0 --index-url https://download.pytorch.org/whl/cu121
!pip3 install --upgrade "autoawq>=0.1.6" "transformers>=4.35.0"

# Record audio from microphone

from IPython.display import Javascript
from google.colab import output
from base64 import b64decode

RECORD = """
const sleep = time => new Promise(resolve => setTimeout(resolve, time))
const b2text = blob => new Promise(resolve => {
  const reader = new FileReader()
  reader.onloadend = e => resolve(e.srcElement.result)
  reader.readAsDataURL(blob)
})
var record = time => new Promise(async resolve => {
  stream = await navigator.mediaDevices.getUserMedia({ audio: true })
  recorder = new MediaRecorder(stream)
  chunks = []
  recorder.ondataavailable = e => chunks.push(e.data)
  recorder.start()
  await sleep(time)
  recorder.onstop = async ()=>{
    blob = new Blob(chunks)
    text = await b2text(blob)
    resolve(text)
  }
  recorder.stop()
})
"""

def record(sec, filename='audio.wav'):
  display(Javascript(RECORD))
  s = output.eval_js('record(%d)' % (sec * 1000))
  b = b64decode(s.split(',')[1])
  with open(filename, 'wb+') as f:
    f.write(b)

# Install faster whisper
!pip install faster-whisper
from faster_whisper import WhisperModel
# model_size = "small"
# model_size = "base"
model_size = "large-v2"

# Run on GPU with FP16
wpmodel = WhisperModel(model_size, device="cuda", compute_type="float16")

# Install Style-Bert-VITS2
!git clone https://github.com/litagin02/Style-Bert-VITS2.git
%cd Style-Bert-VITS2/
!pip install -r requirements.txt
!apt install libcublas11
!python initialize.py --skip_jvnv

# 音声出力
import numpy as np
import IPython.display

# Install LLM (Swallow13B-AWQ)

from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
model_name_or_path = "TheBloke/Swallow-13B-Instruct-AWQ"

tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
model = AutoModelForCausalLM.from_pretrained(
    model_name_or_path,
    low_cpu_mem_usage=True,
    device_map="cuda:0"
)

# Using the text streamer to stream output one token at a time
streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)

#@title Style-Bert-VITS2　の設定
import argparse
import datetime
import json
import os
import sys
from typing import Optional
import torch
import yaml

# 音声ファイル出力
from scipy.io import wavfile

from common.constants import (
    DEFAULT_ASSIST_TEXT_WEIGHT,
    DEFAULT_LENGTH,
    DEFAULT_LINE_SPLIT,
    DEFAULT_NOISE,
    DEFAULT_NOISEW,
    DEFAULT_SDP_RATIO,
    DEFAULT_SPLIT_INTERVAL,
    DEFAULT_STYLE,
    DEFAULT_STYLE_WEIGHT,
    GRADIO_THEME,
    LATEST_VERSION,
    Languages,
)
from common.log import logger
from common.tts_model import ModelHolder
from infer import InvalidToneError
from text.japanese import g2kata_tone, kata_tone2phone_tone, text_normalize

# Get path settings
with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f:
    path_config: dict[str, str] = yaml.safe_load(f.read())
    # dataset_root = path_config["dataset_root"]
    assets_root = path_config["assets_root"]

languages = [l.value for l in Languages]

def tts_fn(
    to_speak
):

    # style, tts_button, speaker = model_holder.load_model_gr(model_name, model_path)
    use_tone = False
    kata_tone = None
    language = "JP"

    wrong_tone_message = ""
    kata_tone: Optional[list[tuple[str, int]]] = None
    if use_tone and kata_tone_json_str != "":
        if language != "JP":
            logger.warning("Only Japanese is supported for tone generation.")
            wrong_tone_message = "アクセント指定は現在日本語のみ対応しています。"
        if line_split:
            logger.warning("Tone generation is not supported for line split.")
            wrong_tone_message = (
                "アクセント指定は改行で分けて生成を使わない場合のみ対応しています。"
            )
        try:
            kata_tone = []
            json_data = json.loads(kata_tone_json_str)
            # tupleを使うように変換
            for kana, tone in json_data:
                assert isinstance(kana, str) and tone in (0, 1), f"{kana}, {tone}"
                kata_tone.append((kana, tone))
        except Exception as e:
            logger.warning(f"Error occurred when parsing kana_tone_json: {e}")
            wrong_tone_message = f"アクセント指定が不正です: {e}"
            kata_tone = None

   # toneは実際に音声合成に代入される際のみnot Noneになる
    tone: Optional[list[int]] = None
    if kata_tone is not None:
        phone_tone = kata_tone2phone_tone(kata_tone)
        tone = [t for _, t in phone_tone]

    speaker = model_name
    print(speaker)
    speaker_id = model_holder.current_model.spk2id[speaker]
    print(speaker_id)

    start_time = datetime.datetime.now()

    try:
        sr, audio = model_holder.current_model.infer(
            text=to_speak,
            language="JP",
            reference_audio_path=None,
            sdp_ratio=DEFAULT_SDP_RATIO,
            noise=DEFAULT_NOISE,
            noisew=DEFAULT_NOISEW,
            length=DEFAULT_LENGTH,
            line_split=DEFAULT_LINE_SPLIT,
            split_interval=DEFAULT_SPLIT_INTERVAL,
            assist_text=None,
            assist_text_weight=DEFAULT_ASSIST_TEXT_WEIGHT,
            use_assist_text=False,
            style= "Neutral",
            style_weight=DEFAULT_STYLE_WEIGHT,
            given_tone=False,
            sid=speaker_id,
        )

        # 音声ファイル保存
        wavfile.write("/content/output.wav", sr, audio)

    except InvalidToneError as e:
        logger.error(f"Tone error: {e}")
        return f"Error: アクセント指定が不正です:\n{e}", None, kata_tone_json_str
    except ValueError as e:
        logger.error(f"Value error: {e}")
        return f"Error: {e}", None, kata_tone_json_str

    end_time = datetime.datetime.now()
    duration = (end_time - start_time).total_seconds()
    return

# ここからメインルーチン

# モデルの場所をフルパスで記載
model_dir = "/content/drive/MyDrive/Style-Bert-VITS2/model_assets"

# CPU or GPU
device = "cuda" if torch.cuda.is_available() else "cpu"

model_holder = ModelHolder(model_dir, device)

model_names = model_holder.model_names
if len(model_names) == 0:
    logger.error(
        f"モデルが見つかりませんでした。{model_dir}にモデルを置いてください。"
    )
    sys.exit(1)
initial_id = 0
initial_pth_files = model_holder.model_files_dict[model_names[initial_id]]

# load model
model_path = initial_pth_files[0]
style, tts_button, speaker = model_holder.load_model_gr(model_name, model_path)
# style = DEFAULT_STYLE
style_weight = DEFAULT_STYLE_WEIGHT

to_speak = "準備が出来ました"
tts_fn(to_speak)

# 音声ファイルの再生
file_path = '/content/output.wav'
IPython.display.Audio(file_path, autoplay=True)

#@title RUN

import locale
locale.getpreferredencoding = lambda: "UTF-8"

llm_flag="LLM使用" #@param ["LLM使用", "LLM不使用"]

# input = "" #@param {type:"string"}
system = "\u3042\u306A\u305F\u306FAI\u3068\u3044\u3046\u5973\u306E\u5B50\u3067\u3059\u3002\u300012\u624D\u306E\u5973\u306E\u5B50\u306E\u611F\u3058\u3067\u56DE\u7B54\u3057\u3066\u304F\u3060\u3055\u3044\u3002" #@param {type:"string"}

audiofile = "/content/talk.wav"
second = 5        # マイク入力の秒数はこの値で変更[sec]
print("音声入力中　")
record(second, audiofile)
print("音声入力完了")

print("音声認識中")
segments, info = wpmodel.transcribe("/content/talk.wav", beam_size=5)

print("Detected language '%s' with probability %f" % (info.language, info.language_probability))

voicein = ""
for segment in segments:
  print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
  voicein = voicein + segment.text
print(voicein)
to_speak = voicein

if llm_flag == "LLM使用":
  print("Swallow 13B 考え中")

  prompt = "\'"
  prompt = prompt + system + " "
  prompt = prompt + "\n\n### 指示:\n" + voicein
  prompt = prompt + " \n\n### 応答:"
  print(prompt)
  print("\n")
  prompt = prompt + "\'"
  prompt_template = prompt

  # Convert prompt to tokens
  tokens = tokenizer(
      prompt_template,
      return_tensors='pt'
  ).input_ids.cuda()

  generation_params = {
      "do_sample": True,
      "temperature": 0.7,
      "top_p": 0.95,
      "top_k": 40,
      "max_new_tokens": 512,
      "repetition_penalty": 1.1
  }

  # Generation without a streamer, which will include the prompt in the output
  generation_output = model.generate(
      tokens,
      **generation_params
  )

  # Get the tokens from the output, decode them, print them
  token_output = generation_output[0]
  text_output = tokenizer.decode(token_output)
  # print("model.generate output: ", text_output)

  output1 = text_output
  # print(output1)

  startpos = output1.index("### 応答:") + 8

  try:
    endpos =  output1.index("</s>")
  except:
    endpos = len(output1)

  output2 = output1[startpos:endpos]

  try:
    endpos = output2.index("###")
  except:
    endpos = len(output2)

  output3 = output2[:endpos]
  print(output3)
  to_speak = output3

locale.getpreferredencoding = lambda: "UTF-8"
print("音声合成中")

tts_fn(to_speak)

# 音声ファイルの再生
file_path = '/content/output.wav'
IPython.display.Audio(file_path, autoplay=True)