#APIキー入力
import os
os.environ["OPENAI_API_KEY"] = "ChatGPT APIのAPIキー"
os.environ["PINECONE_API_KEY"] = "PineconeconeのAPIキー"
あわせてRAG Fusionに使用するモジュールも、以下のコードを使ってインポートします。
#モジュールインポート
from langchain.document_loaders import TextLoader
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.llms import OpenAI
from langchain.text_splitter import CharacterTextSplitter
from langchain_pinecone import PineconeVectorStore
from langchain.schema.output_parser import StrOutputParser
from langchain.schema.runnable import RunnablePassthrough, RunnableLambda
from langchain.prompts import ChatPromptTemplate
from langchain.prompts.chat import (
AIMessagePromptTemplate,
HumanMessagePromptTemplate,
)
#複数クエリの生成
from langchain_core.output_parsers import StrOutputParser
from langchain_openai import ChatOpenAI
from langchain import hub
prompt = hub.pull("langchain-ai/rag-fusion-query-generation")
generate_queries = (
prompt | ChatOpenAI(temperature=0) | StrOutputParser() | (lambda x: x.split("\n"))
)
#関連度の高い順にドキュメントを返すようにする
from langchain.load import dumps, loads
def reciprocal_rank_fusion(results: list[list], k=60):
fused_scores = {}
for docs in results:
for rank, doc in enumerate(docs):
doc_str = dumps(doc)
if doc_str not in fused_scores:
fused_scores[doc_str] = 0
previous_score = fused_scores[doc_str]
fused_scores[doc_str] += 1 / (rank + k)
reranked_results = [
(loads(doc), score)
for doc, score in sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)
]
return reranked_results
# 10個の関連文書を取ってくるようにする
retriever = vectorstore.as_retriever(search_kwargs={'k': 10})
#関連する質問を生成させるためのプロンプト
template = """You are an expert of world knowledge. I am going to ask you a question. Your response should be comprehensive and not contradicted with the following context if they are relevant. Otherwise, ignore them if they are not relevant.
{context}
Question: {question}
"""
prompt2 = ChatPromptTemplate.from_messages(
[
HumanMessagePromptTemplate.from_template(template),
AIMessagePromptTemplate.from_template(""),
]
)
retrieve_chain = (
{"original_query": lambda x: x}
| generate_queries
| retriever.map()
| reciprocal_rank_fusion
)
chain = (
{
"context": retrieve_chain,
"question": lambda x: x,
}
| prompt2
| ChatOpenAI(temperature=0)
| StrOutputParser()
)
