استرجاع النصوص من ويكيبيديا العربية باستخدام Gemma2 (الجزء الأول)

!pip install langchain langchainhub langchain_community langchain-huggingface faiss-gpu transformers accelerate datasets bitsandbytes langchain-text-splitters sentence-transformers huggingface_hub chromadb gradio > /dev/null 2>&1

import os
import torch
from langchain.document_loaders import HuggingFaceDatasetLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
from langchain_huggingface import HuggingFacePipeline
from langchain import hub
from langchain.schema import Document
from huggingface_hub import login
from concurrent.futures import ThreadPoolExecutor, as_completed
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
import gradio as gr

from google.colab import userdata

hf_token = userdata.get('gemma2')
login(token=hf_token)

# تحميل مجموعة البيانات
dataset_name = "wikimedia/wikipedia"
page_content_column = "text"
name = "20231101.ar"
loader = HuggingFaceDatasetLoader(dataset_name, page_content_column, name)
data = loader.load()

# اختيار أول 20 إدخالًا للتوضيح
documents = data[:20]

# تقسيم النص إلى أجزاء أصغر
text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=0, length_function=len, is_separator_regex=False)

def process_document(document):
    chunks = text_splitter.split_text(document.page_content)
    split_docs = []
    for chunk in chunks:
        try:
            decoded_content = chunk.encode().decode('unicode_escape')
        except UnicodeDecodeError:
            decoded_content = chunk
        split_docs.append(Document(page_content=decoded_content, metadata=document.metadata))
    return split_docs

split_documents = []

with ThreadPoolExecutor() as executor:
    futures = [executor.submit(process_document, doc) for doc in documents]
    for future in as_completed(futures):
        split_documents.extend(future.result())

split_documents[:2]

model_path = "sentence-transformers/all-MiniLM-L12-v2"
model_kwargs = {'device': 'cuda'}  
encode_kwargs = {'normalize_embeddings': False}
embeddings = HuggingFaceEmbeddings(model_name=model_path, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs)

text = split_documents[0].page_content
query_result = embeddings.embed_query(text)
query_result[:3]

vector_db = FAISS.from_documents(split_documents, embeddings)
vector_db.save_local("/faiss_index")

base_model = "google/gemma-2-9b-it"
tokenizer = AutoTokenizer.from_pretrained(base_model)
model = AutoModelForCausalLM.from_pretrained(base_model, return_dict=True, low_cpu_mem_usage=True,
                                            torch_dtype=torch.float16, device_map="auto", trust_remote_code=True)

pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=20)
llm = HuggingFacePipeline(pipeline=pipe)

retriever = vector_db.as_retriever()

rag_prompt = hub.pull("rlm/rag-prompt")
qa_chain = ({"context": retriever, "question": RunnablePassthrough()} | rag_prompt | llm | StrOutputParser())

def extract_answer(qa_chain_output):
    lines = qa_chain_output.split('\n')
    for line in lines:
        if line.startswith('Answer:'):
            return line.split(':', 1)[1].strip()
    return None

question = "ما هي النماذج اللغوية؟"
result = qa_chain.invoke(question)
extract_answer(result)

def chatbot_response(question):
    result = qa_chain.invoke(question)
    return result.split("Answer: ")[1]  # استخراج جزء الإجابة

chatbot = gr.Interface(
    fn=chatbot_response,
    inputs="text",
    outputs="text",
    live=False,
    title="روبوت دردشة Gemma 2",
    description="اسألني أي شيء"
)

chatbot.launch(debug=True)