pip install gpt4all

from gpt4all import GPT4All
model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf") # downloads / loads a 4.66GB LLM
with model.chat_session():
    print(model.generate("How can I run LLMs efficiently on my laptop?", max_tokens=1024))

from gpt4all import GPT4All

def output_with_stream_control(prompt: str, model_name: str = "Meta-Llama-3-8B-Instruct.Q4_0.gguf", max_tokens: int = 1024, streaming: bool = False):
    model = GPT4All(model_name)  # 加载指定模型

    # 创建一个对话会话
    with model.chat_session() as session:
        response_buffer = ""

        if streaming:
            # 启用流式输出
            for chunk in model.generate(prompt, max_tokens=max_tokens, streaming=streaming):
                response_buffer += chunk
                print(chunk, end='', flush=True)  # 实时输出生成的文本
            print("\n\n生成的完整答案：", response_buffer)
        else:
            # 批量输出（等待完整生成后返回）
            response_buffer = model.generate(prompt, max_tokens=max_tokens, streaming=streaming)
            print("\n生成的完整答案：", response_buffer)

        return response_buffer

if __name__ == "__main__":
    prompt = "用中文回答，什么是化学？"
    result = output_with_stream_control(prompt, streaming=False)
    print('result:', result)

from nomic import embed
embeddings = embed.text(["String 1", "String 2"], inference_mode="local")['embeddings']
print("Number of embeddings created:", len(embeddings))
print("Number of dimensions per embedding:", len(embeddings[0]))