Falcon 模型提供推論服務使用 🤗 TGI¶

AutoModelForCausalLM.from_pretrained(<model>, device_map="auto")

# or

AutoModelForSeq2SeqLM.from_pretrained(<model>, device_map="auto")

docker pull ghcr.io/huggingface/text-generation-inference:1.0.3

docker run ghcr.io/huggingface/text-generation-inference:1.0.3 --help

# share a volume with the Docker container to avoid 
# downloading weights every run
volume=$PWD/data

# model on 🤗 Hub
model=tiiuae/falcon-7b-instruct

# apply quant to reduce gpu consume
quantize=bitsandbytes

num_shard=1

--trust-remote-code

--sharded true --num-shard NUM_GPUS

--quantize bitsandbytes-nf4

# or

--quantize bitsandbytes-fp4

docker run --gpus all --shm-size 1g -p 8080:80 \ 
        -v $volume:/data \ 
        ghcr.io/huggingface/text-generation-inference:latest \
        --model-id $model --num-shard $num_shard \
        --quantize $quantize

docker run --shm-size 1g -p 8080:80   -v $volume:/data   ghcr.io/huggingface/text-generation-inference:1.0.3   --model-id $model

# share a volume with the Docker container to avoid 
# downloading weights every run
volume=$PWD/data

# model on 🤗 Hub
model=bigscience/bloom-560m

docker run --gpus all --shm-size 1g -p 8080:80 \
  -v $volume:/data \
  ghcr.io/huggingface/text-generation-inference:1.0.3 \
  --model-id $model \
  --disable-custom-kernels \
  --sharded false \
  --quantize bitsandbytes-nf4 \
  --port 80

curl 127.0.0.1:8080/generate \
     -X POST \
     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17}}' \
     -H 'Content-Type: application/json'

curl 127.0.0.1:8080/generate_stream \
    -X POST \
    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17}}' \
    -H 'Content-Type: application/json'

curl 127.0.0.1:8080/ \
    -X POST \
    -d '{"inputs":"What is Deep Learning?", "parameters":{"max_new_tokens":17}, "stream":true}' \
    -H 'Content-Type: application/json'

!pip install text-generation

from text_generation import Client

# Generate
client = Client("http://127.0.0.1:8080")
print(client.generate("What is Deep Learning?", max_new_tokens=17).generated_text)

# Generate stream
text = ""
for response in client.generate_stream("What is Deep Learning?", max_new_tokens=17):
    if not response.token.special:
        text += response.token.text
print(text)

# Some error in colab. fix with
import locale
locale.getpreferredencoding = lambda: "UTF-8"

!pip install langchain transformers

# Wrapper to TGI client with langchain

from langchain.llms import HuggingFaceTextGenInference

inference_server_url_local = "http://127.0.0.1:8080"

llm_local = HuggingFaceTextGenInference(
    inference_server_url=inference_server_url_local,
    max_new_tokens=400,
    top_k=10,
    top_p=0.95,
    typical_p=0.95,
    temperature=0.7,
    repetition_penalty=1.03,
)

from langchain import PromptTemplate, LLMChain

template = """Question: {question}
Answer: Let's think step by step."""

prompt = PromptTemplate(
    template=template, 
    input_variables= ["question"]
)

llm_chain_local = LLMChain(prompt=prompt, llm=llm_local)

# llm_chain_local("your question")

llm_chain_local("What is Deep Learning?")