使用 4-bit quantization 與 6GB GPU 運行 Falcon-7B-Instruct¶

!pip install -q -U bitsandbytes
!pip install -q -U git+https://github.com/huggingface/transformers.git
!pip install -q -U git+https://github.com/huggingface/peft.git
!pip install -q -U git+https://github.com/huggingface/accelerate.git
!pip install -q -U einops
!pip install -q -U safetensors

import torch
from transformers import BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True,
)

from transformers import AutoModelForCausalLM, AutoTokenizer,pipeline

# smaller chunks on safetensors for low RAM environments
model_id = "vilsonrodrigues/falcon-7b-instruct-sharded"

model_4bit = AutoModelForCausalLM.from_pretrained(
        model_id,
        device_map="auto",
        quantization_config=quantization_config,
        trust_remote_code=True)

tokenizer = AutoTokenizer.from_pretrained(model_id)

print(model_4bit)

FalconForCausalLM(
  (transformer): FalconModel(
    (word_embeddings): Embedding(65024, 4544)
    (h): ModuleList(
      (0-31): 32 x FalconDecoderLayer(
        (self_attention): FalconAttention(
          (maybe_rotary): FalconRotaryEmbedding()
          (query_key_value): Linear4bit(in_features=4544, out_features=4672, bias=False)
          (dense): Linear4bit(in_features=4544, out_features=4544, bias=False)
          (attention_dropout): Dropout(p=0.0, inplace=False)
        )
        (mlp): FalconMLP(
          (dense_h_to_4h): Linear4bit(in_features=4544, out_features=18176, bias=False)
          (act): GELU(approximate='none')
          (dense_4h_to_h): Linear4bit(in_features=18176, out_features=4544, bias=False)
        )
        (input_layernorm): LayerNorm((4544,), eps=1e-05, elementwise_affine=True)
      )
    )
    (ln_f): LayerNorm((4544,), eps=1e-05, elementwise_affine=True)
  )
  (lm_head): Linear(in_features=4544, out_features=65024, bias=False)
)

import torch
import transformers

pipeline = transformers.pipeline(
        "text-generation", # 要進行的 task 類型
        model=model_4bit,  # 要使用的 llm model
        tokenizer=tokenizer, # 要使用的 tokenizer
        use_cache=True,
        device_map="auto", # 定義設備
        max_length=296,
        do_sample=True,
        top_k=10,
        num_return_sequences=1,
        eos_token_id=tokenizer.eos_token_id,
        pad_token_id=tokenizer.eos_token_id,
)

sequences = pipeline(
   "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:"
   )

[{'generated_text': "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron: Ahhh, hello Daniel!\nDaniel: How's it goin' today?\nGirafatron: I'm a-giraf-in' just fine, thank you so much for asking!\nI'll be there soon\nGirafatron - a-giraf-in\nI'll be there soon.\nGirafatron, the giraffe of all the giraffes."}]

# Some error in colab. fix with
import locale
locale.getpreferredencoding = lambda: "UTF-8"

!pip install langchain

from langchain import HuggingFacePipeline
from langchain import PromptTemplate, LLMChain

llm = HuggingFacePipeline(pipeline=pipeline)

template = """Question: {question}
Answer: Let's think step by step."""

prompt = PromptTemplate(
    template=template, 
    input_variables= ["question"]
)

llm_chain = LLMChain(prompt=prompt, llm=llm)

llm_chain("How to prepare eggs?")

{'question': 'How to prepare eggs?',
 'text': " First, we need eggs from hens. Next, you have to keep the hens comfortable. They don't lay if they're cold, so you need to have a warm environment. You also need to keep them away from other hens and predators. Then, the eggs need to be laid in a nesting box or nesting tray. You can keep the eggs clean by using a nesting box liner. Finally, after the eggs are laid, you need to keep the hens comfortable by providing a clean, dry nest."}

template2 = """Question: /n {question}. Answer: """

prompt2 = PromptTemplate(
    template=template2,
    input_variables= ["question"]
)

llm_chain_2 = LLMChain(prompt=prompt2, llm=llm)

result_explanation = llm_chain_2("Explain antibiotics")

result_explanation['text']

/n
Antibiotics are medications that are used to treat bacterial infections in the body. They work by preventing bacteria from growing and spreading, thus killing the bacteria or reducing their numbers. There are several classes of antibiotics, such as tetracycline, macrolide, penicillin, cephalosporin, and fluoroquinolones. Antioxidants, such as vitamin C, can also be taken along with antibiotics to reduce the risk of side effects.