git clone https://huggingface.co/huggyllama/llama-13b
git clone https://huggingface.co/lmsys/vicuna-13b-v1.3

git clone https://huggingface.co/lmsys/vicuna-7b-v1.1

git clone https://github.com/lm-sys/FastChat

docker pull huggingface/transformers-pytorch-gpu:4.27.4

 docker run -it huggingface/transformers-pytorch-gpu:4.27.4 /bin/bash

pip install fschat

set PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:32

python3 -m fastchat.serve.cli --model-path lmsys/vicuna-13b-v1.3

python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3

--device cpu

# 指定 local，不然會跑去 OpenAI
export OPENAI_API_KEY=YOUR_API_KEY
export OPENAI_API_BASE=http://localhost:8000/v1

# 執行 controller
python -m fastchat.serve.controller --host 0.0.0.0 --port 21001 

# 執行 Worker
python -m fastchat.serve.model_worker \
		--model-name 'gpt-3.5-turbo,text-davinci-003,text-embedding-ada-002,vicuna-13b' \
		--model-path lmsys/vicuna-13b-v1.3 --host 0.0.0.0 --controller-address http://localhost:21001 --port 21002 --device cpu

# 執行 API Server
python -m fastchat.serve.openai_api_server --host 0.0.0.0 --port 8000 --controller-address http://localhost:21001