Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 10:35:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Right-truncate long chat completion prompts instead of left-truncating

Browse source 
Instructions are usually at the beginning of the prompt.
This commit is contained in:
oobabooga 2024-03-07 08:50:24 -08:00
parent 104573f7d4
commit bae14c8f13
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
modules/chat.py
View file

@ -197,16 +197,16 @@ def generate_chat_prompt(user_input, state, **kwargs):
while right - left > 1: while right - left > 1:
mid = (left + right) // 2 mid = (left + right) // 2
messages[-1]['content'] = user_message[mid:] messages[-1]['content'] = user_message[:mid]
prompt = make_prompt(messages) prompt = make_prompt(messages)
encoded_length = get_encoded_length(prompt) encoded_length = get_encoded_length(prompt)
if encoded_length <= max_length: if encoded_length <= max_length:
right = mid
else:
left = mid left = mid
else:
right = mid
messages[-1]['content'] = user_message[right:] messages[-1]['content'] = user_message[:left]
prompt = make_prompt(messages) prompt = make_prompt(messages)
encoded_length = get_encoded_length(prompt) encoded_length = get_encoded_length(prompt)
if encoded_length > max_length: if encoded_length > max_length: